From 9ec18f6523375563679866d66462ce5d4211fe29 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:41:02 -0400 Subject: [PATCH 001/108] test(cli): freeze beads compatibility contract (age-nw28h.2.1) --- .../families/beads/case.json | 21 ++++++++ .../families/beads/ownership.json | 53 +++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/beads/case.json create mode 100644 cli/testdata/compatibility-baseline/families/beads/ownership.json diff --git a/cli/testdata/compatibility-baseline/families/beads/case.json b/cli/testdata/compatibility-baseline/families/beads/case.json new file mode 100644 index 000000000..891b7b156 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/beads/case.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "family": "beads", + "checks": [ + { + "id": "beads-black-box", + "command": "tmp=$(mktemp -d /tmp/ao-beads.XXXXXX); trap 'rm -r \"$tmp\"' EXIT; (cd cli && go build -o \"$tmp/ao\" ./cmd/ao); \"$tmp/ao\" beads --help >\"$tmp/help\"; grep -q 'Commands that help maintain the bd issue tracker' \"$tmp/help\"; if \"$tmp/ao\" beads verify >\"$tmp/rejected.out\" 2>\"$tmp/rejected.err\"; then exit 1; fi; test ! -s \"$tmp/rejected.out\"; grep -qx 'Error: accepts 1 arg(s), received 0' \"$tmp/rejected.err\"; mkdir -p \"$tmp/repo/.beads\" \"$tmp/bin\"; git -C \"$tmp/repo\" init -q; printf '%s\\n' '#!/usr/bin/env bash' 'printf \"%s\\n\" \"$*\" >>\"$TRACE_FILE\"' 'printf \"[]\\n\"' >\"$tmp/bin/bd\"; chmod +x \"$tmp/bin/bd\"; (cd \"$tmp/repo\" && PATH=\"$tmp/bin:$PATH\" TRACE_FILE=\"$tmp/trace\" \"$tmp/ao\" beads tracker --json >\"$tmp/tracker.json\"); jq -e '.tracker == \"bd\" and .source == \"ledger\" and (.ledger_dir | endswith(\"/.beads\"))' \"$tmp/tracker.json\" >/dev/null; (cd \"$tmp/repo\" && PATH=\"$tmp/bin:$PATH\" TRACE_FILE=\"$tmp/trace\" \"$tmp/ao\" beads exec ready --json >\"$tmp/exec.json\"); grep -qx 'ready --json' \"$tmp/trace\"; jq -e 'type == \"array\" and length == 0' \"$tmp/exec.json\" >/dev/null" + } + ], + "dimensions": { + "path": {"status": "evidence", "evidence": ["beads-black-box"]}, + "aliases": {"status": "not_applicable", "reason": "The ao beads command declares no aliases."}, + "accepted_args": {"status": "evidence", "evidence": ["beads-black-box"]}, + "rejected_args": {"status": "evidence", "evidence": ["beads-black-box"]}, + "stdout": {"status": "evidence", "evidence": ["beads-black-box"]}, + "stderr": {"status": "evidence", "evidence": ["beads-black-box"]}, + "exit_classes": {"status": "evidence", "evidence": ["beads-black-box"]}, + "tracker_selection": {"status": "evidence", "evidence": ["beads-black-box"]}, + "ordered_effects": {"status": "evidence", "evidence": ["beads-black-box"]} + } +} diff --git a/cli/testdata/compatibility-baseline/families/beads/ownership.json b/cli/testdata/compatibility-baseline/families/beads/ownership.json new file mode 100644 index 000000000..f107c4430 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/beads/ownership.json @@ -0,0 +1,53 @@ +{ + "schema_version": 1, + "family": "beads", + "profiles": { + "default": "present", + "flywheel": "present", + "legacy": "present", + "combined": "present" + }, + "legacy_symbols": [ + "beadsExitError", + "execBD", + "bdAvailable", + "beadsCmd", + "beadsDirCmd", + "beadsTrackerCmd", + "beadsVerifyCmd", + "beadsLintCmd", + "beadsHarvestCmd", + "beadsAuditCmd", + "beadsClusterCmd", + "beadsExecCmd", + "beadsResumeCmd", + "beadsScenariosCmd", + "beadsStaleCmd", + "beadsEpicStatusCmd", + "runBeadsDir", + "runBeadsTrackerCmd", + "runBeadsVerify", + "runBeadsLint", + "runBeadsHarvest", + "runBeadsAudit", + "runBeadsCluster", + "runBeadsExec", + "runBeadsResume", + "runBeadsScenariosExtract", + "runBeadsScenariosValidate", + "runBeadsStale", + "runBeadsEpicStatus", + "runBeadsVerifyAcceptance", + "newBeadsVerifyAcceptanceCmd" + ], + "live_owner": "cli/internal/commands/beads", + "allowed_paths": [ + "cli/cmd/ao/beads*.go", + "cli/cmd/ao/root.go", + "cli/cmd/ao/root_test.go", + "cli/internal/adapters/beads/**", + "cli/internal/beads/**", + "cli/internal/commands/beads/**", + "cli/testdata/compatibility-baseline/families/beads/lineage.json" + ] +} From 057774da4ecc4375974869ba6559e8707399dc75 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:41:44 -0400 Subject: [PATCH 002/108] test(cli): bind beads migration lineage (age-nw28h.2.1) --- .../compatibility-baseline/families/beads/lineage.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/beads/lineage.json diff --git a/cli/testdata/compatibility-baseline/families/beads/lineage.json b/cli/testdata/compatibility-baseline/families/beads/lineage.json new file mode 100644 index 000000000..9708d5197 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/beads/lineage.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "family": "beads", + "old_implementation_sha": "5acf5f45b30caf6f8d0557198499487def7b05d1", + "freeze_sha": "9ec18f6523375563679866d66462ce5d4211fe29", + "fixture_sha256": "fa3f72bc3b0a325d5b662a2b56ad0ccc92ac6b669871061be99ef34974b67473", + "ownership_sha256": "c4baf423279f90d923030854688bc547a3170afcc093a31f0dc72c00720adccb", + "scope_sha256": "55272fba3517bec8c3a80503fad1f30a551a1f76e8731e73617c8063da21215a", + "migration_state": "migrating" +} From e7ea6ec348ae7325780f8106b4a2bf002060f6c4 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:43:02 -0400 Subject: [PATCH 003/108] refactor(cli): decouple root from beads exit type (age-nw28h.2.1) --- cli/cmd/ao/root.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cli/cmd/ao/root.go b/cli/cmd/ao/root.go index b03806666..f29f8c2ed 100644 --- a/cli/cmd/ao/root.go +++ b/cli/cmd/ao/root.go @@ -158,14 +158,6 @@ func Execute() { // propagate the code with no extra cobra noise. os.Exit(landErr.ExitCode()) } - var beadsErr *beadsExitError - if errors.As(err, &beadsErr) { - // The exit code IS the verdict in `ao beads verify|lint|audit`: - // 1 means stale/flagged beads found. The verdict already went to - // stdout and the command silenced cobra's error print, so there is - // nothing more to surface here — just map to the process exit code. - os.Exit(beadsErr.ExitCode()) - } var tickErr *tickExitError if errors.As(err, &tickErr) { if tickErr.Error() != "" { @@ -196,12 +188,24 @@ func Execute() { // surface — just map to the process exit code. os.Exit(wikiHealthErr.ExitCode()) } + var commandExit commandExitError + if errors.As(err, &commandExit) { + // Family modules can return a typed verdict without making the root + // executable import their concrete error type. Command output already + // carries the explanation; the root owns only process-code mapping. + os.Exit(commandExit.ExitCode()) + } printRemovedCommandHint(os.Stderr, rootCmd, err) printRequiredFlagHint(executedCmd, err) os.Exit(1) } } +type commandExitError interface { + error + ExitCode() int +} + func init() { // Command groups for organized help output rootCmd.AddGroup( From c510fc645cba422b16fb732f5cd6fb05763a03b8 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:45:08 -0400 Subject: [PATCH 004/108] refactor(beads): extract stale-claim policy (age-nw28h.2.1) --- cli/cmd/ao/beads_stale.go | 79 ++++------------------------------ cli/internal/beads/stale.go | 84 +++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 72 deletions(-) create mode 100644 cli/internal/beads/stale.go diff --git a/cli/cmd/ao/beads_stale.go b/cli/cmd/ao/beads_stale.go index daed0ea62..156597f55 100644 --- a/cli/cmd/ao/beads_stale.go +++ b/cli/cmd/ao/beads_stale.go @@ -18,10 +18,11 @@ import ( "errors" "fmt" "os/exec" - "sort" "time" "github.com/spf13/cobra" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" ) var ( @@ -54,34 +55,13 @@ func init() { } // staleBeadRecord is the subset of `br list --json` output we care about. -type staleBeadRecord struct { - ID string `json:"id"` - Status string `json:"status"` - Assignee string `json:"assignee"` - UpdatedAt string `json:"updated_at"` -} +type staleBeadRecord = beadsapp.StaleBeadRecord // staleEvent mirrors schemas/stale-claim-event.v1.schema.json for // event_type="stale_detected". JSON tags lowercase + snake_case to match. -type staleEvent struct { - SchemaVersion int `json:"schema_version"` - EventType string `json:"event_type"` - BeadID string `json:"bead_id"` - DetectedAt string `json:"detected_at"` - OriginalClaimant staleAgent `json:"original_claimant"` - Evidence staleEvidence `json:"evidence"` -} - -type staleAgent struct { - ID string `json:"id"` -} - -type staleEvidence struct { - LastTouchTS string `json:"last_touch_ts,omitempty"` - ClaimAgeHours float64 `json:"claim_age_hours,omitempty"` - ThresholdHours float64 `json:"threshold_hours,omitempty"` - LastEvidenceEvent string `json:"last_evidence_event,omitempty"` -} +type staleEvent = beadsapp.StaleEvent +type staleAgent = beadsapp.StaleAgent +type staleEvidence = beadsapp.StaleEvidence func runBeadsStale(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) @@ -153,50 +133,5 @@ var beadsStaleFetchCmd = func(ctx context.Context) ([]byte, error) { // in_progress beads. Pure function — no exec, no clock, no FS — so it's // trivially table-testable. func computeStaleEvents(beads []staleBeadRecord, now time.Time, thresholdHours float64) []staleEvent { - var events []staleEvent - for _, b := range beads { - if b.Status != "in_progress" { - continue - } - if b.UpdatedAt == "" { - continue - } - updated, err := time.Parse(time.RFC3339, b.UpdatedAt) - if err != nil { - continue - } - ageHours := now.Sub(updated).Hours() - if ageHours < thresholdHours { - continue - } - claimant := b.Assignee - if claimant == "" { - claimant = "unknown" - } - events = append(events, staleEvent{ - SchemaVersion: 1, - EventType: "stale_detected", - BeadID: b.ID, - DetectedAt: now.UTC().Format(time.RFC3339), - OriginalClaimant: staleAgent{ID: claimant}, - Evidence: staleEvidence{ - LastTouchTS: b.UpdatedAt, - ClaimAgeHours: roundFloat(ageHours, 1), - ThresholdHours: thresholdHours, - }, - }) - } - // Stable order by descending age — oldest claims first. - sort.SliceStable(events, func(i, j int) bool { - return events[i].Evidence.ClaimAgeHours > events[j].Evidence.ClaimAgeHours - }) - return events -} - -func roundFloat(v float64, decimals int) float64 { - factor := 1.0 - for i := 0; i < decimals; i++ { - factor *= 10 - } - return float64(int(v*factor+0.5)) / factor + return beadsapp.ComputeStaleEvents(beads, now, thresholdHours) } diff --git a/cli/internal/beads/stale.go b/cli/internal/beads/stale.go new file mode 100644 index 000000000..ca0d217ae --- /dev/null +++ b/cli/internal/beads/stale.go @@ -0,0 +1,84 @@ +// Package beads owns tracker-independent application behavior for the +// top-level beads command family. +package beads + +import ( + "sort" + "time" +) + +// StaleBeadRecord is the subset of tracker list output needed for staleness. +type StaleBeadRecord struct { + ID string `json:"id"` + Status string `json:"status"` + Assignee string `json:"assignee"` + UpdatedAt string `json:"updated_at"` +} + +// StaleEvent mirrors stale-claim-event.v1 for stale_detected events. +type StaleEvent struct { + SchemaVersion int `json:"schema_version"` + EventType string `json:"event_type"` + BeadID string `json:"bead_id"` + DetectedAt string `json:"detected_at"` + OriginalClaimant StaleAgent `json:"original_claimant"` + Evidence StaleEvidence `json:"evidence"` +} + +type StaleAgent struct { + ID string `json:"id"` +} + +type StaleEvidence struct { + LastTouchTS string `json:"last_touch_ts,omitempty"` + ClaimAgeHours float64 `json:"claim_age_hours,omitempty"` + ThresholdHours float64 `json:"threshold_hours,omitempty"` + LastEvidenceEvent string `json:"last_evidence_event,omitempty"` +} + +// ComputeStaleEvents derives a stable oldest-first event set without reading +// clocks, files, environment, or tracker state. +func ComputeStaleEvents(records []StaleBeadRecord, now time.Time, thresholdHours float64) []StaleEvent { + var events []StaleEvent + for _, record := range records { + if record.Status != "in_progress" || record.UpdatedAt == "" { + continue + } + updated, err := time.Parse(time.RFC3339, record.UpdatedAt) + if err != nil { + continue + } + ageHours := now.Sub(updated).Hours() + if ageHours < thresholdHours { + continue + } + claimant := record.Assignee + if claimant == "" { + claimant = "unknown" + } + events = append(events, StaleEvent{ + SchemaVersion: 1, + EventType: "stale_detected", + BeadID: record.ID, + DetectedAt: now.UTC().Format(time.RFC3339), + OriginalClaimant: StaleAgent{ID: claimant}, + Evidence: StaleEvidence{ + LastTouchTS: record.UpdatedAt, + ClaimAgeHours: round(ageHours, 1), + ThresholdHours: thresholdHours, + }, + }) + } + sort.SliceStable(events, func(left, right int) bool { + return events[left].Evidence.ClaimAgeHours > events[right].Evidence.ClaimAgeHours + }) + return events +} + +func round(value float64, decimals int) float64 { + factor := 1.0 + for range decimals { + factor *= 10 + } + return float64(int(value*factor+0.5)) / factor +} From 8c5f0f2848eb2e875f8fb09d5928df0e7c7a6b32 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:51:40 -0400 Subject: [PATCH 005/108] refactor(beads): extract epic membership policy (age-nw28h.2.1) --- cli/cmd/ao/beads_epic_status.go | 102 ++------------------------ cli/internal/beads/epic_status.go | 115 ++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 97 deletions(-) create mode 100644 cli/internal/beads/epic_status.go diff --git a/cli/cmd/ao/beads_epic_status.go b/cli/cmd/ao/beads_epic_status.go index bc218ac1d..2d38545a9 100644 --- a/cli/cmd/ao/beads_epic_status.go +++ b/cli/cmd/ao/beads_epic_status.go @@ -26,9 +26,9 @@ import ( "fmt" "os" "path/filepath" - "sort" "strings" + beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/boshu2/agentops/cli/internal/epicstatus" "github.com/spf13/cobra" ) @@ -72,21 +72,11 @@ func init() { } // ledgerBead is the subset of an issues.jsonl record the predicate needs. -type ledgerBead struct { - ID string `json:"id"` - Status string `json:"status"` - IssueType string `json:"issue_type"` - Labels []string `json:"labels"` - Dependencies []ledgerDep `json:"dependencies"` -} +type ledgerBead = beadsapp.LedgerBead // ledgerDep is one dependency edge on a bead. A parent-child edge points from a // child UP to its parent/epic (depends_on_id == epic). -type ledgerDep struct { - IssueID string `json:"issue_id"` - DependsOnID string `json:"depends_on_id"` - Type string `json:"type"` -} +type ledgerDep = beadsapp.LedgerDep // beadsEpicStatusReadLedger is the seam for tests: it returns the raw // newline-delimited issues.jsonl bytes for the resolved ledger dir. Tests @@ -136,19 +126,7 @@ func runBeadsEpicStatus(cmd *cobra.Command, args []string) error { // Blank lines are skipped; a malformed line is a hard error (fail closed — // "unreachable is not absent"). func parseLedger(raw []byte) ([]ledgerBead, error) { - var beads []ledgerBead - for i, line := range strings.Split(string(raw), "\n") { - trimmed := strings.TrimSpace(line) - if trimmed == "" { - continue - } - var b ledgerBead - if err := json.Unmarshal([]byte(trimmed), &b); err != nil { - return nil, fmt.Errorf("line %d: %w", i+1, err) - } - beads = append(beads, b) - } - return beads, nil + return beadsapp.ParseLedger(raw) } // buildMembers computes an epic's member set from a ledger snapshot and reports @@ -159,77 +137,7 @@ func parseLedger(raw []byte) ([]ledgerBead, error) { // referenced by a dependency edge but absent from the ledger is a dangling // reference → an unresolved (Present=false) member, so guard 1 fires. func buildMembers(epic string, beads []ledgerBead) (members []epicstatus.Member, epicPresent bool) { - prefix := epic + "." - byID := make(map[string]ledgerBead, len(beads)) - for _, b := range beads { - byID[b.ID] = b - if b.ID == epic { - epicPresent = true - } - } - if !epicPresent { - return nil, false - } - - memberIDs := map[string]bool{} - for _, b := range beads { - if b.ID == epic { - continue - } - if strings.HasPrefix(b.ID, prefix) || hasParentChildEdgeTo(b, epic) { - memberIDs[b.ID] = true - } - } - - // Dangling family references: a ".*" id named by any in-family edge - // that has no ledger record. Scan the epic and every resolved member. - missing := map[string]bool{} - scan := func(b ledgerBead) { - for _, d := range b.Dependencies { - for _, ref := range []string{d.DependsOnID, d.IssueID} { - if ref == "" || ref == epic || memberIDs[ref] { - continue - } - if strings.HasPrefix(ref, prefix) { - if _, ok := byID[ref]; !ok { - missing[ref] = true - } - } - } - } - } - scan(byID[epic]) - for id := range memberIDs { - scan(byID[id]) - } - - for id := range memberIDs { - b := byID[id] - members = append(members, epicstatus.Member{ - ID: b.ID, - Present: true, - Status: b.Status, - IssueType: b.IssueType, - Labels: b.Labels, - }) - } - for id := range missing { - members = append(members, epicstatus.Member{ID: id, Present: false}) - } - // Deterministic order (Evaluate re-sorts too, but keep the seam stable). - sort.SliceStable(members, func(i, j int) bool { return members[i].ID < members[j].ID }) - return members, true -} - -// hasParentChildEdgeTo reports whether bead b has a parent-child dependency -// edge pointing at parentID (the child→parent/epic direction). -func hasParentChildEdgeTo(b ledgerBead, parentID string) bool { - for _, d := range b.Dependencies { - if d.Type == "parent-child" && d.DependsOnID == parentID { - return true - } - } - return false + return beadsapp.BuildMembers(epic, beads) } func emitEpicStatus(cmd *cobra.Command, r epicstatus.Result) { diff --git a/cli/internal/beads/epic_status.go b/cli/internal/beads/epic_status.go new file mode 100644 index 000000000..81a54ef31 --- /dev/null +++ b/cli/internal/beads/epic_status.go @@ -0,0 +1,115 @@ +package beads + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/boshu2/agentops/cli/internal/epicstatus" +) + +// LedgerBead is the subset of an issues.jsonl record needed to derive epic +// membership. +type LedgerBead struct { + ID string `json:"id"` + Status string `json:"status"` + IssueType string `json:"issue_type"` + Labels []string `json:"labels"` + Dependencies []LedgerDep `json:"dependencies"` +} + +type LedgerDep struct { + IssueID string `json:"issue_id"` + DependsOnID string `json:"depends_on_id"` + Type string `json:"type"` +} + +// ParseLedger parses newline-delimited issues and fails closed on any malformed +// non-blank line. +func ParseLedger(raw []byte) ([]LedgerBead, error) { + var records []LedgerBead + for index, line := range strings.Split(string(raw), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + var record LedgerBead + if err := json.Unmarshal([]byte(trimmed), &record); err != nil { + return nil, fmt.Errorf("line %d: %w", index+1, err) + } + records = append(records, record) + } + return records, nil +} + +// BuildMembers derives the deterministic union of prefix children and +// parent-child edges, including unresolved in-family references. +func BuildMembers(epic string, records []LedgerBead) (members []epicstatus.Member, epicPresent bool) { + prefix := epic + "." + byID := make(map[string]LedgerBead, len(records)) + for _, record := range records { + byID[record.ID] = record + if record.ID == epic { + epicPresent = true + } + } + if !epicPresent { + return nil, false + } + + memberIDs := map[string]bool{} + for _, record := range records { + if record.ID == epic { + continue + } + if strings.HasPrefix(record.ID, prefix) || hasParentChildEdgeTo(record, epic) { + memberIDs[record.ID] = true + } + } + + missing := map[string]bool{} + scan := func(record LedgerBead) { + for _, dependency := range record.Dependencies { + for _, reference := range []string{dependency.DependsOnID, dependency.IssueID} { + if reference == "" || reference == epic || memberIDs[reference] { + continue + } + if strings.HasPrefix(reference, prefix) { + if _, exists := byID[reference]; !exists { + missing[reference] = true + } + } + } + } + } + scan(byID[epic]) + for id := range memberIDs { + scan(byID[id]) + } + + for id := range memberIDs { + record := byID[id] + members = append(members, epicstatus.Member{ + ID: record.ID, + Present: true, + Status: record.Status, + IssueType: record.IssueType, + Labels: record.Labels, + }) + } + for id := range missing { + members = append(members, epicstatus.Member{ID: id, Present: false}) + } + sort.SliceStable(members, func(left, right int) bool { return members[left].ID < members[right].ID }) + return members, true +} + +func hasParentChildEdgeTo(record LedgerBead, parentID string) bool { + for _, dependency := range record.Dependencies { + if dependency.Type == "parent-child" && dependency.DependsOnID == parentID { + return true + } + } + return false +} From 89025b31074f96ee37c6ee9ba41d6ed2b10db418 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:53:55 -0400 Subject: [PATCH 006/108] refactor(beads): extract tracker client adapter (age-nw28h.2.1) --- cli/cmd/ao/beads.go | 21 ++--- cli/internal/adapters/beads/tracker.go | 89 +++++++++++++++++++++ cli/internal/adapters/beads/tracker_test.go | 61 ++++++++++++++ cli/internal/beads/tracker.go | 36 +++++++++ 4 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 cli/internal/adapters/beads/tracker.go create mode 100644 cli/internal/adapters/beads/tracker_test.go create mode 100644 cli/internal/beads/tracker.go diff --git a/cli/cmd/ao/beads.go b/cli/cmd/ao/beads.go index 0c3139f48..b1e14782b 100644 --- a/cli/cmd/ao/beads.go +++ b/cli/cmd/ao/beads.go @@ -22,6 +22,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -34,6 +35,8 @@ import ( "time" "github.com/spf13/cobra" + + beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" ) // beadsExitError carries a verdict exit code out through cobra's RunE so @@ -61,21 +64,19 @@ func (e *beadsExitError) ExitCode() int { return e.code } // this to avoid a hard dependency on the real binary. Production code calls // `bd` via PATH; if absent, the caller emits a graceful warning and returns. var execBD = func(args ...string) ([]byte, error) { - cwd, _ := os.Getwd() - resolution, err := resolveTracker(cwd, os.Environ()) - if err != nil { - return nil, err - } - cmd := exec.Command(resolution.Binary, args...) // #nosec G204 -- selected br|bd binary. - return cmd.Output() + return currentBeadsTracker().Output(context.Background(), args...) } // bdAvailable reports whether the bd binary is reachable via PATH. Tests // override this for deterministic behaviour. var bdAvailable = func() bool { - cwd, _ := os.Getwd() - _, err := resolveTracker(cwd, os.Environ()) - return err == nil + return currentBeadsTracker().Available() +} + +func currentBeadsTracker() *beadsadapter.Tracker { + return beadsadapter.NewTrackerWith(os.Getwd, os.Environ, func(name string) (string, error) { + return trackerLookPath(name) + }) } // ------------------------------------------------------------------------ diff --git a/cli/internal/adapters/beads/tracker.go b/cli/internal/adapters/beads/tracker.go new file mode 100644 index 000000000..02c1d4301 --- /dev/null +++ b/cli/internal/adapters/beads/tracker.go @@ -0,0 +1,89 @@ +// Package beads implements the beads command family's driven effects. +package beads + +import ( + "context" + "fmt" + "os" + "os/exec" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" + "github.com/boshu2/agentops/cli/internal/trackerresolve" +) + +type Tracker struct { + workingDirectory func() (string, error) + environment func() []string + lookPath trackerresolve.LookPath +} + +func NewTracker() *Tracker { + return NewTrackerWith(os.Getwd, os.Environ, exec.LookPath) +} + +func NewTrackerWith(workingDirectory func() (string, error), environment func() []string, lookPath trackerresolve.LookPath) *Tracker { + return &Tracker{workingDirectory: workingDirectory, environment: environment, lookPath: lookPath} +} + +func (tracker *Tracker) Resolve() (beadsapp.TrackerResolution, error) { + cwd, env, err := tracker.context() + if err != nil { + return beadsapp.TrackerResolution{}, err + } + resolved, err := trackerresolve.ResolveWithLookPath(cwd, env, tracker.lookPath) + if err != nil { + return beadsapp.TrackerResolution{}, err + } + return beadsapp.TrackerResolution{ + Tracker: resolved.Tracker, + Binary: resolved.Binary, + LedgerDir: resolved.LedgerDir, + Source: resolved.Source, + WorkDir: resolved.WorkDir, + ChildEnv: append([]string(nil), resolved.ChildEnv...), + }, nil +} + +func (tracker *Tracker) BRLedger() (beadsapp.LedgerResolution, error) { + cwd, env, err := tracker.context() + if err != nil { + return beadsapp.LedgerResolution{}, err + } + resolved := trackerresolve.ResolveLedger(cwd, env, trackerresolve.BR) + return beadsapp.LedgerResolution{Path: resolved.Path, Source: resolved.Source}, nil +} + +func (tracker *Tracker) BeadsDirOverride() bool { + _, ok := trackerresolve.BeadsDirValue(tracker.environment()) + return ok +} + +func (tracker *Tracker) Available() bool { + _, err := tracker.Resolve() + return err == nil +} + +func (tracker *Tracker) Output(ctx context.Context, args ...string) ([]byte, error) { + resolved, err := tracker.Resolve() + if err != nil { + return nil, err + } + command := exec.CommandContext(ctx, resolved.Binary, args...) // #nosec G204 -- trackerresolve constrains the binary to br|bd. + command.Dir = resolved.WorkDir + command.Env = append([]string(nil), resolved.ChildEnv...) + return command.Output() +} + +func (tracker *Tracker) context() (string, []string, error) { + if tracker == nil || tracker.workingDirectory == nil || tracker.environment == nil || tracker.lookPath == nil { + return "", nil, fmt.Errorf("beads tracker adapter is not configured") + } + cwd, err := tracker.workingDirectory() + if err != nil { + return "", nil, err + } + return cwd, tracker.environment(), nil +} + +var _ beadsapp.TrackerResolver = (*Tracker)(nil) +var _ beadsapp.TrackerClient = (*Tracker)(nil) diff --git a/cli/internal/adapters/beads/tracker_test.go b/cli/internal/adapters/beads/tracker_test.go new file mode 100644 index 000000000..38d0eaa84 --- /dev/null +++ b/cli/internal/adapters/beads/tracker_test.go @@ -0,0 +1,61 @@ +package beads + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestTrackerPreservesBDLedgerSelectionAndInvocation(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, ".beads"), 0o755); err != nil { + t.Fatal(err) + } + bin := filepath.Join(root, "bd") + if err := os.WriteFile(bin, []byte("#!/bin/sh\nprintf '%s\\n' \"$*\"\n"), 0o755); err != nil { + t.Fatal(err) + } + tracker := NewTrackerWith( + func() (string, error) { return root, nil }, + func() []string { return []string{"PATH=" + root, "BEADS_DIR=/foreign/br"} }, + func(name string) (string, error) { + if name == "bd" { + return bin, nil + } + return "", errors.New("not found") + }, + ) + resolved, err := tracker.Resolve() + if err != nil { + t.Fatal(err) + } + if resolved.Tracker != "bd" || resolved.LedgerDir != filepath.Join(root, ".beads") { + t.Fatalf("resolution = %+v", resolved) + } + for _, entry := range resolved.ChildEnv { + if strings.HasPrefix(entry, "BEADS_DIR=") { + t.Fatalf("bd child inherited foreign BEADS_DIR: %v", resolved.ChildEnv) + } + } + output, err := tracker.Output(context.Background(), "ready", "--json") + if err != nil { + t.Fatal(err) + } + if string(output) != "ready --json\n" { + t.Fatalf("output = %q", output) + } +} + +func TestTrackerFailsClosedWhenWorkingDirectoryFails(t *testing.T) { + tracker := NewTrackerWith( + func() (string, error) { return "", errors.New("cwd unavailable") }, + func() []string { return nil }, + func(string) (string, error) { return "", nil }, + ) + if _, err := tracker.Resolve(); err == nil || !strings.Contains(err.Error(), "cwd unavailable") { + t.Fatalf("Resolve() error = %v", err) + } +} diff --git a/cli/internal/beads/tracker.go b/cli/internal/beads/tracker.go new file mode 100644 index 000000000..cf151fa16 --- /dev/null +++ b/cli/internal/beads/tracker.go @@ -0,0 +1,36 @@ +package beads + +import "context" + +const ( + TrackerBR = "br" + TrackerBD = "bd" +) + +// TrackerResolution is the command-family view of a selected tracker. It is +// deliberately independent of trackerresolve so driving adapters depend on a +// port, not the concrete workspace adapter. +type TrackerResolution struct { + Tracker string `json:"tracker"` + Binary string `json:"binary"` + LedgerDir string `json:"ledger_dir"` + Source string `json:"source"` + WorkDir string `json:"-"` + ChildEnv []string `json:"-"` +} + +type LedgerResolution struct { + Path string + Source string +} + +type TrackerResolver interface { + Resolve() (TrackerResolution, error) + BRLedger() (LedgerResolution, error) + BeadsDirOverride() bool +} + +type TrackerClient interface { + Available() bool + Output(context.Context, ...string) ([]byte, error) +} From 779d31c3f32a475947e54b69a324640e737f0f03 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:54:31 -0400 Subject: [PATCH 007/108] refactor(beads): route tracker report through port (age-nw28h.2.1) --- cli/cmd/ao/beads.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cli/cmd/ao/beads.go b/cli/cmd/ao/beads.go index b1e14782b..0f266b65d 100644 --- a/cli/cmd/ao/beads.go +++ b/cli/cmd/ao/beads.go @@ -284,11 +284,7 @@ func runBeadsDir(cmd *cobra.Command, _ []string) error { // environment: which tracker (bd|br), its resolved binary, its ledger // directory, and how it was selected (age-fvr8). func runBeadsTrackerCmd(cmd *cobra.Command, _ []string) error { - cwd, err := os.Getwd() - if err != nil { - return err - } - res, err := resolveTracker(cwd, os.Environ()) + res, err := currentBeadsTracker().Resolve() if err != nil { return err } From 0d6263136b304964c8f5dbe576a074d27ff5f00f Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:56:35 -0400 Subject: [PATCH 008/108] refactor(beads): isolate ledger directory probing (age-nw28h.2.1) --- cli/cmd/ao/beads.go | 64 ++++---------------------- cli/internal/adapters/beads/tracker.go | 23 +++++++++ cli/internal/beads/tracker.go | 47 ++++++++++++++++++- cli/internal/beads/tracker_test.go | 24 ++++++++++ 4 files changed, 103 insertions(+), 55 deletions(-) create mode 100644 cli/internal/beads/tracker_test.go diff --git a/cli/cmd/ao/beads.go b/cli/cmd/ao/beads.go index 0f266b65d..f4b11a67a 100644 --- a/cli/cmd/ao/beads.go +++ b/cli/cmd/ao/beads.go @@ -37,6 +37,7 @@ import ( "github.com/spf13/cobra" beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" + beadsapp "github.com/boshu2/agentops/cli/internal/beads" ) // beadsExitError carries a verdict exit code out through cobra's RunE so @@ -238,18 +239,15 @@ func init() { } func runBeadsDir(cmd *cobra.Command, _ []string) error { - cwd, err := os.Getwd() - if err != nil { - return err - } + tracker := currentBeadsTracker() // BEADS_DIR is br's explicit ledger override; when it is set we preserve the // historical br resolution verbatim (backward compatibility). Otherwise we // consult the dual-tracker resolver so a bd-tracked repo returns its .beads // directory instead of assuming _beads (age-fvr8). - if _, override := beadsEnvValue(os.Environ()); !override { - if tr, terr := resolveTracker(cwd, os.Environ()); terr == nil && tr.Tracker == trackerBD { + if !tracker.BeadsDirOverride() { + if tr, terr := tracker.Resolve(); terr == nil && tr.Tracker == trackerBD { if beadsDirRequire { - if reason := bdLedgerMissing(tr.LedgerDir); reason != "" { + if reason := beadsapp.LedgerMissing(beadsapp.TrackerBD, tracker.InspectLedger(tr.LedgerDir)); reason != "" { return fmt.Errorf("beads dir --require: %s (resolved %s for tracker bd via %s); refusing to print a path a bd write could silently fall back from", reason, tr.LedgerDir, tr.Source) } } @@ -264,9 +262,12 @@ func runBeadsDir(cmd *cobra.Command, _ []string) error { } } // br path — unchanged historical behavior (also the both-present tie-break). - resolved := resolveBeadsDir(cwd, os.Environ()) + resolved, err := tracker.BRLedger() + if err != nil { + return err + } if beadsDirRequire { - if reason := beadsDirLedgerMissing(resolved.Path); reason != "" { + if reason := beadsapp.LedgerMissing(beadsapp.TrackerBR, tracker.InspectLedger(resolved.Path)); reason != "" { return fmt.Errorf("beads dir --require: %s (resolved %s via %s); refusing to print a path a br write could silently fall back from", reason, resolved.Path, resolved.Source) } } @@ -299,51 +300,6 @@ func runBeadsTrackerCmd(cmd *cobra.Command, _ []string) error { return nil } -// beadsDirLedgerMissing reports why path does not hold a usable br ledger -// (empty string = it does). A ledger dir must exist and contain at least one -// of the artifacts br itself maintains. -func beadsDirLedgerMissing(path string) string { - info, err := os.Stat(path) - if err != nil { - return "resolved path does not exist" - } - if !info.IsDir() { - return "resolved path is not a directory" - } - for _, artifact := range []string{"issues.jsonl", "beads.db"} { - if _, err := os.Stat(filepath.Join(path, artifact)); err == nil { - return "" - } - } - return "no ledger artifact (issues.jsonl or beads.db) in resolved directory" -} - -// bdLedgerMissing reports why path does not hold a usable bd ledger (empty -// string = it does). bd maintains a SQLite db, a jsonl mirror, and a config -// file in its .beads directory; any one of those is enough to treat the dir as -// a real bd ledger rather than an empty stub. -func bdLedgerMissing(path string) string { - info, err := os.Stat(path) - if err != nil { - return "resolved path does not exist" - } - if !info.IsDir() { - return "resolved path is not a directory" - } - entries, err := os.ReadDir(path) - if err != nil { - return "resolved directory is unreadable" - } - for _, e := range entries { - name := e.Name() - if strings.HasSuffix(name, ".db") || strings.HasSuffix(name, ".jsonl") || - name == "config.yaml" || name == "config.json" || name == "beads.json" { - return "" - } - } - return "no ledger artifact (*.db, *.jsonl, or config) in resolved directory" -} - // ------------------------------------------------------------------------ // verify // ------------------------------------------------------------------------ diff --git a/cli/internal/adapters/beads/tracker.go b/cli/internal/adapters/beads/tracker.go index 02c1d4301..ffa87c5d0 100644 --- a/cli/internal/adapters/beads/tracker.go +++ b/cli/internal/adapters/beads/tracker.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/exec" + "sort" beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/boshu2/agentops/cli/internal/trackerresolve" @@ -58,6 +59,27 @@ func (tracker *Tracker) BeadsDirOverride() bool { return ok } +func (tracker *Tracker) InspectLedger(path string) beadsapp.LedgerSnapshot { + info, err := os.Stat(path) + if err != nil { + return beadsapp.LedgerSnapshot{} + } + snapshot := beadsapp.LedgerSnapshot{Exists: true, Directory: info.IsDir()} + if !info.IsDir() { + return snapshot + } + entries, err := os.ReadDir(path) + if err != nil { + return snapshot + } + snapshot.Readable = true + for _, entry := range entries { + snapshot.Entries = append(snapshot.Entries, entry.Name()) + } + sort.Strings(snapshot.Entries) + return snapshot +} + func (tracker *Tracker) Available() bool { _, err := tracker.Resolve() return err == nil @@ -87,3 +109,4 @@ func (tracker *Tracker) context() (string, []string, error) { var _ beadsapp.TrackerResolver = (*Tracker)(nil) var _ beadsapp.TrackerClient = (*Tracker)(nil) +var _ beadsapp.LedgerInspector = (*Tracker)(nil) diff --git a/cli/internal/beads/tracker.go b/cli/internal/beads/tracker.go index cf151fa16..beff1cd24 100644 --- a/cli/internal/beads/tracker.go +++ b/cli/internal/beads/tracker.go @@ -1,6 +1,9 @@ package beads -import "context" +import ( + "context" + "strings" +) const ( TrackerBR = "br" @@ -24,6 +27,13 @@ type LedgerResolution struct { Source string } +type LedgerSnapshot struct { + Exists bool + Directory bool + Readable bool + Entries []string +} + type TrackerResolver interface { Resolve() (TrackerResolution, error) BRLedger() (LedgerResolution, error) @@ -34,3 +44,38 @@ type TrackerClient interface { Available() bool Output(context.Context, ...string) ([]byte, error) } + +type LedgerInspector interface { + InspectLedger(string) LedgerSnapshot +} + +// LedgerMissing preserves the deliberately different BR and BD artifact +// policies while keeping filesystem probing behind LedgerInspector. +func LedgerMissing(tracker string, snapshot LedgerSnapshot) string { + if !snapshot.Exists { + return "resolved path does not exist" + } + if !snapshot.Directory { + return "resolved path is not a directory" + } + if tracker == TrackerBD && !snapshot.Readable { + return "resolved directory is unreadable" + } + for _, name := range snapshot.Entries { + switch tracker { + case TrackerBR: + if name == "issues.jsonl" || name == "beads.db" { + return "" + } + case TrackerBD: + if strings.HasSuffix(name, ".db") || strings.HasSuffix(name, ".jsonl") || + name == "config.yaml" || name == "config.json" || name == "beads.json" { + return "" + } + } + } + if tracker == TrackerBD { + return "no ledger artifact (*.db, *.jsonl, or config) in resolved directory" + } + return "no ledger artifact (issues.jsonl or beads.db) in resolved directory" +} diff --git a/cli/internal/beads/tracker_test.go b/cli/internal/beads/tracker_test.go new file mode 100644 index 000000000..75a477118 --- /dev/null +++ b/cli/internal/beads/tracker_test.go @@ -0,0 +1,24 @@ +package beads + +import "testing" + +func TestLedgerMissingPreservesBRAndBDArtifactPolicies(t *testing.T) { + for name, test := range map[string]struct { + tracker string + snapshot LedgerSnapshot + want string + }{ + "missing": {tracker: TrackerBR, want: "resolved path does not exist"}, + "file": {tracker: TrackerBR, snapshot: LedgerSnapshot{Exists: true}, want: "resolved path is not a directory"}, + "br jsonl": {tracker: TrackerBR, snapshot: LedgerSnapshot{Exists: true, Directory: true, Readable: true, Entries: []string{"issues.jsonl"}}}, + "br rejects config": {tracker: TrackerBR, snapshot: LedgerSnapshot{Exists: true, Directory: true, Readable: true, Entries: []string{"config.yaml"}}, want: "no ledger artifact (issues.jsonl or beads.db) in resolved directory"}, + "bd config": {tracker: TrackerBD, snapshot: LedgerSnapshot{Exists: true, Directory: true, Readable: true, Entries: []string{"config.yaml"}}}, + "bd unreadable": {tracker: TrackerBD, snapshot: LedgerSnapshot{Exists: true, Directory: true}, want: "resolved directory is unreadable"}, + } { + t.Run(name, func(t *testing.T) { + if got := LedgerMissing(test.tracker, test.snapshot); got != test.want { + t.Fatalf("LedgerMissing() = %q, want %q", got, test.want) + } + }) + } +} From c234364d91974d6d65cce99910917bc5d5f0eb43 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:58:41 -0400 Subject: [PATCH 009/108] refactor(beads): extract tracker exec policy (age-nw28h.2.1) --- cli/cmd/ao/beads_exec.go | 136 +++---------------------------------- cli/internal/beads/exec.go | 133 ++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 125 deletions(-) create mode 100644 cli/internal/beads/exec.go diff --git a/cli/cmd/ao/beads_exec.go b/cli/cmd/ao/beads_exec.go index d95183bc6..ca50a8577 100644 --- a/cli/cmd/ao/beads_exec.go +++ b/cli/cmd/ao/beads_exec.go @@ -28,15 +28,15 @@ package main import ( "bytes" - "encoding/json" "errors" "fmt" "os" "os/exec" - "path/filepath" "strings" "github.com/spf13/cobra" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" ) var beadsExecCmd = &cobra.Command{ @@ -153,18 +153,7 @@ func execTracker(cmd *cobra.Command, res trackerResolution, cwd string, args []s // its working directory — and any inherited BEADS_DIR is stripped so it cannot // mislead bd. func beadsExecChildEnv(res trackerResolution, _ string) []string { - base := os.Environ() - out := make([]string, 0, len(base)+1) - for _, e := range base { - if strings.HasPrefix(e, "BEADS_DIR=") { - continue - } - out = append(out, e) - } - if res.Tracker == trackerBR { - out = append(out, "BEADS_DIR="+res.LedgerDir) - } - return out + return beadsapp.ChildEnvironment(os.Environ(), beadsapp.TrackerResolution{Tracker: res.Tracker, LedgerDir: res.LedgerDir}) } // beadsExecChildDir returns the child's working directory. bd resolves its @@ -172,26 +161,7 @@ func beadsExecChildEnv(res trackerResolution, _ string) []string { // the resolved .beads dir, worktree-aware). br takes its ledger from BEADS_DIR, // so it runs from the caller's cwd unchanged. func beadsExecChildDir(res trackerResolution, cwd string) string { - if res.Tracker == trackerBD { - return filepath.Dir(res.LedgerDir) - } - return cwd -} - -// brShowDependent is the subset of a dependent entry in `br show --json`. -type brShowDependent struct { - ID string `json:"id"` - DependencyType string `json:"dependency_type"` -} - -// brShowIssue is the subset of a `br show --json` array element we read. -// `br show` emits a JSON ARRAY of matched issues, each carrying a `dependents` -// array. (The full-output SHAPE divergence between the tracker `--json` outputs -// is now normalized for the READ verbs by execTrackerBDReadJSON — age-f07z; this -// command only makes the child-id SET available for both trackers.) -type brShowIssue struct { - ID string `json:"id"` - Dependents []brShowDependent `json:"dependents"` + return beadsapp.ChildDirectory(beadsapp.TrackerResolution{Tracker: res.Tracker, LedgerDir: res.LedgerDir}, cwd) } // runBeadsExecChildrenBR synthesizes `children` for br from @@ -224,17 +194,13 @@ func runBeadsExecChildrenBR(cmd *cobra.Command, res trackerResolution, cwd strin } return fmt.Errorf("br show %s --json: %w", epic, err) } - var issues []brShowIssue - if err := json.Unmarshal(stdout.Bytes(), &issues); err != nil { + children, err := beadsapp.BRChildren(stdout.Bytes()) + if err != nil { return fmt.Errorf("parse br show %s --json: %w", epic, err) } out := cmd.OutOrStdout() - for _, iss := range issues { - for _, d := range iss.Dependents { - if d.DependencyType == "parent-child" { - fmt.Fprintln(out, d.ID) - } - } + for _, child := range children { + fmt.Fprintln(out, child) } return nil } @@ -243,22 +209,12 @@ func runBeadsExecChildrenBR(cmd *cobra.Command, res trackerResolution, cwd strin // output shape ao normalizes to the canonical (br) shape. Write verbs and read // verbs without --json are streamed verbatim (age-f07z). func isBeadsReadVerb(verb string) bool { - switch verb { - case "list", "ready", "show": - return true - default: - return false - } + return beadsapp.IsReadVerb(verb) } // argsHaveJSONFlag reports whether the forwarded args request JSON output. func argsHaveJSONFlag(args []string) bool { - for _, a := range args { - if a == "--json" { - return true - } - } - return false + return beadsapp.ArgsHaveJSONFlag(args) } // execTrackerBDReadJSON runs a bd READ command with --json, CAPTURES its stdout @@ -301,12 +257,6 @@ func execTrackerBDReadJSON(cmd *cobra.Command, res trackerResolution, cwd string return nil } -// canonicalIssueKeys are the fields every shape-parsing skill selector relies -// on. bd omits an empty description (and, for `show`, the dependents[] array — -// it reports only a dependent_count), so the reshape guarantees these keys -// EXIST (as null / []) and jq selectors never break. -var canonicalIssueKeys = []string{"id", "title", "description", "priority", "status"} - // canonicalizeBDReadJSON reshapes bd's ` --json` stdout into the canonical // br shape: // @@ -322,69 +272,5 @@ var canonicalIssueKeys = []string{"id", "title", "description", "priority", "sta // on bd the reshape guarantees an EMPTY dependents[]; bd child enumeration // should use `ao beads exec children` (bd-native). func canonicalizeBDReadJSON(verb string, raw []byte) ([]byte, error) { - elems, err := decodeIssueArray(raw) - if err != nil { - return nil, err - } - if elems == nil { - // A missing/empty payload becomes an empty set, never JSON null — so - // `.issues | length` and `length` resolve to 0 instead of erroring. - elems = []map[string]json.RawMessage{} - } - isShow := verb == "show" - for _, e := range elems { - ensureCanonicalIssueKeys(e, isShow) - } - if verb == "list" { - return json.Marshal(map[string]any{"issues": elems}) - } - // ready + show are bare arrays in the canonical shape. - return json.Marshal(elems) -} - -// decodeIssueArray decodes a bd read payload into issue objects, tolerating a -// bare JSON array (bd list/ready/show) or a {"issues":[...]} envelope. Each -// element is kept as a raw field map so no field is lost or retyped. -func decodeIssueArray(raw []byte) ([]map[string]json.RawMessage, error) { - trimmed := bytes.TrimSpace(raw) - if len(trimmed) == 0 { - return nil, fmt.Errorf("empty bd --json payload") - } - switch trimmed[0] { - case '[': - var arr []map[string]json.RawMessage - if err := json.Unmarshal(trimmed, &arr); err != nil { - return nil, err - } - return arr, nil - case '{': - var obj struct { - Issues []map[string]json.RawMessage `json:"issues"` - } - if err := json.Unmarshal(trimmed, &obj); err != nil { - return nil, err - } - return obj.Issues, nil - default: - return nil, fmt.Errorf("unexpected bd --json payload (not an array or object): %.32q", trimmed) - } -} - -// ensureCanonicalIssueKeys adds any missing canonical field to elem as null and -// (for `show`) guarantees a dependents[] array, so consumer jq selectors resolve -// against a stable shape regardless of which fields bd emitted. -func ensureCanonicalIssueKeys(elem map[string]json.RawMessage, isShow bool) { - if elem == nil { - return - } - for _, k := range canonicalIssueKeys { - if _, ok := elem[k]; !ok { - elem[k] = json.RawMessage("null") - } - } - if isShow { - if _, ok := elem["dependents"]; !ok { - elem["dependents"] = json.RawMessage("[]") - } - } + return beadsapp.CanonicalizeBDReadJSON(verb, raw) } diff --git a/cli/internal/beads/exec.go b/cli/internal/beads/exec.go new file mode 100644 index 000000000..e122e6e30 --- /dev/null +++ b/cli/internal/beads/exec.go @@ -0,0 +1,133 @@ +package beads + +import ( + "bytes" + "encoding/json" + "fmt" + "path/filepath" + "strings" +) + +func ChildEnvironment(base []string, resolution TrackerResolution) []string { + child := make([]string, 0, len(base)+1) + for _, entry := range base { + if !strings.HasPrefix(entry, "BEADS_DIR=") { + child = append(child, entry) + } + } + if resolution.Tracker == TrackerBR { + child = append(child, "BEADS_DIR="+resolution.LedgerDir) + } + return child +} + +func ChildDirectory(resolution TrackerResolution, callerDirectory string) string { + if resolution.Tracker == TrackerBD { + return filepath.Dir(resolution.LedgerDir) + } + return callerDirectory +} + +func IsReadVerb(verb string) bool { + switch verb { + case "list", "ready", "show": + return true + default: + return false + } +} + +func ArgsHaveJSONFlag(args []string) bool { + for _, argument := range args { + if argument == "--json" { + return true + } + } + return false +} + +type brShowDependent struct { + ID string `json:"id"` + DependencyType string `json:"dependency_type"` +} + +type brShowIssue struct { + Dependents []brShowDependent `json:"dependents"` +} + +func BRChildren(raw []byte) ([]string, error) { + var issues []brShowIssue + if err := json.Unmarshal(raw, &issues); err != nil { + return nil, err + } + var children []string + for _, issue := range issues { + for _, dependent := range issue.Dependents { + if dependent.DependencyType == "parent-child" { + children = append(children, dependent.ID) + } + } + } + return children, nil +} + +var canonicalIssueKeys = []string{"id", "title", "description", "priority", "status"} + +func CanonicalizeBDReadJSON(verb string, raw []byte) ([]byte, error) { + elements, err := decodeIssueArray(raw) + if err != nil { + return nil, err + } + if elements == nil { + elements = []map[string]json.RawMessage{} + } + isShow := verb == "show" + for _, element := range elements { + ensureCanonicalIssueKeys(element, isShow) + } + if verb == "list" { + return json.Marshal(map[string]any{"issues": elements}) + } + return json.Marshal(elements) +} + +func decodeIssueArray(raw []byte) ([]map[string]json.RawMessage, error) { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 { + return nil, fmt.Errorf("empty bd --json payload") + } + switch trimmed[0] { + case '[': + var array []map[string]json.RawMessage + if err := json.Unmarshal(trimmed, &array); err != nil { + return nil, err + } + return array, nil + case '{': + var envelope struct { + Issues []map[string]json.RawMessage `json:"issues"` + } + if err := json.Unmarshal(trimmed, &envelope); err != nil { + return nil, err + } + return envelope.Issues, nil + default: + return nil, fmt.Errorf("unexpected bd --json payload (not an array or object): %.32q", trimmed) + } +} + +func ensureCanonicalIssueKeys(element map[string]json.RawMessage, isShow bool) { + if element == nil { + return + } + for _, key := range canonicalIssueKeys { + if _, exists := element[key]; !exists { + element[key] = json.RawMessage("null") + } + } + if isShow { + if _, exists := element["dependents"]; !exists { + element["dependents"] = json.RawMessage("[]") + } + } +} From d338ea5439ccd8b7b0048da362606d5c50667527 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:01:40 -0400 Subject: [PATCH 010/108] refactor(beads): move tracker execution behind adapter (age-nw28h.2.1) --- cli/cmd/ao/beads_exec.go | 232 +++--------------------- cli/internal/adapters/beads/executor.go | 116 ++++++++++++ cli/internal/beads/exec.go | 19 ++ 3 files changed, 161 insertions(+), 206 deletions(-) create mode 100644 cli/internal/adapters/beads/executor.go diff --git a/cli/cmd/ao/beads_exec.go b/cli/cmd/ao/beads_exec.go index ca50a8577..9c0f3bffd 100644 --- a/cli/cmd/ao/beads_exec.go +++ b/cli/cmd/ao/beads_exec.go @@ -1,41 +1,16 @@ -// `ao beads exec [args...]` — the ONE tracker-agnostic CRUD passthrough -// (age-3mdu, dual-support increment 2, building on age-fvr8's resolveTracker). -// -// AgentOps is a product. Most end users track their beads with bd (beads, Go); -// this repo tracks with br (beads_rust). Skills and callers should not hardcode -// `br` or `bd` — they call `ao beads exec ...` and this command resolves -// the right tracker, sets its ledger correctly, forwards the args verbatim, and -// propagates the child's exit code unchanged. So `ao beads exec ready`, -// `ao beads exec close -r "..."`, `ao beads exec update --status -// in_progress`, `ao beads exec create ...`, and `ao beads exec list --json` all -// work against EITHER tracker. -// -// Ledger wiring per tracker: -// - br: export BEADS_DIR= for the child (br's explicit -// ledger override; worktree-aware via resolveTracker/resolveBeadsDir). -// - bd: set the child's working directory to the repo root so its .beads/ -// auto-discovery resolves; no BEADS_DIR (any inherited value is stripped so -// it cannot mislead bd). -// -// Hard divergence handled here: `bd children ` exists; br has NO children -// subcommand. `ao beads exec children ` works for BOTH — bd's native -// `children` is forwarded verbatim, and for br the child-id list is synthesized -// from `br show --json` (its dependents[] with a parent-child edge). -// -// practices: [tdd] - +// `ao beads exec [args...]` is the one tracker-agnostic CRUD passthrough. +// Most users track with bd; this repository tracks with br. The adapter keeps +// their intentional ledger, child-enumeration, and JSON-shape differences. package main import ( - "bytes" + "context" "errors" - "fmt" "os" - "os/exec" - "strings" "github.com/spf13/cobra" + beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" beadsapp "github.com/boshu2/agentops/cli/internal/beads" ) @@ -69,12 +44,6 @@ Children (hard divergence — bd has 'children', br does not): All flags are forwarded to the tracker verbatim (flag parsing is disabled), so tracker flags never collide with ao's. Only -h/--help is intercepted here.`, - // DisableFlagParsing so every flag reaches the tracker unchanged. The trap: - // cobra then forwards -h/--help into RunE, and if we resolved the tracker (or - // exec'd) before intercepting it, the command-surface doc generator would bake - // a path-dependent runtime string into cli/docs/COMMANDS.md and fail - // derived.changed-scope non-deterministically. Intercept it FIRST. (Mirrors - // membrane.go runMembraneCalibrate.) DisableFlagParsing: true, RunE: runBeadsExec, } @@ -84,193 +53,44 @@ func init() { } func runBeadsExec(cmd *cobra.Command, args []string) error { - // --help/-h is for THIS command, intercepted BEFORE resolving the tracker or - // exec-ing so help stays a static, path-independent string (see the command - // doc comment above). - for _, a := range args { - if a == "--help" || a == "-h" { + for _, argument := range args { + if argument == "--help" || argument == "-h" { return cmd.Help() } } - - cwd, err := os.Getwd() - if err != nil { - return err + err := currentBeadsExecutor().Execute(context.Background(), args, beadsapp.ExecStreams{ + Stdin: cmd.InOrStdin(), + Stdout: cmd.OutOrStdout(), + Stderr: cmd.ErrOrStderr(), + }) + var adapterExit *beadsapp.ExitError + if errors.As(err, &adapterExit) { + err = &beadsExitError{code: adapterExit.ExitCode()} } - res, err := resolveTracker(cwd, os.Environ()) - if err != nil { - return err + var exitErr interface{ ExitCode() int } + if errors.As(err, &exitErr) { + cmd.SilenceUsage = true + cmd.SilenceErrors = true } - - // Hard divergence: br has no `children` subcommand. Synthesize it from - // `br show --json`. bd's `children` is native — fall through to the - // verbatim passthrough, which forwards `children ` to bd unchanged. - if res.Tracker == trackerBR && len(args) >= 1 && args[0] == "children" { - return runBeadsExecChildrenBR(cmd, res, cwd, args[1:]) - } - - // Shape divergence (age-f07z — closes the dual-support epic age-zcvn): bd and - // br emit DIFFERENT `--json` shapes for the READ verbs (list/ready/show), so - // shape-parsing skills could not safely pipe - // `ao beads exec --json | jq ''` across trackers. CANONICAL - // shape = br's (this repo is br; skills already parse it). br's output IS - // canonical — stream it verbatim, unchanged (no reorder/mutation). bd's is - // reshaped to canonical here. ONLY the read+`--json` case is special-cased; - // every other verb (and read verbs WITHOUT --json) streams verbatim. - if res.Tracker == trackerBD && len(args) >= 1 && isBeadsReadVerb(args[0]) && argsHaveJSONFlag(args) { - return execTrackerBDReadJSON(cmd, res, cwd, args) - } - - return execTracker(cmd, res, cwd, args) + return err } -// execTracker forwards args verbatim to the resolved tracker binary, streaming -// stdin/stdout/stderr, and maps a non-zero child exit to a beadsExitError so -// Execute() propagates the code unchanged (no verdict = not done). -func execTracker(cmd *cobra.Command, res trackerResolution, cwd string, args []string) error { - c := exec.Command(res.Binary, args...) // #nosec G204 -- res.Binary is resolved by resolveTracker (bd|br); args are operator-supplied bead CRUD flags forwarded verbatim. - c.Env = beadsExecChildEnv(res, cwd) - c.Dir = beadsExecChildDir(res, cwd) - c.Stdin = cmd.InOrStdin() - c.Stdout = cmd.OutOrStdout() - c.Stderr = cmd.ErrOrStderr() - runErr := c.Run() - if runErr == nil { - return nil - } - cmd.SilenceUsage = true - cmd.SilenceErrors = true - var exitErr *exec.ExitError - if errors.As(runErr, &exitErr) { - return &beadsExitError{code: exitErr.ExitCode()} - } - return runErr +func currentBeadsExecutor() *beadsadapter.Executor { + return beadsadapter.NewExecutor(currentBeadsTracker()) } -// beadsExecChildEnv builds the child environment. For br, BEADS_DIR is the -// explicit ledger override (set to the resolved ledger dir), replacing any -// inherited value. For bd, no BEADS_DIR is set — bd auto-discovers .beads/ from -// its working directory — and any inherited BEADS_DIR is stripped so it cannot -// mislead bd. +// These pure compatibility delegates remain until the yield family moves its +// tracker-child formatting onto the shared application policy. func beadsExecChildEnv(res trackerResolution, _ string) []string { return beadsapp.ChildEnvironment(os.Environ(), beadsapp.TrackerResolution{Tracker: res.Tracker, LedgerDir: res.LedgerDir}) } -// beadsExecChildDir returns the child's working directory. bd resolves its -// .beads/ ledger relative to cwd, so it runs from the repo root (the parent of -// the resolved .beads dir, worktree-aware). br takes its ledger from BEADS_DIR, -// so it runs from the caller's cwd unchanged. func beadsExecChildDir(res trackerResolution, cwd string) string { return beadsapp.ChildDirectory(beadsapp.TrackerResolution{Tracker: res.Tracker, LedgerDir: res.LedgerDir}, cwd) } -// runBeadsExecChildrenBR synthesizes `children` for br from -// `br show --json`: it emits the id of every dependent with a -// parent-child edge, one per line, preserving br's order. bd's native -// `bd children` (the bd branch of runBeadsExec) prints bd's own list format; -// the READ-verb (list/ready/show) `--json` shapes are normalized to one -// machine-readable form by execTrackerBDReadJSON (age-f07z). -func runBeadsExecChildrenBR(cmd *cobra.Command, res trackerResolution, cwd string, rest []string) error { - if len(rest) < 1 || strings.TrimSpace(rest[0]) == "" { - return fmt.Errorf("ao beads exec children: an epic id is required") - } - epic := rest[0] - c := exec.Command(res.Binary, "show", epic, "--json") // #nosec G204 -- res.Binary is the resolved br binary; fixed subcommand + operator-supplied epic id. - c.Env = beadsExecChildEnv(res, cwd) - c.Dir = cwd - var stdout, stderr bytes.Buffer - c.Stdout = &stdout - c.Stderr = &stderr - if err := c.Run(); err != nil { - // Surface br's own stderr for diagnostics, then propagate its exit code - // UNCHANGED — matching the exec passthrough's contract (line ~131) rather than - // collapsing every br failure into the generic ao error path (age-3mdu refute-fix). - if msg := strings.TrimSpace(stderr.String()); msg != "" { - fmt.Fprintln(cmd.ErrOrStderr(), msg) - } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return &beadsExitError{code: exitErr.ExitCode()} - } - return fmt.Errorf("br show %s --json: %w", epic, err) - } - children, err := beadsapp.BRChildren(stdout.Bytes()) - if err != nil { - return fmt.Errorf("parse br show %s --json: %w", epic, err) - } - out := cmd.OutOrStdout() - for _, child := range children { - fmt.Fprintln(out, child) - } - return nil -} - -// isBeadsReadVerb reports whether verb is a bead READ command whose `--json` -// output shape ao normalizes to the canonical (br) shape. Write verbs and read -// verbs without --json are streamed verbatim (age-f07z). -func isBeadsReadVerb(verb string) bool { - return beadsapp.IsReadVerb(verb) -} - -// argsHaveJSONFlag reports whether the forwarded args request JSON output. -func argsHaveJSONFlag(args []string) bool { - return beadsapp.ArgsHaveJSONFlag(args) -} - -// execTrackerBDReadJSON runs a bd READ command with --json, CAPTURES its stdout -// (rather than streaming), reshapes bd's shape into the canonical br shape, and -// emits the canonical JSON — so shape-parsing skills can pipe -// `ao beads exec --json | jq ''` identically for bd and br. -// stderr and the child's exit code propagate EXACTLY as the streaming -// passthrough (execTracker) does — no verdict = not done (age-f07z). -func execTrackerBDReadJSON(cmd *cobra.Command, res trackerResolution, cwd string, args []string) error { - c := exec.Command(res.Binary, args...) // #nosec G204 -- res.Binary is the resolved bd binary; args are operator-supplied read flags forwarded verbatim. - c.Env = beadsExecChildEnv(res, cwd) - c.Dir = beadsExecChildDir(res, cwd) - c.Stdin = cmd.InOrStdin() - var stdout, stderr bytes.Buffer - c.Stdout = &stdout - c.Stderr = &stderr - runErr := c.Run() - // Surface bd's stderr for diagnostics regardless of exit status (matches the - // children path), then propagate a non-zero exit code UNCHANGED. - if msg := strings.TrimSpace(stderr.String()); msg != "" { - fmt.Fprintln(cmd.ErrOrStderr(), msg) - } - if runErr != nil { - cmd.SilenceUsage = true - cmd.SilenceErrors = true - var exitErr *exec.ExitError - if errors.As(runErr, &exitErr) { - return &beadsExitError{code: exitErr.ExitCode()} - } - return runErr - } - canonical, err := canonicalizeBDReadJSON(args[0], stdout.Bytes()) - if err != nil { - // Unexpected bd output shape — never make a working passthrough worse: - // emit bd's raw stdout UNCHANGED (pre-normalization behavior). - _, _ = cmd.OutOrStdout().Write(stdout.Bytes()) - return nil - } - fmt.Fprintln(cmd.OutOrStdout(), string(canonical)) - return nil -} - -// canonicalizeBDReadJSON reshapes bd's ` --json` stdout into the canonical -// br shape: -// -// list -> {"issues":[ , ... ]} (bd emits a bare array -> wrap) -// ready -> [ , ... ] (bd already emits a bare array) -// show -> [ , ... ] (bd already emits a bare array) -// -// It is field-PRESERVING: it NEVER drops bd's extra fields (issue_type, -// close_reason, dependencies, comment_count, ...); it only normalizes the -// ENVELOPE and guarantees the canonical keys exist. Fields bd omits are emitted -// as null (scalars) or [] (dependents) so consumer jq selectors never break. -// bd's default `show` carries no dependents[] array (only a dependent_count), so -// on bd the reshape guarantees an EMPTY dependents[]; bd child enumeration -// should use `ao beads exec children` (bd-native). +// Kept as a package-main test seam until the legacy white-box tests move with +// their final owner. func canonicalizeBDReadJSON(verb string, raw []byte) ([]byte, error) { return beadsapp.CanonicalizeBDReadJSON(verb, raw) } diff --git a/cli/internal/adapters/beads/executor.go b/cli/internal/adapters/beads/executor.go new file mode 100644 index 000000000..9c744336e --- /dev/null +++ b/cli/internal/adapters/beads/executor.go @@ -0,0 +1,116 @@ +package beads + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "strings" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" +) + +type Executor struct { + tracker *Tracker +} + +func NewExecutor(tracker *Tracker) *Executor { + return &Executor{tracker: tracker} +} + +func (executor *Executor) Execute(ctx context.Context, args []string, streams beadsapp.ExecStreams) error { + if executor == nil || executor.tracker == nil { + return fmt.Errorf("beads tracker executor is not configured") + } + resolution, err := executor.tracker.Resolve() + if err != nil { + return err + } + if resolution.Tracker == beadsapp.TrackerBR && len(args) > 0 && args[0] == "children" { + return executor.childrenBR(ctx, resolution, args[1:], streams) + } + if resolution.Tracker == beadsapp.TrackerBD && len(args) > 0 && beadsapp.IsReadVerb(args[0]) && beadsapp.ArgsHaveJSONFlag(args) { + return executor.bdReadJSON(ctx, resolution, args, streams) + } + return executor.stream(ctx, resolution, args, streams) +} + +func (executor *Executor) stream(ctx context.Context, resolution beadsapp.TrackerResolution, args []string, streams beadsapp.ExecStreams) error { + command := exec.CommandContext(ctx, resolution.Binary, args...) // #nosec G204 -- trackerresolve constrains the binary to br|bd. + configureCommand(command, resolution, streams) + return exitError(command.Run()) +} + +func (executor *Executor) childrenBR(ctx context.Context, resolution beadsapp.TrackerResolution, args []string, streams beadsapp.ExecStreams) error { + if len(args) < 1 || strings.TrimSpace(args[0]) == "" { + return fmt.Errorf("ao beads exec children: an epic id is required") + } + epic := args[0] + command := exec.CommandContext(ctx, resolution.Binary, "show", epic, "--json") // #nosec G204 -- trackerresolve constrains the binary to br. + var stdout, stderr bytes.Buffer + configureCommand(command, resolution, beadsapp.ExecStreams{Stdout: &stdout, Stderr: &stderr}) + if err := command.Run(); err != nil { + writeTrimmedLine(streams.Stderr, stderr.String()) + if converted := exitError(err); converted != err { + return converted + } + return fmt.Errorf("br show %s --json: %w", epic, err) + } + children, err := beadsapp.BRChildren(stdout.Bytes()) + if err != nil { + return fmt.Errorf("parse br show %s --json: %w", epic, err) + } + for _, child := range children { + fmt.Fprintln(streams.Stdout, child) + } + return nil +} + +func (executor *Executor) bdReadJSON(ctx context.Context, resolution beadsapp.TrackerResolution, args []string, streams beadsapp.ExecStreams) error { + command := exec.CommandContext(ctx, resolution.Binary, args...) // #nosec G204 -- trackerresolve constrains the binary to bd. + var stdout, stderr bytes.Buffer + configureCommand(command, resolution, beadsapp.ExecStreams{Stdin: streams.Stdin, Stdout: &stdout, Stderr: &stderr}) + runErr := command.Run() + writeTrimmedLine(streams.Stderr, stderr.String()) + if runErr != nil { + return exitError(runErr) + } + canonical, err := beadsapp.CanonicalizeBDReadJSON(args[0], stdout.Bytes()) + if err != nil { + _, _ = streams.Stdout.Write(stdout.Bytes()) + return nil + } + fmt.Fprintln(streams.Stdout, string(canonical)) + return nil +} + +func configureCommand(command *exec.Cmd, resolution beadsapp.TrackerResolution, streams beadsapp.ExecStreams) { + command.Dir = resolution.WorkDir + command.Env = append([]string(nil), resolution.ChildEnv...) + command.Stdin = streams.Stdin + command.Stdout = streams.Stdout + command.Stderr = streams.Stderr +} + +func exitError(err error) error { + if err == nil { + return nil + } + var processExit *exec.ExitError + if errors.As(err, &processExit) { + return &beadsapp.ExitError{Code: processExit.ExitCode()} + } + return err +} + +func writeTrimmedLine(writer interface{ Write([]byte) (int, error) }, value string) { + if writer == nil { + return + } + if message := strings.TrimSpace(value); message != "" { + _, _ = fmt.Fprintln(writer, message) + } +} + +var _ beadsapp.TrackerExecutor = (*Executor)(nil) diff --git a/cli/internal/beads/exec.go b/cli/internal/beads/exec.go index e122e6e30..da9110b3c 100644 --- a/cli/internal/beads/exec.go +++ b/cli/internal/beads/exec.go @@ -2,12 +2,31 @@ package beads import ( "bytes" + "context" "encoding/json" "fmt" + "io" "path/filepath" "strings" ) +type ExecStreams struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +type TrackerExecutor interface { + Execute(context.Context, []string, ExecStreams) error +} + +type ExitError struct { + Code int +} + +func (err *ExitError) Error() string { return "" } +func (err *ExitError) ExitCode() int { return err.Code } + func ChildEnvironment(base []string, resolution TrackerResolution) []string { child := make([]string, 0, len(base)+1) for _, entry := range base { From 574a377e56e4f6d546eedda7c134343fdeec687c Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:03:16 -0400 Subject: [PATCH 011/108] refactor(beads): delegate stale discovery through port (age-nw28h.2.1) --- cli/cmd/ao/beads_stale.go | 27 +++++--------------------- cli/internal/adapters/beads/tracker.go | 14 +++++++++++++ cli/internal/beads/stale.go | 25 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/cli/cmd/ao/beads_stale.go b/cli/cmd/ao/beads_stale.go index 156597f55..2ca77c077 100644 --- a/cli/cmd/ao/beads_stale.go +++ b/cli/cmd/ao/beads_stale.go @@ -15,9 +15,7 @@ package main import ( "context" "encoding/json" - "errors" "fmt" - "os/exec" "time" "github.com/spf13/cobra" @@ -67,15 +65,6 @@ func runBeadsStale(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() - raw, err := beadsStaleFetchCmd(ctx) - if err != nil { - return fmt.Errorf("br list: %w", err) - } - var beads []staleBeadRecord - if err := json.Unmarshal(raw, &beads); err != nil { - return fmt.Errorf("parse br list: %w", err) - } - now := time.Now().UTC() if beadsStaleNowOverride != "" { parsed, err := time.Parse(time.RFC3339, beadsStaleNowOverride) @@ -85,7 +74,10 @@ func runBeadsStale(cmd *cobra.Command, args []string) error { now = parsed.UTC() } - events := computeStaleEvents(beads, now, beadsStaleThresholdHours) + events, err := beadsapp.DetectStale(ctx, beadsapp.StaleSourceFunc(beadsStaleFetchCmd), now, beadsStaleThresholdHours) + if err != nil { + return fmt.Errorf("br list: %w", err) + } if beadsStaleJSON { out, err := json.Marshal(events) @@ -117,16 +109,7 @@ func runBeadsStale(cmd *cobra.Command, args []string) error { // beadsStaleFetchCmd is the seam for tests. Tests overwrite it to inject // canned `br list` output without touching a real br binary. var beadsStaleFetchCmd = func(ctx context.Context) ([]byte, error) { - cmd := beadsTrackerCommandContext(ctx, "list", "--status", "in_progress", "--json", "--limit", "500") - out, err := cmd.Output() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return nil, fmt.Errorf("br list exited %d: %s", exitErr.ExitCode(), string(exitErr.Stderr)) - } - return nil, err - } - return out, nil + return currentBeadsTracker().ListInProgress(ctx) } // computeStaleEvents derives the stale_detected event set for a slice of diff --git a/cli/internal/adapters/beads/tracker.go b/cli/internal/adapters/beads/tracker.go index ffa87c5d0..ef1753321 100644 --- a/cli/internal/adapters/beads/tracker.go +++ b/cli/internal/adapters/beads/tracker.go @@ -3,6 +3,7 @@ package beads import ( "context" + "errors" "fmt" "os" "os/exec" @@ -96,6 +97,18 @@ func (tracker *Tracker) Output(ctx context.Context, args ...string) ([]byte, err return command.Output() } +func (tracker *Tracker) ListInProgress(ctx context.Context) ([]byte, error) { + output, err := tracker.Output(ctx, "list", "--status", "in_progress", "--json", "--limit", "500") + if err == nil { + return output, nil + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return nil, fmt.Errorf("br list exited %d: %s", exitErr.ExitCode(), string(exitErr.Stderr)) + } + return nil, err +} + func (tracker *Tracker) context() (string, []string, error) { if tracker == nil || tracker.workingDirectory == nil || tracker.environment == nil || tracker.lookPath == nil { return "", nil, fmt.Errorf("beads tracker adapter is not configured") @@ -110,3 +123,4 @@ func (tracker *Tracker) context() (string, []string, error) { var _ beadsapp.TrackerResolver = (*Tracker)(nil) var _ beadsapp.TrackerClient = (*Tracker)(nil) var _ beadsapp.LedgerInspector = (*Tracker)(nil) +var _ beadsapp.StaleSource = (*Tracker)(nil) diff --git a/cli/internal/beads/stale.go b/cli/internal/beads/stale.go index ca0d217ae..bf93ed074 100644 --- a/cli/internal/beads/stale.go +++ b/cli/internal/beads/stale.go @@ -3,10 +3,35 @@ package beads import ( + "context" + "encoding/json" + "fmt" "sort" "time" ) +type StaleSource interface { + ListInProgress(context.Context) ([]byte, error) +} + +type StaleSourceFunc func(context.Context) ([]byte, error) + +func (function StaleSourceFunc) ListInProgress(ctx context.Context) ([]byte, error) { + return function(ctx) +} + +func DetectStale(ctx context.Context, source StaleSource, now time.Time, thresholdHours float64) ([]StaleEvent, error) { + raw, err := source.ListInProgress(ctx) + if err != nil { + return nil, err + } + var records []StaleBeadRecord + if err := json.Unmarshal(raw, &records); err != nil { + return nil, fmt.Errorf("parse br list: %w", err) + } + return ComputeStaleEvents(records, now.UTC(), thresholdHours), nil +} + // StaleBeadRecord is the subset of tracker list output needed for staleness. type StaleBeadRecord struct { ID string `json:"id"` From 921f532846013c4b45e8e032bc840fa8bae61928 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:04:36 -0400 Subject: [PATCH 012/108] refactor(beads): extract claim-transfer policy (age-nw28h.2.1) --- cli/cmd/ao/beads_resume.go | 94 +++--------------------------------- cli/internal/beads/resume.go | 80 ++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 88 deletions(-) create mode 100644 cli/internal/beads/resume.go diff --git a/cli/cmd/ao/beads_resume.go b/cli/cmd/ao/beads_resume.go index 6b3a89b35..9ffda0462 100644 --- a/cli/cmd/ao/beads_resume.go +++ b/cli/cmd/ao/beads_resume.go @@ -23,6 +23,8 @@ import ( "time" "github.com/spf13/cobra" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" ) var ( @@ -71,23 +73,7 @@ var beadsResumeShowFunc = func(ctx context.Context, beadID string) (staleBeadRec } return staleBeadRecord{}, err } - // br show --json may emit either an object or a 1-element array. - trimmed := bytes_trim_leading_ws(out) - if len(trimmed) > 0 && trimmed[0] == '[' { - var arr []staleBeadRecord - if err := json.Unmarshal(out, &arr); err != nil { - return staleBeadRecord{}, fmt.Errorf("parse br show array: %w", err) - } - if len(arr) == 0 { - return staleBeadRecord{}, fmt.Errorf("br show %s returned empty array", beadID) - } - return arr[0], nil - } - var rec staleBeadRecord - if err := json.Unmarshal(out, &rec); err != nil { - return staleBeadRecord{}, fmt.Errorf("parse br show object: %w", err) - } - return rec, nil + return beadsapp.ParseShownBead(out, beadID) } // beadsResumeClaimFunc is the test seam for performing the atomic update. @@ -180,33 +166,7 @@ func runBeadsResume(cmd *cobra.Command, args []string) error { if priorAgent == "" { priorAgent = "unknown" } - event := staleEvent{ - SchemaVersion: 1, - EventType: "claim_transferred", - BeadID: beadID, - DetectedAt: now.Format(time.RFC3339), - OriginalClaimant: staleAgent{ID: priorAgent}, - Evidence: staleEvidence{ - LastTouchTS: prior.UpdatedAt, - LastEvidenceEvent: "br update --claim", - }, - } - // Extra fields for the transferred variant are emitted via a wrapper — - // stale_detected and claim_transferred share the same Go type plus - // these post-fields. - transferred := struct { - staleEvent - NewClaimant staleAgent `json:"new_claimant"` - Transfer transferInfo `json:"transfer"` - }{ - staleEvent: event, - NewClaimant: staleAgent{ID: agent}, - Transfer: transferInfo{ - PriorRevision: fingerprint(prior), - NewRevision: fingerprint(posterior), - NotesAppended: false, - }, - } + transferred := beadsapp.BuildTransferredEvent(beadID, agent, prior, posterior, now) // 7. Resolve ledger path relative to repo root. cwd, err := os.Getwd() @@ -244,53 +204,11 @@ func runBeadsResume(cmd *cobra.Command, args []string) error { } // transferInfo mirrors the `transfer` sub-object in stale-claim-event.v1. -type transferInfo struct { - PriorRevision string `json:"prior_revision"` - NewRevision string `json:"new_revision"` - NotesAppended bool `json:"notes_appended"` -} +type transferInfo = beadsapp.TransferInfo // fingerprint produces a compact, stable revision token from (assignee, // updated_at). br itself does not expose an etag; (assignee, updated_at) // changes on every claim/update so it serves as the audit fingerprint. func fingerprint(r staleBeadRecord) string { - if r.Assignee == "" && r.UpdatedAt == "" { - return "unset" - } - a := r.Assignee - if a == "" { - a = "_" - } - u := r.UpdatedAt - if u == "" { - u = "_" - } - return a + "@" + u -} - -// bytes_trim_leading_ws / trailing_ws — tiny local helpers to avoid pulling -// extra packages. Whitespace-trim only (no full unicode). -func bytes_trim_leading_ws(b []byte) []byte { - i := 0 - for i < len(b) { - c := b[i] - if c == ' ' || c == '\t' || c == '\n' || c == '\r' { - i++ - continue - } - break - } - return b[i:] -} -func bytes_trim_trailing_ws(b []byte) []byte { - j := len(b) - for j > 0 { - c := b[j-1] - if c == ' ' || c == '\t' || c == '\n' || c == '\r' { - j-- - continue - } - break - } - return b[:j] + return beadsapp.Fingerprint(r) } diff --git a/cli/internal/beads/resume.go b/cli/internal/beads/resume.go new file mode 100644 index 000000000..c7a62ab36 --- /dev/null +++ b/cli/internal/beads/resume.go @@ -0,0 +1,80 @@ +package beads + +import ( + "bytes" + "encoding/json" + "fmt" + "time" +) + +type TransferInfo struct { + PriorRevision string `json:"prior_revision"` + NewRevision string `json:"new_revision"` + NotesAppended bool `json:"notes_appended"` +} + +type TransferredEvent struct { + StaleEvent + NewClaimant StaleAgent `json:"new_claimant"` + Transfer TransferInfo `json:"transfer"` +} + +func ParseShownBead(raw []byte, beadID string) (StaleBeadRecord, error) { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) > 0 && trimmed[0] == '[' { + var records []StaleBeadRecord + if err := json.Unmarshal(raw, &records); err != nil { + return StaleBeadRecord{}, fmt.Errorf("parse br show array: %w", err) + } + if len(records) == 0 { + return StaleBeadRecord{}, fmt.Errorf("br show %s returned empty array", beadID) + } + return records[0], nil + } + var record StaleBeadRecord + if err := json.Unmarshal(raw, &record); err != nil { + return StaleBeadRecord{}, fmt.Errorf("parse br show object: %w", err) + } + return record, nil +} + +func BuildTransferredEvent(beadID, agent string, prior, posterior StaleBeadRecord, now time.Time) TransferredEvent { + priorAgent := prior.Assignee + if priorAgent == "" { + priorAgent = "unknown" + } + return TransferredEvent{ + StaleEvent: StaleEvent{ + SchemaVersion: 1, + EventType: "claim_transferred", + BeadID: beadID, + DetectedAt: now.UTC().Format(time.RFC3339), + OriginalClaimant: StaleAgent{ID: priorAgent}, + Evidence: StaleEvidence{ + LastTouchTS: prior.UpdatedAt, + LastEvidenceEvent: "br update --claim", + }, + }, + NewClaimant: StaleAgent{ID: agent}, + Transfer: TransferInfo{ + PriorRevision: Fingerprint(prior), + NewRevision: Fingerprint(posterior), + NotesAppended: false, + }, + } +} + +func Fingerprint(record StaleBeadRecord) string { + if record.Assignee == "" && record.UpdatedAt == "" { + return "unset" + } + assignee := record.Assignee + if assignee == "" { + assignee = "_" + } + updatedAt := record.UpdatedAt + if updatedAt == "" { + updatedAt = "_" + } + return assignee + "@" + updatedAt +} From d701f004a4530f6f53a08d1c1c829db4ba0df796 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:05:46 -0400 Subject: [PATCH 013/108] refactor(beads): isolate claim-transfer effects (age-nw28h.2.1) --- cli/cmd/ao/beads_resume.go | 58 +++++--------------------- cli/internal/adapters/beads/runtime.go | 45 ++++++++++++++++++++ cli/internal/adapters/beads/tracker.go | 31 ++++++++++++++ 3 files changed, 86 insertions(+), 48 deletions(-) create mode 100644 cli/internal/adapters/beads/runtime.go diff --git a/cli/cmd/ao/beads_resume.go b/cli/cmd/ao/beads_resume.go index 9ffda0462..dae0b4b44 100644 --- a/cli/cmd/ao/beads_resume.go +++ b/cli/cmd/ao/beads_resume.go @@ -15,15 +15,12 @@ package main import ( "context" "encoding/json" - "errors" "fmt" - "os" - "os/exec" - "path/filepath" "time" "github.com/spf13/cobra" + beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" beadsapp "github.com/boshu2/agentops/cli/internal/beads" ) @@ -65,30 +62,13 @@ func init() { // (assignee + updated_at) BEFORE the claim transfer, so we can record the // prior revision. Production: shells out to `br show --json`. var beadsResumeShowFunc = func(ctx context.Context, beadID string) (staleBeadRecord, error) { - out, err := beadsTrackerCommandContext(ctx, "show", beadID, "--json").Output() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return staleBeadRecord{}, fmt.Errorf("br show %s exited %d: %s", beadID, exitErr.ExitCode(), string(exitErr.Stderr)) - } - return staleBeadRecord{}, err - } - return beadsapp.ParseShownBead(out, beadID) + return currentBeadsTracker().Show(ctx, beadID) } // beadsResumeClaimFunc is the test seam for performing the atomic update. // Production: `br update --claim --actor `. var beadsResumeClaimFunc = func(ctx context.Context, beadID, agent string) error { - args := []string{"update", beadID, "--claim"} - if agent != "" { - args = append(args, "--actor", agent) - } - cmd := beadsTrackerCommandContext(ctx, args...) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("br update --claim failed: %w: %s", err, string(out)) - } - return nil + return currentBeadsTracker().Claim(ctx, beadID, agent) } // beadsResumeAppendLedger is the test seam for writing the provenance event. @@ -97,21 +77,11 @@ var beadsResumeClaimFunc = func(ctx context.Context, beadID, agent string) error // extends staleEvent with new_claimant + transfer) without us introducing // an interface. var beadsResumeAppendLedger = func(ledgerPath string, event any) error { - if err := os.MkdirAll(filepath.Dir(ledgerPath), 0o755); err != nil { - return fmt.Errorf("mkdir ledger dir: %w", err) - } - f, err := os.OpenFile(ledgerPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - return fmt.Errorf("open ledger: %w", err) - } - defer func() { _ = f.Close() }() - enc := json.NewEncoder(f) - if err := enc.Encode(event); err != nil { - return fmt.Errorf("encode event: %w", err) - } - return nil + return currentBeadsRuntime().AppendEvent(ledgerPath, event) } +func currentBeadsRuntime() beadsadapter.Runtime { return beadsadapter.NewRuntime() } + func runBeadsResume(cmd *cobra.Command, args []string) error { beadID := args[0] if beadID == "" { @@ -131,7 +101,7 @@ func runBeadsResume(cmd *cobra.Command, args []string) error { } // 2. Compute now (test override OK). - now := time.Now().UTC() + now := currentBeadsRuntime().Now() if beadsResumeNowOverride != "" { parsed, err := time.Parse(time.RFC3339, beadsResumeNowOverride) if err != nil { @@ -143,7 +113,7 @@ func runBeadsResume(cmd *cobra.Command, args []string) error { // 3. Resolve the new claimant id. agent := beadsResumeAgentID if agent == "" { - agent = os.Getenv("BEADS_ACTOR") + agent = currentBeadsRuntime().Actor() } if agent == "" { agent = "ao-beads-resume" @@ -169,17 +139,9 @@ func runBeadsResume(cmd *cobra.Command, args []string) error { transferred := beadsapp.BuildTransferredEvent(beadID, agent, prior, posterior, now) // 7. Resolve ledger path relative to repo root. - cwd, err := os.Getwd() + ledger, err := currentBeadsRuntime().ResolveRepoPath(beadsResumeLedgerPath) if err != nil { - return fmt.Errorf("resolve cwd: %w", err) - } - root, err := repoRootForBeads(cwd) - if err != nil { - return fmt.Errorf("resolve repo root: %w", err) - } - ledger := beadsResumeLedgerPath - if !filepath.IsAbs(ledger) { - ledger = filepath.Join(root, ledger) + return err } // Write the full claim_transferred shape (with new_claimant + transfer). diff --git a/cli/internal/adapters/beads/runtime.go b/cli/internal/adapters/beads/runtime.go new file mode 100644 index 000000000..60a6e4d61 --- /dev/null +++ b/cli/internal/adapters/beads/runtime.go @@ -0,0 +1,45 @@ +package beads + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/boshu2/agentops/cli/internal/trackerresolve" +) + +type Runtime struct{} + +func NewRuntime() Runtime { return Runtime{} } + +func (Runtime) Now() time.Time { return time.Now().UTC() } + +func (Runtime) Actor() string { return os.Getenv("BEADS_ACTOR") } + +func (Runtime) ResolveRepoPath(path string) (string, error) { + if filepath.IsAbs(path) { + return path, nil + } + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("resolve cwd: %w", err) + } + return filepath.Join(trackerresolve.RepoRoot(cwd), path), nil +} + +func (Runtime) AppendEvent(path string, event any) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("mkdir ledger dir: %w", err) + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return fmt.Errorf("open ledger: %w", err) + } + defer func() { _ = file.Close() }() + if err := json.NewEncoder(file).Encode(event); err != nil { + return fmt.Errorf("encode event: %w", err) + } + return nil +} diff --git a/cli/internal/adapters/beads/tracker.go b/cli/internal/adapters/beads/tracker.go index ef1753321..a3c6b038f 100644 --- a/cli/internal/adapters/beads/tracker.go +++ b/cli/internal/adapters/beads/tracker.go @@ -109,6 +109,37 @@ func (tracker *Tracker) ListInProgress(ctx context.Context) ([]byte, error) { return nil, err } +func (tracker *Tracker) Show(ctx context.Context, beadID string) (beadsapp.StaleBeadRecord, error) { + output, err := tracker.Output(ctx, "show", beadID, "--json") + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return beadsapp.StaleBeadRecord{}, fmt.Errorf("br show %s exited %d: %s", beadID, exitErr.ExitCode(), string(exitErr.Stderr)) + } + return beadsapp.StaleBeadRecord{}, err + } + return beadsapp.ParseShownBead(output, beadID) +} + +func (tracker *Tracker) Claim(ctx context.Context, beadID, agent string) error { + resolved, err := tracker.Resolve() + if err != nil { + return err + } + args := []string{"update", beadID, "--claim"} + if agent != "" { + args = append(args, "--actor", agent) + } + command := exec.CommandContext(ctx, resolved.Binary, args...) // #nosec G204 -- trackerresolve constrains the binary to br|bd. + command.Dir = resolved.WorkDir + command.Env = append([]string(nil), resolved.ChildEnv...) + output, err := command.CombinedOutput() + if err != nil { + return fmt.Errorf("br update --claim failed: %w: %s", err, string(output)) + } + return nil +} + func (tracker *Tracker) context() (string, []string, error) { if tracker == nil || tracker.workingDirectory == nil || tracker.environment == nil || tracker.lookPath == nil { return "", nil, fmt.Errorf("beads tracker adapter is not configured") From 4f52df71f545cfbcbd5b5d73a6feee1fa1c0c982 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:06:27 -0400 Subject: [PATCH 014/108] refactor(beads): isolate epic ledger read (age-nw28h.2.1) --- cli/cmd/ao/beads_epic_status.go | 9 ++++++--- cli/internal/adapters/beads/runtime.go | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cli/cmd/ao/beads_epic_status.go b/cli/cmd/ao/beads_epic_status.go index 2d38545a9..43f9b257f 100644 --- a/cli/cmd/ao/beads_epic_status.go +++ b/cli/cmd/ao/beads_epic_status.go @@ -24,7 +24,6 @@ package main import ( "encoding/json" "fmt" - "os" "path/filepath" "strings" @@ -82,7 +81,7 @@ type ledgerDep = beadsapp.LedgerDep // newline-delimited issues.jsonl bytes for the resolved ledger dir. Tests // override it to inject an in-memory ledger without touching a br/ao binary. var beadsEpicStatusReadLedger = func(dir string) ([]byte, error) { - return os.ReadFile(filepath.Join(dir, "issues.jsonl")) + return currentBeadsRuntime().ReadFile(filepath.Join(dir, "issues.jsonl")) } func runBeadsEpicStatus(cmd *cobra.Command, args []string) error { @@ -91,7 +90,11 @@ func runBeadsEpicStatus(cmd *cobra.Command, args []string) error { return fmt.Errorf("epic id is required") } - dir := resolveBeadsDir("", os.Environ()).Path + ledger, err := currentBeadsTracker().BRLedger() + if err != nil { + return err + } + dir := ledger.Path raw, err := beadsEpicStatusReadLedger(dir) if err != nil { return fmt.Errorf("read ledger %s: %w", filepath.Join(dir, "issues.jsonl"), err) diff --git a/cli/internal/adapters/beads/runtime.go b/cli/internal/adapters/beads/runtime.go index 60a6e4d61..893905822 100644 --- a/cli/internal/adapters/beads/runtime.go +++ b/cli/internal/adapters/beads/runtime.go @@ -43,3 +43,5 @@ func (Runtime) AppendEvent(path string, event any) error { } return nil } + +func (Runtime) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } From 64a7ab99fab161b997ad88f3102b0f23b5cb90b0 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:11:14 -0400 Subject: [PATCH 015/108] refactor(beads): extract knowledge policy (age-nw28h.2.1) --- cli/cmd/ao/beads.go | 250 ++------------------------------ cli/internal/beads/knowledge.go | 239 ++++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+), 234 deletions(-) create mode 100644 cli/internal/beads/knowledge.go diff --git a/cli/cmd/ao/beads.go b/cli/cmd/ao/beads.go index f4b11a67a..54adba732 100644 --- a/cli/cmd/ao/beads.go +++ b/cli/cmd/ao/beads.go @@ -24,13 +24,11 @@ package main import ( "context" "encoding/json" - "errors" "fmt" "os" "os/exec" "path/filepath" "regexp" - "sort" "strings" "time" @@ -306,34 +304,19 @@ func runBeadsTrackerCmd(cmd *cobra.Command, _ []string) error { // CitationStatus is the three-valued verdict for a single citation extracted // from a bead description. -type CitationStatus string +type CitationStatus = beadsapp.CitationStatus const ( - CitationFresh CitationStatus = "FRESH" - CitationStale CitationStatus = "STALE" - CitationUnknown CitationStatus = "UNKNOWN" + CitationFresh = beadsapp.CitationFresh + CitationStale = beadsapp.CitationStale + CitationUnknown = beadsapp.CitationUnknown ) // Citation is a single verifiable reference pulled from a bead description. -type Citation struct { - Kind string `json:"kind"` // "file", "function", "symbol" - Raw string `json:"raw"` // verbatim text from description - Status CitationStatus `json:"status"` // FRESH / STALE / UNKNOWN - Reason string `json:"reason"` // human-readable explanation - Resolved string `json:"resolved"` // HEAD location if resolved differently -} +type Citation = beadsapp.Citation // VerifyReport is the structured result of `ao beads verify`. -type VerifyReport struct { - BeadID string `json:"bead_id"` - Title string `json:"title"` - Status string `json:"status"` - Citations []Citation `json:"citations"` - StaleCount int `json:"stale_count"` - FreshCount int `json:"fresh_count"` - TotalCount int `json:"total_count"` - BDAvailable bool `json:"bd_available"` -} +type VerifyReport = beadsapp.VerifyReport func runBeadsVerify(cmd *cobra.Command, args []string) error { beadID := args[0] @@ -405,22 +388,7 @@ func verifyBead(beadID string) (*VerifyReport, error) { // typically hides the original description and surfaces the operator's // `Close reason:` line instead. `harvest` wants the close reason; `verify` // wants whichever is present. The Body accessor returns the first non-empty. -type bdShowParsed struct { - ID string - Title string - Status string - Description string // DESCRIPTION-heading body (open beads, empty on closed) - CloseReason string // "Close reason:" line (closed beads only) -} - -// Body returns the non-empty body to use for citation extraction / harvest. -// Prefers CloseReason (closed beads) and falls back to Description. -func (p *bdShowParsed) Body() string { - if p.CloseReason != "" { - return p.CloseReason - } - return p.Description -} +type bdShowParsed = beadsapp.ParsedBead // parseBDShow parses the human-readable `bd show ` output. Observed // formats (2026-04-11): @@ -442,46 +410,7 @@ func (p *bdShowParsed) Body() string { // captured into CloseReason. The DESCRIPTION body is captured into // Description. func parseBDShow(raw string) (*bdShowParsed, error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return nil, errors.New("empty bd show output") - } - out := &bdShowParsed{} - lines := strings.Split(raw, "\n") - headerRe := regexp.MustCompile(`^[○●✓]?\s*(\S+)\s*·\s*(.*?)\s*\[([^\[\]]*)\]\s*$`) - for i, line := range lines { - if out.ID == "" { - if m := headerRe.FindStringSubmatch(line); m != nil { - out.ID = strings.TrimSpace(m[1]) - out.Title = strings.TrimSpace(m[2]) - out.Status = strings.TrimSpace(m[3]) - continue - } - } - if strings.HasPrefix(line, "Close reason:") { - out.CloseReason = strings.TrimSpace(strings.TrimPrefix(line, "Close reason:")) - continue - } - if strings.HasPrefix(line, "DESCRIPTION") { - // Everything until EOF or [rerun: ...] sentinel. - tail := strings.Join(lines[i+1:], "\n") - if idx := strings.LastIndex(tail, "\n[rerun:"); idx >= 0 { - tail = tail[:idx] - } - // Also strip a leading [rerun:] if the description body is empty - // (closed beads often have just this marker). - tail = strings.TrimSpace(tail) - if strings.HasPrefix(tail, "[rerun:") { - tail = "" - } - out.Description = tail - break - } - } - if out.ID == "" && out.Description == "" && out.CloseReason == "" { - return nil, fmt.Errorf("could not parse bd show output: %q", beadTruncate(raw, 80)) - } - return out, nil + return beadsapp.ParseBDShow(raw) } // extractCitations pulls verifiable references out of a description body. @@ -490,66 +419,7 @@ func parseBDShow(raw string) (*bdShowParsed, error) { // - Go function references (`func Name(` or `type.Method(`) // - Backticked symbols that look like identifiers func extractCitations(desc string) []Citation { - var out []Citation - seen := make(map[string]bool) - - // File paths. Accept common source/doc extensions. Allow an optional - // ":line" suffix. Lead-dot paths (.agents/, .github/) are captured by - // requiring the character before the match to be non-ident, then - // including the dot in the path prefix. - fileRe := regexp.MustCompile(`(?:^|[^\w.])([.\w][\w./-]*\.(?:go|py|sh|md|yaml|yml|json|ts|tsx|js|jsx|rs|toml))(?::(\d+))?`) - for _, m := range fileRe.FindAllStringSubmatch(desc, -1) { - path := m[1] - line := m[2] - raw := path - if line != "" { - raw = path + ":" + line - } - key := "file:" + raw - if seen[key] { - continue - } - seen[key] = true - out = append(out, Citation{Kind: "file", Raw: raw}) - } - - // Go function citations: `func Name` or `func (r *T) Name`. - funcRe := regexp.MustCompile(`\bfunc\s+(?:\([^)]*\)\s*)?([A-Za-z_]\w*)`) - for _, m := range funcRe.FindAllStringSubmatch(desc, -1) { - raw := "func " + m[1] - key := "func:" + m[1] - if seen[key] { - continue - } - seen[key] = true - out = append(out, Citation{Kind: "function", Raw: raw}) - } - - // Backticked symbols that look like identifiers (not arbitrary code blocks). - // Require at least one alpha char and only ident-safe chars. - backtickRe := regexp.MustCompile("`([A-Za-z_][\\w.]{2,})`") - for _, m := range backtickRe.FindAllStringSubmatch(desc, -1) { - sym := m[1] - // Skip things that look like file paths (already handled) or numbers. - if strings.ContainsAny(sym, "/") { - continue - } - key := "sym:" + sym - if seen[key] { - continue - } - seen[key] = true - out = append(out, Citation{Kind: "symbol", Raw: "`" + sym + "`"}) - } - - // Deterministic ordering for test stability. - sort.SliceStable(out, func(i, j int) bool { - if out[i].Kind != out[j].Kind { - return out[i].Kind < out[j].Kind - } - return out[i].Raw < out[j].Raw - }) - return out + return beadsapp.ExtractCitations(desc) } // verifyCitationInPlace checks a single citation against the repo state at @@ -748,14 +618,7 @@ func emitVerifyHuman(w *os.File, r *VerifyReport, verbose bool) { // ------------------------------------------------------------------------ // LintReport is the aggregate result of `ao beads lint`. -type LintReport struct { - StatusFilter string `json:"status_filter"` - TotalBeads int `json:"total_beads"` - CleanBeads int `json:"clean_beads"` - StaleBeads int `json:"stale_beads"` - ErrorBeads int `json:"error_beads"` - PerBead []VerifyReport `json:"per_bead"` -} +type LintReport = beadsapp.LintReport func runBeadsLint(cmd *cobra.Command, args []string) error { if !bdAvailable() { @@ -814,31 +677,7 @@ func listBeadIDs(status string) ([]string, error) { if err != nil { return nil, fmt.Errorf("bd list: %w", err) } - // Bead ID grammar: 2-4 letter rig, dash, then ident chars. Examples: - // na-h61, na-348.1, ocpcm2-abc. - idRe := regexp.MustCompile(`\b([a-z]{2,6}-[0-9a-z][\w.]*)\b`) - seen := make(map[string]bool) - var ids []string - for _, line := range strings.Split(string(raw), "\n") { - // Skip header/footer lines that might match incidentally — we want - // only lines that start with a bullet or tree char. - trimmed := strings.TrimLeft(line, " \t├─└│") - if trimmed == "" { - continue - } - firstRune := []rune(trimmed)[0] - if firstRune != '○' && firstRune != '●' && firstRune != '✓' { - continue - } - if m := idRe.FindStringSubmatch(line); m != nil { - id := m[1] - if !seen[id] { - seen[id] = true - ids = append(ids, id) - } - } - } - return ids, nil + return beadsapp.ParseBeadIDs(raw), nil } func emitLintHuman(w *os.File, r *LintReport) { @@ -865,15 +704,7 @@ func emitLintHuman(w *os.File, r *LintReport) { // LearningFrontmatter is the yaml frontmatter block written to the top of // each materialised learning file. Intentionally minimal — downstream // reducers handle enrichment. -type LearningFrontmatter struct { - Title string `json:"title" yaml:"title"` - BeadID string `json:"bead_id" yaml:"bead_id"` - Source string `json:"source" yaml:"source"` // "bd-close" - Date string `json:"date" yaml:"date"` - Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` - Maturity string `json:"maturity" yaml:"maturity"` // "provisional" — fresh harvest - Provenance string `json:"provenance" yaml:"provenance"` -} +type LearningFrontmatter = beadsapp.LearningFrontmatter func runBeadsHarvest(cmd *cobra.Command, args []string) error { beadID := args[0] @@ -932,66 +763,20 @@ func runBeadsHarvest(cmd *cobra.Command, args []string) error { // including YAML frontmatter and the closure reason as the primary learning // content. func renderLearningBody(fm LearningFrontmatter, parsed *bdShowParsed) string { - var b strings.Builder - b.WriteString("---\n") - fmt.Fprintf(&b, "title: %q\n", fm.Title) - fmt.Fprintf(&b, "bead_id: %s\n", fm.BeadID) - fmt.Fprintf(&b, "source: %s\n", fm.Source) - fmt.Fprintf(&b, "date: %s\n", fm.Date) - fmt.Fprintf(&b, "maturity: %s\n", fm.Maturity) - fmt.Fprintf(&b, "provenance: %q\n", fm.Provenance) - b.WriteString("tags:\n") - for _, t := range fm.Tags { - fmt.Fprintf(&b, " - %s\n", t) - } - b.WriteString("---\n\n") - fmt.Fprintf(&b, "# %s\n\n", fm.Title) - fmt.Fprintf(&b, "Harvested from closed bead [%s] on %s.\n\n", fm.BeadID, fm.Date) - b.WriteString("## Closure reason\n\n") - b.WriteString(parsed.Body()) - b.WriteString("\n") - return b.String() + return beadsapp.RenderLearningBody(fm, parsed) } // isClosedStatus is tolerant of the various ways bd might render a closed // state. Uses substring matching because the real status field often // includes priority and type tokens (e.g., "● P2 · CLOSED" or "CLOSED P1 task"). func isClosedStatus(status string) bool { - s := strings.ToUpper(strings.TrimSpace(status)) - for _, tok := range []string{"CLOSED", "DONE", "RESOLVED"} { - if strings.Contains(s, tok) { - return true - } - } - return false + return beadsapp.IsClosedStatus(status) } // beadSlugify converts a free-text title into a filesystem-safe kebab-case slug // capped at maxLen characters. func beadSlugify(title string, maxLen int) string { - s := strings.ToLower(title) - var b strings.Builder - lastDash := false - for _, r := range s { - switch { - case r >= 'a' && r <= 'z', r >= '0' && r <= '9': - b.WriteRune(r) - lastDash = false - default: - if !lastDash && b.Len() > 0 { - b.WriteRune('-') - lastDash = true - } - } - } - out := strings.TrimRight(b.String(), "-") - if len(out) > maxLen { - out = strings.TrimRight(out[:maxLen], "-") - } - if out == "" { - out = "untitled" - } - return out + return beadsapp.Slugify(title, maxLen) } // ------------------------------------------------------------------------ @@ -1005,8 +790,5 @@ func emitJSON(w *os.File, v any) error { } func beadTruncate(s string, n int) string { - if len(s) <= n { - return s - } - return s[:n] + "..." + return beadsapp.Truncate(s, n) } diff --git a/cli/internal/beads/knowledge.go b/cli/internal/beads/knowledge.go new file mode 100644 index 000000000..6149c28ac --- /dev/null +++ b/cli/internal/beads/knowledge.go @@ -0,0 +1,239 @@ +package beads + +import ( + "errors" + "fmt" + "regexp" + "sort" + "strings" +) + +type CitationStatus string + +const ( + CitationFresh CitationStatus = "FRESH" + CitationStale CitationStatus = "STALE" + CitationUnknown CitationStatus = "UNKNOWN" +) + +type Citation struct { + Kind string `json:"kind"` + Raw string `json:"raw"` + Status CitationStatus `json:"status"` + Reason string `json:"reason"` + Resolved string `json:"resolved"` +} + +type VerifyReport struct { + BeadID string `json:"bead_id"` + Title string `json:"title"` + Status string `json:"status"` + Citations []Citation `json:"citations"` + StaleCount int `json:"stale_count"` + FreshCount int `json:"fresh_count"` + TotalCount int `json:"total_count"` + BDAvailable bool `json:"bd_available"` +} + +type ParsedBead struct { + ID string + Title string + Status string + Description string + CloseReason string +} + +func (bead *ParsedBead) Body() string { + if bead.CloseReason != "" { + return bead.CloseReason + } + return bead.Description +} + +func ParseBDShow(raw string) (*ParsedBead, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, errors.New("empty bd show output") + } + parsed := &ParsedBead{} + lines := strings.Split(raw, "\n") + header := regexp.MustCompile(`^[○●✓]?\s*(\S+)\s*·\s*(.*?)\s*\[([^\[\]]*)\]\s*$`) + for index, line := range lines { + if parsed.ID == "" { + if match := header.FindStringSubmatch(line); match != nil { + parsed.ID = strings.TrimSpace(match[1]) + parsed.Title = strings.TrimSpace(match[2]) + parsed.Status = strings.TrimSpace(match[3]) + continue + } + } + if strings.HasPrefix(line, "Close reason:") { + parsed.CloseReason = strings.TrimSpace(strings.TrimPrefix(line, "Close reason:")) + continue + } + if strings.HasPrefix(line, "DESCRIPTION") { + tail := strings.Join(lines[index+1:], "\n") + if marker := strings.LastIndex(tail, "\n[rerun:"); marker >= 0 { + tail = tail[:marker] + } + tail = strings.TrimSpace(tail) + if strings.HasPrefix(tail, "[rerun:") { + tail = "" + } + parsed.Description = tail + break + } + } + if parsed.ID == "" && parsed.Description == "" && parsed.CloseReason == "" { + return nil, fmt.Errorf("could not parse bd show output: %q", Truncate(raw, 80)) + } + return parsed, nil +} + +func ExtractCitations(description string) []Citation { + var citations []Citation + seen := make(map[string]bool) + filePattern := regexp.MustCompile(`(?:^|[^\w.])([.\w][\w./-]*\.(?:go|py|sh|md|yaml|yml|json|ts|tsx|js|jsx|rs|toml))(?::(\d+))?`) + for _, match := range filePattern.FindAllStringSubmatch(description, -1) { + raw := match[1] + if match[2] != "" { + raw += ":" + match[2] + } + key := "file:" + raw + if !seen[key] { + seen[key] = true + citations = append(citations, Citation{Kind: "file", Raw: raw}) + } + } + functionPattern := regexp.MustCompile(`\bfunc\s+(?:\([^)]*\)\s*)?([A-Za-z_]\w*)`) + for _, match := range functionPattern.FindAllStringSubmatch(description, -1) { + key := "func:" + match[1] + if !seen[key] { + seen[key] = true + citations = append(citations, Citation{Kind: "function", Raw: "func " + match[1]}) + } + } + backtickPattern := regexp.MustCompile("`([A-Za-z_][\\w.]{2,})`") + for _, match := range backtickPattern.FindAllStringSubmatch(description, -1) { + symbol := match[1] + if strings.Contains(symbol, "/") { + continue + } + key := "sym:" + symbol + if !seen[key] { + seen[key] = true + citations = append(citations, Citation{Kind: "symbol", Raw: "`" + symbol + "`"}) + } + } + sort.SliceStable(citations, func(left, right int) bool { + if citations[left].Kind != citations[right].Kind { + return citations[left].Kind < citations[right].Kind + } + return citations[left].Raw < citations[right].Raw + }) + return citations +} + +type LintReport struct { + StatusFilter string `json:"status_filter"` + TotalBeads int `json:"total_beads"` + CleanBeads int `json:"clean_beads"` + StaleBeads int `json:"stale_beads"` + ErrorBeads int `json:"error_beads"` + PerBead []VerifyReport `json:"per_bead"` +} + +func ParseBeadIDs(raw []byte) []string { + pattern := regexp.MustCompile(`\b([a-z]{2,6}-[0-9a-z][\w.]*)\b`) + seen := make(map[string]bool) + var ids []string + for _, line := range strings.Split(string(raw), "\n") { + trimmed := strings.TrimLeft(line, " \t├─└│") + if trimmed == "" { + continue + } + first := []rune(trimmed)[0] + if first != '○' && first != '●' && first != '✓' { + continue + } + if match := pattern.FindStringSubmatch(line); match != nil && !seen[match[1]] { + seen[match[1]] = true + ids = append(ids, match[1]) + } + } + return ids +} + +type LearningFrontmatter struct { + Title string `json:"title" yaml:"title"` + BeadID string `json:"bead_id" yaml:"bead_id"` + Source string `json:"source" yaml:"source"` + Date string `json:"date" yaml:"date"` + Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` + Maturity string `json:"maturity" yaml:"maturity"` + Provenance string `json:"provenance" yaml:"provenance"` +} + +func RenderLearningBody(frontmatter LearningFrontmatter, parsed *ParsedBead) string { + var body strings.Builder + body.WriteString("---\n") + fmt.Fprintf(&body, "title: %q\n", frontmatter.Title) + fmt.Fprintf(&body, "bead_id: %s\n", frontmatter.BeadID) + fmt.Fprintf(&body, "source: %s\n", frontmatter.Source) + fmt.Fprintf(&body, "date: %s\n", frontmatter.Date) + fmt.Fprintf(&body, "maturity: %s\n", frontmatter.Maturity) + fmt.Fprintf(&body, "provenance: %q\n", frontmatter.Provenance) + body.WriteString("tags:\n") + for _, tag := range frontmatter.Tags { + fmt.Fprintf(&body, " - %s\n", tag) + } + body.WriteString("---\n\n") + fmt.Fprintf(&body, "# %s\n\n", frontmatter.Title) + fmt.Fprintf(&body, "Harvested from closed bead [%s] on %s.\n\n", frontmatter.BeadID, frontmatter.Date) + body.WriteString("## Closure reason\n\n") + body.WriteString(parsed.Body()) + body.WriteString("\n") + return body.String() +} + +func IsClosedStatus(status string) bool { + normalized := strings.ToUpper(strings.TrimSpace(status)) + for _, token := range []string{"CLOSED", "DONE", "RESOLVED"} { + if strings.Contains(normalized, token) { + return true + } + } + return false +} + +func Slugify(title string, maxLength int) string { + var slug strings.Builder + lastDash := false + for _, character := range strings.ToLower(title) { + switch { + case character >= 'a' && character <= 'z', character >= '0' && character <= '9': + slug.WriteRune(character) + lastDash = false + default: + if !lastDash && slug.Len() > 0 { + slug.WriteRune('-') + lastDash = true + } + } + } + value := strings.TrimRight(slug.String(), "-") + if len(value) > maxLength { + value = strings.TrimRight(value[:maxLength], "-") + } + if value == "" { + return "untitled" + } + return value +} + +func Truncate(value string, length int) string { + if len(value) <= length { + return value + } + return value[:length] + "..." +} From 84e5926b70bbd347816bdaf9a1058958bd0cdb9b Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:17:10 -0400 Subject: [PATCH 016/108] refactor(beads): introduce explicit command module (age-nw28h.2.1) --- cli/internal/commands/beads/module.go | 273 +++++++++++++++++++++ cli/internal/commands/beads/module_test.go | 59 +++++ 2 files changed, 332 insertions(+) create mode 100644 cli/internal/commands/beads/module.go create mode 100644 cli/internal/commands/beads/module_test.go diff --git a/cli/internal/commands/beads/module.go b/cli/internal/commands/beads/module.go new file mode 100644 index 000000000..66b4d3439 --- /dev/null +++ b/cli/internal/commands/beads/module.go @@ -0,0 +1,273 @@ +// Package beads owns the Cobra presentation for the beads command family. +// Behavior is delegated through Runner so tracker, filesystem, process, clock, +// and environment effects remain behind driven adapters. +package beads + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/boshu2/agentops/cli/internal/clicontract" +) + +type Operation string + +const ( + OperationDir Operation = "dir" + OperationTracker Operation = "tracker" + OperationVerify Operation = "verify" + OperationLint Operation = "lint" + OperationHarvest Operation = "harvest" + OperationAudit Operation = "audit" + OperationCluster Operation = "cluster" + OperationExec Operation = "exec" + OperationResume Operation = "resume" + OperationScenariosExtract Operation = "scenarios.extract" + OperationScenariosCheck Operation = "scenarios.validate" + OperationStaleClaims Operation = "stale-claims" + OperationEpicStatus Operation = "epic-status" + OperationAcceptance Operation = "verify-acceptance" +) + +type Invocation struct { + Operation Operation + Args []string + Options Options +} + +type Options struct { + JSON bool + Require bool + Verbose bool + Status string + OutputDirectory string + DryRun bool + Strict bool + AutoClose bool + Apply bool + Agent string + Ledger string + Force bool + Write bool + ThresholdHours float64 + Terminal bool +} + +// Runner is the command family's inbound application port. The command module +// owns parsing; the implementation owns use-case orchestration and effects. +type Runner interface { + Run(*cobra.Command, Invocation) error +} + +type Module struct { + runner Runner +} + +func NewModule(runner Runner) Module { + return Module{runner: runner} +} + +func (Module) Contract() clicontract.CommandContract { + return clicontract.CommandContract{ + ID: "ao.beads", + Profiles: clicontract.ProfileDefault | + clicontract.ProfileFlywheel | + clicontract.ProfileLegacy | + clicontract.ProfileCombined, + Args: clicontract.ArgsPolicy{Name: "subcommands-only", Validate: cobra.NoArgs}, + Output: clicontract.OutputNone, + Effects: clicontract.EffectPure, + ExitClasses: map[int]clicontract.ExitClass{0: clicontract.ExitSuccess, 1: clicontract.ExitFailure}, + } +} + +func (module Module) Command() *cobra.Command { + root := &cobra.Command{ + Use: "beads", + Short: "Complementary tooling for the bd (beads) issue tracker", + Args: cobra.NoArgs, + Long: `Commands that help maintain the bd issue tracker alongside the main +bd CLI. These tools focus on catching stale descriptions before a new +session acts on them and harvesting closure reasons into durable learnings. + +None of these commands replace bd itself — they complement it.`, + } + root.AddCommand( + module.dirCommand(), + module.trackerCommand(), + module.verifyCommand(), + module.lintCommand(), + module.harvestCommand(), + module.auditCommand(), + module.clusterCommand(), + module.execCommand(), + module.resumeCommand(), + module.scenariosCommand(), + module.staleCommand(), + module.epicStatusCommand(), + module.acceptanceCommand(), + ) + return root +} + +func (module Module) invoke(command *cobra.Command, operation Operation, args []string, options Options) error { + if module.runner == nil { + return fmt.Errorf("beads command runner is not configured") + } + return module.runner.Run(command, Invocation{Operation: operation, Args: append([]string(nil), args...), Options: options}) +} + +func (module Module) dirCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "dir", Short: "Print the resolved live br ledger directory", Args: cobra.NoArgs} + command.Flags().BoolVar(&options.JSON, "json", false, "Emit {beads_dir, source} as JSON") + command.Flags().BoolVar(&options.Require, "require", false, "Fail closed: exit non-zero (printing nothing to stdout) unless the resolved directory holds a br ledger") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationDir, args, options) + } + return command +} + +func (module Module) trackerCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "tracker", Short: "Print the resolved beads tracker (bd or br) for this environment", Args: cobra.NoArgs} + command.Flags().BoolVar(&options.JSON, "json", false, "Emit {tracker, binary, ledger_dir, source} as JSON") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationTracker, args, options) + } + return command +} + +func (module Module) verifyCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "verify ", Short: "Detect stale citations in a bead description (files, functions, symbols)", Args: cobra.ExactArgs(1)} + command.Flags().BoolVar(&options.JSON, "json", false, "Emit verification report as JSON instead of human-readable text") + command.Flags().BoolVar(&options.Verbose, "verbose", false, "Include FRESH citations in the output (default: stale only)") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationVerify, args, options) + } + return command +} + +func (module Module) lintCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "lint", Short: "Batch-verify every open bead (or filtered set) against HEAD"} + command.Flags().StringVar(&options.Status, "status", "open", "bd status filter (open, closed, all)") + command.Flags().BoolVar(&options.JSON, "json", false, "Emit lint report as JSON") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationLint, args, options) + } + return command +} + +func (module Module) harvestCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "harvest ", Short: "Materialize a closed bead's reason as a structured learning file", Args: cobra.ExactArgs(1)} + command.Flags().StringVar(&options.OutputDirectory, "out-dir", ".agents/learnings", "Directory to write the learning file into") + command.Flags().BoolVar(&options.DryRun, "dry-run", false, "Print the learning content to stdout without writing a file") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationHarvest, args, options) + } + return command +} + +func (module Module) auditCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "audit", Short: "Audit open beads for likely-fixed, stale, or consolidatable work"} + command.Flags().BoolVar(&options.JSON, "json", false, "Emit audit report as JSON") + command.Flags().BoolVar(&options.Strict, "strict", false, "Exit 1 when any likely-fixed, likely-stale, or consolidatable bead is found") + command.Flags().BoolVar(&options.AutoClose, "auto-close", false, "Close likely-fixed beads when commit or file-change evidence is found") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationAudit, args, options) + } + return command +} + +func (module Module) clusterCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "cluster", Short: "Suggest consolidation clusters for overlapping open beads"} + command.Flags().BoolVar(&options.JSON, "json", false, "Emit cluster report as JSON") + command.Flags().BoolVar(&options.Apply, "apply", false, "Reparent non-representative beads under the cluster representative") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationCluster, args, options) + } + return command +} + +func (module Module) execCommand() *cobra.Command { + command := &cobra.Command{ + Use: "exec [args...]", + Short: "Run a bead CRUD command against whichever tracker (bd or br) this environment uses", + DisableFlagParsing: true, + } + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationExec, args, Options{}) + } + return command +} + +func (module Module) resumeCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "resume ", Short: "Atomically transfer an in_progress claim from a stale agent to this one", Args: cobra.ExactArgs(1)} + command.Flags().StringVar(&options.Agent, "agent", "", "New claimant id (defaults to BEADS_ACTOR env var, else ao-beads-resume).") + command.Flags().StringVar(&options.Ledger, "ledger", "docs/provenance/ledger.jsonl", "Path to the provenance ledger (relative to repo root).") + command.Flags().BoolVar(&options.JSON, "json", false, "Emit the claim_transferred event to stdout (always written to ledger).") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationResume, args, options) + } + return command +} + +func (module Module) scenariosCommand() *cobra.Command { + root := &cobra.Command{Use: "scenarios", Short: "[DEPRECATED — use 'ao beads verify-acceptance'] Convert bead acceptance criteria into Gherkin scenarios", Args: cobra.NoArgs} + var extract Options + extractCommand := &cobra.Command{Use: "extract ", Short: "Print a candidate Gherkin '## Scenarios' block from a bead's acceptance (dry-run)", Args: cobra.ExactArgs(1)} + extractCommand.Flags().BoolVar(&extract.JSON, "json", false, "Emit extracted scenarios as JSON (data on stdout) instead of a Gherkin block") + extractCommand.Flags().BoolVar(&extract.Force, "force", false, "Extract even when the bead already has a '## Scenarios' block") + extractCommand.Flags().BoolVar(&extract.Write, "write", false, "After printing the block and an operator y/N confirmation, append it to the bead via 'bd update'") + extractCommand.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationScenariosExtract, args, extract) + } + var check Options + checkCommand := &cobra.Command{Use: "validate ", Short: "Check that a bead's authored '## Scenarios' block is well-formed Gherkin", Args: cobra.ExactArgs(1)} + checkCommand.Flags().BoolVar(&check.JSON, "json", false, "Emit a structured validation verdict as JSON on stdout") + checkCommand.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationScenariosCheck, args, check) + } + root.AddCommand(extractCommand, checkCommand) + return root +} + +func (module Module) staleCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "stale-claims", Short: "List in_progress beads whose claim looks stale"} + command.Flags().Float64Var(&options.ThresholdHours, "threshold", 4, "Staleness threshold in hours (claim updated more than N hours ago).") + command.Flags().BoolVar(&options.JSON, "json", false, "Emit JSON array conforming to stale-claim-event.v1 (event_type: stale_detected).") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationStaleClaims, args, options) + } + return command +} + +func (module Module) epicStatusCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "epic-status ", Short: "Deterministic group-terminality verdict for an epic/wave", Args: cobra.ExactArgs(1)} + command.Flags().BoolVar(&options.Terminal, "terminal", false, "Map the verdict to the process exit code (0 terminal / 2 not-terminal / 3 skipped).") + command.Flags().BoolVar(&options.JSON, "json", false, "Emit the verdict as a JSON object instead of a human-readable line.") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationEpicStatus, args, options) + } + return command +} + +func (module Module) acceptanceCommand() *cobra.Command { + var options Options + command := &cobra.Command{Use: "verify-acceptance ...", Short: "Assert each bead carries the acceptance contract for its type (br-native)", Args: cobra.MinimumNArgs(1)} + command.Flags().BoolVar(&options.Strict, "strict", false, "Exit non-zero on any FAIL or UNDEFINED verdict") + command.Flags().BoolVar(&options.JSON, "json", false, "Emit verdicts as JSON") + command.RunE = func(command *cobra.Command, args []string) error { + return module.invoke(command, OperationAcceptance, args, options) + } + return command +} diff --git a/cli/internal/commands/beads/module_test.go b/cli/internal/commands/beads/module_test.go new file mode 100644 index 000000000..34ae18d40 --- /dev/null +++ b/cli/internal/commands/beads/module_test.go @@ -0,0 +1,59 @@ +package beads + +import ( + "reflect" + "testing" + + "github.com/spf13/cobra" +) + +type recordingRunner struct { + invocations []Invocation +} + +func (runner *recordingRunner) Run(_ *cobra.Command, invocation Invocation) error { + runner.invocations = append(runner.invocations, invocation) + return nil +} + +func TestModuleBuildsFreshCompleteTrees(t *testing.T) { + runner := &recordingRunner{} + module := NewModule(runner) + first := module.Command() + second := module.Command() + if first == second { + t.Fatal("Command reused a Cobra tree") + } + want := []string{ + "audit", "cluster", "dir", "epic-status", "exec", "harvest", "lint", + "resume", "scenarios", "stale-claims", "tracker", "verify", "verify-acceptance", + } + var got []string + for _, command := range first.Commands() { + got = append(got, command.Name()) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("commands = %v, want %v", got, want) + } +} + +func TestModuleParsesTypedInvocation(t *testing.T) { + runner := &recordingRunner{} + command := NewModule(runner).Command() + command.SetArgs([]string{"resume", "age-123", "--agent", "codex", "--json"}) + if err := command.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + want := Invocation{ + Operation: OperationResume, + Args: []string{"age-123"}, + Options: Options{ + Agent: "codex", + Ledger: "docs/provenance/ledger.jsonl", + JSON: true, + }, + } + if len(runner.invocations) != 1 || !reflect.DeepEqual(runner.invocations[0], want) { + t.Fatalf("invocations = %#v, want %#v", runner.invocations, want) + } +} From f1bcca226e66bf8a3b8b38ef1a0daa35524897c6 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:23:30 -0400 Subject: [PATCH 017/108] refactor(beads): cut over to explicit command owner (age-nw28h.2.1) --- cli/cmd/ao/beads.go | 198 +++------------------ cli/cmd/ao/beads_audit_cluster.go | 65 +------ cli/cmd/ao/beads_audit_cluster_test.go | 20 +-- cli/cmd/ao/beads_command_legacy_test.go | 45 +++++ cli/cmd/ao/beads_dual_tracker_test.go | 18 +- cli/cmd/ao/beads_epic_status.go | 39 +--- cli/cmd/ao/beads_epic_status_test.go | 14 +- cli/cmd/ao/beads_exec.go | 42 +---- cli/cmd/ao/beads_exec_test.go | 76 ++++---- cli/cmd/ao/beads_exit_test.go | 62 +++---- cli/cmd/ao/beads_module.go | 73 ++++++++ cli/cmd/ao/beads_recovery_test.go | 16 +- cli/cmd/ao/beads_resume.go | 29 +-- cli/cmd/ao/beads_resume_test.go | 26 +-- cli/cmd/ao/beads_scenarios.go | 79 +------- cli/cmd/ao/beads_scenarios_test.go | 20 +-- cli/cmd/ao/beads_stale.go | 25 +-- cli/cmd/ao/beads_stale_test.go | 18 +- cli/cmd/ao/beads_test.go | 76 ++++---- cli/cmd/ao/beads_tracker_test.go | 22 +-- cli/cmd/ao/beads_verify_acceptance.go | 50 +----- cli/cmd/ao/beads_verify_acceptance_test.go | 16 +- 22 files changed, 362 insertions(+), 667 deletions(-) create mode 100644 cli/cmd/ao/beads_command_legacy_test.go create mode 100644 cli/cmd/ao/beads_module.go diff --git a/cli/cmd/ao/beads.go b/cli/cmd/ao/beads.go index 54adba732..e146aaa4e 100644 --- a/cli/cmd/ao/beads.go +++ b/cli/cmd/ao/beads.go @@ -38,7 +38,7 @@ import ( beadsapp "github.com/boshu2/agentops/cli/internal/beads" ) -// beadsExitError carries a verdict exit code out through cobra's RunE so +// beadsVerdictError carries a verdict exit code out through cobra's RunE so // Execute() can map it to os.Exit() WITHOUT calling os.Exit() mid-command — // which would skip deferred cleanup (temp files, lock release) and kill the // test binary, making these paths untestable. @@ -50,25 +50,25 @@ import ( // cmd.SilenceErrors so cobra emits no spurious "Error:" line, while genuine // errors returned elsewhere in those commands still print normally. Mirrors // gateExitError (validate.go) and doctorExitError (doctor_surface.go). -type beadsExitError struct { +type beadsVerdictError struct { code int } -func (e *beadsExitError) Error() string { return "" } +func (e *beadsVerdictError) Error() string { return "" } // ExitCode returns the process exit code this verdict maps to. -func (e *beadsExitError) ExitCode() int { return e.code } +func (e *beadsVerdictError) ExitCode() int { return e.code } -// execBD is the single entry point for shelling out to bd. Tests override +// beadsTrackerOutput is the single entry point for shelling out to bd. Tests override // this to avoid a hard dependency on the real binary. Production code calls // `bd` via PATH; if absent, the caller emits a graceful warning and returns. -var execBD = func(args ...string) ([]byte, error) { +var beadsTrackerOutput = func(args ...string) ([]byte, error) { return currentBeadsTracker().Output(context.Background(), args...) } -// bdAvailable reports whether the bd binary is reachable via PATH. Tests +// beadsTrackerAvailable reports whether the bd binary is reachable via PATH. Tests // override this for deterministic behaviour. -var bdAvailable = func() bool { +var beadsTrackerAvailable = func() bool { return currentBeadsTracker().Available() } @@ -78,165 +78,19 @@ func currentBeadsTracker() *beadsadapter.Tracker { }) } -// ------------------------------------------------------------------------ -// Command wiring -// ------------------------------------------------------------------------ - -var beadsCmd = &cobra.Command{ - Use: "beads", - Short: "Complementary tooling for the bd (beads) issue tracker", - Args: cobra.NoArgs, - Long: `Commands that help maintain the bd issue tracker alongside the main -bd CLI. These tools focus on catching stale descriptions before a new -session acts on them and harvesting closure reasons into durable learnings. - -None of these commands replace bd itself — they complement it.`, -} - -var ( - beadsDirJSON bool - beadsDirRequire bool - beadsTrackerJSON bool -) - -var beadsDirCmd = &cobra.Command{ - Use: "dir", - Short: "Print the resolved live br ledger directory", - Long: `Print the BEADS_DIR path AgentOps will use for br subprocesses. - -In linked git worktrees, this resolves through git's common directory back to -the canonical private _beads ledger instead of assuming $PWD/_beads exists. - -With --require, the resolved path must be an existing directory that contains -a ledger artifact (issues.jsonl or beads.db); otherwise nothing is printed to -stdout and the command exits non-zero. Use this before br WRITE commands so a -failed resolution cannot silently fall back to the wrong tracker (age-gstf): - - BEADS_DIR="$(ao beads dir --require)" && export BEADS_DIR && br close -r "Done"`, - Args: cobra.NoArgs, - RunE: runBeadsDir, -} - -var beadsTrackerCmd = &cobra.Command{ - Use: "tracker", - Short: "Print the resolved beads tracker (bd or br) for this environment", - Long: `Detect which beads tracker AgentOps will drive here and how it was -selected. AgentOps skills and the ao CLI are tracker-agnostic: most end users -track with bd (beads, Go); this repo tracks with br (beads_rust). - -Resolution precedence (first match wins): - 1. AGENTOPS_TRACKER=bd|br env override (over config) - 2. tracker: in .agentops/config.yaml project config over home - 3. Ledger present (worktree-aware): _beads => br, .beads => bd; both => br - 4. Available binary: br first, then bd - 5. Otherwise a non-zero error naming both install paths - -Prints tracker, binary, ledger_dir, and source (table by default, or --json).`, - Args: cobra.NoArgs, - RunE: runBeadsTrackerCmd, -} - var ( + beadsDirJSON bool + beadsDirRequire bool + beadsTrackerJSON bool beadsVerifyJSON bool beadsVerifyVerbose bool -) - -var beadsVerifyCmd = &cobra.Command{ - Use: "verify ", - Short: "Detect stale citations in a bead description (files, functions, symbols)", - Long: `Reads a bead description via 'bd show ' and checks every file -path, function reference, and backticked symbol against HEAD. Reports each -citation as FRESH, STALE, or UNKNOWN with a per-citation reason. - -Intended use: before acting on a deferred bead or handoff reference, run -'ao beads verify ' to catch drift. See the planning rule at -skills/plan/references/stale-scope-validation.md for when this applies. - -Exit codes: - 0 — all citations fresh (or bd unavailable; graceful degradation) - 1 — at least one stale citation detected - 2 — error invoking bd or parsing output`, - Args: cobra.ExactArgs(1), - RunE: runBeadsVerify, -} - -var ( - beadsLintStatus string - beadsLintJSON bool -) - -var beadsLintCmd = &cobra.Command{ - Use: "lint", - Short: "Batch-verify every open bead (or filtered set) against HEAD", - Long: `Runs 'ao beads verify' on every bead matching a status filter and -aggregates the results. Useful as a weekly audit or pre-release gate. - -Exit codes: - 0 — all beads have fresh citations (or bd unavailable) - 1 — at least one bead has stale citations - 2 — error invoking bd`, - RunE: runBeadsLint, -} - -var ( + beadsLintStatus string + beadsLintJSON bool beadsHarvestOutDir string beadsHarvestDryRun bool ) -var beadsHarvestCmd = &cobra.Command{ - Use: "harvest ", - Short: "Materialize a closed bead's reason as a structured learning file", - Long: `Reads a closed bead via 'bd show ' and writes its closure reason -to .agents/learnings/YYYY-MM-DD--.md with frontmatter so the -learning can be picked up by the knowledge flywheel. - -Only works on beads in CLOSED state. Use after 'bd close --reason "..."' -to promote the reason into the learnings pool. - -Exit codes: - 0 — learning written (or already exists, or bd unavailable) - 2 — bead is not closed / error`, - Args: cobra.ExactArgs(1), - RunE: runBeadsHarvest, -} - -func init() { - beadsCmd.GroupID = "knowledge" - rootCmd.AddCommand(beadsCmd) - beadsCmd.AddCommand(beadsDirCmd) - beadsCmd.AddCommand(beadsTrackerCmd) // age-fvr8 dual-tracker detection - beadsCmd.AddCommand(beadsVerifyCmd) - beadsCmd.AddCommand(beadsLintCmd) - beadsCmd.AddCommand(beadsHarvestCmd) - beadsCmd.AddCommand(beadsStaleCmd) // soc-vuu6.27 slice 2 - beadsCmd.AddCommand(beadsResumeCmd) // soc-vuu6.27 slice 3 - beadsCmd.AddCommand(beadsEpicStatusCmd) // age-gascity-port-slate-irye.4 - - beadsDirCmd.Flags().BoolVar(&beadsDirJSON, "json", false, - "Emit {beads_dir, source} as JSON") - beadsDirCmd.Flags().BoolVar(&beadsDirRequire, "require", false, - "Fail closed: exit non-zero (printing nothing to stdout) unless the resolved directory holds a br ledger") - - beadsTrackerCmd.Flags().BoolVar(&beadsTrackerJSON, "json", false, - "Emit {tracker, binary, ledger_dir, source} as JSON") - - beadsVerifyCmd.Flags().BoolVar(&beadsVerifyJSON, "json", false, - "Emit verification report as JSON instead of human-readable text") - beadsVerifyCmd.Flags().BoolVar(&beadsVerifyVerbose, "verbose", false, - "Include FRESH citations in the output (default: stale only)") - - beadsLintCmd.Flags().StringVar(&beadsLintStatus, "status", "open", - "bd status filter (open, closed, all)") - beadsLintCmd.Flags().BoolVar(&beadsLintJSON, "json", false, - "Emit lint report as JSON") - - beadsHarvestCmd.Flags().StringVar(&beadsHarvestOutDir, "out-dir", ".agents/learnings", - "Directory to write the learning file into") - beadsHarvestCmd.Flags().BoolVar(&beadsHarvestDryRun, "dry-run", false, - "Print the learning content to stdout without writing a file") -} - -func runBeadsDir(cmd *cobra.Command, _ []string) error { +func executeBeadsDir(cmd *cobra.Command, _ []string) error { tracker := currentBeadsTracker() // BEADS_DIR is br's explicit ledger override; when it is set we preserve the // historical br resolution verbatim (backward compatibility). Otherwise we @@ -282,7 +136,7 @@ func runBeadsDir(cmd *cobra.Command, _ []string) error { // runBeadsTracker prints the resolved beads tracker for the current // environment: which tracker (bd|br), its resolved binary, its ledger // directory, and how it was selected (age-fvr8). -func runBeadsTrackerCmd(cmd *cobra.Command, _ []string) error { +func executeBeadsTracker(cmd *cobra.Command, _ []string) error { res, err := currentBeadsTracker().Resolve() if err != nil { return err @@ -318,7 +172,7 @@ type Citation = beadsapp.Citation // VerifyReport is the structured result of `ao beads verify`. type VerifyReport = beadsapp.VerifyReport -func runBeadsVerify(cmd *cobra.Command, args []string) error { +func executeBeadsVerify(cmd *cobra.Command, args []string) error { beadID := args[0] report, err := verifyBead(beadID) if err != nil { @@ -336,7 +190,7 @@ func runBeadsVerify(cmd *cobra.Command, args []string) error { if cmd != nil { cmd.SilenceErrors = true } - return &beadsExitError{code: 1} + return &beadsVerdictError{code: 1} } return nil } @@ -345,10 +199,10 @@ func runBeadsVerify(cmd *cobra.Command, args []string) error { // and verifies each against HEAD. Returns a report regardless of verdict; // callers decide what to do with StaleCount. func verifyBead(beadID string) (*VerifyReport, error) { - if !bdAvailable() { + if !beadsTrackerAvailable() { return &VerifyReport{BeadID: beadID, BDAvailable: false}, nil } - raw, err := execBD("show", beadID) + raw, err := beadsTrackerOutput("show", beadID) if err != nil { return nil, fmt.Errorf("bd show %s: %w", beadID, err) } @@ -620,8 +474,8 @@ func emitVerifyHuman(w *os.File, r *VerifyReport, verbose bool) { // LintReport is the aggregate result of `ao beads lint`. type LintReport = beadsapp.LintReport -func runBeadsLint(cmd *cobra.Command, args []string) error { - if !bdAvailable() { +func executeBeadsLint(cmd *cobra.Command, args []string) error { + if !beadsTrackerAvailable() { fmt.Fprintln(os.Stderr, "WARN: bd not on PATH — skipping lint (graceful degradation)") return nil } @@ -654,7 +508,7 @@ func runBeadsLint(cmd *cobra.Command, args []string) error { if cmd != nil { cmd.SilenceErrors = true } - return &beadsExitError{code: 1} + return &beadsVerdictError{code: 1} } return nil } @@ -673,7 +527,7 @@ func runBeadsLint(cmd *cobra.Command, args []string) error { // on the line after optional tree chars + bullet — this is robust to // future bd output tweaks. func listBeadIDs(status string) ([]string, error) { - raw, err := execBD("list", "--status", status) + raw, err := beadsTrackerOutput("list", "--status", status) if err != nil { return nil, fmt.Errorf("bd list: %w", err) } @@ -706,13 +560,13 @@ func emitLintHuman(w *os.File, r *LintReport) { // reducers handle enrichment. type LearningFrontmatter = beadsapp.LearningFrontmatter -func runBeadsHarvest(cmd *cobra.Command, args []string) error { +func executeBeadsHarvest(cmd *cobra.Command, args []string) error { beadID := args[0] - if !bdAvailable() { + if !beadsTrackerAvailable() { fmt.Fprintln(os.Stderr, "WARN: bd not on PATH — skipping harvest (graceful degradation)") return nil } - raw, err := execBD("show", beadID) + raw, err := beadsTrackerOutput("show", beadID) if err != nil { return fmt.Errorf("bd show %s: %w", beadID, err) } diff --git a/cli/cmd/ao/beads_audit_cluster.go b/cli/cmd/ao/beads_audit_cluster.go index 61fce0241..f8092b3ea 100644 --- a/cli/cmd/ao/beads_audit_cluster.go +++ b/cli/cmd/ao/beads_audit_cluster.go @@ -27,53 +27,6 @@ var ( beadsClusterApply bool ) -var beadsAuditCmd = &cobra.Command{ - Use: "audit", - Short: "Audit open beads for likely-fixed, stale, or consolidatable work", - Long: `Audits open and in-progress beads for backlog hygiene issues. - -The audit checks for: -- bead IDs already referenced by git commits -- bead descriptions whose cited files changed since the bead was created -- bead descriptions whose referenced patterns no longer appear in the repo -- multiple beads that mention the same file path - -This is the native Go equivalent of scripts/bd-audit.sh. The shell script is -kept as a compatibility entrypoint for existing hooks and skill guidance.`, - RunE: runBeadsAudit, -} - -var beadsClusterCmd = &cobra.Command{ - Use: "cluster", - Short: "Suggest consolidation clusters for overlapping open beads", - Long: `Analyzes open beads for domain overlap and suggests consolidation groups. - -The scorer compares title/body keywords, mentioned file paths, and labels. It -prefers an existing epic as the cluster representative when one exists, falling -back to the lexicographically smallest bead ID. - -This is the native Go equivalent of scripts/bd-cluster.sh. The shell script is -kept as a compatibility entrypoint for existing hooks and skill guidance.`, - RunE: runBeadsCluster, -} - -func init() { - beadsCmd.AddCommand(beadsAuditCmd) - beadsCmd.AddCommand(beadsClusterCmd) - - beadsAuditCmd.Flags().BoolVar(&beadsAuditJSON, "json", false, - "Emit audit report as JSON") - beadsAuditCmd.Flags().BoolVar(&beadsAuditStrict, "strict", false, - "Exit 1 when any likely-fixed, likely-stale, or consolidatable bead is found") - beadsAuditCmd.Flags().BoolVar(&beadsAuditAutoClose, "auto-close", false, - "Close likely-fixed beads when commit or file-change evidence is found") - - beadsClusterCmd.Flags().BoolVar(&beadsClusterJSON, "json", false, - "Emit cluster report as JSON") - beadsClusterCmd.Flags().BoolVar(&beadsClusterApply, "apply", false, - "Reparent non-representative beads under the cluster representative") -} - type beadRecord struct { ID string `json:"id"` Title string `json:"title"` @@ -190,7 +143,7 @@ type AuditReport struct { Error string `json:"error,omitempty"` } -func runBeadsAudit(cmd *cobra.Command, args []string) error { +func executeBeadsAudit(cmd *cobra.Command, args []string) error { report, err := auditBeads(beadsAuditAutoClose) if err != nil { return err @@ -213,7 +166,7 @@ func runBeadsAudit(cmd *cobra.Command, args []string) error { if cmd != nil { cmd.SilenceErrors = true } - return &beadsExitError{code: 1} + return &beadsVerdictError{code: 1} } return nil } @@ -223,7 +176,7 @@ func auditBeads(autoClose bool) (*AuditReport, error) { LikelyFixed: []AuditFinding{}, LikelyStale: []AuditFinding{}, Consolidatable: []AuditConsolidation{}, - BDAvailable: bdAvailable(), + BDAvailable: beadsTrackerAvailable(), } if !report.BDAvailable { report.Error = "bd CLI not found" @@ -366,7 +319,7 @@ func collectAuditBeads() ([]beadRecord, error) { } func listBDRecordsByStatus(status string) ([]beadRecord, error) { - raw, err := execBD("list", "--status", status, "--json") + raw, err := beadsTrackerOutput("list", "--status", status, "--json") if err != nil { return nil, fmt.Errorf("bd list --status %s --json: %w", status, err) } @@ -509,7 +462,7 @@ func fileChangesSinceCommits(commits []auditCommit, createdAt string, paths []st // returns any error from the bd update so callers can surface a failed state // mutation instead of silently dropping it. func autoCloseLikelyFixed(id, note string) error { - if _, err := execBD("update", id, "--status", "closed", "--append-notes", note); err != nil { + if _, err := beadsTrackerOutput("update", id, "--status", "closed", "--append-notes", note); err != nil { return fmt.Errorf("auto-close bead %s: %w", id, err) } return nil @@ -679,7 +632,7 @@ type ClusterReport struct { Error string `json:"error,omitempty"` } -func runBeadsCluster(cmd *cobra.Command, args []string) error { +func executeBeadsCluster(cmd *cobra.Command, args []string) error { report, err := clusterBeads(beadsClusterApply) if err != nil { return err @@ -702,7 +655,7 @@ func clusterBeads(apply bool) (*ClusterReport, error) { report := &ClusterReport{ Clusters: []BeadCluster{}, Unclustered: []ClusterBead{}, - BDAvailable: bdAvailable(), + BDAvailable: beadsTrackerAvailable(), } if !report.BDAvailable { report.Error = "bd CLI not found" @@ -729,7 +682,7 @@ func clusterBeads(apply bool) (*ClusterReport, error) { if bead.ID == cluster.Representative { continue } - if _, err := execBD("update", bead.ID, "--parent", cluster.Representative); err != nil { + if _, err := beadsTrackerOutput("update", bead.ID, "--parent", cluster.Representative); err != nil { report.ApplyErrors = append(report.ApplyErrors, fmt.Sprintf("%s -> %s: %v", bead.ID, cluster.Representative, err)) continue @@ -745,7 +698,7 @@ func enrichBeadRecord(record beadRecord) beadRecord { if record.ID == "" { return record } - raw, err := execBD("show", record.ID, "--json") + raw, err := beadsTrackerOutput("show", record.ID, "--json") if err != nil { return record } diff --git a/cli/cmd/ao/beads_audit_cluster_test.go b/cli/cmd/ao/beads_audit_cluster_test.go index 3e334dd5c..6f1760a78 100644 --- a/cli/cmd/ao/beads_audit_cluster_test.go +++ b/cli/cmd/ao/beads_audit_cluster_test.go @@ -124,12 +124,12 @@ func TestPatternExistsInIndex(t *testing.T) { // wrapped error, on success it returns nil, and it shells out with the // close arguments. func TestAutoCloseLikelyFixed_SurfacesError(t *testing.T) { - origExec := execBD - t.Cleanup(func() { execBD = origExec }) + origExec := beadsTrackerOutput + t.Cleanup(func() { beadsTrackerOutput = origExec }) var gotArgs []string bdErr := errors.New("bd update failed: server unreachable") - execBD = func(args ...string) ([]byte, error) { + beadsTrackerOutput = func(args ...string) ([]byte, error) { gotArgs = args return nil, bdErr } @@ -143,16 +143,16 @@ func TestAutoCloseLikelyFixed_SurfacesError(t *testing.T) { } wantArgs := []string{"update", "soc-aaa", "--status", "closed", "--append-notes", "note text"} if len(gotArgs) != len(wantArgs) { - t.Fatalf("execBD args = %v, want %v", gotArgs, wantArgs) + t.Fatalf("beadsTrackerOutput args = %v, want %v", gotArgs, wantArgs) } for i := range wantArgs { if gotArgs[i] != wantArgs[i] { - t.Errorf("execBD arg[%d] = %q, want %q", i, gotArgs[i], wantArgs[i]) + t.Errorf("beadsTrackerOutput arg[%d] = %q, want %q", i, gotArgs[i], wantArgs[i]) } } // Success path returns nil. - execBD = func(args ...string) ([]byte, error) { return []byte("ok"), nil } + beadsTrackerOutput = func(args ...string) ([]byte, error) { return []byte("ok"), nil } if err := autoCloseLikelyFixed("soc-aaa", "note"); err != nil { t.Errorf("autoCloseLikelyFixed on success = %v, want nil", err) } @@ -162,11 +162,11 @@ func TestAutoCloseLikelyFixed_SurfacesError(t *testing.T) { // triggers the (now error-returning) tracker close when autoClose is on, and // that a failing close does not panic or abort the report-recording flow. func TestRecordLikelyFixed_AutoCloseInvokesBD(t *testing.T) { - origExec := execBD - t.Cleanup(func() { execBD = origExec }) + origExec := beadsTrackerOutput + t.Cleanup(func() { beadsTrackerOutput = origExec }) var closeCount int - execBD = func(args ...string) ([]byte, error) { + beadsTrackerOutput = func(args ...string) ([]byte, error) { closeCount++ return nil, errors.New("bd update failed") // failure must not be swallowed silently } @@ -180,7 +180,7 @@ func TestRecordLikelyFixed_AutoCloseInvokesBD(t *testing.T) { t.Fatal("recordLikelyFixedAuditFinding did not record commit evidence") } if closeCount != 1 { - t.Errorf("autoClose invoked execBD %d times, want 1", closeCount) + t.Errorf("autoClose invoked beadsTrackerOutput %d times, want 1", closeCount) } if len(report.LikelyFixed) != 1 || report.LikelyFixed[0].ID != "soc-aaa" { t.Errorf("report.LikelyFixed = %+v, want one finding for soc-aaa", report.LikelyFixed) diff --git a/cli/cmd/ao/beads_command_legacy_test.go b/cli/cmd/ao/beads_command_legacy_test.go new file mode 100644 index 000000000..17da2f3e7 --- /dev/null +++ b/cli/cmd/ao/beads_command_legacy_test.go @@ -0,0 +1,45 @@ +package main + +import "github.com/spf13/cobra" + +// White-box behavior tests keep private throwaway commands for stream and flag +// control. Production command ownership lives in internal/commands/beads. +var legacyBeadsDirCommand = &cobra.Command{Use: "dir", RunE: executeBeadsDir} +var legacyBeadsTrackerCommand = &cobra.Command{Use: "tracker", RunE: executeBeadsTracker} +var legacyBeadsExecCommand = &cobra.Command{ + Use: "exec [args...]", + DisableFlagParsing: true, + RunE: executeBeadsExec, +} + +var legacyBeadsResumeCommand = func() *cobra.Command { + command := &cobra.Command{Use: "resume ", RunE: executeBeadsResume} + command.Flags().StringVar(&beadsResumeAgentID, "agent", "", "") + command.Flags().StringVar(&beadsResumeLedgerPath, "ledger", "docs/provenance/ledger.jsonl", "") + command.Flags().BoolVar(&beadsResumeJSON, "json", false, "") + return command +}() + +var legacyBeadsStaleCommand = func() *cobra.Command { + command := &cobra.Command{Use: "stale-claims", RunE: executeBeadsStale} + command.Flags().Float64Var(&beadsStaleThresholdHours, "threshold", 4, "") + command.Flags().BoolVar(&beadsStaleJSON, "json", false, "") + return command +}() + +var beadsScenariosExtractCmd = &cobra.Command{Use: "extract ", RunE: executeBeadsScenariosExtract} +var beadsScenariosValidateCmd = &cobra.Command{Use: "validate ", RunE: executeBeadsScenariosValidate} + +func newLegacyBeadsVerifyAcceptanceCommand() *cobra.Command { + var strict, asJSON bool + command := &cobra.Command{ + Use: "verify-acceptance ...", + Args: cobra.MinimumNArgs(1), + RunE: func(command *cobra.Command, args []string) error { + return executeBeadsVerifyAcceptance(command, args, strict, asJSON) + }, + } + command.Flags().BoolVar(&strict, "strict", false, "") + command.Flags().BoolVar(&asJSON, "json", false, "") + return command +} diff --git a/cli/cmd/ao/beads_dual_tracker_test.go b/cli/cmd/ao/beads_dual_tracker_test.go index 705a3228b..c15f8a96b 100644 --- a/cli/cmd/ao/beads_dual_tracker_test.go +++ b/cli/cmd/ao/beads_dual_tracker_test.go @@ -205,11 +205,11 @@ func TestRunBeadsTrackerCmdJSON(t *testing.T) { t.Chdir(root) var out strings.Builder - cmd := beadsTrackerCmd + cmd := legacyBeadsTrackerCommand cmd.SetOut(&out) t.Cleanup(func() { cmd.SetOut(nil) }) - if err := runBeadsTrackerCmd(cmd, nil); err != nil { - t.Fatalf("runBeadsTrackerCmd: %v", err) + if err := executeBeadsTracker(cmd, nil); err != nil { + t.Fatalf("executeBeadsTracker: %v", err) } var res trackerResolution if err := json.Unmarshal([]byte(out.String()), &res); err != nil { @@ -263,11 +263,11 @@ func TestRunBeadsDirBdRepoReturnsBeadsDir(t *testing.T) { beadsDirJSON = false var out strings.Builder - cmd := beadsDirCmd + cmd := legacyBeadsDirCommand cmd.SetOut(&out) t.Cleanup(func() { cmd.SetOut(nil) }) - if err := runBeadsDir(cmd, nil); err != nil { - t.Fatalf("runBeadsDir: %v", err) + if err := executeBeadsDir(cmd, nil); err != nil { + t.Fatalf("executeBeadsDir: %v", err) } if got := strings.TrimSpace(out.String()); got != beadsDir { t.Fatalf("beads dir in bd repo = %q, want %q", got, beadsDir) @@ -297,11 +297,11 @@ func TestRunBeadsDirBothPresentStaysBr(t *testing.T) { beadsDirJSON = true var out strings.Builder - cmd := beadsDirCmd + cmd := legacyBeadsDirCommand cmd.SetOut(&out) t.Cleanup(func() { cmd.SetOut(nil) }) - if err := runBeadsDir(cmd, nil); err != nil { - t.Fatalf("runBeadsDir: %v", err) + if err := executeBeadsDir(cmd, nil); err != nil { + t.Fatalf("executeBeadsDir: %v", err) } var payload map[string]string if err := json.Unmarshal([]byte(out.String()), &payload); err != nil { diff --git a/cli/cmd/ao/beads_epic_status.go b/cli/cmd/ao/beads_epic_status.go index 43f9b257f..4d6efddb3 100644 --- a/cli/cmd/ao/beads_epic_status.go +++ b/cli/cmd/ao/beads_epic_status.go @@ -37,39 +37,6 @@ var ( beadsEpicStatusJSON bool ) -var beadsEpicStatusCmd = &cobra.Command{ - Use: "epic-status ", - Short: "Deterministic group-terminality verdict for an epic/wave", - Long: `Emit a deterministic "is this epic/wave actually done" verdict, replacing -agent self-report. Members are the epic's id-prefix children (.N) UNION -any bead with a parent-child dependency edge to the epic. - -Three guards (all must hold for a terminal/done verdict): - 1. an unresolved/missing member becomes an unknown-status placeholder that - NEVER counts as done; - 2. a group with a deliberately-open descendant (a human-gate / checkpoint - bead) is NOT complete; - 3. a zero-descendant, still-materializing group is skipped, not done. - -Output: a human-readable reason by default, or a machine-readable verdict with ---json. With --terminal the verdict also maps to the exit code: - - 0 — terminal (done) 2 — not terminal 3 — skipped (materializing) - 1 — error (epic not found, ledger unreadable) - -Reads the live br ledger resolved via 'ao beads dir' (works from linked -worktrees). Read-only.`, - Args: cobra.ExactArgs(1), - RunE: runBeadsEpicStatus, -} - -func init() { - beadsEpicStatusCmd.Flags().BoolVar(&beadsEpicStatusTerminal, "terminal", false, - "Map the verdict to the process exit code (0 terminal / 2 not-terminal / 3 skipped).") - beadsEpicStatusCmd.Flags().BoolVar(&beadsEpicStatusJSON, "json", false, - "Emit the verdict as a JSON object instead of a human-readable line.") -} - // ledgerBead is the subset of an issues.jsonl record the predicate needs. type ledgerBead = beadsapp.LedgerBead @@ -84,7 +51,7 @@ var beadsEpicStatusReadLedger = func(dir string) ([]byte, error) { return currentBeadsRuntime().ReadFile(filepath.Join(dir, "issues.jsonl")) } -func runBeadsEpicStatus(cmd *cobra.Command, args []string) error { +func executeBeadsEpicStatus(cmd *cobra.Command, args []string) error { epic := strings.TrimSpace(args[0]) if epic == "" { return fmt.Errorf("epic id is required") @@ -116,10 +83,10 @@ func runBeadsEpicStatus(cmd *cobra.Command, args []string) error { switch result.Verdict { case epicstatus.NotTerminal: cmd.SilenceUsage, cmd.SilenceErrors = true, true - return &beadsExitError{code: 2} + return &beadsVerdictError{code: 2} case epicstatus.Skipped: cmd.SilenceUsage, cmd.SilenceErrors = true, true - return &beadsExitError{code: 3} + return &beadsVerdictError{code: 3} } } return nil diff --git a/cli/cmd/ao/beads_epic_status_test.go b/cli/cmd/ao/beads_epic_status_test.go index e262e7872..255735fbf 100644 --- a/cli/cmd/ao/beads_epic_status_test.go +++ b/cli/cmd/ao/beads_epic_status_test.go @@ -137,7 +137,7 @@ func runEpicStatus(t *testing.T, epic string) (string, error) { cmd := &cobra.Command{} var buf bytes.Buffer cmd.SetOut(&buf) - err := runBeadsEpicStatus(cmd, []string{epic}) + err := executeBeadsEpicStatus(cmd, []string{epic}) return buf.String(), err } @@ -171,7 +171,7 @@ func TestRunBeadsEpicStatus_TerminalJSON(t *testing.T) { } // TestRunBeadsEpicStatus_NotTerminalExitCode pins that --terminal maps a -// not-terminal verdict to exit code 2 via beadsExitError. +// not-terminal verdict to exit code 2 via beadsVerdictError. func TestRunBeadsEpicStatus_NotTerminalExitCode(t *testing.T) { setEpicStatusFlags(t, true, false) stubLedger(t, jsonLedger(t, []ledgerBead{ @@ -182,9 +182,9 @@ func TestRunBeadsEpicStatus_NotTerminalExitCode(t *testing.T) { }), nil) out, err := runEpicStatus(t, "age-x") - var exitErr *beadsExitError + var exitErr *beadsVerdictError if !errors.As(err, &exitErr) { - t.Fatalf("err = %v, want *beadsExitError", err) + t.Fatalf("err = %v, want *beadsVerdictError", err) } if exitErr.ExitCode() != 2 { t.Errorf("exit code = %d, want 2", exitErr.ExitCode()) @@ -203,9 +203,9 @@ func TestRunBeadsEpicStatus_SkippedExitCode(t *testing.T) { }), nil) _, err := runEpicStatus(t, "age-x") - var exitErr *beadsExitError + var exitErr *beadsVerdictError if !errors.As(err, &exitErr) { - t.Fatalf("err = %v, want *beadsExitError", err) + t.Fatalf("err = %v, want *beadsVerdictError", err) } if exitErr.ExitCode() != 3 { t.Errorf("exit code = %d, want 3", exitErr.ExitCode()) @@ -222,7 +222,7 @@ func TestRunBeadsEpicStatus_EpicNotFound(t *testing.T) { if err == nil { t.Fatal("err = nil, want epic-not-found error") } - var exitErr *beadsExitError + var exitErr *beadsVerdictError if errors.As(err, &exitErr) { t.Fatalf("epic-not-found mapped to verdict exit %d; want a plain error", exitErr.ExitCode()) } diff --git a/cli/cmd/ao/beads_exec.go b/cli/cmd/ao/beads_exec.go index 9c0f3bffd..3f8313ca0 100644 --- a/cli/cmd/ao/beads_exec.go +++ b/cli/cmd/ao/beads_exec.go @@ -14,45 +14,7 @@ import ( beadsapp "github.com/boshu2/agentops/cli/internal/beads" ) -var beadsExecCmd = &cobra.Command{ - Use: "exec [args...]", - Short: "Run a bead CRUD command against whichever tracker (bd or br) this environment uses", - Long: `Forward a bead command verbatim to the resolved beads tracker (bd or br), -setting that tracker's ledger correctly and propagating its exit code unchanged. - -This is the ONE tracker-agnostic entry point: skills and callers use -'ao beads exec ...' instead of hardcoding 'br' or 'bd'. The tracker is -resolved exactly as 'ao beads tracker' reports it (AGENTOPS_TRACKER > config > -ledger > binary). Examples (all work against either tracker): - - ao beads exec ready - ao beads exec close -r "Done" - ao beads exec update --status in_progress - ao beads exec create "title" --type task - ao beads exec list --json - -Ledger wiring: - br — BEADS_DIR is set to the resolved ledger dir (worktree-aware). - bd — the child runs from the repo root so .beads/ auto-discovery resolves; - no BEADS_DIR is set. - -Children (hard divergence — bd has 'children', br does not): - ao beads exec children - bd: forwarded to 'bd children ' (bd's native list output). - br: synthesized from 'br show --json' — the id of every dependent - with a parent-child edge, one per line. - -All flags are forwarded to the tracker verbatim (flag parsing is disabled), so -tracker flags never collide with ao's. Only -h/--help is intercepted here.`, - DisableFlagParsing: true, - RunE: runBeadsExec, -} - -func init() { - beadsCmd.AddCommand(beadsExecCmd) -} - -func runBeadsExec(cmd *cobra.Command, args []string) error { +func executeBeadsExec(cmd *cobra.Command, args []string) error { for _, argument := range args { if argument == "--help" || argument == "-h" { return cmd.Help() @@ -65,7 +27,7 @@ func runBeadsExec(cmd *cobra.Command, args []string) error { }) var adapterExit *beadsapp.ExitError if errors.As(err, &adapterExit) { - err = &beadsExitError{code: adapterExit.ExitCode()} + err = &beadsVerdictError{code: adapterExit.ExitCode()} } var exitErr interface{ ExitCode() int } if errors.As(err, &exitErr) { diff --git a/cli/cmd/ao/beads_exec_test.go b/cli/cmd/ao/beads_exec_test.go index 39bbafc75..37d1f7fd2 100644 --- a/cli/cmd/ao/beads_exec_test.go +++ b/cli/cmd/ao/beads_exec_test.go @@ -32,8 +32,8 @@ func writeTrackerShim(t *testing.T, name, body string) { } // newBeadsExecCmd returns a throwaway cobra command wired to buffers so a test -// can drive runBeadsExec and read what the child wrote, without mutating the -// shared beadsExecCmd global. +// can drive executeBeadsExec and read what the child wrote, without mutating the +// shared legacyBeadsExecCommand global. func newBeadsExecCmd(out, errbuf *strings.Builder) *cobra.Command { c := &cobra.Command{Use: "exec"} c.SetOut(out) @@ -60,8 +60,8 @@ echo "CWD=$(/bin/pwd)" var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - if err := runBeadsExec(cmd, []string{"ready"}); err != nil { - t.Fatalf("runBeadsExec: %v (stderr=%s)", err, errbuf.String()) + if err := executeBeadsExec(cmd, []string{"ready"}); err != nil { + t.Fatalf("executeBeadsExec: %v (stderr=%s)", err, errbuf.String()) } got := out.String() if !strings.Contains(got, "ARGS=ready") { @@ -94,8 +94,8 @@ echo "CWD=$(/bin/pwd)" var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - if err := runBeadsExec(cmd, []string{"ready"}); err != nil { - t.Fatalf("runBeadsExec: %v (stderr=%s)", err, errbuf.String()) + if err := executeBeadsExec(cmd, []string{"ready"}); err != nil { + t.Fatalf("executeBeadsExec: %v (stderr=%s)", err, errbuf.String()) } got := out.String() if !strings.Contains(got, "ARGS=ready") { @@ -125,12 +125,12 @@ func TestRunBeadsExec_HelpIsStatic(t *testing.T) { for _, flag := range []string{"--help", "-h"} { t.Run(flag, func(t *testing.T) { var out, errbuf strings.Builder - cmd := beadsExecCmd + cmd := legacyBeadsExecCommand cmd.SetOut(&out) cmd.SetErr(&errbuf) t.Cleanup(func() { cmd.SetOut(nil); cmd.SetErr(nil) }) - if err := runBeadsExec(cmd, []string{flag}); err != nil { - t.Fatalf("runBeadsExec %s = %v, want nil (static help)", flag, err) + if err := executeBeadsExec(cmd, []string{flag}); err != nil { + t.Fatalf("executeBeadsExec %s = %v, want nil (static help)", flag, err) } help := out.String() + errbuf.String() if !strings.Contains(help, "Usage:") { @@ -171,8 +171,8 @@ exit 9 var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - if err := runBeadsExec(cmd, []string{"children", "age-x"}); err != nil { - t.Fatalf("runBeadsExec children: %v (stderr=%s)", err, errbuf.String()) + if err := executeBeadsExec(cmd, []string{"children", "age-x"}); err != nil { + t.Fatalf("executeBeadsExec children: %v (stderr=%s)", err, errbuf.String()) } lines := splitNonEmptyLines(out.String()) want := []string{"age-x.1", "age-x.2"} // the related dependent is excluded @@ -198,8 +198,8 @@ func TestRunBeadsExec_ChildrenBD(t *testing.T) { var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - if err := runBeadsExec(cmd, []string{"children", "age-x"}); err != nil { - t.Fatalf("runBeadsExec children (bd): %v (stderr=%s)", err, errbuf.String()) + if err := executeBeadsExec(cmd, []string{"children", "age-x"}); err != nil { + t.Fatalf("executeBeadsExec children (bd): %v (stderr=%s)", err, errbuf.String()) } got := strings.TrimSpace(out.String()) if got != "ARGS=children age-x" { @@ -208,7 +208,7 @@ func TestRunBeadsExec_ChildrenBD(t *testing.T) { } // TestRunBeadsExec_PropagatesExitCode: a non-zero tracker exit surfaces as a -// beadsExitError carrying the same code (Execute maps it to os.Exit). +// beadsVerdictError carrying the same code (Execute maps it to os.Exit). func TestRunBeadsExec_PropagatesExitCode(t *testing.T) { root := makeGitRepoForTracker(t) if err := os.Mkdir(filepath.Join(root, "_beads"), 0o755); err != nil { @@ -224,13 +224,13 @@ exit 7 var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - err := runBeadsExec(cmd, []string{"close", "age-x", "-r", "nope"}) + err := executeBeadsExec(cmd, []string{"close", "age-x", "-r", "nope"}) if err == nil { - t.Fatal("runBeadsExec with failing tracker = nil, want beadsExitError") + t.Fatal("executeBeadsExec with failing tracker = nil, want beadsVerdictError") } - var exitErr *beadsExitError + var exitErr *beadsVerdictError if !errors.As(err, &exitErr) { - t.Fatalf("error type = %T, want *beadsExitError", err) + t.Fatalf("error type = %T, want *beadsVerdictError", err) } if exitErr.ExitCode() != 7 { t.Errorf("exit code = %d, want 7 (propagated unchanged)", exitErr.ExitCode()) @@ -256,13 +256,13 @@ echo "unexpected verb: $*" var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - err := runBeadsExec(cmd, []string{"children", "age-epic"}) + err := executeBeadsExec(cmd, []string{"children", "age-epic"}) if err == nil { - t.Fatal("children on a failing br show = nil, want beadsExitError") + t.Fatal("children on a failing br show = nil, want beadsVerdictError") } - var exitErr *beadsExitError + var exitErr *beadsVerdictError if !errors.As(err, &exitErr) { - t.Fatalf("error type = %T, want *beadsExitError", err) + t.Fatalf("error type = %T, want *beadsVerdictError", err) } if exitErr.ExitCode() != 7 { t.Errorf("exit code = %d, want 7 (br show status propagated unchanged)", exitErr.ExitCode()) @@ -296,8 +296,8 @@ echo "UNEXPECTED $*" >&2; exit 9 var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - if err := runBeadsExec(cmd, []string{"list", "--json"}); err != nil { - t.Fatalf("runBeadsExec list --json (br): %v (stderr=%s)", err, errbuf.String()) + if err := executeBeadsExec(cmd, []string{"list", "--json"}); err != nil { + t.Fatalf("executeBeadsExec list --json (br): %v (stderr=%s)", err, errbuf.String()) } var obj map[string]json.RawMessage if err := json.Unmarshal([]byte(out.String()), &obj); err != nil { @@ -334,8 +334,8 @@ echo "UNEXPECTED $*" >&2; exit 9 var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - if err := runBeadsExec(cmd, []string{"list", "--json"}); err != nil { - t.Fatalf("runBeadsExec list --json (bd): %v (stderr=%s)", err, errbuf.String()) + if err := executeBeadsExec(cmd, []string{"list", "--json"}); err != nil { + t.Fatalf("executeBeadsExec list --json (bd): %v (stderr=%s)", err, errbuf.String()) } var obj struct { Issues []map[string]json.RawMessage `json:"issues"` @@ -381,8 +381,8 @@ echo "UNEXPECTED $*" >&2; exit 9 var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - if err := runBeadsExec(cmd, []string{"show", "bd-ep", "--json"}); err != nil { - t.Fatalf("runBeadsExec show --json (bd): %v (stderr=%s)", err, errbuf.String()) + if err := executeBeadsExec(cmd, []string{"show", "bd-ep", "--json"}); err != nil { + t.Fatalf("executeBeadsExec show --json (bd): %v (stderr=%s)", err, errbuf.String()) } var arr []map[string]json.RawMessage if err := json.Unmarshal([]byte(out.String()), &arr); err != nil { @@ -426,8 +426,8 @@ echo "UNEXPECTED $*" >&2; exit 9 var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - if err := runBeadsExec(cmd, []string{"ready", "--json"}); err != nil { - t.Fatalf("runBeadsExec ready --json (bd): %v (stderr=%s)", err, errbuf.String()) + if err := executeBeadsExec(cmd, []string{"ready", "--json"}); err != nil { + t.Fatalf("executeBeadsExec ready --json (bd): %v (stderr=%s)", err, errbuf.String()) } var arr []map[string]json.RawMessage if err := json.Unmarshal([]byte(out.String()), &arr); err != nil { @@ -450,8 +450,8 @@ func TestRunBeadsExec_BDReadWithoutJSONStreamsVerbatim(t *testing.T) { var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - if err := runBeadsExec(cmd, []string{"list"}); err != nil { - t.Fatalf("runBeadsExec list (bd, no --json): %v (stderr=%s)", err, errbuf.String()) + if err := executeBeadsExec(cmd, []string{"list"}); err != nil { + t.Fatalf("executeBeadsExec list (bd, no --json): %v (stderr=%s)", err, errbuf.String()) } if got := strings.TrimSpace(out.String()); got != "HUMAN: list" { t.Errorf("bd read without --json must stream verbatim; got %q", got) @@ -470,8 +470,8 @@ func TestRunBeadsExec_BDWriteVerbWithJSONStreamsVerbatim(t *testing.T) { var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - if err := runBeadsExec(cmd, []string{"create", "title", "--json"}); err != nil { - t.Fatalf("runBeadsExec create --json (bd): %v (stderr=%s)", err, errbuf.String()) + if err := executeBeadsExec(cmd, []string{"create", "title", "--json"}); err != nil { + t.Fatalf("executeBeadsExec create --json (bd): %v (stderr=%s)", err, errbuf.String()) } if got := strings.TrimSpace(out.String()); got != "RAW: create title --json" { t.Errorf("bd write verb must stream verbatim (no reshape); got %q", got) @@ -493,13 +493,13 @@ exit 7 var out, errbuf strings.Builder cmd := newBeadsExecCmd(&out, &errbuf) - err := runBeadsExec(cmd, []string{"list", "--json"}) + err := executeBeadsExec(cmd, []string{"list", "--json"}) if err == nil { - t.Fatal("bd read+--json with a failing tracker = nil, want beadsExitError") + t.Fatal("bd read+--json with a failing tracker = nil, want beadsVerdictError") } - var exitErr *beadsExitError + var exitErr *beadsVerdictError if !errors.As(err, &exitErr) { - t.Fatalf("error type = %T, want *beadsExitError", err) + t.Fatalf("error type = %T, want *beadsVerdictError", err) } if exitErr.ExitCode() != 7 { t.Errorf("exit code = %d, want 7 (propagated unchanged)", exitErr.ExitCode()) diff --git a/cli/cmd/ao/beads_exit_test.go b/cli/cmd/ao/beads_exit_test.go index 378c06032..bb9fd3c2d 100644 --- a/cli/cmd/ao/beads_exit_test.go +++ b/cli/cmd/ao/beads_exit_test.go @@ -2,7 +2,7 @@ // // These commands previously called os.Exit(1) mid-RunE to signal a stale / // flagged verdict, which skipped deferred cleanup and killed the test binary, -// making the verdict path untestable. They now return a *beadsExitError that +// making the verdict path untestable. They now return a *beadsVerdictError that // Execute() maps to the process exit code. These tests pin that contract. // practices: [dora-metrics, lean-startup] @@ -26,7 +26,7 @@ forcing a stale citation regardless of the test's working directory. ` func TestBeadsExitError_Contract(t *testing.T) { - e := &beadsExitError{code: 1} + e := &beadsVerdictError{code: 1} // Error() is intentionally empty: the verdict text already went to stdout // and the command silences cobra's error print. if e.Error() != "" { @@ -36,27 +36,27 @@ func TestBeadsExitError_Contract(t *testing.T) { t.Fatalf("ExitCode() = %d, want 1", e.ExitCode()) } // It must be discoverable via errors.As, the way Execute() maps it. - var target *beadsExitError + var target *beadsVerdictError if !errors.As(error(e), &target) { - t.Fatalf("errors.As failed to match *beadsExitError") + t.Fatalf("errors.As failed to match *beadsVerdictError") } } func TestRunBeadsVerify_ReturnsExitErrorOnStale(t *testing.T) { - origBD, origAvail := execBD, bdAvailable - defer func() { execBD, bdAvailable = origBD, origAvail }() + origBD, origAvail := beadsTrackerOutput, beadsTrackerAvailable + defer func() { beadsTrackerOutput, beadsTrackerAvailable = origBD, origAvail }() - bdAvailable = func() bool { return true } - execBD = func(args ...string) ([]byte, error) { + beadsTrackerAvailable = func() bool { return true } + beadsTrackerOutput = func(args ...string) ([]byte, error) { return []byte(staleBeadShow), nil } cmd := &cobra.Command{} - err := runBeadsVerify(cmd, []string{"na-stale"}) + err := executeBeadsVerify(cmd, []string{"na-stale"}) - var bxErr *beadsExitError + var bxErr *beadsVerdictError if !errors.As(err, &bxErr) { - t.Fatalf("runBeadsVerify on stale bead: got %v, want *beadsExitError", err) + t.Fatalf("executeBeadsVerify on stale bead: got %v, want *beadsVerdictError", err) } if bxErr.ExitCode() != 1 { t.Fatalf("verdict exit code = %d, want 1", bxErr.ExitCode()) @@ -68,28 +68,28 @@ func TestRunBeadsVerify_ReturnsExitErrorOnStale(t *testing.T) { } func TestRunBeadsVerify_NilCmdDoesNotPanicOnStale(t *testing.T) { - origBD, origAvail := execBD, bdAvailable - defer func() { execBD, bdAvailable = origBD, origAvail }() + origBD, origAvail := beadsTrackerOutput, beadsTrackerAvailable + defer func() { beadsTrackerOutput, beadsTrackerAvailable = origBD, origAvail }() - bdAvailable = func() bool { return true } - execBD = func(args ...string) ([]byte, error) { + beadsTrackerAvailable = func() bool { return true } + beadsTrackerOutput = func(args ...string) ([]byte, error) { return []byte(staleBeadShow), nil } // Direct callers (and older tests) pass a nil cmd; the nil-guard must hold. - err := runBeadsVerify(nil, []string{"na-stale"}) - var bxErr *beadsExitError + err := executeBeadsVerify(nil, []string{"na-stale"}) + var bxErr *beadsVerdictError if !errors.As(err, &bxErr) { - t.Fatalf("runBeadsVerify(nil, ...) on stale: got %v, want *beadsExitError", err) + t.Fatalf("executeBeadsVerify(nil, ...) on stale: got %v, want *beadsVerdictError", err) } } func TestRunBeadsLint_ReturnsExitErrorOnStale(t *testing.T) { - origBD, origAvail := execBD, bdAvailable - defer func() { execBD, bdAvailable = origBD, origAvail }() + origBD, origAvail := beadsTrackerOutput, beadsTrackerAvailable + defer func() { beadsTrackerOutput, beadsTrackerAvailable = origBD, origAvail }() - bdAvailable = func() bool { return true } - execBD = func(args ...string) ([]byte, error) { + beadsTrackerAvailable = func() bool { return true } + beadsTrackerOutput = func(args ...string) ([]byte, error) { if len(args) > 0 && args[0] == "list" { return []byte("○ na-stale · Stale bead [● P2 · OPEN]\n"), nil } @@ -98,11 +98,11 @@ func TestRunBeadsLint_ReturnsExitErrorOnStale(t *testing.T) { } cmd := &cobra.Command{} - err := runBeadsLint(cmd, []string{}) + err := executeBeadsLint(cmd, []string{}) - var bxErr *beadsExitError + var bxErr *beadsVerdictError if !errors.As(err, &bxErr) { - t.Fatalf("runBeadsLint with a stale bead: got %v, want *beadsExitError", err) + t.Fatalf("executeBeadsLint with a stale bead: got %v, want *beadsVerdictError", err) } if bxErr.ExitCode() != 1 { t.Fatalf("verdict exit code = %d, want 1", bxErr.ExitCode()) @@ -113,11 +113,11 @@ func TestRunBeadsLint_ReturnsExitErrorOnStale(t *testing.T) { } func TestRunBeadsLint_CleanReturnsNil(t *testing.T) { - origBD, origAvail := execBD, bdAvailable - defer func() { execBD, bdAvailable = origBD, origAvail }() + origBD, origAvail := beadsTrackerOutput, beadsTrackerAvailable + defer func() { beadsTrackerOutput, beadsTrackerAvailable = origBD, origAvail }() - bdAvailable = func() bool { return true } - execBD = func(args ...string) ([]byte, error) { + beadsTrackerAvailable = func() bool { return true } + beadsTrackerOutput = func(args ...string) ([]byte, error) { if len(args) > 0 && args[0] == "list" { return []byte("○ na-clean · Clean bead [● P2 · OPEN]\n"), nil } @@ -125,7 +125,7 @@ func TestRunBeadsLint_CleanReturnsNil(t *testing.T) { return []byte("○ na-clean · Clean bead [● P2 · OPEN]\nOwner: Test · Type: task\n\nDESCRIPTION\nNo citations here.\n"), nil } - if err := runBeadsLint(&cobra.Command{}, []string{}); err != nil { - t.Fatalf("runBeadsLint with a clean bead: got %v, want nil", err) + if err := executeBeadsLint(&cobra.Command{}, []string{}); err != nil { + t.Fatalf("executeBeadsLint with a clean bead: got %v, want nil", err) } } diff --git a/cli/cmd/ao/beads_module.go b/cli/cmd/ao/beads_module.go new file mode 100644 index 000000000..3ea13b06d --- /dev/null +++ b/cli/cmd/ao/beads_module.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + + "github.com/boshu2/agentops/cli/internal/clicontract" + beadscommands "github.com/boshu2/agentops/cli/internal/commands/beads" + "github.com/spf13/cobra" +) + +// beadsModuleRunner is the composition seam between the explicit driving +// adapter and the existing family use cases. The module owns Cobra parsing; +// this bridge transfers typed options while the remaining use cases move out +// of cmd/ao behind their application ports. +type beadsModuleRunner struct{} + +func (beadsModuleRunner) Run(command *cobra.Command, invocation beadscommands.Invocation) error { + options := invocation.Options + switch invocation.Operation { + case beadscommands.OperationDir: + beadsDirJSON, beadsDirRequire = options.JSON, options.Require + return executeBeadsDir(command, invocation.Args) + case beadscommands.OperationTracker: + beadsTrackerJSON = options.JSON + return executeBeadsTracker(command, invocation.Args) + case beadscommands.OperationVerify: + beadsVerifyJSON, beadsVerifyVerbose = options.JSON, options.Verbose + return executeBeadsVerify(command, invocation.Args) + case beadscommands.OperationLint: + beadsLintStatus, beadsLintJSON = options.Status, options.JSON + return executeBeadsLint(command, invocation.Args) + case beadscommands.OperationHarvest: + beadsHarvestOutDir, beadsHarvestDryRun = options.OutputDirectory, options.DryRun + return executeBeadsHarvest(command, invocation.Args) + case beadscommands.OperationAudit: + beadsAuditJSON, beadsAuditStrict, beadsAuditAutoClose = options.JSON, options.Strict, options.AutoClose + return executeBeadsAudit(command, invocation.Args) + case beadscommands.OperationCluster: + beadsClusterJSON, beadsClusterApply = options.JSON, options.Apply + return executeBeadsCluster(command, invocation.Args) + case beadscommands.OperationExec: + return executeBeadsExec(command, invocation.Args) + case beadscommands.OperationResume: + beadsResumeAgentID, beadsResumeLedgerPath, beadsResumeJSON = options.Agent, options.Ledger, options.JSON + return executeBeadsResume(command, invocation.Args) + case beadscommands.OperationScenariosExtract: + beadsScenariosJSON, beadsScenariosForce, beadsScenariosWrite = options.JSON, options.Force, options.Write + return executeBeadsScenariosExtract(command, invocation.Args) + case beadscommands.OperationScenariosCheck: + beadsScenariosValidateJSON = options.JSON + return executeBeadsScenariosValidate(command, invocation.Args) + case beadscommands.OperationStaleClaims: + beadsStaleThresholdHours, beadsStaleJSON = options.ThresholdHours, options.JSON + return executeBeadsStale(command, invocation.Args) + case beadscommands.OperationEpicStatus: + beadsEpicStatusTerminal, beadsEpicStatusJSON = options.Terminal, options.JSON + return executeBeadsEpicStatus(command, invocation.Args) + case beadscommands.OperationAcceptance: + return executeBeadsVerifyAcceptance(command, invocation.Args, options.Strict, options.JSON) + default: + return fmt.Errorf("unsupported beads operation %q", invocation.Operation) + } +} + +func init() { + module := beadscommands.NewModule(beadsModuleRunner{}) + command := module.Command() + command.GroupID = "knowledge" + if err := clicontract.Attach(command, module.Contract()); err != nil { + panic(err) + } + rootCmd.AddCommand(command) +} diff --git a/cli/cmd/ao/beads_recovery_test.go b/cli/cmd/ao/beads_recovery_test.go index 6b59ece25..15fca53e7 100644 --- a/cli/cmd/ao/beads_recovery_test.go +++ b/cli/cmd/ao/beads_recovery_test.go @@ -108,10 +108,10 @@ esac beadsStaleNowOverride = "2026-05-20T12:00:00Z" staleBuf := &bytes.Buffer{} - beadsStaleCmd.SetOut(staleBuf) - defer beadsStaleCmd.SetOut(nil) - if err := runBeadsStale(beadsStaleCmd, nil); err != nil { - t.Fatalf("runBeadsStale: %v", err) + legacyBeadsStaleCommand.SetOut(staleBuf) + defer legacyBeadsStaleCommand.SetOut(nil) + if err := executeBeadsStale(legacyBeadsStaleCommand, nil); err != nil { + t.Fatalf("executeBeadsStale: %v", err) } var staleEvents []staleEvent if err := json.Unmarshal(staleBuf.Bytes(), &staleEvents); err != nil { @@ -130,10 +130,10 @@ esac beadsResumeNowOverride = "2026-05-20T12:30:00Z" resumeBuf := &bytes.Buffer{} - beadsResumeCmd.SetOut(resumeBuf) - defer beadsResumeCmd.SetOut(nil) - if err := runBeadsResume(beadsResumeCmd, []string{"ag-recover"}); err != nil { - t.Fatalf("runBeadsResume: %v", err) + legacyBeadsResumeCommand.SetOut(resumeBuf) + defer legacyBeadsResumeCommand.SetOut(nil) + if err := executeBeadsResume(legacyBeadsResumeCommand, []string{"ag-recover"}); err != nil { + t.Fatalf("executeBeadsResume: %v", err) } assignee, err := os.ReadFile(statePath) if err != nil { diff --git a/cli/cmd/ao/beads_resume.go b/cli/cmd/ao/beads_resume.go index dae0b4b44..114874f21 100644 --- a/cli/cmd/ao/beads_resume.go +++ b/cli/cmd/ao/beads_resume.go @@ -31,33 +31,6 @@ var ( beadsResumeNowOverride string // test seam ) -var beadsResumeCmd = &cobra.Command{ - Use: "resume ", - Short: "Atomically transfer an in_progress claim from a stale agent to this one", - Long: `Transfers a stale claim via 'br update --claim', then appends a -claim_transferred event (matching schemas/stale-claim-event.v1.schema.json) -to docs/provenance/ledger.jsonl. The bead's prior + new revision (assignee -and updated_at hash) is captured in the event for audit. - -Use 'ao beads stale-claims' (slice 2) to find candidates first. - ---agent: explicit new claimant id. Defaults to BEADS_ACTOR env var, else "ao-beads-resume". ---ledger: provenance ledger path (default docs/provenance/ledger.jsonl). ---json: emit the event to stdout in addition to the ledger.`, - Args: cobra.ExactArgs(1), - RunE: runBeadsResume, -} - -func init() { - beadsResumeCmd.Flags().StringVar(&beadsResumeAgentID, "agent", "", - "New claimant id (defaults to BEADS_ACTOR env var, else ao-beads-resume).") - beadsResumeCmd.Flags().StringVar(&beadsResumeLedgerPath, "ledger", - "docs/provenance/ledger.jsonl", - "Path to the provenance ledger (relative to repo root).") - beadsResumeCmd.Flags().BoolVar(&beadsResumeJSON, "json", false, - "Emit the claim_transferred event to stdout (always written to ledger).") -} - // beadsResumeShowFunc is the test seam for fetching a bead's current state // (assignee + updated_at) BEFORE the claim transfer, so we can record the // prior revision. Production: shells out to `br show --json`. @@ -82,7 +55,7 @@ var beadsResumeAppendLedger = func(ledgerPath string, event any) error { func currentBeadsRuntime() beadsadapter.Runtime { return beadsadapter.NewRuntime() } -func runBeadsResume(cmd *cobra.Command, args []string) error { +func executeBeadsResume(cmd *cobra.Command, args []string) error { beadID := args[0] if beadID == "" { return fmt.Errorf("bead id is required") diff --git a/cli/cmd/ao/beads_resume_test.go b/cli/cmd/ao/beads_resume_test.go index cb404fb9e..42a0492e2 100644 --- a/cli/cmd/ao/beads_resume_test.go +++ b/cli/cmd/ao/beads_resume_test.go @@ -1,6 +1,6 @@ // Tests for ao beads resume (soc-vuu6.27 slice 3). // -// Exercise the runBeadsResume flow via the three seams (show / claim / +// Exercise the executeBeadsResume flow via the three seams (show / claim / // append-ledger) so tests never depend on a real bd binary or the live // provenance ledger. @@ -74,11 +74,11 @@ func TestRunBeadsResume_HappyPath_WritesClaimTransferredEvent(t *testing.T) { beadsResumeJSON = true buf := &bytes.Buffer{} - beadsResumeCmd.SetOut(buf) - defer beadsResumeCmd.SetOut(nil) + legacyBeadsResumeCommand.SetOut(buf) + defer legacyBeadsResumeCommand.SetOut(nil) - if err := runBeadsResume(beadsResumeCmd, []string{"x"}); err != nil { - t.Fatalf("runBeadsResume: %v", err) + if err := executeBeadsResume(legacyBeadsResumeCommand, []string{"x"}); err != nil { + t.Fatalf("executeBeadsResume: %v", err) } if len(*appends) != 1 { t.Fatalf("expected 1 ledger append; got %d", len(*appends)) @@ -126,7 +126,7 @@ func TestRunBeadsResume_RefusesNonInProgress(t *testing.T) { defer teardown() beadsResumeAgentID = "bob" - err := runBeadsResume(beadsResumeCmd, []string{"x"}) + err := executeBeadsResume(legacyBeadsResumeCommand, []string{"x"}) if err == nil { t.Fatalf("expected error on non-in-progress bead") } @@ -144,7 +144,7 @@ func TestRunBeadsResume_ClaimErrorBubblesUp(t *testing.T) { defer teardown() beadsResumeAgentID = "bob" - err := runBeadsResume(beadsResumeCmd, []string{"x"}) + err := executeBeadsResume(legacyBeadsResumeCommand, []string{"x"}) if err == nil { t.Fatalf("expected error when bd claim fails") } @@ -165,8 +165,8 @@ func TestRunBeadsResume_AgentResolution_FlagWinsOverEnv(t *testing.T) { t.Setenv("BEADS_ACTOR", "from-env") beadsResumeAgentID = "from-flag" - if err := runBeadsResume(beadsResumeCmd, []string{"x"}); err != nil { - t.Fatalf("runBeadsResume: %v", err) + if err := executeBeadsResume(legacyBeadsResumeCommand, []string{"x"}); err != nil { + t.Fatalf("executeBeadsResume: %v", err) } raw, _ := json.Marshal((*appends)[0].event) var got map[string]any @@ -186,8 +186,8 @@ func TestRunBeadsResume_AgentResolution_EnvWhenNoFlag(t *testing.T) { t.Setenv("BEADS_ACTOR", "from-env") beadsResumeAgentID = "" - if err := runBeadsResume(beadsResumeCmd, []string{"x"}); err != nil { - t.Fatalf("runBeadsResume: %v", err) + if err := executeBeadsResume(legacyBeadsResumeCommand, []string{"x"}); err != nil { + t.Fatalf("executeBeadsResume: %v", err) } raw, _ := json.Marshal((*appends)[0].event) var got map[string]any @@ -207,8 +207,8 @@ func TestRunBeadsResume_AgentResolution_DefaultWhenNothingSet(t *testing.T) { t.Setenv("BEADS_ACTOR", "") beadsResumeAgentID = "" - if err := runBeadsResume(beadsResumeCmd, []string{"x"}); err != nil { - t.Fatalf("runBeadsResume: %v", err) + if err := executeBeadsResume(legacyBeadsResumeCommand, []string{"x"}); err != nil { + t.Fatalf("executeBeadsResume: %v", err) } raw, _ := json.Marshal((*appends)[0].event) var got map[string]any diff --git a/cli/cmd/ao/beads_scenarios.go b/cli/cmd/ao/beads_scenarios.go index a668b5d32..ac1d3d603 100644 --- a/cli/cmd/ao/beads_scenarios.go +++ b/cli/cmd/ao/beads_scenarios.go @@ -48,84 +48,17 @@ type fetchedBead struct { // beadsScenariosFetch fetches a bead's acceptance and description text. It is a // package var so tests can inject a fake without shelling out to bd. var beadsScenariosFetch = func(id string) (fetchedBead, error) { - out, err := execBD("show", id, "--json") + out, err := beadsTrackerOutput("show", id, "--json") if err != nil { return fetchedBead{}, fmt.Errorf("bd show %s --json: %w", id, err) } return parseBeadFromBDJSON(out) } -var beadsScenariosCmd = &cobra.Command{ - Use: "scenarios", - Short: "[DEPRECATED — use 'ao beads verify-acceptance'] Convert bead acceptance criteria into Gherkin scenarios", - Args: cobra.NoArgs, - Long: `Turn a bead's free-text acceptance criteria into structured Gherkin -Given/When/Then scenarios. - -DEPRECATED: this surface shells the retired 'bd' binary and applies a single -Gherkin contract to every bead. Prefer 'ao beads verify-acceptance', which is -br-native and applies the right acceptance contract per issue_type. - -The 'extract' subcommand is a dry-run: it prints a candidate '## Scenarios' -block to stdout for review and never modifies the bead. The 'validate' -subcommand checks that an authored '## Scenarios' block is well-formed Gherkin -and exits non-zero (naming the parse error) when it is not.`, -} - -var beadsScenariosExtractCmd = &cobra.Command{ - Use: "extract ", - Short: "Print a candidate Gherkin '## Scenarios' block from a bead's acceptance (dry-run)", - Args: cobra.ExactArgs(1), - Long: `Read a bead's acceptance criteria via 'bd show --json', convert the -free-text bullets into Given/When/Then scenarios using deterministic rules, -and print a candidate '## Scenarios' block to stdout. - -By default this is a dry-run — the bead is never modified. Review the output -and author it into the bead manually. With --json the scenarios are emitted as -structured data on stdout instead of a Gherkin block. With --write the block is -appended to the bead's description via 'bd update', but only after you confirm -the printed block at a y/N prompt; declining leaves the bead unchanged. - -If the bead already carries a '## Scenarios' block, extract refuses (nothing to -do) unless --force is passed, to avoid generating noise over already-shaped -acceptance.`, - RunE: runBeadsScenariosExtract, -} - -var beadsScenariosValidateCmd = &cobra.Command{ - Use: "validate ", - Short: "Check that a bead's authored '## Scenarios' block is well-formed Gherkin", - Args: cobra.ExactArgs(1), - Long: `Read a bead via 'bd show --json' and validate its authored -'## Scenarios' block. Each scenario must declare a name and a Given/When/Then -step (in that order) with a non-empty body; And/But lines are accepted as -continuations. - -Exits 0 when the block is well-formed. Exits non-zero, naming the parse error, -when the block is missing or malformed. With --json a verdict object is emitted -on stdout ({"bead_id","valid","scenarios"} on success, or -{"bead_id","valid":false,"error"} on failure) while diagnostics go to stderr.`, - RunE: runBeadsScenariosValidate, -} - -func init() { - beadsCmd.AddCommand(beadsScenariosCmd) - beadsScenariosCmd.AddCommand(beadsScenariosExtractCmd) - beadsScenariosCmd.AddCommand(beadsScenariosValidateCmd) - beadsScenariosExtractCmd.Flags().BoolVar(&beadsScenariosJSON, "json", false, - "Emit extracted scenarios as JSON (data on stdout) instead of a Gherkin block") - beadsScenariosExtractCmd.Flags().BoolVar(&beadsScenariosForce, "force", false, - "Extract even when the bead already has a '## Scenarios' block") - beadsScenariosExtractCmd.Flags().BoolVar(&beadsScenariosWrite, "write", false, - "After printing the block and an operator y/N confirmation, append it to the bead via 'bd update'") - beadsScenariosValidateCmd.Flags().BoolVar(&beadsScenariosValidateJSON, "json", false, - "Emit a structured validation verdict as JSON on stdout") -} - -func runBeadsScenariosExtract(cmd *cobra.Command, args []string) error { +func executeBeadsScenariosExtract(cmd *cobra.Command, args []string) error { id := args[0] - if !bdAvailable() { + if !beadsTrackerAvailable() { fmt.Fprintln(cmd.ErrOrStderr(), "warning: bd not found on PATH; cannot fetch bead. Install bd or author scenarios manually.") return nil @@ -183,7 +116,7 @@ func writeExtractedScenarios(cmd *cobra.Command, id string, bead fetchedBead, ex } newDesc := composeDescriptionWithScenarios(bead.Description, rendered) - if _, err := execBD("update", id, "--description", newDesc); err != nil { + if _, err := beadsTrackerOutput("update", id, "--description", newDesc); err != nil { return fmt.Errorf("write '## Scenarios' into %s: %w (inspect with 'bd show %s')", id, err, id) } @@ -216,10 +149,10 @@ func composeDescriptionWithScenarios(desc, rendered string) string { return base + "\n\n" + block + "\n" } -func runBeadsScenariosValidate(cmd *cobra.Command, args []string) error { +func executeBeadsScenariosValidate(cmd *cobra.Command, args []string) error { id := args[0] - if !bdAvailable() { + if !beadsTrackerAvailable() { fmt.Fprintln(cmd.ErrOrStderr(), "warning: bd not found on PATH; cannot fetch bead. Install bd or author scenarios manually.") return nil diff --git a/cli/cmd/ao/beads_scenarios_test.go b/cli/cmd/ao/beads_scenarios_test.go index e575e0f65..8141412be 100644 --- a/cli/cmd/ao/beads_scenarios_test.go +++ b/cli/cmd/ao/beads_scenarios_test.go @@ -8,17 +8,17 @@ import ( "testing" ) -// withStubbedBD swaps execBD/bdAvailable for the duration of a test and +// withStubbedBD swaps beadsTrackerOutput/beadsTrackerAvailable for the duration of a test and // restores them afterward. The stub records every bd argv so tests can assert // the dry-run contract (no `bd update`). func withStubbedBD(t *testing.T, available bool, handler func(args ...string) ([]byte, error)) *[][]string { t.Helper() - origExec, origAvail := execBD, bdAvailable - t.Cleanup(func() { execBD, bdAvailable = origExec, origAvail }) + origExec, origAvail := beadsTrackerOutput, beadsTrackerAvailable + t.Cleanup(func() { beadsTrackerOutput, beadsTrackerAvailable = origExec, origAvail }) var calls [][]string - bdAvailable = func() bool { return available } - execBD = func(args ...string) ([]byte, error) { + beadsTrackerAvailable = func() bool { return available } + beadsTrackerOutput = func(args ...string) ([]byte, error) { calls = append(calls, args) return handler(args...) } @@ -38,7 +38,7 @@ func runScenariosExtract(t *testing.T, jsonOut bool, id string) (stdout, stderr beadsScenariosExtractCmd.SetErr(nil) }) - err = runBeadsScenariosExtract(beadsScenariosExtractCmd, []string{id}) + err = executeBeadsScenariosExtract(beadsScenariosExtractCmd, []string{id}) return out.String(), errBuf.String(), err } @@ -55,7 +55,7 @@ func runScenariosValidate(t *testing.T, jsonOut bool, id string) (stdout, stderr beadsScenariosValidateCmd.SetErr(nil) }) - err = runBeadsScenariosValidate(beadsScenariosValidateCmd, []string{id}) + err = executeBeadsScenariosValidate(beadsScenariosValidateCmd, []string{id}) return out.String(), errBuf.String(), err } @@ -76,7 +76,7 @@ func runScenariosExtractWrite(t *testing.T, id, stdin string) (stdout, stderr st beadsScenariosExtractCmd.SetIn(nil) }) - err = runBeadsScenariosExtract(beadsScenariosExtractCmd, []string{id}) + err = executeBeadsScenariosExtract(beadsScenariosExtractCmd, []string{id}) return out.String(), errBuf.String(), err } @@ -245,7 +245,7 @@ func TestRunBeadsScenariosValidate_JSONFailureVerdictOnStdout(t *testing.T) { func TestRunBeadsScenariosValidate_BDUnavailableWarnsAndSucceeds(t *testing.T) { withStubbedBD(t, false, func(args ...string) ([]byte, error) { - t.Fatalf("execBD must not be called when bd is unavailable: %v", args) + t.Fatalf("beadsTrackerOutput must not be called when bd is unavailable: %v", args) return nil, nil }) @@ -337,7 +337,7 @@ func TestRunBeadsScenariosExtract_UnparseableIsError(t *testing.T) { func TestRunBeadsScenariosExtract_BDUnavailableWarnsAndSucceeds(t *testing.T) { withStubbedBD(t, false, func(args ...string) ([]byte, error) { - t.Fatalf("execBD must not be called when bd is unavailable: %v", args) + t.Fatalf("beadsTrackerOutput must not be called when bd is unavailable: %v", args) return nil, nil }) diff --git a/cli/cmd/ao/beads_stale.go b/cli/cmd/ao/beads_stale.go index 2ca77c077..aefeacbe4 100644 --- a/cli/cmd/ao/beads_stale.go +++ b/cli/cmd/ao/beads_stale.go @@ -29,29 +29,6 @@ var ( beadsStaleNowOverride string // used by tests to make detected_at deterministic ) -var beadsStaleCmd = &cobra.Command{ - Use: "stale-claims", - Short: "List in_progress beads whose claim looks stale", - Long: `Lists in_progress beads whose claim activity is older than --threshold. - -A bead is "stale" when its updated_at timestamp is older than the threshold -(default 4h). Future signals (worktree quietness, session-bootstrap heartbeat -expiry) will be added when the enabling primitives (soc-vuu6.25) ship. - -Output: human-readable table by default, or JSON array (matching -schemas/stale-claim-event.v1.schema.json shape) with --json. Read-only: -this command DOES NOT transfer claims. Use 'ao beads resume ' for that -(soc-vuu6.27 slice 3).`, - RunE: runBeadsStale, -} - -func init() { - beadsStaleCmd.Flags().Float64Var(&beadsStaleThresholdHours, "threshold", 4.0, - "Staleness threshold in hours (claim updated more than N hours ago).") - beadsStaleCmd.Flags().BoolVar(&beadsStaleJSON, "json", false, - "Emit JSON array conforming to stale-claim-event.v1 (event_type: stale_detected).") -} - // staleBeadRecord is the subset of `br list --json` output we care about. type staleBeadRecord = beadsapp.StaleBeadRecord @@ -61,7 +38,7 @@ type staleEvent = beadsapp.StaleEvent type staleAgent = beadsapp.StaleAgent type staleEvidence = beadsapp.StaleEvidence -func runBeadsStale(cmd *cobra.Command, args []string) error { +func executeBeadsStale(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() diff --git a/cli/cmd/ao/beads_stale_test.go b/cli/cmd/ao/beads_stale_test.go index d0a516dca..f1af5f9fa 100644 --- a/cli/cmd/ao/beads_stale_test.go +++ b/cli/cmd/ao/beads_stale_test.go @@ -169,11 +169,11 @@ func TestRunBeadsStale_JSON_OutputShape(t *testing.T) { beadsStaleJSON = true buf := &bytes.Buffer{} - beadsStaleCmd.SetOut(buf) - defer beadsStaleCmd.SetOut(nil) + legacyBeadsStaleCommand.SetOut(buf) + defer legacyBeadsStaleCommand.SetOut(nil) - if err := runBeadsStale(beadsStaleCmd, nil); err != nil { - t.Fatalf("runBeadsStale: %v", err) + if err := executeBeadsStale(legacyBeadsStaleCommand, nil); err != nil { + t.Fatalf("executeBeadsStale: %v", err) } var got []staleEvent @@ -206,11 +206,11 @@ func TestRunBeadsStale_HumanOutput_HasZeroMessage(t *testing.T) { beadsStaleJSON = false buf := &bytes.Buffer{} - beadsStaleCmd.SetOut(buf) - defer beadsStaleCmd.SetOut(nil) + legacyBeadsStaleCommand.SetOut(buf) + defer legacyBeadsStaleCommand.SetOut(nil) - if err := runBeadsStale(beadsStaleCmd, nil); err != nil { - t.Fatalf("runBeadsStale: %v", err) + if err := executeBeadsStale(legacyBeadsStaleCommand, nil); err != nil { + t.Fatalf("executeBeadsStale: %v", err) } if !strings.Contains(buf.String(), "none") { t.Errorf("expected zero-state message containing 'none'; got %q", buf.String()) @@ -223,7 +223,7 @@ func TestRunBeadsStale_BdMalformed(t *testing.T) { beadsStaleFetchCmd = func(_ context.Context) ([]byte, error) { return []byte("not json"), nil } - err := runBeadsStale(beadsStaleCmd, nil) + err := executeBeadsStale(legacyBeadsStaleCommand, nil) if err == nil { t.Fatalf("expected error on malformed br-list JSON; got nil") } diff --git a/cli/cmd/ao/beads_test.go b/cli/cmd/ao/beads_test.go index e5fd30027..abf63b31b 100644 --- a/cli/cmd/ao/beads_test.go +++ b/cli/cmd/ao/beads_test.go @@ -1,6 +1,6 @@ // Tests for `ao beads` subcommands — focused on pure-function logic // (parsing, extraction, verification wiring) rather than end-to-end bd -// invocations. Tests that need bd override execBD/bdAvailable with fakes. +// invocations. Tests that need bd override beadsTrackerOutput/beadsTrackerAvailable with fakes. // practices: [dora-metrics, lean-startup] package main @@ -260,10 +260,10 @@ func TestIsClosedStatus_RecognisesCommonSpellings(t *testing.T) { } func TestListBeadIDs_ParsesFlatAndTreeOutput(t *testing.T) { - origBD, origAvail := execBD, bdAvailable - defer func() { execBD, bdAvailable = origBD, origAvail }() - bdAvailable = func() bool { return true } - execBD = func(args ...string) ([]byte, error) { + origBD, origAvail := beadsTrackerOutput, beadsTrackerAvailable + defer func() { beadsTrackerOutput, beadsTrackerAvailable = origBD, origAvail }() + beadsTrackerAvailable = func() bool { return true } + beadsTrackerOutput = func(args ...string) ([]byte, error) { return []byte(`✓ na-0g5 ● P1 task Integrate behavioral discipline ✓ na-348 ● P1 epic Flywheel provenance-first live corpus control loop ├── ✓ na-348.1 ● P1 task Retro quick-capture provenance defaults @@ -287,12 +287,12 @@ not a bead line — should be skipped } } -// ---------- bdAvailable degradation ---------- +// ---------- beadsTrackerAvailable degradation ---------- func TestVerifyBead_DegradesWhenBDAbsent(t *testing.T) { - origAvail := bdAvailable - defer func() { bdAvailable = origAvail }() - bdAvailable = func() bool { return false } + origAvail := beadsTrackerAvailable + defer func() { beadsTrackerAvailable = origAvail }() + beadsTrackerAvailable = func() bool { return false } report, err := verifyBead("na-nothing") if err != nil { @@ -307,9 +307,9 @@ func TestVerifyBead_DegradesWhenBDAbsent(t *testing.T) { } func TestAuditBeads_DegradesWhenBDAbsent(t *testing.T) { - origAvail := bdAvailable - defer func() { bdAvailable = origAvail }() - bdAvailable = func() bool { return false } + origAvail := beadsTrackerAvailable + defer func() { beadsTrackerAvailable = origAvail }() + beadsTrackerAvailable = func() bool { return false } report, err := auditBeads(false) if err != nil { @@ -324,15 +324,15 @@ func TestAuditBeads_DegradesWhenBDAbsent(t *testing.T) { } func TestAuditBeads_ClassifiesStaleAndConsolidatable(t *testing.T) { - origBD, origAvail := execBD, bdAvailable + origBD, origAvail := beadsTrackerOutput, beadsTrackerAvailable origGit, origPattern := execGitLog, repoPatternExists defer func() { - execBD, bdAvailable = origBD, origAvail + beadsTrackerOutput, beadsTrackerAvailable = origBD, origAvail execGitLog, repoPatternExists = origGit, origPattern }() - bdAvailable = func() bool { return true } - execBD = func(args ...string) ([]byte, error) { + beadsTrackerAvailable = func() bool { return true } + beadsTrackerOutput = func(args ...string) ([]byte, error) { if len(args) >= 4 && args[0] == "list" && args[3] == "--json" { switch args[2] { case "open": @@ -442,9 +442,9 @@ func TestPatternExistsInRepoSearchesScopedRoots(t *testing.T) { } func TestClusterBeads_DegradesWhenBDAbsent(t *testing.T) { - origAvail := bdAvailable - defer func() { bdAvailable = origAvail }() - bdAvailable = func() bool { return false } + origAvail := beadsTrackerAvailable + defer func() { beadsTrackerAvailable = origAvail }() + beadsTrackerAvailable = func() bool { return false } report, err := clusterBeads(false) if err != nil { @@ -497,14 +497,14 @@ func TestClusterBeadRecords_GroupsSharedPathAndPrefersEpic(t *testing.T) { } } -// ---------- verifyBead with execBD fake (end-to-end smoke) ---------- +// ---------- verifyBead with beadsTrackerOutput fake (end-to-end smoke) ---------- func TestVerifyBead_ParsesAndClassifiesCitations(t *testing.T) { - origBD, origAvail := execBD, bdAvailable - defer func() { execBD, bdAvailable = origBD, origAvail }() + origBD, origAvail := beadsTrackerOutput, beadsTrackerAvailable + defer func() { beadsTrackerOutput, beadsTrackerAvailable = origBD, origAvail }() - bdAvailable = func() bool { return true } - execBD = func(args ...string) ([]byte, error) { + beadsTrackerAvailable = func() bool { return true } + beadsTrackerOutput = func(args ...string) ([]byte, error) { return []byte(`○ na-fake · Fake bead for testing [● P2 · OPEN] Owner: Test · Type: task @@ -823,34 +823,34 @@ func TestEmitLintHuman_AllCleanProducesHeaderOnly(t *testing.T) { } } -// ---------- runBeadsVerify / runBeadsLint graceful-degradation paths ---------- +// ---------- executeBeadsVerify / executeBeadsLint graceful-degradation paths ---------- // Cycle 77 / soc-wxh5.1.1 — fourth climb-back. Both functions are cobra // Run handlers. Their graceful-degradation path (bd not on PATH) is -// testable by stubbing the bdAvailable var. Sibling pattern: +// testable by stubbing the beadsTrackerAvailable var. Sibling pattern: // TestVerifyBead_DegradesWhenBDAbsent (this file, line 292). func TestRunBeadsVerify_DegradesGracefullyWhenBDAbsent(t *testing.T) { - origAvail := bdAvailable - defer func() { bdAvailable = origAvail }() - bdAvailable = func() bool { return false } + origAvail := beadsTrackerAvailable + defer func() { beadsTrackerAvailable = origAvail }() + beadsTrackerAvailable = func() bool { return false } - // runBeadsVerify reads args[0] for the beadID; cmd is unused on + // executeBeadsVerify reads args[0] for the beadID; cmd is unused on // the degradation path. - err := runBeadsVerify(nil, []string{"soc-graceful-test"}) + err := executeBeadsVerify(nil, []string{"soc-graceful-test"}) if err != nil { - t.Fatalf("runBeadsVerify should return nil on graceful degradation, got %v", err) + t.Fatalf("executeBeadsVerify should return nil on graceful degradation, got %v", err) } } func TestRunBeadsLint_DegradesGracefullyWhenBDAbsent(t *testing.T) { - origAvail := bdAvailable - defer func() { bdAvailable = origAvail }() - bdAvailable = func() bool { return false } + origAvail := beadsTrackerAvailable + defer func() { beadsTrackerAvailable = origAvail }() + beadsTrackerAvailable = func() bool { return false } - // runBeadsLint takes no positional args; cmd is unused on the + // executeBeadsLint takes no positional args; cmd is unused on the // degradation path. - err := runBeadsLint(nil, []string{}) + err := executeBeadsLint(nil, []string{}) if err != nil { - t.Fatalf("runBeadsLint should return nil on graceful degradation, got %v", err) + t.Fatalf("executeBeadsLint should return nil on graceful degradation, got %v", err) } } diff --git a/cli/cmd/ao/beads_tracker_test.go b/cli/cmd/ao/beads_tracker_test.go index 70d0054cd..5baa34521 100644 --- a/cli/cmd/ao/beads_tracker_test.go +++ b/cli/cmd/ao/beads_tracker_test.go @@ -72,10 +72,10 @@ func TestRunBeadsDirJSON(t *testing.T) { t.Chdir(lane) var out strings.Builder - cmd := beadsDirCmd + cmd := legacyBeadsDirCommand cmd.SetOut(&out) t.Cleanup(func() { cmd.SetOut(nil) }) - if err := runBeadsDir(cmd, nil); err != nil { + if err := executeBeadsDir(cmd, nil); err != nil { t.Fatal(err) } var payload map[string]string @@ -109,12 +109,12 @@ func TestRunBeadsDirRequireFailsClosedWhenNoLedger(t *testing.T) { setBeadsDirRequire(t, true) var out strings.Builder - cmd := beadsDirCmd + cmd := legacyBeadsDirCommand cmd.SetOut(&out) t.Cleanup(func() { cmd.SetOut(nil) }) - err := runBeadsDir(cmd, nil) + err := executeBeadsDir(cmd, nil) if err == nil { - t.Fatalf("runBeadsDir(--require) = nil error for ledgerless dir; want fail-closed error") + t.Fatalf("executeBeadsDir(--require) = nil error for ledgerless dir; want fail-closed error") } if !strings.Contains(err.Error(), "no ledger artifact") { t.Fatalf("error = %q, want the no-ledger-artifact reason", err) @@ -130,12 +130,12 @@ func TestRunBeadsDirRequireFailsClosedWhenPathMissing(t *testing.T) { setBeadsDirRequire(t, true) var out strings.Builder - cmd := beadsDirCmd + cmd := legacyBeadsDirCommand cmd.SetOut(&out) t.Cleanup(func() { cmd.SetOut(nil) }) - err := runBeadsDir(cmd, nil) + err := executeBeadsDir(cmd, nil) if err == nil || !strings.Contains(err.Error(), "does not exist") { - t.Fatalf("runBeadsDir(--require) err = %v, want does-not-exist refusal", err) + t.Fatalf("executeBeadsDir(--require) err = %v, want does-not-exist refusal", err) } if out.String() != "" { t.Fatalf("stdout = %q, want empty on refusal", out.String()) @@ -155,11 +155,11 @@ func TestRunBeadsDirRequirePassesWithLedgerArtifact(t *testing.T) { setBeadsDirRequire(t, true) var out strings.Builder - cmd := beadsDirCmd + cmd := legacyBeadsDirCommand cmd.SetOut(&out) t.Cleanup(func() { cmd.SetOut(nil) }) - if err := runBeadsDir(cmd, nil); err != nil { - t.Fatalf("runBeadsDir(--require) = %v for a dir with issues.jsonl; want success", err) + if err := executeBeadsDir(cmd, nil); err != nil { + t.Fatalf("executeBeadsDir(--require) = %v for a dir with issues.jsonl; want success", err) } if got := strings.TrimSpace(out.String()); got != ledger { t.Fatalf("stdout = %q, want resolved ledger path %q", got, ledger) diff --git a/cli/cmd/ao/beads_verify_acceptance.go b/cli/cmd/ao/beads_verify_acceptance.go index 0050520f4..d56390c6b 100644 --- a/cli/cmd/ao/beads_verify_acceptance.go +++ b/cli/cmd/ao/beads_verify_acceptance.go @@ -38,7 +38,7 @@ import ( // execBR is the single entry point for shelling out to br (beads_rust), the // sanctioned tracker. It is a package var so tests inject a fake without a live -// ledger — the br-native analog of execBD. BEADS_DIR is inherited from the +// ledger — the br-native analog of beadsTrackerOutput. BEADS_DIR is inherited from the // process environment (exec.Command keeps os.Environ by default). var execBR = func(args ...string) ([]byte, error) { cwd, _ := os.Getwd() @@ -513,44 +513,6 @@ func validateAcceptanceCriteriaContent(desc string) []string { return problems } -func newBeadsVerifyAcceptanceCmd() *cobra.Command { - var strict, asJSON bool - cmd := &cobra.Command{ - Use: "verify-acceptance ...", - Short: "Assert each bead carries the acceptance contract for its type (br-native)", - Long: `Read beads via br (never the retired bd) and check each bead carries the -acceptance contract that fits its issue_type: - - feature -> Gherkin '## Scenarios' + a TDD/test signal - spike -> decision-criteria - design -> formal-spec - test -> assertion-inventory - cutover -> migration-checklist - task/epic -> N/A (structural, no per-item acceptance) - other -> UNDEFINED (no contract defined; reported, never silently skipped) - -Advisory by default (prints verdicts, exits 0). With --strict, any FAIL or -UNDEFINED maps to a non-zero exit — the gate posture for new/changed beads. - -Scope: this verifies the contract SHAPE for each type (the right section -headings / structural tokens) AND its CONTENT QUALITY (age-xmkn): a section or -criterion that is structurally present but only a placeholder ("TBD") FAILs, and -a fenced acceptance_criteria block is parsed and each criterion validated against -the canonical authored contract (skills/plan/SKILL.md) — a measurable -description, a valid check_type, a check_command for runnable check_types, and -agent_judge when check_type == custom_rubric. The lifted fields (weight / -optional / evidence_required) are added by /discovery, not authored, so they are -not required here.`, - Args: cobra.MinimumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - return runBeadsVerifyAcceptance(cmd, args, strict, asJSON) - }, - } - cmd.Flags().BoolVar(&strict, "strict", false, "Exit non-zero on any FAIL or UNDEFINED verdict") - cmd.Flags().BoolVar(&asJSON, "json", false, "Emit verdicts as JSON") - return cmd -} - type acceptanceResult struct { BeadID string `json:"bead_id"` IssueType string `json:"issue_type"` @@ -558,7 +520,7 @@ type acceptanceResult struct { Missing []string `json:"missing,omitempty"` } -func runBeadsVerifyAcceptance(cmd *cobra.Command, ids []string, strict, asJSON bool) error { +func executeBeadsVerifyAcceptance(cmd *cobra.Command, ids []string, strict, asJSON bool) error { out, err := execBR(append([]string{"show", "--format", "json"}, ids...)...) if err != nil { return fmt.Errorf("br show %v: %w", ids, err) @@ -613,16 +575,12 @@ func runBeadsVerifyAcceptance(cmd *cobra.Command, ids []string, strict, asJSON b } if strict && nonPass { - return &beadsExitError{code: 1} + return &beadsVerdictError{code: 1} } return nil } // asBeadsExit is a thin errors.As wrapper used by tests. -func asBeadsExit(err error, target **beadsExitError) bool { +func asBeadsExit(err error, target **beadsVerdictError) bool { return errors.As(err, target) } - -func init() { - beadsCmd.AddCommand(newBeadsVerifyAcceptanceCmd()) -} diff --git a/cli/cmd/ao/beads_verify_acceptance_test.go b/cli/cmd/ao/beads_verify_acceptance_test.go index 99d97a2c5..6030387d4 100644 --- a/cli/cmd/ao/beads_verify_acceptance_test.go +++ b/cli/cmd/ao/beads_verify_acceptance_test.go @@ -191,7 +191,7 @@ func TestRunBeadsVerifyAcceptance_AdvisoryExitsZero(t *testing.T) { return []byte(`[{"id":"age-x","issue_type":"feature","description":"prose only"}]`), nil }) var out bytes.Buffer - cmd := newBeadsVerifyAcceptanceCmd() + cmd := newLegacyBeadsVerifyAcceptanceCommand() cmd.SetOut(&out) cmd.SetArgs([]string{"age-x"}) if err := cmd.Execute(); err != nil { @@ -207,16 +207,16 @@ func TestRunBeadsVerifyAcceptance_StrictExitsNonZero(t *testing.T) { return []byte(`[{"id":"age-x","issue_type":"feature","description":"prose only"}]`), nil }) var out bytes.Buffer - cmd := newBeadsVerifyAcceptanceCmd() + cmd := newLegacyBeadsVerifyAcceptanceCommand() cmd.SetOut(&out) cmd.SetArgs([]string{"--strict", "age-x"}) err := cmd.Execute() - var exitErr *beadsExitError + var exitErr *beadsVerdictError if err == nil { - t.Fatal("strict mode with a FAIL should return a non-zero beadsExitError") + t.Fatal("strict mode with a FAIL should return a non-zero beadsVerdictError") } if !asBeadsExit(err, &exitErr) || exitErr.ExitCode() != 1 { - t.Fatalf("want beadsExitError code 1, got: %v", err) + t.Fatalf("want beadsVerdictError code 1, got: %v", err) } } @@ -225,7 +225,7 @@ func TestRunBeadsVerifyAcceptance_StrictUndefinedIsNonZero(t *testing.T) { return []byte(`[{"id":"age-b","issue_type":"bug","description":"x"}]`), nil }) var out bytes.Buffer - cmd := newBeadsVerifyAcceptanceCmd() + cmd := newLegacyBeadsVerifyAcceptanceCommand() cmd.SetOut(&out) cmd.SetArgs([]string{"--strict", "age-b"}) if err := cmd.Execute(); err == nil { @@ -241,7 +241,7 @@ func TestRunBeadsVerifyAcceptance_PartialResponseErrors(t *testing.T) { // Requested two ids; br returns only one (the hypothetical partial array). return []byte(`[{"id":"age-x","issue_type":"task","description":"## Acceptance Criteria\n- done"}]`), nil }) - cmd := newBeadsVerifyAcceptanceCmd() + cmd := newLegacyBeadsVerifyAcceptanceCommand() cmd.SetOut(&bytes.Buffer{}) cmd.SetArgs([]string{"age-x", "age-missing"}) err := cmd.Execute() @@ -258,7 +258,7 @@ func TestRunBeadsVerifyAcceptance_PassExitsZeroStrict(t *testing.T) { // A task that carries a measurable acceptance signal passes even in strict mode. return []byte(`[{"id":"age-t","issue_type":"task","description":"do X\n\n## Acceptance Criteria\nacceptance_criteria:\n - id: ac-t.1\n description: X is verified by the test\n check_type: test_pass\n check_command: go test"}]`), nil }) - cmd := newBeadsVerifyAcceptanceCmd() + cmd := newLegacyBeadsVerifyAcceptanceCommand() cmd.SetOut(&bytes.Buffer{}) cmd.SetArgs([]string{"--strict", "age-t"}) if err := cmd.Execute(); err != nil { From 66f91c9058766dcbb9d0f1835745c05e45fd45cf Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:28:23 -0400 Subject: [PATCH 018/108] refactor(beads): internalize tracker command handlers (age-nw28h.2.1) --- cli/cmd/ao/beads.go | 62 -------------- cli/cmd/ao/beads_command_legacy_test.go | 59 ++++++++++++- cli/cmd/ao/beads_exec.go | 32 -------- cli/cmd/ao/beads_module.go | 12 +-- cli/internal/commands/beads/module.go | 96 ++++++++++++++++++++-- cli/internal/commands/beads/module_test.go | 90 +++++++++++++++++++- 6 files changed, 237 insertions(+), 114 deletions(-) diff --git a/cli/cmd/ao/beads.go b/cli/cmd/ao/beads.go index e146aaa4e..e05c937a2 100644 --- a/cli/cmd/ao/beads.go +++ b/cli/cmd/ao/beads.go @@ -90,68 +90,6 @@ var ( beadsHarvestDryRun bool ) -func executeBeadsDir(cmd *cobra.Command, _ []string) error { - tracker := currentBeadsTracker() - // BEADS_DIR is br's explicit ledger override; when it is set we preserve the - // historical br resolution verbatim (backward compatibility). Otherwise we - // consult the dual-tracker resolver so a bd-tracked repo returns its .beads - // directory instead of assuming _beads (age-fvr8). - if !tracker.BeadsDirOverride() { - if tr, terr := tracker.Resolve(); terr == nil && tr.Tracker == trackerBD { - if beadsDirRequire { - if reason := beadsapp.LedgerMissing(beadsapp.TrackerBD, tracker.InspectLedger(tr.LedgerDir)); reason != "" { - return fmt.Errorf("beads dir --require: %s (resolved %s for tracker bd via %s); refusing to print a path a bd write could silently fall back from", reason, tr.LedgerDir, tr.Source) - } - } - if beadsDirJSON { - return json.NewEncoder(cmd.OutOrStdout()).Encode(map[string]string{ - "beads_dir": tr.LedgerDir, - "source": tr.Source, - }) - } - fmt.Fprintln(cmd.OutOrStdout(), tr.LedgerDir) - return nil - } - } - // br path — unchanged historical behavior (also the both-present tie-break). - resolved, err := tracker.BRLedger() - if err != nil { - return err - } - if beadsDirRequire { - if reason := beadsapp.LedgerMissing(beadsapp.TrackerBR, tracker.InspectLedger(resolved.Path)); reason != "" { - return fmt.Errorf("beads dir --require: %s (resolved %s via %s); refusing to print a path a br write could silently fall back from", reason, resolved.Path, resolved.Source) - } - } - if beadsDirJSON { - return json.NewEncoder(cmd.OutOrStdout()).Encode(map[string]string{ - "beads_dir": resolved.Path, - "source": resolved.Source, - }) - } - fmt.Fprintln(cmd.OutOrStdout(), resolved.Path) - return nil -} - -// runBeadsTracker prints the resolved beads tracker for the current -// environment: which tracker (bd|br), its resolved binary, its ledger -// directory, and how it was selected (age-fvr8). -func executeBeadsTracker(cmd *cobra.Command, _ []string) error { - res, err := currentBeadsTracker().Resolve() - if err != nil { - return err - } - if beadsTrackerJSON { - return json.NewEncoder(cmd.OutOrStdout()).Encode(res) - } - w := cmd.OutOrStdout() - fmt.Fprintf(w, "tracker %s\n", res.Tracker) - fmt.Fprintf(w, "binary %s\n", res.Binary) - fmt.Fprintf(w, "ledger_dir %s\n", res.LedgerDir) - fmt.Fprintf(w, "source %s\n", res.Source) - return nil -} - // ------------------------------------------------------------------------ // verify // ------------------------------------------------------------------------ diff --git a/cli/cmd/ao/beads_command_legacy_test.go b/cli/cmd/ao/beads_command_legacy_test.go index 17da2f3e7..cc5af7773 100644 --- a/cli/cmd/ao/beads_command_legacy_test.go +++ b/cli/cmd/ao/beads_command_legacy_test.go @@ -1,6 +1,13 @@ package main -import "github.com/spf13/cobra" +import ( + "errors" + + beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" + beadsapp "github.com/boshu2/agentops/cli/internal/beads" + beadscommands "github.com/boshu2/agentops/cli/internal/commands/beads" + "github.com/spf13/cobra" +) // White-box behavior tests keep private throwaway commands for stream and flag // control. Production command ownership lives in internal/commands/beads. @@ -43,3 +50,53 @@ func newLegacyBeadsVerifyAcceptanceCommand() *cobra.Command { command.Flags().BoolVar(&asJSON, "json", false, "") return command } + +func executeBeadsDir(command *cobra.Command, _ []string) error { + child := newTestBeadsCommand("dir") + child.SetOut(command.OutOrStdout()) + var flags []string + if beadsDirJSON { + flags = append(flags, "--json") + } + if beadsDirRequire { + flags = append(flags, "--require") + } + if err := child.ParseFlags(flags); err != nil { + return err + } + return child.RunE(child, nil) +} + +func executeBeadsTracker(command *cobra.Command, _ []string) error { + child := newTestBeadsCommand("tracker") + child.SetOut(command.OutOrStdout()) + if beadsTrackerJSON { + if err := child.ParseFlags([]string{"--json"}); err != nil { + return err + } + } + return child.RunE(child, nil) +} + +func executeBeadsExec(command *cobra.Command, args []string) error { + child := newTestBeadsCommand("exec") + child.SetIn(command.InOrStdin()) + child.SetOut(command.OutOrStdout()) + child.SetErr(command.ErrOrStderr()) + err := child.RunE(child, args) + var exitError *beadsapp.ExitError + if errors.As(err, &exitError) { + return &beadsVerdictError{code: exitError.ExitCode()} + } + return err +} + +func newTestBeadsCommand(name string) *cobra.Command { + tracker := currentBeadsTracker() + root := beadscommands.NewModule(nil, tracker, tracker, beadsadapter.NewExecutor(tracker)).Command() + child, _, err := root.Find([]string{name}) + if err != nil || child == nil { + panic("missing test beads command " + name) + } + return child +} diff --git a/cli/cmd/ao/beads_exec.go b/cli/cmd/ao/beads_exec.go index 3f8313ca0..11dc3a427 100644 --- a/cli/cmd/ao/beads_exec.go +++ b/cli/cmd/ao/beads_exec.go @@ -4,43 +4,11 @@ package main import ( - "context" - "errors" "os" - "github.com/spf13/cobra" - - beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" beadsapp "github.com/boshu2/agentops/cli/internal/beads" ) -func executeBeadsExec(cmd *cobra.Command, args []string) error { - for _, argument := range args { - if argument == "--help" || argument == "-h" { - return cmd.Help() - } - } - err := currentBeadsExecutor().Execute(context.Background(), args, beadsapp.ExecStreams{ - Stdin: cmd.InOrStdin(), - Stdout: cmd.OutOrStdout(), - Stderr: cmd.ErrOrStderr(), - }) - var adapterExit *beadsapp.ExitError - if errors.As(err, &adapterExit) { - err = &beadsVerdictError{code: adapterExit.ExitCode()} - } - var exitErr interface{ ExitCode() int } - if errors.As(err, &exitErr) { - cmd.SilenceUsage = true - cmd.SilenceErrors = true - } - return err -} - -func currentBeadsExecutor() *beadsadapter.Executor { - return beadsadapter.NewExecutor(currentBeadsTracker()) -} - // These pure compatibility delegates remain until the yield family moves its // tracker-child formatting onto the shared application policy. func beadsExecChildEnv(res trackerResolution, _ string) []string { diff --git a/cli/cmd/ao/beads_module.go b/cli/cmd/ao/beads_module.go index 3ea13b06d..5f7af3de8 100644 --- a/cli/cmd/ao/beads_module.go +++ b/cli/cmd/ao/beads_module.go @@ -3,6 +3,7 @@ package main import ( "fmt" + beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" "github.com/boshu2/agentops/cli/internal/clicontract" beadscommands "github.com/boshu2/agentops/cli/internal/commands/beads" "github.com/spf13/cobra" @@ -17,12 +18,6 @@ type beadsModuleRunner struct{} func (beadsModuleRunner) Run(command *cobra.Command, invocation beadscommands.Invocation) error { options := invocation.Options switch invocation.Operation { - case beadscommands.OperationDir: - beadsDirJSON, beadsDirRequire = options.JSON, options.Require - return executeBeadsDir(command, invocation.Args) - case beadscommands.OperationTracker: - beadsTrackerJSON = options.JSON - return executeBeadsTracker(command, invocation.Args) case beadscommands.OperationVerify: beadsVerifyJSON, beadsVerifyVerbose = options.JSON, options.Verbose return executeBeadsVerify(command, invocation.Args) @@ -38,8 +33,6 @@ func (beadsModuleRunner) Run(command *cobra.Command, invocation beadscommands.In case beadscommands.OperationCluster: beadsClusterJSON, beadsClusterApply = options.JSON, options.Apply return executeBeadsCluster(command, invocation.Args) - case beadscommands.OperationExec: - return executeBeadsExec(command, invocation.Args) case beadscommands.OperationResume: beadsResumeAgentID, beadsResumeLedgerPath, beadsResumeJSON = options.Agent, options.Ledger, options.JSON return executeBeadsResume(command, invocation.Args) @@ -63,7 +56,8 @@ func (beadsModuleRunner) Run(command *cobra.Command, invocation beadscommands.In } func init() { - module := beadscommands.NewModule(beadsModuleRunner{}) + tracker := currentBeadsTracker() + module := beadscommands.NewModule(beadsModuleRunner{}, tracker, tracker, beadsadapter.NewExecutor(tracker)) command := module.Command() command.GroupID = "knowledge" if err := clicontract.Attach(command, module.Contract()); err != nil { diff --git a/cli/internal/commands/beads/module.go b/cli/internal/commands/beads/module.go index 66b4d3439..a476f6da9 100644 --- a/cli/internal/commands/beads/module.go +++ b/cli/internal/commands/beads/module.go @@ -4,10 +4,14 @@ package beads import ( + "context" + "encoding/json" + "errors" "fmt" "github.com/spf13/cobra" + beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/boshu2/agentops/cli/internal/clicontract" ) @@ -61,11 +65,14 @@ type Runner interface { } type Module struct { - runner Runner + runner Runner + resolver beadsapp.TrackerResolver + inspector beadsapp.LedgerInspector + executor beadsapp.TrackerExecutor } -func NewModule(runner Runner) Module { - return Module{runner: runner} +func NewModule(runner Runner, resolver beadsapp.TrackerResolver, inspector beadsapp.LedgerInspector, executor beadsapp.TrackerExecutor) Module { + return Module{runner: runner, resolver: resolver, inspector: inspector, executor: executor} } func (Module) Contract() clicontract.CommandContract { @@ -123,8 +130,8 @@ func (module Module) dirCommand() *cobra.Command { command := &cobra.Command{Use: "dir", Short: "Print the resolved live br ledger directory", Args: cobra.NoArgs} command.Flags().BoolVar(&options.JSON, "json", false, "Emit {beads_dir, source} as JSON") command.Flags().BoolVar(&options.Require, "require", false, "Fail closed: exit non-zero (printing nothing to stdout) unless the resolved directory holds a br ledger") - command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationDir, args, options) + command.RunE = func(command *cobra.Command, _ []string) error { + return module.runDir(command, options) } return command } @@ -133,8 +140,8 @@ func (module Module) trackerCommand() *cobra.Command { var options Options command := &cobra.Command{Use: "tracker", Short: "Print the resolved beads tracker (bd or br) for this environment", Args: cobra.NoArgs} command.Flags().BoolVar(&options.JSON, "json", false, "Emit {tracker, binary, ledger_dir, source} as JSON") - command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationTracker, args, options) + command.RunE = func(command *cobra.Command, _ []string) error { + return module.runTracker(command, options) } return command } @@ -202,11 +209,84 @@ func (module Module) execCommand() *cobra.Command { DisableFlagParsing: true, } command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationExec, args, Options{}) + return module.runExec(command, args) } return command } +func (module Module) runDir(command *cobra.Command, options Options) error { + if module.resolver == nil || module.inspector == nil { + return fmt.Errorf("beads directory ports are not configured") + } + if !module.resolver.BeadsDirOverride() { + if resolved, err := module.resolver.Resolve(); err == nil && resolved.Tracker == beadsapp.TrackerBD { + if options.Require { + if reason := beadsapp.LedgerMissing(beadsapp.TrackerBD, module.inspector.InspectLedger(resolved.LedgerDir)); reason != "" { + return fmt.Errorf("beads dir --require: %s (resolved %s for tracker bd via %s); refusing to print a path a bd write could silently fall back from", reason, resolved.LedgerDir, resolved.Source) + } + } + return writeDirectory(command, options.JSON, resolved.LedgerDir, resolved.Source) + } + } + resolved, err := module.resolver.BRLedger() + if err != nil { + return err + } + if options.Require { + if reason := beadsapp.LedgerMissing(beadsapp.TrackerBR, module.inspector.InspectLedger(resolved.Path)); reason != "" { + return fmt.Errorf("beads dir --require: %s (resolved %s via %s); refusing to print a path a br write could silently fall back from", reason, resolved.Path, resolved.Source) + } + } + return writeDirectory(command, options.JSON, resolved.Path, resolved.Source) +} + +func writeDirectory(command *cobra.Command, asJSON bool, path, source string) error { + if asJSON { + return json.NewEncoder(command.OutOrStdout()).Encode(map[string]string{"beads_dir": path, "source": source}) + } + _, err := fmt.Fprintln(command.OutOrStdout(), path) + return err +} + +func (module Module) runTracker(command *cobra.Command, options Options) error { + if module.resolver == nil { + return fmt.Errorf("beads tracker resolver is not configured") + } + resolved, err := module.resolver.Resolve() + if err != nil { + return err + } + if options.JSON { + return json.NewEncoder(command.OutOrStdout()).Encode(resolved) + } + output := command.OutOrStdout() + fmt.Fprintf(output, "tracker %s\n", resolved.Tracker) + fmt.Fprintf(output, "binary %s\n", resolved.Binary) + fmt.Fprintf(output, "ledger_dir %s\n", resolved.LedgerDir) + fmt.Fprintf(output, "source %s\n", resolved.Source) + return nil +} + +func (module Module) runExec(command *cobra.Command, args []string) error { + for _, argument := range args { + if argument == "--help" || argument == "-h" { + return command.Help() + } + } + if module.executor == nil { + return fmt.Errorf("beads tracker executor is not configured") + } + err := module.executor.Execute(context.Background(), args, beadsapp.ExecStreams{ + Stdin: command.InOrStdin(), Stdout: command.OutOrStdout(), Stderr: command.ErrOrStderr(), + }) + var exitError interface{ ExitCode() int } + if errors.As(err, &exitError) { + command.SilenceUsage = true + command.SilenceErrors = true + } + return err +} + func (module Module) resumeCommand() *cobra.Command { var options Options command := &cobra.Command{Use: "resume ", Short: "Atomically transfer an in_progress claim from a stale agent to this one", Args: cobra.ExactArgs(1)} diff --git a/cli/internal/commands/beads/module_test.go b/cli/internal/commands/beads/module_test.go index 34ae18d40..9424c68b9 100644 --- a/cli/internal/commands/beads/module_test.go +++ b/cli/internal/commands/beads/module_test.go @@ -1,9 +1,13 @@ package beads import ( + "bytes" + "context" + "errors" "reflect" "testing" + beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/spf13/cobra" ) @@ -18,7 +22,7 @@ func (runner *recordingRunner) Run(_ *cobra.Command, invocation Invocation) erro func TestModuleBuildsFreshCompleteTrees(t *testing.T) { runner := &recordingRunner{} - module := NewModule(runner) + module := NewModule(runner, nil, nil, nil) first := module.Command() second := module.Command() if first == second { @@ -39,7 +43,7 @@ func TestModuleBuildsFreshCompleteTrees(t *testing.T) { func TestModuleParsesTypedInvocation(t *testing.T) { runner := &recordingRunner{} - command := NewModule(runner).Command() + command := NewModule(runner, nil, nil, nil).Command() command.SetArgs([]string{"resume", "age-123", "--agent", "codex", "--json"}) if err := command.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -57,3 +61,85 @@ func TestModuleParsesTypedInvocation(t *testing.T) { t.Fatalf("invocations = %#v, want %#v", runner.invocations, want) } } + +type fakeTrackerPorts struct { + resolution beadsapp.TrackerResolution + ledger beadsapp.LedgerResolution + snapshot beadsapp.LedgerSnapshot + override bool + executed bool + execErr error +} + +func (fake *fakeTrackerPorts) Resolve() (beadsapp.TrackerResolution, error) { + return fake.resolution, nil +} +func (fake *fakeTrackerPorts) BRLedger() (beadsapp.LedgerResolution, error) { + return fake.ledger, nil +} +func (fake *fakeTrackerPorts) BeadsDirOverride() bool { return fake.override } +func (fake *fakeTrackerPorts) InspectLedger(string) beadsapp.LedgerSnapshot { + return fake.snapshot +} +func (fake *fakeTrackerPorts) Execute(_ context.Context, _ []string, streams beadsapp.ExecStreams) error { + fake.executed = true + _, _ = streams.Stdout.Write([]byte("executed\n")) + return fake.execErr +} + +func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { + ports := &fakeTrackerPorts{ + resolution: beadsapp.TrackerResolution{Tracker: beadsapp.TrackerBD, Binary: "/bin/bd", LedgerDir: "/repo/.beads", Source: "ledger"}, + snapshot: beadsapp.LedgerSnapshot{Exists: true, Directory: true, Readable: true, Entries: []string{"beads.db"}}, + } + + t.Run("directory JSON", func(t *testing.T) { + var output bytes.Buffer + root := NewModule(nil, ports, ports, ports).Command() + root.SetOut(&output) + root.SetArgs([]string{"dir", "--require", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if got := output.String(); got != "{\"beads_dir\":\"/repo/.beads\",\"source\":\"ledger\"}\n" { + t.Fatalf("output = %q", got) + } + }) + + t.Run("tracker text", func(t *testing.T) { + var output bytes.Buffer + root := NewModule(nil, ports, ports, ports).Command() + root.SetOut(&output) + root.SetArgs([]string{"tracker"}) + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if got := output.String(); got != "tracker bd\nbinary /bin/bd\nledger_dir /repo/.beads\nsource ledger\n" { + t.Fatalf("output = %q", got) + } + }) + + t.Run("exec help precedes resolution", func(t *testing.T) { + ports.executed = false + root := NewModule(nil, ports, ports, ports).Command() + root.SetArgs([]string{"exec", "--help"}) + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if ports.executed { + t.Fatal("executor called for static help") + } + }) + + t.Run("exec preserves typed exit", func(t *testing.T) { + ports.executed = false + ports.execErr = &beadsapp.ExitError{Code: 7} + root := NewModule(nil, ports, ports, ports).Command() + root.SetArgs([]string{"exec", "close", "age-x"}) + err := root.Execute() + var exitError *beadsapp.ExitError + if !errors.As(err, &exitError) || exitError.ExitCode() != 7 { + t.Fatalf("error = %v, want ExitError(7)", err) + } + }) +} From 80a29757aa8cf90cee96aa1a51a34e08875685c3 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:32:09 -0400 Subject: [PATCH 019/108] refactor(beads): internalize recovery command handlers (age-nw28h.2.1) --- cli/cmd/ao/beads_command_legacy_test.go | 3 +- ...us.go => beads_epic_status_legacy_test.go} | 0 cli/cmd/ao/beads_module.go | 15 +- ..._resume.go => beads_resume_legacy_test.go} | 0 ...ds_stale.go => beads_stale_legacy_test.go} | 0 cli/internal/beads/epic_status.go | 4 + cli/internal/beads/resume.go | 13 ++ cli/internal/beads/stale.go | 4 + cli/internal/commands/beads/module.go | 161 +++++++++++++++++- cli/internal/commands/beads/module_test.go | 103 ++++++++++- 10 files changed, 281 insertions(+), 22 deletions(-) rename cli/cmd/ao/{beads_epic_status.go => beads_epic_status_legacy_test.go} (100%) rename cli/cmd/ao/{beads_resume.go => beads_resume_legacy_test.go} (100%) rename cli/cmd/ao/{beads_stale.go => beads_stale_legacy_test.go} (100%) diff --git a/cli/cmd/ao/beads_command_legacy_test.go b/cli/cmd/ao/beads_command_legacy_test.go index cc5af7773..e2817760b 100644 --- a/cli/cmd/ao/beads_command_legacy_test.go +++ b/cli/cmd/ao/beads_command_legacy_test.go @@ -93,7 +93,8 @@ func executeBeadsExec(command *cobra.Command, args []string) error { func newTestBeadsCommand(name string) *cobra.Command { tracker := currentBeadsTracker() - root := beadscommands.NewModule(nil, tracker, tracker, beadsadapter.NewExecutor(tracker)).Command() + runtime := beadsadapter.NewRuntime() + root := beadscommands.NewModule(nil, tracker, tracker, beadsadapter.NewExecutor(tracker), tracker, tracker, runtime, runtime).Command() child, _, err := root.Find([]string{name}) if err != nil || child == nil { panic("missing test beads command " + name) diff --git a/cli/cmd/ao/beads_epic_status.go b/cli/cmd/ao/beads_epic_status_legacy_test.go similarity index 100% rename from cli/cmd/ao/beads_epic_status.go rename to cli/cmd/ao/beads_epic_status_legacy_test.go diff --git a/cli/cmd/ao/beads_module.go b/cli/cmd/ao/beads_module.go index 5f7af3de8..fb5867407 100644 --- a/cli/cmd/ao/beads_module.go +++ b/cli/cmd/ao/beads_module.go @@ -33,21 +33,12 @@ func (beadsModuleRunner) Run(command *cobra.Command, invocation beadscommands.In case beadscommands.OperationCluster: beadsClusterJSON, beadsClusterApply = options.JSON, options.Apply return executeBeadsCluster(command, invocation.Args) - case beadscommands.OperationResume: - beadsResumeAgentID, beadsResumeLedgerPath, beadsResumeJSON = options.Agent, options.Ledger, options.JSON - return executeBeadsResume(command, invocation.Args) case beadscommands.OperationScenariosExtract: beadsScenariosJSON, beadsScenariosForce, beadsScenariosWrite = options.JSON, options.Force, options.Write return executeBeadsScenariosExtract(command, invocation.Args) case beadscommands.OperationScenariosCheck: beadsScenariosValidateJSON = options.JSON return executeBeadsScenariosValidate(command, invocation.Args) - case beadscommands.OperationStaleClaims: - beadsStaleThresholdHours, beadsStaleJSON = options.ThresholdHours, options.JSON - return executeBeadsStale(command, invocation.Args) - case beadscommands.OperationEpicStatus: - beadsEpicStatusTerminal, beadsEpicStatusJSON = options.Terminal, options.JSON - return executeBeadsEpicStatus(command, invocation.Args) case beadscommands.OperationAcceptance: return executeBeadsVerifyAcceptance(command, invocation.Args, options.Strict, options.JSON) default: @@ -57,7 +48,11 @@ func (beadsModuleRunner) Run(command *cobra.Command, invocation beadscommands.In func init() { tracker := currentBeadsTracker() - module := beadscommands.NewModule(beadsModuleRunner{}, tracker, tracker, beadsadapter.NewExecutor(tracker)) + runtime := beadsadapter.NewRuntime() + module := beadscommands.NewModule( + beadsModuleRunner{}, tracker, tracker, beadsadapter.NewExecutor(tracker), + tracker, tracker, runtime, runtime, + ) command := module.Command() command.GroupID = "knowledge" if err := clicontract.Attach(command, module.Contract()); err != nil { diff --git a/cli/cmd/ao/beads_resume.go b/cli/cmd/ao/beads_resume_legacy_test.go similarity index 100% rename from cli/cmd/ao/beads_resume.go rename to cli/cmd/ao/beads_resume_legacy_test.go diff --git a/cli/cmd/ao/beads_stale.go b/cli/cmd/ao/beads_stale_legacy_test.go similarity index 100% rename from cli/cmd/ao/beads_stale.go rename to cli/cmd/ao/beads_stale_legacy_test.go diff --git a/cli/internal/beads/epic_status.go b/cli/internal/beads/epic_status.go index 81a54ef31..dcb0c877c 100644 --- a/cli/internal/beads/epic_status.go +++ b/cli/internal/beads/epic_status.go @@ -9,6 +9,10 @@ import ( "github.com/boshu2/agentops/cli/internal/epicstatus" ) +type LedgerReader interface { + ReadFile(string) ([]byte, error) +} + // LedgerBead is the subset of an issues.jsonl record needed to derive epic // membership. type LedgerBead struct { diff --git a/cli/internal/beads/resume.go b/cli/internal/beads/resume.go index c7a62ab36..f647ab014 100644 --- a/cli/internal/beads/resume.go +++ b/cli/internal/beads/resume.go @@ -2,11 +2,24 @@ package beads import ( "bytes" + "context" "encoding/json" "fmt" "time" ) +type ClaimStore interface { + Show(context.Context, string) (StaleBeadRecord, error) + Claim(context.Context, string, string) error +} + +type ResumeRuntime interface { + Clock + Actor() string + ResolveRepoPath(string) (string, error) + AppendEvent(string, any) error +} + type TransferInfo struct { PriorRevision string `json:"prior_revision"` NewRevision string `json:"new_revision"` diff --git a/cli/internal/beads/stale.go b/cli/internal/beads/stale.go index bf93ed074..9fc176b9f 100644 --- a/cli/internal/beads/stale.go +++ b/cli/internal/beads/stale.go @@ -14,6 +14,10 @@ type StaleSource interface { ListInProgress(context.Context) ([]byte, error) } +type Clock interface { + Now() time.Time +} + type StaleSourceFunc func(context.Context) ([]byte, error) func (function StaleSourceFunc) ListInProgress(ctx context.Context) ([]byte, error) { diff --git a/cli/internal/commands/beads/module.go b/cli/internal/commands/beads/module.go index a476f6da9..8ff7bc8eb 100644 --- a/cli/internal/commands/beads/module.go +++ b/cli/internal/commands/beads/module.go @@ -8,11 +8,15 @@ import ( "encoding/json" "errors" "fmt" + "path/filepath" + "strings" + "time" "github.com/spf13/cobra" beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/boshu2/agentops/cli/internal/clicontract" + "github.com/boshu2/agentops/cli/internal/epicstatus" ) type Operation string @@ -69,10 +73,26 @@ type Module struct { resolver beadsapp.TrackerResolver inspector beadsapp.LedgerInspector executor beadsapp.TrackerExecutor + stale beadsapp.StaleSource + claims beadsapp.ClaimStore + runtime beadsapp.ResumeRuntime + reader beadsapp.LedgerReader } -func NewModule(runner Runner, resolver beadsapp.TrackerResolver, inspector beadsapp.LedgerInspector, executor beadsapp.TrackerExecutor) Module { - return Module{runner: runner, resolver: resolver, inspector: inspector, executor: executor} +func NewModule( + runner Runner, + resolver beadsapp.TrackerResolver, + inspector beadsapp.LedgerInspector, + executor beadsapp.TrackerExecutor, + stale beadsapp.StaleSource, + claims beadsapp.ClaimStore, + runtime beadsapp.ResumeRuntime, + reader beadsapp.LedgerReader, +) Module { + return Module{ + runner: runner, resolver: resolver, inspector: inspector, executor: executor, + stale: stale, claims: claims, runtime: runtime, reader: reader, + } } func (Module) Contract() clicontract.CommandContract { @@ -294,7 +314,7 @@ func (module Module) resumeCommand() *cobra.Command { command.Flags().StringVar(&options.Ledger, "ledger", "docs/provenance/ledger.jsonl", "Path to the provenance ledger (relative to repo root).") command.Flags().BoolVar(&options.JSON, "json", false, "Emit the claim_transferred event to stdout (always written to ledger).") command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationResume, args, options) + return module.runResume(command, args, options) } return command } @@ -325,7 +345,7 @@ func (module Module) staleCommand() *cobra.Command { command.Flags().Float64Var(&options.ThresholdHours, "threshold", 4, "Staleness threshold in hours (claim updated more than N hours ago).") command.Flags().BoolVar(&options.JSON, "json", false, "Emit JSON array conforming to stale-claim-event.v1 (event_type: stale_detected).") command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationStaleClaims, args, options) + return module.runStale(command, options) } return command } @@ -336,11 +356,142 @@ func (module Module) epicStatusCommand() *cobra.Command { command.Flags().BoolVar(&options.Terminal, "terminal", false, "Map the verdict to the process exit code (0 terminal / 2 not-terminal / 3 skipped).") command.Flags().BoolVar(&options.JSON, "json", false, "Emit the verdict as a JSON object instead of a human-readable line.") command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationEpicStatus, args, options) + return module.runEpicStatus(command, args, options) } return command } +func (module Module) runStale(command *cobra.Command, options Options) error { + if module.stale == nil || module.runtime == nil { + return fmt.Errorf("beads stale-claims ports are not configured") + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + events, err := beadsapp.DetectStale(ctx, module.stale, module.runtime.Now(), options.ThresholdHours) + if err != nil { + return fmt.Errorf("br list: %w", err) + } + if options.JSON { + encoded, err := json.Marshal(events) + if err != nil { + return fmt.Errorf("marshal events: %w", err) + } + _, err = fmt.Fprintln(command.OutOrStdout(), string(encoded)) + return err + } + if len(events) == 0 { + _, err := fmt.Fprintf(command.OutOrStdout(), "ao beads stale-claims: none — all in_progress beads touched within %.1fh\n", options.ThresholdHours) + return err + } + fmt.Fprintf(command.OutOrStdout(), "ao beads stale-claims: %d in_progress bead(s) stale (threshold %.1fh)\n", len(events), options.ThresholdHours) + for _, event := range events { + fmt.Fprintf(command.OutOrStdout(), " %-22s claim_age=%.1fh last_touch=%s claimant=%s\n", event.BeadID, event.Evidence.ClaimAgeHours, event.Evidence.LastTouchTS, event.OriginalClaimant.ID) + } + return nil +} + +func (module Module) runResume(command *cobra.Command, args []string, options Options) error { + if module.claims == nil || module.runtime == nil { + return fmt.Errorf("beads resume ports are not configured") + } + beadID := args[0] + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + prior, err := module.claims.Show(ctx, beadID) + if err != nil { + return fmt.Errorf("fetch prior state: %w", err) + } + if prior.Status != "in_progress" { + return fmt.Errorf("bead %s is %q, not in_progress — resume only handles in_progress claims", beadID, prior.Status) + } + now := module.runtime.Now().UTC() + agent := options.Agent + if agent == "" { + agent = module.runtime.Actor() + } + if agent == "" { + agent = "ao-beads-resume" + } + if err := module.claims.Claim(ctx, beadID, agent); err != nil { + return fmt.Errorf("claim transfer: %w", err) + } + posterior, err := module.claims.Show(ctx, beadID) + if err != nil { + posterior = beadsapp.StaleBeadRecord{ID: beadID, Status: "in_progress", Assignee: agent, UpdatedAt: now.Format(time.RFC3339)} + } + event := beadsapp.BuildTransferredEvent(beadID, agent, prior, posterior, now) + ledger, err := module.runtime.ResolveRepoPath(options.Ledger) + if err != nil { + return err + } + if err := module.runtime.AppendEvent(ledger, event); err != nil { + return fmt.Errorf("append ledger (claim already transferred): %w", err) + } + if options.JSON { + encoded, err := json.Marshal(event) + if err != nil { + return fmt.Errorf("marshal transferred claim: %w", err) + } + _, err = fmt.Fprintln(command.OutOrStdout(), string(encoded)) + return err + } + priorAgent := prior.Assignee + if priorAgent == "" { + priorAgent = "unknown" + } + _, err = fmt.Fprintf(command.OutOrStdout(), "ao beads resume: %s transferred from %q to %q (prior_rev=%s, new_rev=%s)\n", beadID, priorAgent, agent, event.Transfer.PriorRevision, event.Transfer.NewRevision) + return err +} + +func (module Module) runEpicStatus(command *cobra.Command, args []string, options Options) error { + if module.resolver == nil || module.reader == nil { + return fmt.Errorf("beads epic-status ports are not configured") + } + epic := strings.TrimSpace(args[0]) + ledger, err := module.resolver.BRLedger() + if err != nil { + return err + } + path := filepath.Join(ledger.Path, "issues.jsonl") + raw, err := module.reader.ReadFile(path) + if err != nil { + return fmt.Errorf("read ledger %s: %w", path, err) + } + records, err := beadsapp.ParseLedger(raw) + if err != nil { + return fmt.Errorf("parse ledger: %w", err) + } + members, present := beadsapp.BuildMembers(epic, records) + if !present { + return fmt.Errorf("epic %s not found in ledger %s", epic, ledger.Path) + } + result := epicstatus.Evaluate(epic, members) + if options.JSON { + encoder := json.NewEncoder(command.OutOrStdout()) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(result); err != nil { + return err + } + } else { + fmt.Fprintf(command.OutOrStdout(), "%s: %s [%s]\n", result.Group, strings.ToUpper(string(result.Verdict)), result.Code) + fmt.Fprintf(command.OutOrStdout(), " %s\n", result.Reason) + for _, blocker := range result.Blockers { + fmt.Fprintf(command.OutOrStdout(), " - blocker %s (%s): %s\n", blocker.ID, blocker.Status, blocker.Class) + } + } + if options.Terminal { + switch result.Verdict { + case epicstatus.NotTerminal: + command.SilenceUsage, command.SilenceErrors = true, true + return &beadsapp.ExitError{Code: 2} + case epicstatus.Skipped: + command.SilenceUsage, command.SilenceErrors = true, true + return &beadsapp.ExitError{Code: 3} + } + } + return nil +} + func (module Module) acceptanceCommand() *cobra.Command { var options Options command := &cobra.Command{Use: "verify-acceptance ...", Short: "Assert each bead carries the acceptance contract for its type (br-native)", Args: cobra.MinimumNArgs(1)} diff --git a/cli/internal/commands/beads/module_test.go b/cli/internal/commands/beads/module_test.go index 9424c68b9..cd3987628 100644 --- a/cli/internal/commands/beads/module_test.go +++ b/cli/internal/commands/beads/module_test.go @@ -6,6 +6,7 @@ import ( "errors" "reflect" "testing" + "time" beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/spf13/cobra" @@ -22,7 +23,7 @@ func (runner *recordingRunner) Run(_ *cobra.Command, invocation Invocation) erro func TestModuleBuildsFreshCompleteTrees(t *testing.T) { runner := &recordingRunner{} - module := NewModule(runner, nil, nil, nil) + module := NewModule(runner, nil, nil, nil, nil, nil, nil, nil) first := module.Command() second := module.Command() if first == second { @@ -43,7 +44,7 @@ func TestModuleBuildsFreshCompleteTrees(t *testing.T) { func TestModuleParsesTypedInvocation(t *testing.T) { runner := &recordingRunner{} - command := NewModule(runner, nil, nil, nil).Command() + command := NewModule(runner, nil, nil, nil, nil, nil, nil, nil).Command() command.SetArgs([]string{"resume", "age-123", "--agent", "codex", "--json"}) if err := command.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -69,6 +70,14 @@ type fakeTrackerPorts struct { override bool executed bool execErr error + listOutput []byte + shown []beadsapp.StaleBeadRecord + showIndex int + calls []string + now time.Time + actor string + appended any + readOutput []byte } func (fake *fakeTrackerPorts) Resolve() (beadsapp.TrackerResolution, error) { @@ -86,6 +95,37 @@ func (fake *fakeTrackerPorts) Execute(_ context.Context, _ []string, streams bea _, _ = streams.Stdout.Write([]byte("executed\n")) return fake.execErr } +func (fake *fakeTrackerPorts) ListInProgress(context.Context) ([]byte, error) { + return fake.listOutput, nil +} +func (fake *fakeTrackerPorts) Show(_ context.Context, beadID string) (beadsapp.StaleBeadRecord, error) { + fake.calls = append(fake.calls, "show:"+beadID) + if fake.showIndex >= len(fake.shown) { + return beadsapp.StaleBeadRecord{}, errors.New("missing shown record") + } + record := fake.shown[fake.showIndex] + fake.showIndex++ + return record, nil +} +func (fake *fakeTrackerPorts) Claim(_ context.Context, beadID, agent string) error { + fake.calls = append(fake.calls, "claim:"+beadID+":"+agent) + return nil +} +func (fake *fakeTrackerPorts) Now() time.Time { return fake.now } +func (fake *fakeTrackerPorts) Actor() string { return fake.actor } +func (fake *fakeTrackerPorts) ResolveRepoPath(path string) (string, error) { + fake.calls = append(fake.calls, "resolve:"+path) + return "/repo/" + path, nil +} +func (fake *fakeTrackerPorts) AppendEvent(path string, event any) error { + fake.calls = append(fake.calls, "append:"+path) + fake.appended = event + return nil +} +func (fake *fakeTrackerPorts) ReadFile(path string) ([]byte, error) { + fake.calls = append(fake.calls, "read:"+path) + return fake.readOutput, nil +} func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { ports := &fakeTrackerPorts{ @@ -95,7 +135,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("directory JSON", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, ports, ports, ports).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"dir", "--require", "--json"}) if err := root.Execute(); err != nil { @@ -108,7 +148,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("tracker text", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, ports, ports, ports).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"tracker"}) if err := root.Execute(); err != nil { @@ -121,7 +161,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("exec help precedes resolution", func(t *testing.T) { ports.executed = false - root := NewModule(nil, ports, ports, ports).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil).Command() root.SetArgs([]string{"exec", "--help"}) if err := root.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -134,7 +174,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("exec preserves typed exit", func(t *testing.T) { ports.executed = false ports.execErr = &beadsapp.ExitError{Code: 7} - root := NewModule(nil, ports, ports, ports).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil).Command() root.SetArgs([]string{"exec", "close", "age-x"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -143,3 +183,54 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { } }) } + +func TestModuleOwnsRecoveryHandlers(t *testing.T) { + now := time.Date(2026, 7, 11, 18, 0, 0, 0, time.UTC) + ports := &fakeTrackerPorts{ + ledger: beadsapp.LedgerResolution{Path: "/ledger", Source: "env"}, + now: now, + actor: "codex", + } + + t.Run("stale claims", func(t *testing.T) { + ports.listOutput = []byte(`[{"id":"age-old","status":"in_progress","assignee":"bo","updated_at":"2026-07-11T10:00:00Z"}]`) + var output bytes.Buffer + root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports).Command() + root.SetOut(&output) + root.SetArgs([]string{"stale-claims", "--threshold", "4", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if !bytes.Contains(output.Bytes(), []byte(`"bead_id":"age-old"`)) { + t.Fatalf("output = %q", output.String()) + } + }) + + t.Run("resume ordered effects", func(t *testing.T) { + ports.calls, ports.showIndex = nil, 0 + ports.shown = []beadsapp.StaleBeadRecord{ + {ID: "age-x", Status: "in_progress", Assignee: "old", UpdatedAt: "2026-07-11T10:00:00Z"}, + {ID: "age-x", Status: "in_progress", Assignee: "codex", UpdatedAt: "2026-07-11T18:00:00Z"}, + } + root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports).Command() + root.SetArgs([]string{"resume", "age-x", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + want := []string{"show:age-x", "claim:age-x:codex", "show:age-x", "resolve:docs/provenance/ledger.jsonl", "append:/repo/docs/provenance/ledger.jsonl"} + if !reflect.DeepEqual(ports.calls, want) { + t.Fatalf("calls = %v, want %v", ports.calls, want) + } + }) + + t.Run("epic terminal exit", func(t *testing.T) { + ports.readOutput = []byte("{\"id\":\"age-e\",\"status\":\"open\",\"issue_type\":\"epic\"}\n{\"id\":\"age-e.1\",\"status\":\"open\",\"issue_type\":\"task\"}\n") + root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports).Command() + root.SetArgs([]string{"epic-status", "age-e", "--terminal"}) + err := root.Execute() + var exitError *beadsapp.ExitError + if !errors.As(err, &exitError) || exitError.ExitCode() != 2 { + t.Fatalf("error = %v, want ExitError(2)", err) + } + }) +} From 88790b93d86e25620e72970ef9a00048cb375bbd Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:39:43 -0400 Subject: [PATCH 020/108] refactor(beads): internalize knowledge command handlers (age-nw28h.2.1) --- cli/cmd/ao/beads_command_legacy_test.go | 2 +- ...eads.go => beads_knowledge_legacy_test.go} | 56 ------ cli/cmd/ao/beads_module.go | 14 +- cli/cmd/ao/beads_support.go | 43 +++++ cli/cmd/ao/citation_port_adapter.go | 19 +- cli/internal/adapters/beads/knowledge.go | 170 ++++++++++++++++++ cli/internal/beads/knowledge_service.go | 135 ++++++++++++++ cli/internal/commands/beads/module.go | 128 ++++++++++++- cli/internal/commands/beads/module_test.go | 94 ++++++++-- 9 files changed, 566 insertions(+), 95 deletions(-) rename cli/cmd/ao/{beads.go => beads_knowledge_legacy_test.go} (88%) create mode 100644 cli/cmd/ao/beads_support.go create mode 100644 cli/internal/adapters/beads/knowledge.go create mode 100644 cli/internal/beads/knowledge_service.go diff --git a/cli/cmd/ao/beads_command_legacy_test.go b/cli/cmd/ao/beads_command_legacy_test.go index e2817760b..648842c68 100644 --- a/cli/cmd/ao/beads_command_legacy_test.go +++ b/cli/cmd/ao/beads_command_legacy_test.go @@ -94,7 +94,7 @@ func executeBeadsExec(command *cobra.Command, args []string) error { func newTestBeadsCommand(name string) *cobra.Command { tracker := currentBeadsTracker() runtime := beadsadapter.NewRuntime() - root := beadscommands.NewModule(nil, tracker, tracker, beadsadapter.NewExecutor(tracker), tracker, tracker, runtime, runtime).Command() + root := beadscommands.NewModule(nil, tracker, tracker, beadsadapter.NewExecutor(tracker), tracker, tracker, runtime, runtime, nil).Command() child, _, err := root.Find([]string{name}) if err != nil || child == nil { panic("missing test beads command " + name) diff --git a/cli/cmd/ao/beads.go b/cli/cmd/ao/beads_knowledge_legacy_test.go similarity index 88% rename from cli/cmd/ao/beads.go rename to cli/cmd/ao/beads_knowledge_legacy_test.go index e05c937a2..4953e9cdb 100644 --- a/cli/cmd/ao/beads.go +++ b/cli/cmd/ao/beads_knowledge_legacy_test.go @@ -22,8 +22,6 @@ package main import ( - "context" - "encoding/json" "fmt" "os" "os/exec" @@ -34,50 +32,9 @@ import ( "github.com/spf13/cobra" - beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" beadsapp "github.com/boshu2/agentops/cli/internal/beads" ) -// beadsVerdictError carries a verdict exit code out through cobra's RunE so -// Execute() can map it to os.Exit() WITHOUT calling os.Exit() mid-command — -// which would skip deferred cleanup (temp files, lock release) and kill the -// test binary, making these paths untestable. -// -// The `ao beads verify|lint|audit` commands use exit-code-as-verdict: exit 1 -// means "stale citations / flagged beads found", a normal diagnostic outcome, -// not an internal failure. The verdict text already went to stdout, so the -// error carries no message (Error() == ""); the returning RunE sets -// cmd.SilenceErrors so cobra emits no spurious "Error:" line, while genuine -// errors returned elsewhere in those commands still print normally. Mirrors -// gateExitError (validate.go) and doctorExitError (doctor_surface.go). -type beadsVerdictError struct { - code int -} - -func (e *beadsVerdictError) Error() string { return "" } - -// ExitCode returns the process exit code this verdict maps to. -func (e *beadsVerdictError) ExitCode() int { return e.code } - -// beadsTrackerOutput is the single entry point for shelling out to bd. Tests override -// this to avoid a hard dependency on the real binary. Production code calls -// `bd` via PATH; if absent, the caller emits a graceful warning and returns. -var beadsTrackerOutput = func(args ...string) ([]byte, error) { - return currentBeadsTracker().Output(context.Background(), args...) -} - -// beadsTrackerAvailable reports whether the bd binary is reachable via PATH. Tests -// override this for deterministic behaviour. -var beadsTrackerAvailable = func() bool { - return currentBeadsTracker().Available() -} - -func currentBeadsTracker() *beadsadapter.Tracker { - return beadsadapter.NewTrackerWith(os.Getwd, os.Environ, func(name string) (string, error) { - return trackerLookPath(name) - }) -} - var ( beadsDirJSON bool beadsDirRequire bool @@ -319,13 +276,6 @@ func findFilesByBasename(cwd, name string) []string { return matches } -func beadMinInt(a, b int) int { - if a < b { - return a - } - return b -} - func verifyFunctionCitation(c *Citation, cwd string) { // c.Raw is "func Name". Grep for it across cli/ and skills/. name := strings.TrimPrefix(c.Raw, "func ") @@ -575,12 +525,6 @@ func beadSlugify(title string, maxLen int) string { // shared helpers // ------------------------------------------------------------------------ -func emitJSON(w *os.File, v any) error { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(v) -} - func beadTruncate(s string, n int) string { return beadsapp.Truncate(s, n) } diff --git a/cli/cmd/ao/beads_module.go b/cli/cmd/ao/beads_module.go index fb5867407..9014fe57f 100644 --- a/cli/cmd/ao/beads_module.go +++ b/cli/cmd/ao/beads_module.go @@ -4,6 +4,7 @@ import ( "fmt" beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" + beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/boshu2/agentops/cli/internal/clicontract" beadscommands "github.com/boshu2/agentops/cli/internal/commands/beads" "github.com/spf13/cobra" @@ -18,15 +19,6 @@ type beadsModuleRunner struct{} func (beadsModuleRunner) Run(command *cobra.Command, invocation beadscommands.Invocation) error { options := invocation.Options switch invocation.Operation { - case beadscommands.OperationVerify: - beadsVerifyJSON, beadsVerifyVerbose = options.JSON, options.Verbose - return executeBeadsVerify(command, invocation.Args) - case beadscommands.OperationLint: - beadsLintStatus, beadsLintJSON = options.Status, options.JSON - return executeBeadsLint(command, invocation.Args) - case beadscommands.OperationHarvest: - beadsHarvestOutDir, beadsHarvestDryRun = options.OutputDirectory, options.DryRun - return executeBeadsHarvest(command, invocation.Args) case beadscommands.OperationAudit: beadsAuditJSON, beadsAuditStrict, beadsAuditAutoClose = options.JSON, options.Strict, options.AutoClose return executeBeadsAudit(command, invocation.Args) @@ -49,9 +41,11 @@ func (beadsModuleRunner) Run(command *cobra.Command, invocation beadscommands.In func init() { tracker := currentBeadsTracker() runtime := beadsadapter.NewRuntime() + repository := beadsadapter.NewKnowledgeRepository() + knowledge := beadsapp.KnowledgeService{Tracker: tracker, Repository: repository, Clock: runtime} module := beadscommands.NewModule( beadsModuleRunner{}, tracker, tracker, beadsadapter.NewExecutor(tracker), - tracker, tracker, runtime, runtime, + tracker, tracker, runtime, runtime, knowledge, ) command := module.Command() command.GroupID = "knowledge" diff --git a/cli/cmd/ao/beads_support.go b/cli/cmd/ao/beads_support.go new file mode 100644 index 000000000..44c66e654 --- /dev/null +++ b/cli/cmd/ao/beads_support.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "encoding/json" + "os" + + beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" +) + +// beadsVerdictError remains only for command families and white-box tests +// that have not yet adopted the shared command exit type. +type beadsVerdictError struct { + code int +} + +func (err *beadsVerdictError) Error() string { return "" } +func (err *beadsVerdictError) ExitCode() int { return err.code } + +var beadsTrackerOutput = func(args ...string) ([]byte, error) { + return currentBeadsTracker().Output(context.Background(), args...) +} + +var beadsTrackerAvailable = func() bool { return currentBeadsTracker().Available() } + +func currentBeadsTracker() *beadsadapter.Tracker { + return beadsadapter.NewTrackerWith(os.Getwd, os.Environ, func(name string) (string, error) { + return trackerLookPath(name) + }) +} + +func beadMinInt(left, right int) int { + if left < right { + return left + } + return right +} + +func emitJSON(file *os.File, value any) error { + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} diff --git a/cli/cmd/ao/citation_port_adapter.go b/cli/cmd/ao/citation_port_adapter.go index c1c081651..856484b2b 100644 --- a/cli/cmd/ao/citation_port_adapter.go +++ b/cli/cmd/ao/citation_port_adapter.go @@ -6,6 +6,8 @@ import ( "fmt" "strings" + beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" + beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/boshu2/agentops/cli/internal/ports" ) @@ -43,20 +45,17 @@ func (a *productionCitationAdapter) Verify(ctx context.Context, req ports.Citati Reason: "empty Raw", }, nil } - c := Citation{Kind: string(req.Kind), Raw: req.Raw} switch req.Kind { - case ports.CitationKindFile: - verifyFileCitation(&c, req.Cwd) - case ports.CitationKindFunction: - verifyFunctionCitation(&c, req.Cwd) - case ports.CitationKindSymbol: - verifySymbolCitation(&c, req.Cwd) + case ports.CitationKindFile, ports.CitationKindFunction, ports.CitationKindSymbol: default: return ports.CitationVerdict{ Status: ports.CitationStatusUnknown, Reason: fmt.Sprintf("unknown citation kind %q (expected file|function|symbol)", req.Kind), }, nil } + repository := beadsadapter.NewKnowledgeRepositoryWithWorkingDirectory(func() (string, error) { return req.Cwd, nil }) + verified := repository.VerifyCitations([]beadsapp.Citation{{Kind: string(req.Kind), Raw: req.Raw}}) + c := verified[0] return ports.CitationVerdict{ Status: translateCitationStatus(c.Status), Reason: c.Reason, @@ -67,11 +66,11 @@ func (a *productionCitationAdapter) Verify(ctx context.Context, req ports.Citati // translateCitationStatus maps the existing CitationStatus values to // the port's CitationStatusResult. The string values are identical, // but the types are distinct — this conversion is the boundary. -func translateCitationStatus(s CitationStatus) ports.CitationStatusResult { +func translateCitationStatus(s beadsapp.CitationStatus) ports.CitationStatusResult { switch s { - case CitationFresh: + case beadsapp.CitationFresh: return ports.CitationStatusFresh - case CitationStale: + case beadsapp.CitationStale: return ports.CitationStatusStale default: return ports.CitationStatusUnknown diff --git a/cli/internal/adapters/beads/knowledge.go b/cli/internal/adapters/beads/knowledge.go new file mode 100644 index 000000000..95e8fdfa9 --- /dev/null +++ b/cli/internal/adapters/beads/knowledge.go @@ -0,0 +1,170 @@ +package beads + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" +) + +type KnowledgeRepository struct { + workingDirectory func() (string, error) +} + +func NewKnowledgeRepository() KnowledgeRepository { + return KnowledgeRepository{workingDirectory: os.Getwd} +} + +func NewKnowledgeRepositoryWithWorkingDirectory(workingDirectory func() (string, error)) KnowledgeRepository { + return KnowledgeRepository{workingDirectory: workingDirectory} +} + +func (repository KnowledgeRepository) VerifyCitations(citations []beadsapp.Citation) []beadsapp.Citation { + cwd, err := repository.workingDirectory() + if err != nil { + for index := range citations { + citations[index].Status = beadsapp.CitationUnknown + citations[index].Reason = "working directory unavailable" + } + return citations + } + for index := range citations { + repository.verifyCitation(&citations[index], cwd) + } + return citations +} + +func (repository KnowledgeRepository) CreateLearning(path string, content []byte) (bool, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return false, fmt.Errorf("mkdir %s: %w", filepath.Dir(path), err) + } + if _, err := os.Stat(path); err == nil { + return false, nil + } else if !os.IsNotExist(err) { + return false, err + } + if err := os.WriteFile(path, content, 0o644); err != nil { + return false, fmt.Errorf("write %s: %w", path, err) + } + return true, nil +} + +func (repository KnowledgeRepository) verifyCitation(citation *beadsapp.Citation, cwd string) { + switch citation.Kind { + case "file": + repository.verifyFile(citation, cwd) + case "function": + repository.verifySymbol(citation, cwd, strings.TrimPrefix(citation.Raw, "func "), true) + case "symbol": + repository.verifySymbol(citation, cwd, strings.Trim(citation.Raw, "`"), false) + default: + citation.Status = beadsapp.CitationUnknown + citation.Reason = "unrecognized citation kind" + } +} + +func (repository KnowledgeRepository) verifyFile(citation *beadsapp.Citation, cwd string) { + path := citation.Raw + if index := strings.LastIndex(path, ":"); index >= 0 { + if _, err := fmt.Sscanf(path[index+1:], "%d", new(int)); err == nil { + path = path[:index] + } + } + if _, err := os.Stat(filepath.Join(cwd, path)); err == nil { + citation.Status, citation.Reason = beadsapp.CitationFresh, "file exists at HEAD" + return + } + if strings.Contains(path, "/") { + citation.Status, citation.Reason = beadsapp.CitationStale, fmt.Sprintf("file %s not found at HEAD", path) + return + } + matches := findFilesByBasename(cwd, path) + switch len(matches) { + case 0: + citation.Status, citation.Reason = beadsapp.CitationStale, fmt.Sprintf("bare filename %q has zero matches at HEAD", path) + case 1: + citation.Status, citation.Reason, citation.Resolved = beadsapp.CitationFresh, "bare filename resolves uniquely", matches[0] + default: + citation.Status = beadsapp.CitationUnknown + citation.Reason = fmt.Sprintf("bare filename %q is ambiguous (%d matches) — cite the full path", path, len(matches)) + citation.Resolved = strings.Join(matches[:min(3, len(matches))], ", ") + } +} + +func (repository KnowledgeRepository) verifySymbol(citation *beadsapp.Citation, cwd, symbol string, definition bool) { + matches := grepSymbol(cwd, symbol) + if len(matches) == 0 { + citation.Status = beadsapp.CitationStale + if definition { + citation.Reason = fmt.Sprintf("function %q has zero definitions at HEAD", symbol) + } else { + citation.Reason = fmt.Sprintf("symbol %q has zero references at HEAD", symbol) + } + return + } + citation.Status = beadsapp.CitationFresh + if definition { + citation.Reason = fmt.Sprintf("function defined at %d location(s)", len(matches)) + if len(matches) == 1 { + citation.Resolved = matches[0] + } + } else { + citation.Reason = fmt.Sprintf("symbol found at %d location(s)", len(matches)) + } +} + +func findFilesByBasename(cwd, name string) []string { + var matches []string + for _, root := range []string{"cli", "skills", "docs", "scripts", ".agents"} { + rootPath := filepath.Join(cwd, root) + if _, err := os.Stat(rootPath); err != nil { + continue + } + _ = filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.IsDir() { + base := filepath.Base(path) + if (strings.HasPrefix(base, ".") && path != rootPath) || base == "node_modules" || base == "vendor" || base == "testdata" { + return filepath.SkipDir + } + return nil + } + if filepath.Base(path) == name { + if relative, relErr := filepath.Rel(cwd, path); relErr == nil { + matches = append(matches, relative) + } + } + return nil + }) + if len(matches) >= 10 { + return matches[:10] + } + } + return matches +} + +func grepSymbol(cwd, symbol string) []string { + if symbol == "" { + return nil + } + command := exec.Command("grep", "-rn", "-l", "--include=*.go", "--include=*.md", "--include=*.py", "--include=*.sh", "--include=*.yaml", "--include=*.yml", "--include=*.json", regexp.QuoteMeta(symbol), filepath.Join(cwd, "cli"), filepath.Join(cwd, "skills"), filepath.Join(cwd, "scripts")) // #nosec G204 -- fixed grep binary and bounded source roots. + output, _ := command.Output() + var matches []string + for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") { + if line = strings.TrimSpace(line); line != "" { + matches = append(matches, line) + } + if len(matches) == 10 { + break + } + } + return matches +} + +var _ beadsapp.KnowledgeRepository = KnowledgeRepository{} diff --git a/cli/internal/beads/knowledge_service.go b/cli/internal/beads/knowledge_service.go new file mode 100644 index 000000000..315f55b42 --- /dev/null +++ b/cli/internal/beads/knowledge_service.go @@ -0,0 +1,135 @@ +package beads + +import ( + "context" + "fmt" + "path/filepath" +) + +type KnowledgeRepository interface { + VerifyCitations([]Citation) []Citation + CreateLearning(string, []byte) (bool, error) +} + +type HarvestResult struct { + Body string + Target string + AlreadyExists bool +} + +type KnowledgeUseCases interface { + Available() bool + Verify(context.Context, string) (*VerifyReport, error) + Lint(context.Context, string) (*LintReport, error) + Harvest(context.Context, string, string, bool) (HarvestResult, error) +} + +type KnowledgeService struct { + Tracker TrackerClient + Repository KnowledgeRepository + Clock Clock +} + +func (service KnowledgeService) Available() bool { + return service.Tracker != nil && service.Tracker.Available() +} + +func (service KnowledgeService) Verify(ctx context.Context, beadID string) (*VerifyReport, error) { + if !service.Available() { + return &VerifyReport{BeadID: beadID, BDAvailable: false}, nil + } + raw, err := service.Tracker.Output(ctx, "show", beadID) + if err != nil { + return nil, fmt.Errorf("bd show %s: %w", beadID, err) + } + parsed, err := ParseBDShow(string(raw)) + if err != nil { + return nil, err + } + citations := ExtractCitations(parsed.Body()) + if service.Repository != nil { + citations = service.Repository.VerifyCitations(citations) + } + report := &VerifyReport{ + BeadID: beadID, Title: parsed.Title, Status: parsed.Status, + Citations: citations, TotalCount: len(citations), BDAvailable: true, + } + for _, citation := range citations { + switch citation.Status { + case CitationFresh: + report.FreshCount++ + case CitationStale: + report.StaleCount++ + } + } + return report, nil +} + +func (service KnowledgeService) Lint(ctx context.Context, status string) (*LintReport, error) { + if !service.Available() { + return &LintReport{StatusFilter: status}, nil + } + raw, err := service.Tracker.Output(ctx, "list", "--status", status) + if err != nil { + return nil, fmt.Errorf("bd list: %w", err) + } + ids := ParseBeadIDs(raw) + report := &LintReport{StatusFilter: status, TotalBeads: len(ids)} + for _, id := range ids { + verified, err := service.Verify(ctx, id) + if err != nil { + report.ErrorBeads++ + continue + } + report.PerBead = append(report.PerBead, *verified) + if verified.StaleCount > 0 { + report.StaleBeads++ + } else { + report.CleanBeads++ + } + } + return report, nil +} + +func (service KnowledgeService) Harvest(ctx context.Context, beadID, outputDirectory string, dryRun bool) (HarvestResult, error) { + if !service.Available() { + return HarvestResult{}, nil + } + raw, err := service.Tracker.Output(ctx, "show", beadID) + if err != nil { + return HarvestResult{}, fmt.Errorf("bd show %s: %w", beadID, err) + } + parsed, err := ParseBDShow(string(raw)) + if err != nil { + return HarvestResult{}, err + } + if !IsClosedStatus(parsed.Status) { + return HarvestResult{}, fmt.Errorf("bead %s is not CLOSED (status=%q) — harvest only materialises closed beads", beadID, parsed.Status) + } + if service.Clock == nil { + return HarvestResult{}, fmt.Errorf("beads harvest clock is not configured") + } + frontmatter := LearningFrontmatter{ + Title: parsed.Title, BeadID: beadID, Source: "bd-close", + Date: service.Clock.Now().UTC().Format("2006-01-02"), + Tags: []string{"bead-closure", "auto-harvested"}, Maturity: "provisional", + Provenance: fmt.Sprintf("bd show %s (harvested via `ao beads harvest`)", beadID), + } + body := RenderLearningBody(frontmatter, parsed) + result := HarvestResult{Body: body} + if dryRun { + return result, nil + } + result.Target = filepath.Join(outputDirectory, fmt.Sprintf("%s-%s-%s.md", frontmatter.Date, beadID, Slugify(parsed.Title, 40))) + if service.Repository == nil { + return HarvestResult{}, fmt.Errorf("beads harvest repository is not configured") + } + created, err := service.Repository.CreateLearning(result.Target, []byte(body)) + if err != nil { + return HarvestResult{}, err + } + result.AlreadyExists = !created + return result, nil +} + +var _ KnowledgeUseCases = KnowledgeService{} diff --git a/cli/internal/commands/beads/module.go b/cli/internal/commands/beads/module.go index 8ff7bc8eb..4b9e4d839 100644 --- a/cli/internal/commands/beads/module.go +++ b/cli/internal/commands/beads/module.go @@ -77,6 +77,7 @@ type Module struct { claims beadsapp.ClaimStore runtime beadsapp.ResumeRuntime reader beadsapp.LedgerReader + knowledge beadsapp.KnowledgeUseCases } func NewModule( @@ -88,10 +89,11 @@ func NewModule( claims beadsapp.ClaimStore, runtime beadsapp.ResumeRuntime, reader beadsapp.LedgerReader, + knowledge beadsapp.KnowledgeUseCases, ) Module { return Module{ runner: runner, resolver: resolver, inspector: inspector, executor: executor, - stale: stale, claims: claims, runtime: runtime, reader: reader, + stale: stale, claims: claims, runtime: runtime, reader: reader, knowledge: knowledge, } } @@ -172,7 +174,7 @@ func (module Module) verifyCommand() *cobra.Command { command.Flags().BoolVar(&options.JSON, "json", false, "Emit verification report as JSON instead of human-readable text") command.Flags().BoolVar(&options.Verbose, "verbose", false, "Include FRESH citations in the output (default: stale only)") command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationVerify, args, options) + return module.runVerify(command, args, options) } return command } @@ -183,7 +185,7 @@ func (module Module) lintCommand() *cobra.Command { command.Flags().StringVar(&options.Status, "status", "open", "bd status filter (open, closed, all)") command.Flags().BoolVar(&options.JSON, "json", false, "Emit lint report as JSON") command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationLint, args, options) + return module.runLint(command, options) } return command } @@ -194,7 +196,7 @@ func (module Module) harvestCommand() *cobra.Command { command.Flags().StringVar(&options.OutputDirectory, "out-dir", ".agents/learnings", "Directory to write the learning file into") command.Flags().BoolVar(&options.DryRun, "dry-run", false, "Print the learning content to stdout without writing a file") command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationHarvest, args, options) + return module.runHarvest(command, args, options) } return command } @@ -307,6 +309,124 @@ func (module Module) runExec(command *cobra.Command, args []string) error { return err } +func (module Module) runVerify(command *cobra.Command, args []string, options Options) error { + if module.knowledge == nil { + return fmt.Errorf("beads knowledge use cases are not configured") + } + report, err := module.knowledge.Verify(context.Background(), args[0]) + if err != nil { + return err + } + if !report.BDAvailable { + _, err := fmt.Fprintln(command.ErrOrStderr(), "WARN: bd not on PATH — skipping verify (graceful degradation)") + return err + } + if options.JSON { + return encodeJSON(command, report) + } + emitVerify(command, report, options.Verbose) + if report.StaleCount > 0 { + command.SilenceErrors = true + return &beadsapp.ExitError{Code: 1} + } + return nil +} + +func emitVerify(command *cobra.Command, report *beadsapp.VerifyReport, verbose bool) { + fmt.Fprintf(command.OutOrStdout(), "bead %s: %s [%s]\n", report.BeadID, report.Title, report.Status) + fmt.Fprintf(command.OutOrStdout(), " citations: %d total, %d fresh, %d stale\n", report.TotalCount, report.FreshCount, report.StaleCount) + for _, citation := range report.Citations { + if citation.Status == beadsapp.CitationFresh && !verbose { + continue + } + marker := " " + switch citation.Status { + case beadsapp.CitationStale: + marker = "[STALE]" + case beadsapp.CitationFresh: + marker = "[FRESH]" + case beadsapp.CitationUnknown: + marker = "[?????]" + } + fmt.Fprintf(command.OutOrStdout(), " %s %s — %s\n", marker, citation.Raw, citation.Reason) + if citation.Resolved != "" { + fmt.Fprintf(command.OutOrStdout(), " → %s\n", citation.Resolved) + } + } +} + +func (module Module) runLint(command *cobra.Command, options Options) error { + if module.knowledge == nil { + return fmt.Errorf("beads knowledge use cases are not configured") + } + if !module.knowledge.Available() { + _, err := fmt.Fprintln(command.ErrOrStderr(), "WARN: bd not on PATH — skipping lint (graceful degradation)") + return err + } + report, err := module.knowledge.Lint(context.Background(), options.Status) + if err != nil { + return err + } + if options.JSON { + if err := encodeJSON(command, report); err != nil { + return err + } + } else { + emitLint(command, report) + } + if report.StaleBeads > 0 { + command.SilenceErrors = true + return &beadsapp.ExitError{Code: 1} + } + return nil +} + +func emitLint(command *cobra.Command, report *beadsapp.LintReport) { + fmt.Fprintf(command.OutOrStdout(), "ao beads lint (status=%s): %d beads\n", report.StatusFilter, report.TotalBeads) + fmt.Fprintf(command.OutOrStdout(), " %d clean, %d stale, %d errors\n", report.CleanBeads, report.StaleBeads, report.ErrorBeads) + for _, verified := range report.PerBead { + if verified.StaleCount == 0 { + continue + } + fmt.Fprintf(command.OutOrStdout(), "\n [STALE] %s: %s\n", verified.BeadID, verified.Title) + for _, citation := range verified.Citations { + if citation.Status == beadsapp.CitationStale { + fmt.Fprintf(command.OutOrStdout(), " - %s: %s\n", citation.Raw, citation.Reason) + } + } + } +} + +func (module Module) runHarvest(command *cobra.Command, args []string, options Options) error { + if module.knowledge == nil { + return fmt.Errorf("beads knowledge use cases are not configured") + } + if !module.knowledge.Available() { + _, err := fmt.Fprintln(command.ErrOrStderr(), "WARN: bd not on PATH — skipping harvest (graceful degradation)") + return err + } + result, err := module.knowledge.Harvest(context.Background(), args[0], options.OutputDirectory, options.DryRun) + if err != nil { + return err + } + if options.DryRun { + _, err := fmt.Fprintln(command.OutOrStdout(), result.Body) + return err + } + if result.AlreadyExists { + _, err := fmt.Fprintf(command.ErrOrStderr(), "learning already exists at %s — not overwriting\n", result.Target) + return err + } + _, err = fmt.Fprintf(command.OutOrStdout(), "harvested bead %s → %s\n", args[0], result.Target) + return err +} + +func encodeJSON(command *cobra.Command, value any) error { + encoder := json.NewEncoder(command.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} + func (module Module) resumeCommand() *cobra.Command { var options Options command := &cobra.Command{Use: "resume ", Short: "Atomically transfer an in_progress claim from a stale agent to this one", Args: cobra.ExactArgs(1)} diff --git a/cli/internal/commands/beads/module_test.go b/cli/internal/commands/beads/module_test.go index cd3987628..7e60d3104 100644 --- a/cli/internal/commands/beads/module_test.go +++ b/cli/internal/commands/beads/module_test.go @@ -23,7 +23,7 @@ func (runner *recordingRunner) Run(_ *cobra.Command, invocation Invocation) erro func TestModuleBuildsFreshCompleteTrees(t *testing.T) { runner := &recordingRunner{} - module := NewModule(runner, nil, nil, nil, nil, nil, nil, nil) + module := NewModule(runner, nil, nil, nil, nil, nil, nil, nil, nil) first := module.Command() second := module.Command() if first == second { @@ -44,17 +44,15 @@ func TestModuleBuildsFreshCompleteTrees(t *testing.T) { func TestModuleParsesTypedInvocation(t *testing.T) { runner := &recordingRunner{} - command := NewModule(runner, nil, nil, nil, nil, nil, nil, nil).Command() - command.SetArgs([]string{"resume", "age-123", "--agent", "codex", "--json"}) + command := NewModule(runner, nil, nil, nil, nil, nil, nil, nil, nil).Command() + command.SetArgs([]string{"audit", "--strict", "--json"}) if err := command.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) } want := Invocation{ - Operation: OperationResume, - Args: []string{"age-123"}, + Operation: OperationAudit, Options: Options{ - Agent: "codex", - Ledger: "docs/provenance/ledger.jsonl", + Strict: true, JSON: true, }, } @@ -80,6 +78,24 @@ type fakeTrackerPorts struct { readOutput []byte } +type fakeKnowledge struct { + available bool + verified *beadsapp.VerifyReport + linted *beadsapp.LintReport + harvested beadsapp.HarvestResult +} + +func (fake fakeKnowledge) Available() bool { return fake.available } +func (fake fakeKnowledge) Verify(context.Context, string) (*beadsapp.VerifyReport, error) { + return fake.verified, nil +} +func (fake fakeKnowledge) Lint(context.Context, string) (*beadsapp.LintReport, error) { + return fake.linted, nil +} +func (fake fakeKnowledge) Harvest(context.Context, string, string, bool) (beadsapp.HarvestResult, error) { + return fake.harvested, nil +} + func (fake *fakeTrackerPorts) Resolve() (beadsapp.TrackerResolution, error) { return fake.resolution, nil } @@ -135,7 +151,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("directory JSON", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"dir", "--require", "--json"}) if err := root.Execute(); err != nil { @@ -148,7 +164,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("tracker text", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"tracker"}) if err := root.Execute(); err != nil { @@ -161,7 +177,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("exec help precedes resolution", func(t *testing.T) { ports.executed = false - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil).Command() root.SetArgs([]string{"exec", "--help"}) if err := root.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -174,7 +190,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("exec preserves typed exit", func(t *testing.T) { ports.executed = false ports.execErr = &beadsapp.ExitError{Code: 7} - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil).Command() root.SetArgs([]string{"exec", "close", "age-x"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -195,7 +211,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { t.Run("stale claims", func(t *testing.T) { ports.listOutput = []byte(`[{"id":"age-old","status":"in_progress","assignee":"bo","updated_at":"2026-07-11T10:00:00Z"}]`) var output bytes.Buffer - root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports).Command() + root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil).Command() root.SetOut(&output) root.SetArgs([]string{"stale-claims", "--threshold", "4", "--json"}) if err := root.Execute(); err != nil { @@ -212,7 +228,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { {ID: "age-x", Status: "in_progress", Assignee: "old", UpdatedAt: "2026-07-11T10:00:00Z"}, {ID: "age-x", Status: "in_progress", Assignee: "codex", UpdatedAt: "2026-07-11T18:00:00Z"}, } - root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports).Command() + root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil).Command() root.SetArgs([]string{"resume", "age-x", "--json"}) if err := root.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -225,7 +241,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { t.Run("epic terminal exit", func(t *testing.T) { ports.readOutput = []byte("{\"id\":\"age-e\",\"status\":\"open\",\"issue_type\":\"epic\"}\n{\"id\":\"age-e.1\",\"status\":\"open\",\"issue_type\":\"task\"}\n") - root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports).Command() + root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil).Command() root.SetArgs([]string{"epic-status", "age-e", "--terminal"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -234,3 +250,53 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { } }) } + +func TestModuleOwnsKnowledgeHandlers(t *testing.T) { + knowledge := fakeKnowledge{ + available: true, + verified: &beadsapp.VerifyReport{ + BeadID: "age-x", Title: "stale", Status: "OPEN", TotalCount: 1, StaleCount: 1, BDAvailable: true, + Citations: []beadsapp.Citation{{Kind: "file", Raw: "gone.go", Status: beadsapp.CitationStale, Reason: "missing"}}, + }, + linted: &beadsapp.LintReport{StatusFilter: "open", TotalBeads: 1, StaleBeads: 1}, + harvested: beadsapp.HarvestResult{Body: "learning", Target: ".agents/learnings/age-x.md"}, + } + + t.Run("verify maps stale verdict", func(t *testing.T) { + var output bytes.Buffer + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge).Command() + root.SetOut(&output) + root.SetArgs([]string{"verify", "age-x"}) + err := root.Execute() + var exitError *beadsapp.ExitError + if !errors.As(err, &exitError) || exitError.ExitCode() != 1 { + t.Fatalf("error = %v, want ExitError(1)", err) + } + if !bytes.Contains(output.Bytes(), []byte("[STALE] gone.go")) { + t.Fatalf("output = %q", output.String()) + } + }) + + t.Run("lint maps stale verdict", func(t *testing.T) { + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge).Command() + root.SetArgs([]string{"lint"}) + err := root.Execute() + var exitError *beadsapp.ExitError + if !errors.As(err, &exitError) || exitError.ExitCode() != 1 { + t.Fatalf("error = %v, want ExitError(1)", err) + } + }) + + t.Run("harvest renders result", func(t *testing.T) { + var output bytes.Buffer + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge).Command() + root.SetOut(&output) + root.SetArgs([]string{"harvest", "age-x"}) + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if got := output.String(); got != "harvested bead age-x → .agents/learnings/age-x.md\n" { + t.Fatalf("output = %q", got) + } + }) +} From 44d5e2747b282f7b6af180056bb6e182a4343325 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:44:56 -0400 Subject: [PATCH 021/108] refactor(beads): internalize hygiene command handlers (age-nw28h.2.1) --- ....go => beads_audit_cluster_legacy_test.go} | 0 cli/cmd/ao/beads_command_legacy_test.go | 2 +- cli/cmd/ao/beads_module.go | 9 +- cli/internal/adapters/beads/hygiene.go | 144 +++++ cli/internal/beads/hygiene.go | 494 ++++++++++++++++++ cli/internal/commands/beads/module.go | 136 ++++- cli/internal/commands/beads/module_test.go | 78 ++- 7 files changed, 840 insertions(+), 23 deletions(-) rename cli/cmd/ao/{beads_audit_cluster.go => beads_audit_cluster_legacy_test.go} (100%) create mode 100644 cli/internal/adapters/beads/hygiene.go create mode 100644 cli/internal/beads/hygiene.go diff --git a/cli/cmd/ao/beads_audit_cluster.go b/cli/cmd/ao/beads_audit_cluster_legacy_test.go similarity index 100% rename from cli/cmd/ao/beads_audit_cluster.go rename to cli/cmd/ao/beads_audit_cluster_legacy_test.go diff --git a/cli/cmd/ao/beads_command_legacy_test.go b/cli/cmd/ao/beads_command_legacy_test.go index 648842c68..5f92e579f 100644 --- a/cli/cmd/ao/beads_command_legacy_test.go +++ b/cli/cmd/ao/beads_command_legacy_test.go @@ -94,7 +94,7 @@ func executeBeadsExec(command *cobra.Command, args []string) error { func newTestBeadsCommand(name string) *cobra.Command { tracker := currentBeadsTracker() runtime := beadsadapter.NewRuntime() - root := beadscommands.NewModule(nil, tracker, tracker, beadsadapter.NewExecutor(tracker), tracker, tracker, runtime, runtime, nil).Command() + root := beadscommands.NewModule(nil, tracker, tracker, beadsadapter.NewExecutor(tracker), tracker, tracker, runtime, runtime, nil, nil).Command() child, _, err := root.Find([]string{name}) if err != nil || child == nil { panic("missing test beads command " + name) diff --git a/cli/cmd/ao/beads_module.go b/cli/cmd/ao/beads_module.go index 9014fe57f..6300cc45d 100644 --- a/cli/cmd/ao/beads_module.go +++ b/cli/cmd/ao/beads_module.go @@ -19,12 +19,6 @@ type beadsModuleRunner struct{} func (beadsModuleRunner) Run(command *cobra.Command, invocation beadscommands.Invocation) error { options := invocation.Options switch invocation.Operation { - case beadscommands.OperationAudit: - beadsAuditJSON, beadsAuditStrict, beadsAuditAutoClose = options.JSON, options.Strict, options.AutoClose - return executeBeadsAudit(command, invocation.Args) - case beadscommands.OperationCluster: - beadsClusterJSON, beadsClusterApply = options.JSON, options.Apply - return executeBeadsCluster(command, invocation.Args) case beadscommands.OperationScenariosExtract: beadsScenariosJSON, beadsScenariosForce, beadsScenariosWrite = options.JSON, options.Force, options.Write return executeBeadsScenariosExtract(command, invocation.Args) @@ -43,9 +37,10 @@ func init() { runtime := beadsadapter.NewRuntime() repository := beadsadapter.NewKnowledgeRepository() knowledge := beadsapp.KnowledgeService{Tracker: tracker, Repository: repository, Clock: runtime} + hygiene := beadsapp.HygieneService{Repository: beadsadapter.NewHygieneRepository(tracker)} module := beadscommands.NewModule( beadsModuleRunner{}, tracker, tracker, beadsadapter.NewExecutor(tracker), - tracker, tracker, runtime, runtime, knowledge, + tracker, tracker, runtime, runtime, knowledge, hygiene, ) command := module.Command() command.GroupID = "knowledge" diff --git a/cli/internal/adapters/beads/hygiene.go b/cli/internal/adapters/beads/hygiene.go new file mode 100644 index 000000000..b1b261185 --- /dev/null +++ b/cli/internal/adapters/beads/hygiene.go @@ -0,0 +1,144 @@ +package beads + +import ( + "context" + "fmt" + "io/fs" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" + "time" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" +) + +type HygieneRepository struct { + tracker *Tracker + once sync.Once + index map[string]string +} + +func NewHygieneRepository(tracker *Tracker) *HygieneRepository { + return &HygieneRepository{tracker: tracker} +} + +func (repository *HygieneRepository) Available() bool { + return repository != nil && repository.tracker != nil && repository.tracker.Available() +} + +func (repository *HygieneRepository) List(status string) ([]beadsapp.BeadRecord, error) { + raw, err := repository.tracker.Output(context.Background(), "list", "--status", status, "--json") + if err != nil { + return nil, fmt.Errorf("bd list --status %s --json: %w", status, err) + } + return beadsapp.ParseBDRecordList(raw) +} + +func (repository *HygieneRepository) Show(id string) (beadsapp.BeadRecord, error) { + raw, err := repository.tracker.Output(context.Background(), "show", id, "--json") + if err != nil { + return beadsapp.BeadRecord{}, err + } + return beadsapp.ParseBDRecord(raw) +} + +func (repository *HygieneRepository) Close(id, note string) error { + if _, err := repository.tracker.Output(context.Background(), "update", id, "--status", "closed", "--append-notes", note); err != nil { + return fmt.Errorf("auto-close bead %s: %w", id, err) + } + return nil +} + +func (repository *HygieneRepository) Reparent(id, parent string) error { + _, err := repository.tracker.Output(context.Background(), "update", id, "--parent", parent) + return err +} + +func (repository *HygieneRepository) Commits() []beadsapp.AuditCommit { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + const recordSeparator, fieldSeparator = "\x1e", "\x1f" + command := exec.CommandContext(ctx, "git", "log", "--all", "--name-only", "--pretty=format:"+recordSeparator+"%h"+fieldSeparator+"%cI"+fieldSeparator+"%s"+fieldSeparator+"%b"+fieldSeparator) // #nosec G204 -- fixed read-only git invocation. + output, err := command.Output() + if err != nil { + return nil + } + var commits []beadsapp.AuditCommit + for _, rawRecord := range strings.Split(string(output), recordSeparator) { + rawRecord = strings.TrimLeft(rawRecord, "\n") + if rawRecord == "" { + continue + } + parts := strings.SplitN(rawRecord, fieldSeparator, 5) + if len(parts) < 5 { + continue + } + commitAt, _ := time.Parse(time.RFC3339, strings.TrimSpace(parts[1])) + commit := beadsapp.AuditCommit{ + ShortSHA: strings.TrimSpace(parts[0]), CommitAt: commitAt, + Subject: strings.TrimSpace(parts[2]), Body: parts[3], Files: make(map[string]struct{}), + } + for _, line := range strings.Split(parts[4], "\n") { + if file := strings.TrimSpace(line); file != "" { + commit.Files[file] = struct{}{} + } + } + commits = append(commits, commit) + } + return commits +} + +func (repository *HygieneRepository) PatternExists(pattern string) bool { + if pattern == "" { + return false + } + repository.once.Do(func() { repository.index = buildHygieneIndex() }) + for _, content := range repository.index { + if strings.Contains(content, pattern) { + return true + } + } + return false +} + +func buildHygieneIndex() map[string]string { + index := make(map[string]string) + for _, root := range []string{"cli", "skills", "skills-codex", "scripts", "docs", "tests"} { + openRoot, err := os.OpenRoot(root) + if err != nil { + continue + } + _ = fs.WalkDir(openRoot.FS(), ".", func(walkPath string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return nil + } + if entry.IsDir() { + switch path.Base(walkPath) { + case ".git", ".beads", ".agents", "node_modules", "vendor", "testdata": + return fs.SkipDir + } + return nil + } + switch filepath.Ext(walkPath) { + case ".go", ".py", ".sh", ".ts", ".js", ".md": + default: + return nil + } + info, err := entry.Info() + if err != nil || info.Size() > 1_000_000 { + return nil + } + if content, err := openRoot.ReadFile(walkPath); err == nil { + index[path.Join(root, walkPath)] = string(content) + } + return nil + }) + _ = openRoot.Close() + } + return index +} + +var _ beadsapp.HygieneRepository = (*HygieneRepository)(nil) diff --git a/cli/internal/beads/hygiene.go b/cli/internal/beads/hygiene.go new file mode 100644 index 000000000..2f28e67ea --- /dev/null +++ b/cli/internal/beads/hygiene.go @@ -0,0 +1,494 @@ +package beads + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + "time" +) + +type BeadRecord struct { + ID string `json:"id"` + Title string `json:"title"` + Subject string `json:"subject"` + Description string `json:"description"` + Body string `json:"body"` + Status string `json:"status"` + IssueType string `json:"issue_type"` + Type string `json:"type"` + Kind string `json:"kind"` + CreatedAt string `json:"created_at"` + Labels []string `json:"labels"` + Children []any `json:"children"` +} + +func (record BeadRecord) DisplayTitle() string { + if record.Title != "" { + return record.Title + } + return record.Subject +} + +func (record BeadRecord) TextBody() string { + if record.Body != "" { + return record.Body + } + return record.Description +} + +func (record BeadRecord) IsEpic() bool { + for _, value := range []string{record.IssueType, record.Type, record.Kind} { + if strings.EqualFold(value, "epic") { + return true + } + } + return len(record.Children) > 0 +} + +type AuditCommit struct { + ShortSHA string + Subject string + Body string + CommitAt time.Time + Files map[string]struct{} +} + +type HygieneRepository interface { + Available() bool + List(string) ([]BeadRecord, error) + Show(string) (BeadRecord, error) + Commits() []AuditCommit + PatternExists(string) bool + Close(string, string) error + Reparent(string, string) error +} + +type AuditFinding struct { + ID string `json:"id"` + Title string `json:"title"` + Reason string `json:"reason"` + Evidence string `json:"evidence,omitempty"` +} + +type AuditConsolidation struct { + File string `json:"file"` + BeadIDs []string `json:"bead_ids"` +} + +type AuditSummary struct { + LikelyFixed int `json:"likely_fixed"` + LikelyStale int `json:"likely_stale"` + Consolidatable int `json:"consolidatable"` + Total int `json:"total"` + FlaggedPct int `json:"flagged_pct,omitempty"` +} + +type AuditReport struct { + LikelyFixed []AuditFinding `json:"likely_fixed"` + LikelyStale []AuditFinding `json:"likely_stale"` + Consolidatable []AuditConsolidation `json:"consolidatable"` + Summary AuditSummary `json:"summary"` + BDAvailable bool `json:"bd_available"` + Error string `json:"error,omitempty"` +} + +func (report *AuditReport) FlaggedCount() int { + return report.Summary.LikelyFixed + report.Summary.LikelyStale + report.Summary.Consolidatable +} + +type ClusterBead struct { + ID string `json:"id"` + Title string `json:"title"` + IsEpic bool `json:"is_epic"` +} + +type BeadCluster struct { + Representative string `json:"representative"` + SharedKeywords []string `json:"shared_keywords"` + Beads []ClusterBead `json:"beads"` +} + +type ClusterReport struct { + Clusters []BeadCluster `json:"clusters"` + Unclustered []ClusterBead `json:"unclustered"` + Message string `json:"message,omitempty"` + BDAvailable bool `json:"bd_available"` + Applied int `json:"applied,omitempty"` + ApplyErrors []string `json:"apply_errors,omitempty"` + Error string `json:"error,omitempty"` +} + +type HygieneUseCases interface { + Audit(bool) (*AuditReport, error) + Cluster(bool) (*ClusterReport, error) +} + +type HygieneService struct { + Repository HygieneRepository +} + +func (service HygieneService) Audit(autoClose bool) (*AuditReport, error) { + report := &AuditReport{LikelyFixed: []AuditFinding{}, LikelyStale: []AuditFinding{}, Consolidatable: []AuditConsolidation{}} + if service.Repository == nil || !service.Repository.Available() { + report.Error = "bd CLI not found" + return report, nil + } + report.BDAvailable = true + open, err := service.Repository.List("open") + if err != nil { + return nil, err + } + inProgress, err := service.Repository.List("in_progress") + if err != nil { + return nil, err + } + records := append(open, inProgress...) + report.Summary.Total = len(records) + commits := service.Repository.Commits() + fileToBeads := make(map[string]map[string]bool) + for _, record := range records { + if record.ID == "" { + continue + } + body := record.TextBody() + if evidence := GrepCommitsForID(commits, record.ID); evidence != "" { + report.LikelyFixed = append(report.LikelyFixed, AuditFinding{ID: record.ID, Title: record.DisplayTitle(), Reason: "commit_match", Evidence: evidence}) + if autoClose { + _ = service.Repository.Close(record.ID, "Auto-closed by ao beads audit: commit evidence found: "+evidence) + } + continue + } + paths := ExtractAuditFilePaths(body, 10) + if record.CreatedAt != "" && len(paths) > 0 { + if evidence := FileChangesSinceCommits(commits, record.CreatedAt, paths); evidence != "" { + report.LikelyFixed = append(report.LikelyFixed, AuditFinding{ID: record.ID, Title: record.DisplayTitle(), Reason: "file_modified_since_creation", Evidence: evidence}) + if autoClose { + _ = service.Repository.Close(record.ID, "Auto-closed by ao beads audit: mentioned files modified since creation.") + } + continue + } + } + for _, path := range paths { + if fileToBeads[path] == nil { + fileToBeads[path] = make(map[string]bool) + } + fileToBeads[path][record.ID] = true + } + patterns := ExtractAuditPatterns(body, 10) + found := false + for _, pattern := range patterns { + if service.Repository.PatternExists(pattern) { + found = true + break + } + } + if len(patterns) > 0 && !found { + report.LikelyStale = append(report.LikelyStale, AuditFinding{ID: record.ID, Title: record.DisplayTitle(), Reason: "referenced_patterns_not_found"}) + } + } + consolidatable := make(map[string]bool) + for path, ids := range fileToBeads { + if len(ids) < 2 { + continue + } + beadIDs := SortedMapKeys(ids) + for _, id := range beadIDs { + consolidatable[id] = true + } + report.Consolidatable = append(report.Consolidatable, AuditConsolidation{File: path, BeadIDs: beadIDs}) + } + sort.Slice(report.Consolidatable, func(left, right int) bool { + return report.Consolidatable[left].File < report.Consolidatable[right].File + }) + report.Summary.LikelyFixed = len(report.LikelyFixed) + report.Summary.LikelyStale = len(report.LikelyStale) + report.Summary.Consolidatable = len(consolidatable) + if report.Summary.Total > 0 { + report.Summary.FlaggedPct = report.FlaggedCount() * 100 / report.Summary.Total + } + return report, nil +} + +func (service HygieneService) Cluster(apply bool) (*ClusterReport, error) { + report := &ClusterReport{Clusters: []BeadCluster{}, Unclustered: []ClusterBead{}} + if service.Repository == nil || !service.Repository.Available() { + report.Error = "bd CLI not found" + return report, nil + } + report.BDAvailable = true + records, err := service.Repository.List("open") + if err != nil { + return nil, err + } + for index, record := range records { + if enriched, showErr := service.Repository.Show(record.ID); showErr == nil && enriched.ID != "" { + if enriched.Title == "" { + enriched.Title = record.Title + } + if len(enriched.Labels) == 0 { + enriched.Labels = record.Labels + } + records[index] = enriched + } + } + if len(records) < 2 { + report.Message = "fewer than 2 open beads — nothing to cluster" + return report, nil + } + report.Clusters, report.Unclustered = ClusterBeadRecords(records) + if apply { + for _, cluster := range report.Clusters { + for _, bead := range cluster.Beads { + if bead.ID == cluster.Representative { + continue + } + if err := service.Repository.Reparent(bead.ID, cluster.Representative); err != nil { + report.ApplyErrors = append(report.ApplyErrors, fmt.Sprintf("%s -> %s: %v", bead.ID, cluster.Representative, err)) + } else { + report.Applied++ + } + } + } + } + return report, nil +} + +func ParseBDRecordList(raw []byte) ([]BeadRecord, error) { + if len(strings.TrimSpace(string(raw))) == 0 { + return nil, nil + } + var records []BeadRecord + if err := json.Unmarshal(raw, &records); err != nil { + return nil, err + } + return records, nil +} + +func ParseBDRecord(raw []byte) (BeadRecord, error) { + trimmed := []byte(strings.TrimSpace(string(raw))) + if len(trimmed) == 0 { + return BeadRecord{}, nil + } + var records []BeadRecord + if json.Unmarshal(trimmed, &records) == nil { + if len(records) == 0 { + return BeadRecord{}, nil + } + return records[0], nil + } + var record BeadRecord + if err := json.Unmarshal(trimmed, &record); err != nil { + return BeadRecord{}, err + } + return record, nil +} + +func ExtractAuditFilePaths(description string, limit int) []string { + pattern := regexp.MustCompile(`[a-zA-Z0-9_./-]+\.[a-zA-Z]{1,6}`) + seen := make(map[string]bool) + var paths []string + for _, match := range pattern.FindAllString(description, -1) { + if !strings.Contains(match, "/") || seen[match] { + continue + } + seen[match] = true + paths = append(paths, match) + if limit > 0 && len(paths) >= limit { + break + } + } + return paths +} + +func ExtractAuditPatterns(description string, limit int) []string { + seen := make(map[string]bool) + var patterns []string + add := func(value string) { + value = strings.TrimSpace(value) + if value != "" && !seen[value] { + seen[value], patterns = true, append(patterns, value) + } + } + for _, match := range regexp.MustCompile("`([^`]+)`").FindAllStringSubmatch(description, -1) { + add(match[1]) + if limit > 0 && len(patterns) >= limit/2 { + break + } + } + for _, match := range regexp.MustCompile(`\b[a-z][a-zA-Z0-9_]{5,}\b`).FindAllString(description, -1) { + add(match) + if limit > 0 && len(patterns) >= limit { + break + } + } + return patterns +} + +func GrepCommitsForID(commits []AuditCommit, id string) string { + var lines []string + for _, commit := range commits { + if strings.Contains(commit.Subject, id) || strings.Contains(commit.Body, id) { + lines = append(lines, commit.ShortSHA+" "+commit.Subject) + if len(lines) == 3 { + break + } + } + } + return strings.Join(lines, "\n") +} + +func FileChangesSinceCommits(commits []AuditCommit, createdAt string, paths []string) string { + since, _ := time.Parse(time.RFC3339, strings.TrimSpace(createdAt)) + var chunks []string + for _, path := range paths { + var lines []string + for _, commit := range commits { + if !since.IsZero() && !commit.CommitAt.After(since) { + continue + } + if _, exists := commit.Files[path]; exists { + lines = append(lines, commit.ShortSHA+" "+commit.Subject) + if len(lines) == 3 { + break + } + } + } + if len(lines) > 0 { + chunks = append(chunks, strings.Join(lines, "\n")) + } + } + return strings.Join(chunks, "\n") +} + +func ClusterBeadRecords(records []BeadRecord) ([]BeadCluster, []ClusterBead) { + if len(records) == 0 { + return []BeadCluster{}, []ClusterBead{} + } + parent := make([]int, len(records)) + for index := range parent { + parent[index] = index + } + var find func(int) int + find = func(index int) int { + if parent[index] != index { + parent[index] = find(parent[index]) + } + return parent[index] + } + for left := range records { + for right := left + 1; right < len(records); right++ { + if ScoreBeadOverlap(records[left], records[right]) >= 2 { + parent[find(right)] = find(left) + } + } + } + groups := make(map[int][]BeadRecord) + for index, record := range records { + groups[find(index)] = append(groups[find(index)], record) + } + var clusters []BeadCluster + var unclustered []ClusterBead + for _, group := range groups { + sort.Slice(group, func(left, right int) bool { return group[left].ID < group[right].ID }) + if len(group) == 1 { + unclustered = append(unclustered, ClusterBead{ID: group[0].ID, Title: group[0].DisplayTitle(), IsEpic: group[0].IsEpic()}) + continue + } + beads := make([]ClusterBead, 0, len(group)) + for _, record := range group { + beads = append(beads, ClusterBead{ID: record.ID, Title: record.DisplayTitle(), IsEpic: record.IsEpic()}) + } + clusters = append(clusters, BeadCluster{Representative: ClusterRepresentative(group), SharedKeywords: SharedClusterKeywords(group), Beads: beads}) + } + sort.Slice(clusters, func(left, right int) bool { return clusters[left].Representative < clusters[right].Representative }) + sort.Slice(unclustered, func(left, right int) bool { return unclustered[left].ID < unclustered[right].ID }) + return clusters, unclustered +} + +var clusterStopWords = map[string]bool{"the": true, "a": true, "an": true, "in": true, "to": true, "for": true, "of": true, "and": true, "or": true, "with": true, "is": true, "are": true, "be": true, "was": true, "were": true, "by": true, "on": true, "at": true, "from": true, "as": true, "this": true, "that": true, "it": true, "its": true, "into": true} + +func ScoreBeadOverlap(left, right BeadRecord) int { + return intersectionCount(tokenSet(left.DisplayTitle()+" "+left.TextBody()), tokenSet(right.DisplayTitle()+" "+right.TextBody())) + + 2*intersectionCount(pathSet(left.TextBody()), pathSet(right.TextBody())) + 3*intersectionCount(stringSet(left.Labels), stringSet(right.Labels)) +} + +func ClusterRepresentative(records []BeadRecord) string { + for _, record := range records { + if record.IsEpic() { + return record.ID + } + } + if len(records) > 0 { + return records[0].ID + } + return "" +} + +func SharedClusterKeywords(records []BeadRecord) []string { + if len(records) == 0 { + return nil + } + shared := tokenSet(records[0].DisplayTitle() + " " + records[0].TextBody()) + for _, record := range records[1:] { + current := tokenSet(record.DisplayTitle() + " " + record.TextBody()) + for keyword := range shared { + if !current[keyword] { + delete(shared, keyword) + } + } + } + return SortedMapKeys(shared) +} + +func tokenSet(input string) map[string]bool { + result := make(map[string]bool) + for _, token := range regexp.MustCompile(`[^a-z0-9/]+`).Split(strings.ToLower(input), -1) { + if len(token) >= 3 && !clusterStopWords[token] { + result[token] = true + } + } + return result +} + +func pathSet(input string) map[string]bool { + result := make(map[string]bool) + for _, match := range regexp.MustCompile(`[a-zA-Z0-9_./-]+/[a-zA-Z0-9_./-]+`).FindAllString(input, -1) { + if strings.Contains(match, ".") { + result[match] = true + } + } + return result +} + +func stringSet(values []string) map[string]bool { + result := make(map[string]bool) + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + result[value] = true + } + } + return result +} + +func intersectionCount(left, right map[string]bool) int { + count := 0 + for value := range left { + if right[value] { + count++ + } + } + return count +} + +func SortedMapKeys(values map[string]bool) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +var _ HygieneUseCases = HygieneService{} diff --git a/cli/internal/commands/beads/module.go b/cli/internal/commands/beads/module.go index 4b9e4d839..18ded02f1 100644 --- a/cli/internal/commands/beads/module.go +++ b/cli/internal/commands/beads/module.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "path/filepath" + "sort" "strings" "time" @@ -78,6 +79,7 @@ type Module struct { runtime beadsapp.ResumeRuntime reader beadsapp.LedgerReader knowledge beadsapp.KnowledgeUseCases + hygiene beadsapp.HygieneUseCases } func NewModule( @@ -90,10 +92,11 @@ func NewModule( runtime beadsapp.ResumeRuntime, reader beadsapp.LedgerReader, knowledge beadsapp.KnowledgeUseCases, + hygiene beadsapp.HygieneUseCases, ) Module { return Module{ runner: runner, resolver: resolver, inspector: inspector, executor: executor, - stale: stale, claims: claims, runtime: runtime, reader: reader, knowledge: knowledge, + stale: stale, claims: claims, runtime: runtime, reader: reader, knowledge: knowledge, hygiene: hygiene, } } @@ -208,7 +211,7 @@ func (module Module) auditCommand() *cobra.Command { command.Flags().BoolVar(&options.Strict, "strict", false, "Exit 1 when any likely-fixed, likely-stale, or consolidatable bead is found") command.Flags().BoolVar(&options.AutoClose, "auto-close", false, "Close likely-fixed beads when commit or file-change evidence is found") command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationAudit, args, options) + return module.runAudit(command, options) } return command } @@ -219,11 +222,138 @@ func (module Module) clusterCommand() *cobra.Command { command.Flags().BoolVar(&options.JSON, "json", false, "Emit cluster report as JSON") command.Flags().BoolVar(&options.Apply, "apply", false, "Reparent non-representative beads under the cluster representative") command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationCluster, args, options) + return module.runCluster(command, options) } return command } +func (module Module) runAudit(command *cobra.Command, options Options) error { + if module.hygiene == nil { + return fmt.Errorf("beads hygiene use cases are not configured") + } + report, err := module.hygiene.Audit(options.AutoClose) + if err != nil { + return err + } + if !report.BDAvailable { + if options.JSON { + return encodeJSON(command, report) + } + _, err := fmt.Fprintln(command.ErrOrStderr(), "WARN: bd not on PATH — skipping audit (graceful degradation)") + return err + } + if options.JSON { + if err := encodeJSON(command, report); err != nil { + return err + } + } else { + emitAudit(command, report) + } + if options.Strict && report.FlaggedCount() > 0 { + command.SilenceErrors = true + return &beadsapp.ExitError{Code: 1} + } + return nil +} + +func emitAudit(command *cobra.Command, report *beadsapp.AuditReport) { + output := command.OutOrStdout() + fmt.Fprintln(output, "=== ao beads audit results ===") + fmt.Fprintf(output, "Total open/in-progress beads: %d\n", report.Summary.Total) + fmt.Fprintf(output, "likely-fixed: %d\n", report.Summary.LikelyFixed) + fmt.Fprintf(output, "likely-stale: %d\n", report.Summary.LikelyStale) + fmt.Fprintf(output, "consolidatable: %d\n", report.Summary.Consolidatable) + if len(report.LikelyFixed) > 0 { + fmt.Fprintf(output, "\nLikely fixed: %s\n", auditFindingIDs(report.LikelyFixed)) + } + if len(report.LikelyStale) > 0 { + fmt.Fprintf(output, "\nLikely stale: %s\n", auditFindingIDs(report.LikelyStale)) + } + if len(report.Consolidatable) > 0 { + fmt.Fprintln(output, "\nConsolidatable:") + for _, consolidation := range report.Consolidatable { + fmt.Fprintf(output, " %s: %s\n", consolidation.File, strings.Join(consolidation.BeadIDs, " ")) + } + } +} + +func auditFindingIDs(findings []beadsapp.AuditFinding) string { + ids := make([]string, 0, len(findings)) + for _, finding := range findings { + ids = append(ids, finding.ID) + } + sort.Strings(ids) + return strings.Join(ids, " ") +} + +func (module Module) runCluster(command *cobra.Command, options Options) error { + if module.hygiene == nil { + return fmt.Errorf("beads hygiene use cases are not configured") + } + report, err := module.hygiene.Cluster(options.Apply) + if err != nil { + return err + } + if !report.BDAvailable { + if options.JSON { + return encodeJSON(command, report) + } + _, err := fmt.Fprintln(command.ErrOrStderr(), "WARN: bd not on PATH — skipping cluster analysis (graceful degradation)") + return err + } + if options.JSON { + return encodeJSON(command, report) + } + emitCluster(command, report) + return nil +} + +func emitCluster(command *cobra.Command, report *beadsapp.ClusterReport) { + output := command.OutOrStdout() + if report.Message != "" { + fmt.Fprintln(output, report.Message) + return + } + if len(report.Clusters) == 0 { + fmt.Fprintf(output, "No clusters found across %d open bead(s).\n", len(report.Unclustered)) + return + } + for index, cluster := range report.Clusters { + label := "overlapping beads" + if len(cluster.SharedKeywords) > 0 { + label = strings.Join(cluster.SharedKeywords[:min(3, len(cluster.SharedKeywords))], " ") + } + fmt.Fprintf(output, "Cluster %d: %q (%d beads)\n", index+1, label, len(cluster.Beads)) + for _, bead := range cluster.Beads { + marker := "" + if bead.IsEpic { + marker = " [epic]" + } + fmt.Fprintf(output, " %s%s: %s\n", bead.ID, marker, bead.Title) + } + if len(cluster.SharedKeywords) == 0 { + fmt.Fprintln(output, " Shared keywords: none") + } else { + fmt.Fprintf(output, " Shared keywords: %s\n", strings.Join(cluster.SharedKeywords, " ")) + } + fmt.Fprintf(output, " Suggestion: Consolidate under %s", cluster.Representative) + for _, bead := range cluster.Beads { + if bead.ID == cluster.Representative && bead.IsEpic { + fmt.Fprint(output, " (existing epic)") + } + } + fmt.Fprintln(output) + fmt.Fprintln(output) + } + fmt.Fprintf(output, "No clusters found for %d remaining bead(s).\n", len(report.Unclustered)) + if report.Applied > 0 || len(report.ApplyErrors) > 0 { + fmt.Fprintf(output, "Applied %d reparenting operation(s).\n", report.Applied) + for _, applyError := range report.ApplyErrors { + fmt.Fprintf(output, "WARN: %s\n", applyError) + } + } +} + func (module Module) execCommand() *cobra.Command { command := &cobra.Command{ Use: "exec [args...]", diff --git a/cli/internal/commands/beads/module_test.go b/cli/internal/commands/beads/module_test.go index 7e60d3104..5d7cff469 100644 --- a/cli/internal/commands/beads/module_test.go +++ b/cli/internal/commands/beads/module_test.go @@ -23,7 +23,7 @@ func (runner *recordingRunner) Run(_ *cobra.Command, invocation Invocation) erro func TestModuleBuildsFreshCompleteTrees(t *testing.T) { runner := &recordingRunner{} - module := NewModule(runner, nil, nil, nil, nil, nil, nil, nil, nil) + module := NewModule(runner, nil, nil, nil, nil, nil, nil, nil, nil, nil) first := module.Command() second := module.Command() if first == second { @@ -44,7 +44,7 @@ func TestModuleBuildsFreshCompleteTrees(t *testing.T) { func TestModuleParsesTypedInvocation(t *testing.T) { runner := &recordingRunner{} - command := NewModule(runner, nil, nil, nil, nil, nil, nil, nil, nil).Command() + command := NewModule(runner, nil, nil, nil, nil, nil, nil, nil, nil, nil).Command() command.SetArgs([]string{"audit", "--strict", "--json"}) if err := command.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -85,6 +85,14 @@ type fakeKnowledge struct { harvested beadsapp.HarvestResult } +type fakeHygiene struct { + audit *beadsapp.AuditReport + cluster *beadsapp.ClusterReport +} + +func (fake fakeHygiene) Audit(bool) (*beadsapp.AuditReport, error) { return fake.audit, nil } +func (fake fakeHygiene) Cluster(bool) (*beadsapp.ClusterReport, error) { return fake.cluster, nil } + func (fake fakeKnowledge) Available() bool { return fake.available } func (fake fakeKnowledge) Verify(context.Context, string) (*beadsapp.VerifyReport, error) { return fake.verified, nil @@ -151,7 +159,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("directory JSON", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"dir", "--require", "--json"}) if err := root.Execute(); err != nil { @@ -164,7 +172,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("tracker text", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"tracker"}) if err := root.Execute(); err != nil { @@ -177,7 +185,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("exec help precedes resolution", func(t *testing.T) { ports.executed = false - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil, nil).Command() root.SetArgs([]string{"exec", "--help"}) if err := root.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -190,7 +198,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("exec preserves typed exit", func(t *testing.T) { ports.executed = false ports.execErr = &beadsapp.ExitError{Code: 7} - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil).Command() + root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil, nil).Command() root.SetArgs([]string{"exec", "close", "age-x"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -211,7 +219,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { t.Run("stale claims", func(t *testing.T) { ports.listOutput = []byte(`[{"id":"age-old","status":"in_progress","assignee":"bo","updated_at":"2026-07-11T10:00:00Z"}]`) var output bytes.Buffer - root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil).Command() + root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"stale-claims", "--threshold", "4", "--json"}) if err := root.Execute(); err != nil { @@ -228,7 +236,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { {ID: "age-x", Status: "in_progress", Assignee: "old", UpdatedAt: "2026-07-11T10:00:00Z"}, {ID: "age-x", Status: "in_progress", Assignee: "codex", UpdatedAt: "2026-07-11T18:00:00Z"}, } - root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil).Command() + root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil, nil).Command() root.SetArgs([]string{"resume", "age-x", "--json"}) if err := root.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -241,7 +249,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { t.Run("epic terminal exit", func(t *testing.T) { ports.readOutput = []byte("{\"id\":\"age-e\",\"status\":\"open\",\"issue_type\":\"epic\"}\n{\"id\":\"age-e.1\",\"status\":\"open\",\"issue_type\":\"task\"}\n") - root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil).Command() + root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil, nil).Command() root.SetArgs([]string{"epic-status", "age-e", "--terminal"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -264,7 +272,7 @@ func TestModuleOwnsKnowledgeHandlers(t *testing.T) { t.Run("verify maps stale verdict", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge, nil).Command() root.SetOut(&output) root.SetArgs([]string{"verify", "age-x"}) err := root.Execute() @@ -278,7 +286,7 @@ func TestModuleOwnsKnowledgeHandlers(t *testing.T) { }) t.Run("lint maps stale verdict", func(t *testing.T) { - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge, nil).Command() root.SetArgs([]string{"lint"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -289,7 +297,7 @@ func TestModuleOwnsKnowledgeHandlers(t *testing.T) { t.Run("harvest renders result", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge, nil).Command() root.SetOut(&output) root.SetArgs([]string{"harvest", "age-x"}) if err := root.Execute(); err != nil { @@ -300,3 +308,49 @@ func TestModuleOwnsKnowledgeHandlers(t *testing.T) { } }) } + +func TestModuleOwnsHygieneHandlers(t *testing.T) { + hygiene := fakeHygiene{ + audit: &beadsapp.AuditReport{ + BDAvailable: true, + LikelyFixed: []beadsapp.AuditFinding{{ID: "age-fixed", Title: "fixed", Reason: "commit_match"}}, + Summary: beadsapp.AuditSummary{Total: 1, LikelyFixed: 1}, + }, + cluster: &beadsapp.ClusterReport{ + BDAvailable: true, + Clusters: []beadsapp.BeadCluster{{ + Representative: "age-e", + SharedKeywords: []string{"tracker"}, + Beads: []beadsapp.ClusterBead{{ID: "age-e", Title: "tracker epic", IsEpic: true}, {ID: "age-x", Title: "tracker task"}}, + }}, + }, + } + + t.Run("strict audit maps findings", func(t *testing.T) { + var output bytes.Buffer + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, hygiene).Command() + root.SetOut(&output) + root.SetArgs([]string{"audit", "--strict"}) + err := root.Execute() + var exitError *beadsapp.ExitError + if !errors.As(err, &exitError) || exitError.ExitCode() != 1 { + t.Fatalf("error = %v, want ExitError(1)", err) + } + if !bytes.Contains(output.Bytes(), []byte("Likely fixed: age-fixed")) { + t.Fatalf("output = %q", output.String()) + } + }) + + t.Run("cluster renders representative", func(t *testing.T) { + var output bytes.Buffer + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, hygiene).Command() + root.SetOut(&output) + root.SetArgs([]string{"cluster"}) + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if !bytes.Contains(output.Bytes(), []byte("Consolidate under age-e (existing epic)")) { + t.Fatalf("output = %q", output.String()) + } + }) +} From 7d1a204709cf73062551d3e5fe3566437c0b0888 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:51:51 -0400 Subject: [PATCH 022/108] refactor(beads): internalize acceptance command handlers (age-nw28h.2.1) --- cli/cmd/ao/beads_command_legacy_test.go | 55 +++++- cli/cmd/ao/beads_module.go | 31 +--- ...rios.go => beads_scenarios_legacy_test.go} | 0 cli/internal/adapters/beads/acceptance.go | 33 ++++ cli/internal/adapters/beads/scenarios.go | 35 ++++ .../beads/acceptance.go} | 155 +++++++---------- cli/internal/beads/scenario_service.go | 133 ++++++++++++++ cli/internal/commands/beads/module.go | 163 ++++++++++++------ cli/internal/commands/beads/module_test.go | 137 ++++++++++----- 9 files changed, 527 insertions(+), 215 deletions(-) rename cli/cmd/ao/{beads_scenarios.go => beads_scenarios_legacy_test.go} (100%) create mode 100644 cli/internal/adapters/beads/acceptance.go create mode 100644 cli/internal/adapters/beads/scenarios.go rename cli/{cmd/ao/beads_verify_acceptance.go => internal/beads/acceptance.go} (85%) create mode 100644 cli/internal/beads/scenario_service.go diff --git a/cli/cmd/ao/beads_command_legacy_test.go b/cli/cmd/ao/beads_command_legacy_test.go index 5f92e579f..cdaff543f 100644 --- a/cli/cmd/ao/beads_command_legacy_test.go +++ b/cli/cmd/ao/beads_command_legacy_test.go @@ -1,7 +1,9 @@ package main import ( + "encoding/json" "errors" + "fmt" beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" beadsapp "github.com/boshu2/agentops/cli/internal/beads" @@ -9,6 +11,57 @@ import ( "github.com/spf13/cobra" ) +type acceptBead = beadsapp.AcceptanceBead +type acceptanceVerdict = beadsapp.AcceptanceVerdict + +const ( + acPass = beadsapp.AcceptancePass + acFail = beadsapp.AcceptanceFail + acUndefined = beadsapp.AcceptanceUndefined +) + +var parseBeadsFromBRJSON = beadsapp.ParseBeadsFromBRJSON +var checkAcceptanceContract = beadsapp.CheckAcceptanceContract +var validateAcceptanceCriteriaContent = beadsapp.ValidateAcceptanceCriteriaContent + +var execBR = func(...string) ([]byte, error) { return nil, errors.New("execBR test seam is not configured") } + +type acceptanceRepositoryFunc func([]string) ([]byte, error) + +func (function acceptanceRepositoryFunc) ShowAcceptance(ids []string) ([]byte, error) { + return function(ids) +} + +func executeBeadsVerifyAcceptance(command *cobra.Command, ids []string, strict, asJSON bool) error { + service := beadsapp.AcceptanceService{Repository: acceptanceRepositoryFunc(func(ids []string) ([]byte, error) { + return execBR(append([]string{"show", "--format", "json"}, ids...)...) + })} + results, nonPass, err := service.VerifyAcceptance(ids) + if err != nil { + return err + } + if asJSON { + encoder := json.NewEncoder(command.OutOrStdout()) + encoder.SetIndent("", " ") + if err := encoder.Encode(results); err != nil { + return err + } + } else { + for _, result := range results { + fmt.Fprintf(command.OutOrStdout(), "%s [%s] %s\n", result.Verdict, result.IssueType, result.BeadID) + for _, missing := range result.Missing { + fmt.Fprintf(command.OutOrStdout(), " missing: %s\n", missing) + } + } + } + if strict && nonPass { + return &beadsVerdictError{code: 1} + } + return nil +} + +func asBeadsExit(err error, target **beadsVerdictError) bool { return errors.As(err, target) } + // White-box behavior tests keep private throwaway commands for stream and flag // control. Production command ownership lives in internal/commands/beads. var legacyBeadsDirCommand = &cobra.Command{Use: "dir", RunE: executeBeadsDir} @@ -94,7 +147,7 @@ func executeBeadsExec(command *cobra.Command, args []string) error { func newTestBeadsCommand(name string) *cobra.Command { tracker := currentBeadsTracker() runtime := beadsadapter.NewRuntime() - root := beadscommands.NewModule(nil, tracker, tracker, beadsadapter.NewExecutor(tracker), tracker, tracker, runtime, runtime, nil, nil).Command() + root := beadscommands.NewModule(tracker, tracker, beadsadapter.NewExecutor(tracker), tracker, tracker, runtime, runtime, nil, nil, nil, nil).Command() child, _, err := root.Find([]string{name}) if err != nil || child == nil { panic("missing test beads command " + name) diff --git a/cli/cmd/ao/beads_module.go b/cli/cmd/ao/beads_module.go index 6300cc45d..43f19acd5 100644 --- a/cli/cmd/ao/beads_module.go +++ b/cli/cmd/ao/beads_module.go @@ -1,46 +1,23 @@ package main import ( - "fmt" - beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/boshu2/agentops/cli/internal/clicontract" beadscommands "github.com/boshu2/agentops/cli/internal/commands/beads" - "github.com/spf13/cobra" ) -// beadsModuleRunner is the composition seam between the explicit driving -// adapter and the existing family use cases. The module owns Cobra parsing; -// this bridge transfers typed options while the remaining use cases move out -// of cmd/ao behind their application ports. -type beadsModuleRunner struct{} - -func (beadsModuleRunner) Run(command *cobra.Command, invocation beadscommands.Invocation) error { - options := invocation.Options - switch invocation.Operation { - case beadscommands.OperationScenariosExtract: - beadsScenariosJSON, beadsScenariosForce, beadsScenariosWrite = options.JSON, options.Force, options.Write - return executeBeadsScenariosExtract(command, invocation.Args) - case beadscommands.OperationScenariosCheck: - beadsScenariosValidateJSON = options.JSON - return executeBeadsScenariosValidate(command, invocation.Args) - case beadscommands.OperationAcceptance: - return executeBeadsVerifyAcceptance(command, invocation.Args, options.Strict, options.JSON) - default: - return fmt.Errorf("unsupported beads operation %q", invocation.Operation) - } -} - func init() { tracker := currentBeadsTracker() runtime := beadsadapter.NewRuntime() repository := beadsadapter.NewKnowledgeRepository() knowledge := beadsapp.KnowledgeService{Tracker: tracker, Repository: repository, Clock: runtime} hygiene := beadsapp.HygieneService{Repository: beadsadapter.NewHygieneRepository(tracker)} + scenario := beadsapp.ScenarioService{Repository: beadsadapter.NewScenarioRepository(tracker)} + acceptance := beadsapp.AcceptanceService{Repository: beadsadapter.NewAcceptanceRepository(tracker)} module := beadscommands.NewModule( - beadsModuleRunner{}, tracker, tracker, beadsadapter.NewExecutor(tracker), - tracker, tracker, runtime, runtime, knowledge, hygiene, + tracker, tracker, beadsadapter.NewExecutor(tracker), tracker, tracker, + runtime, runtime, knowledge, hygiene, scenario, acceptance, ) command := module.Command() command.GroupID = "knowledge" diff --git a/cli/cmd/ao/beads_scenarios.go b/cli/cmd/ao/beads_scenarios_legacy_test.go similarity index 100% rename from cli/cmd/ao/beads_scenarios.go rename to cli/cmd/ao/beads_scenarios_legacy_test.go diff --git a/cli/internal/adapters/beads/acceptance.go b/cli/internal/adapters/beads/acceptance.go new file mode 100644 index 000000000..8dfa3ea8a --- /dev/null +++ b/cli/internal/adapters/beads/acceptance.go @@ -0,0 +1,33 @@ +package beads + +import ( + "context" + "fmt" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" +) + +type AcceptanceRepository struct { + tracker *Tracker +} + +func NewAcceptanceRepository(tracker *Tracker) AcceptanceRepository { + return AcceptanceRepository{tracker: tracker} +} + +func (repository AcceptanceRepository) ShowAcceptance(ids []string) ([]byte, error) { + if repository.tracker == nil { + return nil, fmt.Errorf("beads acceptance tracker is not configured") + } + resolved, err := repository.tracker.Resolve() + if err != nil { + return nil, err + } + if resolved.Tracker != beadsapp.TrackerBR { + return nil, fmt.Errorf("verify-acceptance requires the BR acceptance wire; selected tracker is %s", resolved.Tracker) + } + args := append([]string{"show", "--format", "json"}, ids...) + return repository.tracker.Output(context.Background(), args...) +} + +var _ beadsapp.AcceptanceRepository = AcceptanceRepository{} diff --git a/cli/internal/adapters/beads/scenarios.go b/cli/internal/adapters/beads/scenarios.go new file mode 100644 index 000000000..dcbe01c9a --- /dev/null +++ b/cli/internal/adapters/beads/scenarios.go @@ -0,0 +1,35 @@ +package beads + +import ( + "context" + "fmt" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" +) + +type ScenarioRepository struct { + tracker *Tracker +} + +func NewScenarioRepository(tracker *Tracker) ScenarioRepository { + return ScenarioRepository{tracker: tracker} +} + +func (repository ScenarioRepository) Available() bool { + return repository.tracker != nil && repository.tracker.Available() +} + +func (repository ScenarioRepository) FetchScenarioBead(id string) (beadsapp.FetchedBead, error) { + raw, err := repository.tracker.Output(context.Background(), "show", id, "--json") + if err != nil { + return beadsapp.FetchedBead{}, fmt.Errorf("bd show %s --json: %w", id, err) + } + return beadsapp.ParseScenarioBead(raw) +} + +func (repository ScenarioRepository) UpdateDescription(id, description string) error { + _, err := repository.tracker.Output(context.Background(), "update", id, "--description", description) + return err +} + +var _ beadsapp.ScenarioRepository = ScenarioRepository{} diff --git a/cli/cmd/ao/beads_verify_acceptance.go b/cli/internal/beads/acceptance.go similarity index 85% rename from cli/cmd/ao/beads_verify_acceptance.go rename to cli/internal/beads/acceptance.go index d56390c6b..5f4f1d0d1 100644 --- a/cli/cmd/ao/beads_verify_acceptance.go +++ b/cli/internal/beads/acceptance.go @@ -17,56 +17,35 @@ // It is advisory by default (prints a per-bead verdict, exits 0) so it can run // over today's ledger as an adoption ratchet. With --strict, any FAIL or // UNDEFINED maps to a non-zero exit — the gate posture for new/changed beads. -package main +package beads import ( "bytes" "encoding/json" - "errors" "fmt" - "os" - "os/exec" "regexp" "strings" "unicode" - "github.com/spf13/cobra" "gopkg.in/yaml.v3" "github.com/boshu2/agentops/cli/internal/scenarios" ) -// execBR is the single entry point for shelling out to br (beads_rust), the -// sanctioned tracker. It is a package var so tests inject a fake without a live -// ledger — the br-native analog of beadsTrackerOutput. BEADS_DIR is inherited from the -// process environment (exec.Command keeps os.Environ by default). -var execBR = func(args ...string) ([]byte, error) { - cwd, _ := os.Getwd() - resolution, err := resolveTracker(cwd, os.Environ()) - if err != nil { - return nil, err - } - if resolution.Tracker != trackerBR { - return nil, fmt.Errorf("verify-acceptance requires the BR acceptance wire; selected tracker is %s", resolution.Tracker) - } - cmd := exec.Command(resolution.Binary, args...) // #nosec G204 -- selected BR binary. - return cmd.Output() -} - -// acceptBead is the subset of a br bead the verifier needs. -type acceptBead struct { +// AcceptanceBead is the subset of a br bead the verifier needs. +type AcceptanceBead struct { ID string `json:"id"` IssueType string `json:"issue_type"` Description string `json:"description"` } -// acceptanceVerdict is the per-bead outcome. -type acceptanceVerdict string +// AcceptanceVerdict is the per-bead outcome. +type AcceptanceVerdict string const ( - acPass acceptanceVerdict = "PASS" - acFail acceptanceVerdict = "FAIL" - acUndefined acceptanceVerdict = "UNDEFINED" + AcceptancePass AcceptanceVerdict = "PASS" + AcceptanceFail AcceptanceVerdict = "FAIL" + AcceptanceUndefined AcceptanceVerdict = "UNDEFINED" ) // brErrorEnvelope matches br's `{"error": {...}}` failure shape. br emits this @@ -80,10 +59,10 @@ type brErrorEnvelope struct { } `json:"error"` } -// parseBeadsFromBRJSON parses `br show --format json` output into beads. +// ParseBeadsFromBRJSON parses `br show --format json` output into beads. // br emits a JSON array; it emits an `{"error":...}` object (exit 0) when an id // is not found — that object is rejected, never coerced to a bead. -func parseBeadsFromBRJSON(out []byte) ([]acceptBead, error) { +func ParseBeadsFromBRJSON(out []byte) ([]AcceptanceBead, error) { trimmed := bytes.TrimSpace(out) if len(trimmed) == 0 { return nil, fmt.Errorf("br returned no output") @@ -95,7 +74,7 @@ func parseBeadsFromBRJSON(out []byte) ([]acceptBead, error) { } return nil, fmt.Errorf("br returned an object, not the expected bead array") } - var beads []acceptBead + var beads []AcceptanceBead if err := json.Unmarshal(trimmed, &beads); err != nil { return nil, fmt.Errorf("parse br json array: %w", err) } @@ -164,29 +143,29 @@ func isPlaceholder(s string) bool { // cross-family REFUTE). var acIDPattern = regexp.MustCompile(`^ac-[a-z0-9][a-z0-9._-]*\.[0-9]+$`) -// checkAcceptanceContract applies the per-type acceptance contract and returns +// CheckAcceptanceContract applies the per-type acceptance contract and returns // the verdict plus, for FAIL, the named missing elements. Beyond the per-type // SHAPE check it also runs CONTENT-QUALITY validation (age-xmkn): a section or // criterion that is structurally present but only a placeholder ("TBD"), and a // fenced acceptance_criteria block whose parsed criteria violate the canonical // authored contract, both FAIL — even when the shape check alone would pass. -func checkAcceptanceContract(b acceptBead) (acceptanceVerdict, []string) { +func CheckAcceptanceContract(b AcceptanceBead) (AcceptanceVerdict, []string) { verdict, missing := checkAcceptanceShape(b) // Content-quality only sharpens a would-be PASS into a FAIL; it never rescues // an already-failing or UNDEFINED verdict. A present-but-invalid // acceptance_criteria block is a content defect on any type. - if verdict == acPass { - if problems := validateAcceptanceCriteriaContent(b.Description); len(problems) > 0 { - return acFail, problems + if verdict == AcceptancePass { + if problems := ValidateAcceptanceCriteriaContent(b.Description); len(problems) > 0 { + return AcceptanceFail, problems } } return verdict, missing } // checkAcceptanceShape is the per-type SHAPE contract (the original -// checkAcceptanceContract body): the right section headings / structural tokens +// CheckAcceptanceContract body): the right section headings / structural tokens // for the bead's issue_type, now with placeholder-only bodies rejected. -func checkAcceptanceShape(b acceptBead) (acceptanceVerdict, []string) { +func checkAcceptanceShape(b AcceptanceBead) (AcceptanceVerdict, []string) { desc := b.Description switch strings.ToLower(strings.TrimSpace(b.IssueType)) { case "feature": @@ -198,9 +177,9 @@ func checkAcceptanceShape(b acceptBead) (acceptanceVerdict, []string) { missing = append(missing, "test: a TDD signal (acceptance_criteria / check_command / check_type: test_pass)") } if len(missing) > 0 { - return acFail, missing + return AcceptanceFail, missing } - return acPass, nil + return AcceptancePass, nil case "spike": return single(desc, []string{"decision", "decisions"}, "decision-criteria: a 'Decision' / 'Decision Criteria' section heading") case "design": @@ -218,11 +197,11 @@ func checkAcceptanceShape(b acceptBead) (acceptanceVerdict, []string) { // children. Returning N/A here was a fail-open (cross-family REFUTE): // `--strict` would pass a task/epic with no acceptance at all. if hasAcceptanceSignal(desc) { - return acPass, nil + return AcceptancePass, nil } - return acFail, []string{"acceptance: a measurable acceptance_criteria block or 'Acceptance' section heading (every bead requires acceptance per skills/plan/SKILL.md)"} + return AcceptanceFail, []string{"acceptance: a measurable acceptance_criteria block or 'Acceptance' section heading (every bead requires acceptance per skills/plan/SKILL.md)"} default: - return acUndefined, nil + return AcceptanceUndefined, nil } } @@ -303,11 +282,11 @@ func sectionHasRealContent(desc string, keywords []string) bool { // single passes when a markdown heading matches one of the keywords AND that // section carries real (non-placeholder) body content — not just the heading. -func single(desc string, keywords []string, missingMsg string) (acceptanceVerdict, []string) { +func single(desc string, keywords []string, missingMsg string) (AcceptanceVerdict, []string) { if sectionHasRealContent(desc, keywords) { - return acPass, nil + return AcceptancePass, nil } - return acFail, []string{missingMsg} + return AcceptanceFail, []string{missingMsg} } // sectionBody returns the body lines under the FIRST heading whose words @@ -344,20 +323,20 @@ func sectionBody(desc string, keywords []string) (string, bool) { // checkListContract passes when a matching section heading exists AND at least // one bullet list item appears WITHIN that section. -func checkListContract(desc string, keywords []string, missingMsg string) (acceptanceVerdict, []string) { +func checkListContract(desc string, keywords []string, missingMsg string) (AcceptanceVerdict, []string) { if body, ok := sectionBody(desc, keywords); ok && hasListItem(body) { - return acPass, nil + return AcceptancePass, nil } - return acFail, []string{missingMsg} + return AcceptanceFail, []string{missingMsg} } // checkCheckboxContract passes when a matching section heading exists AND at // least one markdown checkbox appears WITHIN that section. -func checkCheckboxContract(desc string, keywords []string, missingMsg string) (acceptanceVerdict, []string) { +func checkCheckboxContract(desc string, keywords []string, missingMsg string) (AcceptanceVerdict, []string) { if body, ok := sectionBody(desc, keywords); ok && hasCheckbox(body) { - return acPass, nil + return AcceptancePass, nil } - return acFail, []string{missingMsg} + return AcceptanceFail, []string{missingMsg} } // acListItem matches a REAL markdown list item: an unordered (-,*,+) or ordered @@ -465,13 +444,13 @@ func extractAcceptanceCriteriaYAML(desc string) string { return strings.Join(out, "\n") } -// validateAcceptanceCriteriaContent parses the embedded acceptance_criteria block +// ValidateAcceptanceCriteriaContent parses the embedded acceptance_criteria block // and validates each criterion against the canonical authored contract (age-xmkn): // a measurable (non-placeholder) description, a valid check_type, a check_command // for runnable check_types, and agent_judge when check_type == custom_rubric. It // returns content-quality problems (empty when the block is absent or valid). A // present-but-unparseable block is itself a problem (a malformed contract). -func validateAcceptanceCriteriaContent(desc string) []string { +func ValidateAcceptanceCriteriaContent(desc string) []string { raw := extractAcceptanceCriteriaYAML(desc) if raw == "" { return nil // no block present — the SHAPE check owns "block required" @@ -513,21 +492,36 @@ func validateAcceptanceCriteriaContent(desc string) []string { return problems } -type acceptanceResult struct { +type AcceptanceResult struct { BeadID string `json:"bead_id"` IssueType string `json:"issue_type"` - Verdict acceptanceVerdict `json:"verdict"` + Verdict AcceptanceVerdict `json:"verdict"` Missing []string `json:"missing,omitempty"` } -func executeBeadsVerifyAcceptance(cmd *cobra.Command, ids []string, strict, asJSON bool) error { - out, err := execBR(append([]string{"show", "--format", "json"}, ids...)...) - if err != nil { - return fmt.Errorf("br show %v: %w", ids, err) +type AcceptanceRepository interface { + ShowAcceptance([]string) ([]byte, error) +} + +type AcceptanceUseCases interface { + VerifyAcceptance([]string) ([]AcceptanceResult, bool, error) +} + +type AcceptanceService struct { + Repository AcceptanceRepository +} + +func (service AcceptanceService) VerifyAcceptance(ids []string) ([]AcceptanceResult, bool, error) { + if service.Repository == nil { + return nil, false, fmt.Errorf("beads acceptance repository is not configured") } - beads, err := parseBeadsFromBRJSON(out) + out, err := service.Repository.ShowAcceptance(ids) if err != nil { - return fmt.Errorf("read beads: %w", err) + return nil, false, fmt.Errorf("br show %v: %w", ids, err) + } + beads, err := ParseBeadsFromBRJSON(out) + if err != nil { + return nil, false, fmt.Errorf("read beads: %w", err) } // Defense-in-depth (cross-family REFUTE): never silently verify fewer beads @@ -546,41 +540,20 @@ func executeBeadsVerifyAcceptance(cmd *cobra.Command, ids []string, strict, asJS } } if len(absent) > 0 { - return fmt.Errorf("br did not return requested bead(s): %s", strings.Join(absent, ", ")) + return nil, false, fmt.Errorf("br did not return requested bead(s): %s", strings.Join(absent, ", ")) } - results := make([]acceptanceResult, 0, len(beads)) + results := make([]AcceptanceResult, 0, len(beads)) nonPass := false for _, b := range beads { - v, missing := checkAcceptanceContract(b) - results = append(results, acceptanceResult{BeadID: b.ID, IssueType: b.IssueType, Verdict: v, Missing: missing}) - if v == acFail || v == acUndefined { + v, missing := CheckAcceptanceContract(b) + results = append(results, AcceptanceResult{BeadID: b.ID, IssueType: b.IssueType, Verdict: v, Missing: missing}) + if v == AcceptanceFail || v == AcceptanceUndefined { nonPass = true } } - if asJSON { - enc := json.NewEncoder(cmd.OutOrStdout()) - enc.SetIndent("", " ") - if err := enc.Encode(results); err != nil { - return err - } - } else { - for _, r := range results { - fmt.Fprintf(cmd.OutOrStdout(), "%s [%s] %s\n", r.Verdict, r.IssueType, r.BeadID) - for _, m := range r.Missing { - fmt.Fprintf(cmd.OutOrStdout(), " missing: %s\n", m) - } - } - } - - if strict && nonPass { - return &beadsVerdictError{code: 1} - } - return nil + return results, nonPass, nil } -// asBeadsExit is a thin errors.As wrapper used by tests. -func asBeadsExit(err error, target **beadsVerdictError) bool { - return errors.As(err, target) -} +var _ AcceptanceUseCases = AcceptanceService{} diff --git a/cli/internal/beads/scenario_service.go b/cli/internal/beads/scenario_service.go new file mode 100644 index 000000000..fe43d770f --- /dev/null +++ b/cli/internal/beads/scenario_service.go @@ -0,0 +1,133 @@ +package beads + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/boshu2/agentops/cli/internal/scenarios" +) + +type FetchedBead struct { + Acceptance string + Description string +} + +type ScenarioExtraction struct { + BeadID string + Bead FetchedBead + Scenarios []scenarios.Scenario + AlreadyShaped bool +} + +type ScenarioValidation struct { + BeadID string `json:"bead_id"` + Valid bool `json:"valid"` + Scenarios []scenarios.Scenario `json:"scenarios,omitempty"` + Error string `json:"error,omitempty"` +} + +type ScenarioRepository interface { + Available() bool + FetchScenarioBead(string) (FetchedBead, error) + UpdateDescription(string, string) error +} + +type ScenarioUseCases interface { + Available() bool + PrepareScenarios(string, bool) (ScenarioExtraction, error) + ApplyScenarios(ScenarioExtraction) error + ValidateScenarios(string) (ScenarioValidation, error) +} + +type ScenarioService struct { + Repository ScenarioRepository +} + +func (service ScenarioService) Available() bool { + return service.Repository != nil && service.Repository.Available() +} + +func (service ScenarioService) PrepareScenarios(id string, force bool) (ScenarioExtraction, error) { + bead, err := service.Repository.FetchScenarioBead(id) + if err != nil { + return ScenarioExtraction{}, fmt.Errorf("fetch acceptance for %s: %w (inspect with 'bd show %s --json')", id, err, id) + } + extraction := ScenarioExtraction{BeadID: id, Bead: bead} + if !force && (scenarios.HasScenariosBlock(bead.Description) || scenarios.HasScenariosBlock(bead.Acceptance)) { + extraction.AlreadyShaped = true + return extraction, nil + } + extraction.Scenarios, err = scenarios.Extract(bead.Acceptance) + if err != nil { + return ScenarioExtraction{}, fmt.Errorf("extract scenarios from %s: %w; author a '## Scenarios' block manually (see CLAUDE.md acceptance doctrine)", id, err) + } + return extraction, nil +} + +func (service ScenarioService) ApplyScenarios(extraction ScenarioExtraction) error { + rendered := scenarios.Render(extraction.Scenarios) + description := ComposeDescriptionWithScenarios(extraction.Bead.Description, rendered) + if err := service.Repository.UpdateDescription(extraction.BeadID, description); err != nil { + return fmt.Errorf("write '## Scenarios' into %s: %w (inspect with 'bd show %s')", extraction.BeadID, err, extraction.BeadID) + } + return nil +} + +func (service ScenarioService) ValidateScenarios(id string) (ScenarioValidation, error) { + bead, err := service.Repository.FetchScenarioBead(id) + if err != nil { + return ScenarioValidation{}, fmt.Errorf("fetch bead %s: %w (inspect with 'bd show %s --json')", id, err, id) + } + text := bead.Description + if !scenarios.HasScenariosBlock(text) && scenarios.HasScenariosBlock(bead.Acceptance) { + text = bead.Acceptance + } + parsed, err := scenarios.ParseBlock(text) + if err != nil { + return ScenarioValidation{BeadID: id, Error: err.Error()}, fmt.Errorf("validate scenarios for %s: %w; fix the block or regenerate it with 'ao beads scenarios extract %s --force'", id, err, id) + } + return ScenarioValidation{BeadID: id, Valid: true, Scenarios: parsed}, nil +} + +func ParseScenarioBead(out []byte) (FetchedBead, error) { + type bead struct { + AcceptanceCriteria string `json:"acceptance_criteria"` + Description string `json:"description"` + } + trimmed := bytes.TrimSpace(out) + var parsed bead + if len(trimmed) > 0 && trimmed[0] == '[' { + var records []bead + if err := json.Unmarshal(trimmed, &records); err != nil { + return FetchedBead{}, fmt.Errorf("parse bd json array: %w", err) + } + if len(records) == 0 { + return FetchedBead{}, fmt.Errorf("bead not found") + } + parsed = records[0] + } else if err := json.Unmarshal(trimmed, &parsed); err != nil { + return FetchedBead{}, fmt.Errorf("parse bd json: %w", err) + } + description := strings.TrimSpace(parsed.Description) + acceptance := strings.TrimSpace(parsed.AcceptanceCriteria) + if acceptance == "" { + acceptance = description + } + if acceptance == "" { + return FetchedBead{}, fmt.Errorf("bead has no acceptance_criteria or description text") + } + return FetchedBead{Acceptance: acceptance, Description: description}, nil +} + +func ComposeDescriptionWithScenarios(description, rendered string) string { + block := strings.TrimRight(rendered, "\n") + base := strings.TrimRight(description, "\n") + if base == "" { + return block + "\n" + } + return base + "\n\n" + block + "\n" +} + +var _ ScenarioUseCases = ScenarioService{} diff --git a/cli/internal/commands/beads/module.go b/cli/internal/commands/beads/module.go index 18ded02f1..298811d7c 100644 --- a/cli/internal/commands/beads/module.go +++ b/cli/internal/commands/beads/module.go @@ -4,6 +4,7 @@ package beads import ( + "bufio" "context" "encoding/json" "errors" @@ -18,33 +19,9 @@ import ( beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/boshu2/agentops/cli/internal/clicontract" "github.com/boshu2/agentops/cli/internal/epicstatus" + "github.com/boshu2/agentops/cli/internal/scenarios" ) -type Operation string - -const ( - OperationDir Operation = "dir" - OperationTracker Operation = "tracker" - OperationVerify Operation = "verify" - OperationLint Operation = "lint" - OperationHarvest Operation = "harvest" - OperationAudit Operation = "audit" - OperationCluster Operation = "cluster" - OperationExec Operation = "exec" - OperationResume Operation = "resume" - OperationScenariosExtract Operation = "scenarios.extract" - OperationScenariosCheck Operation = "scenarios.validate" - OperationStaleClaims Operation = "stale-claims" - OperationEpicStatus Operation = "epic-status" - OperationAcceptance Operation = "verify-acceptance" -) - -type Invocation struct { - Operation Operation - Args []string - Options Options -} - type Options struct { JSON bool Require bool @@ -63,27 +40,21 @@ type Options struct { Terminal bool } -// Runner is the command family's inbound application port. The command module -// owns parsing; the implementation owns use-case orchestration and effects. -type Runner interface { - Run(*cobra.Command, Invocation) error -} - type Module struct { - runner Runner - resolver beadsapp.TrackerResolver - inspector beadsapp.LedgerInspector - executor beadsapp.TrackerExecutor - stale beadsapp.StaleSource - claims beadsapp.ClaimStore - runtime beadsapp.ResumeRuntime - reader beadsapp.LedgerReader - knowledge beadsapp.KnowledgeUseCases - hygiene beadsapp.HygieneUseCases + resolver beadsapp.TrackerResolver + inspector beadsapp.LedgerInspector + executor beadsapp.TrackerExecutor + stale beadsapp.StaleSource + claims beadsapp.ClaimStore + runtime beadsapp.ResumeRuntime + reader beadsapp.LedgerReader + knowledge beadsapp.KnowledgeUseCases + hygiene beadsapp.HygieneUseCases + scenario beadsapp.ScenarioUseCases + acceptance beadsapp.AcceptanceUseCases } func NewModule( - runner Runner, resolver beadsapp.TrackerResolver, inspector beadsapp.LedgerInspector, executor beadsapp.TrackerExecutor, @@ -93,10 +64,13 @@ func NewModule( reader beadsapp.LedgerReader, knowledge beadsapp.KnowledgeUseCases, hygiene beadsapp.HygieneUseCases, + scenario beadsapp.ScenarioUseCases, + acceptance beadsapp.AcceptanceUseCases, ) Module { return Module{ - runner: runner, resolver: resolver, inspector: inspector, executor: executor, + resolver: resolver, inspector: inspector, executor: executor, stale: stale, claims: claims, runtime: runtime, reader: reader, knowledge: knowledge, hygiene: hygiene, + scenario: scenario, acceptance: acceptance, } } @@ -143,13 +117,6 @@ None of these commands replace bd itself — they complement it.`, return root } -func (module Module) invoke(command *cobra.Command, operation Operation, args []string, options Options) error { - if module.runner == nil { - return fmt.Errorf("beads command runner is not configured") - } - return module.runner.Run(command, Invocation{Operation: operation, Args: append([]string(nil), args...), Options: options}) -} - func (module Module) dirCommand() *cobra.Command { var options Options command := &cobra.Command{Use: "dir", Short: "Print the resolved live br ledger directory", Args: cobra.NoArgs} @@ -577,18 +544,82 @@ func (module Module) scenariosCommand() *cobra.Command { extractCommand.Flags().BoolVar(&extract.Force, "force", false, "Extract even when the bead already has a '## Scenarios' block") extractCommand.Flags().BoolVar(&extract.Write, "write", false, "After printing the block and an operator y/N confirmation, append it to the bead via 'bd update'") extractCommand.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationScenariosExtract, args, extract) + return module.runScenarioExtract(command, args, extract) } var check Options checkCommand := &cobra.Command{Use: "validate ", Short: "Check that a bead's authored '## Scenarios' block is well-formed Gherkin", Args: cobra.ExactArgs(1)} checkCommand.Flags().BoolVar(&check.JSON, "json", false, "Emit a structured validation verdict as JSON on stdout") checkCommand.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationScenariosCheck, args, check) + return module.runScenarioValidation(command, args, check) } root.AddCommand(extractCommand, checkCommand) return root } +func (module Module) runScenarioExtract(command *cobra.Command, args []string, options Options) error { + if module.scenario == nil { + return fmt.Errorf("beads scenario use cases are not configured") + } + if !module.scenario.Available() { + _, err := fmt.Fprintln(command.ErrOrStderr(), "warning: bd not found on PATH; cannot fetch bead. Install bd or author scenarios manually.") + return err + } + extraction, err := module.scenario.PrepareScenarios(args[0], options.Force) + if err != nil { + return err + } + if extraction.AlreadyShaped { + _, err := fmt.Fprintf(command.ErrOrStderr(), "bead %s already has a '## Scenarios' block; nothing to extract. Re-run with --force to extract anyway.\n", args[0]) + return err + } + if options.Write { + rendered := scenarios.Render(extraction.Scenarios) + fmt.Fprintf(command.ErrOrStderr(), "About to append this block to %s's description:\n\n%s\nProceed? [y/N]: ", args[0], rendered) + line, _ := bufio.NewReader(command.InOrStdin()).ReadString('\n') + switch strings.ToLower(strings.TrimSpace(line)) { + case "y", "yes": + default: + _, err := fmt.Fprintf(command.ErrOrStderr(), "aborted; %s left unchanged\n", args[0]) + return err + } + if err := module.scenario.ApplyScenarios(extraction); err != nil { + return err + } + _, err := fmt.Fprintf(command.OutOrStdout(), "%s updated with %d scenario(s)\n", args[0], len(extraction.Scenarios)) + return err + } + if options.JSON { + return encodeJSON(command, struct { + BeadID string `json:"bead_id"` + Scenarios []scenarios.Scenario `json:"scenarios"` + }{BeadID: args[0], Scenarios: extraction.Scenarios}) + } + _, err = fmt.Fprint(command.OutOrStdout(), scenarios.Render(extraction.Scenarios)) + return err +} + +func (module Module) runScenarioValidation(command *cobra.Command, args []string, options Options) error { + if module.scenario == nil { + return fmt.Errorf("beads scenario use cases are not configured") + } + if !module.scenario.Available() { + _, err := fmt.Fprintln(command.ErrOrStderr(), "warning: bd not found on PATH; cannot fetch bead. Install bd or author scenarios manually.") + return err + } + result, err := module.scenario.ValidateScenarios(args[0]) + if err != nil { + if options.JSON { + _ = encodeJSON(command, result) + } + return err + } + if options.JSON { + return encodeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "%s: %d scenario(s) well-formed\n", args[0], len(result.Scenarios)) + return err +} + func (module Module) staleCommand() *cobra.Command { var options Options command := &cobra.Command{Use: "stale-claims", Short: "List in_progress beads whose claim looks stale"} @@ -748,7 +779,33 @@ func (module Module) acceptanceCommand() *cobra.Command { command.Flags().BoolVar(&options.Strict, "strict", false, "Exit non-zero on any FAIL or UNDEFINED verdict") command.Flags().BoolVar(&options.JSON, "json", false, "Emit verdicts as JSON") command.RunE = func(command *cobra.Command, args []string) error { - return module.invoke(command, OperationAcceptance, args, options) + return module.runAcceptance(command, args, options) } return command } + +func (module Module) runAcceptance(command *cobra.Command, ids []string, options Options) error { + if module.acceptance == nil { + return fmt.Errorf("beads acceptance use cases are not configured") + } + results, nonPass, err := module.acceptance.VerifyAcceptance(ids) + if err != nil { + return err + } + if options.JSON { + if err := encodeJSON(command, results); err != nil { + return err + } + } else { + for _, result := range results { + fmt.Fprintf(command.OutOrStdout(), "%s [%s] %s\n", result.Verdict, result.IssueType, result.BeadID) + for _, missing := range result.Missing { + fmt.Fprintf(command.OutOrStdout(), " missing: %s\n", missing) + } + } + } + if options.Strict && nonPass { + return &beadsapp.ExitError{Code: 1} + } + return nil +} diff --git a/cli/internal/commands/beads/module_test.go b/cli/internal/commands/beads/module_test.go index 5d7cff469..e09436dba 100644 --- a/cli/internal/commands/beads/module_test.go +++ b/cli/internal/commands/beads/module_test.go @@ -5,25 +5,16 @@ import ( "context" "errors" "reflect" + "strings" "testing" "time" beadsapp "github.com/boshu2/agentops/cli/internal/beads" - "github.com/spf13/cobra" + "github.com/boshu2/agentops/cli/internal/scenarios" ) -type recordingRunner struct { - invocations []Invocation -} - -func (runner *recordingRunner) Run(_ *cobra.Command, invocation Invocation) error { - runner.invocations = append(runner.invocations, invocation) - return nil -} - func TestModuleBuildsFreshCompleteTrees(t *testing.T) { - runner := &recordingRunner{} - module := NewModule(runner, nil, nil, nil, nil, nil, nil, nil, nil, nil) + module := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) first := module.Command() second := module.Command() if first == second { @@ -42,25 +33,6 @@ func TestModuleBuildsFreshCompleteTrees(t *testing.T) { } } -func TestModuleParsesTypedInvocation(t *testing.T) { - runner := &recordingRunner{} - command := NewModule(runner, nil, nil, nil, nil, nil, nil, nil, nil, nil).Command() - command.SetArgs([]string{"audit", "--strict", "--json"}) - if err := command.Execute(); err != nil { - t.Fatalf("Execute() error = %v", err) - } - want := Invocation{ - Operation: OperationAudit, - Options: Options{ - Strict: true, - JSON: true, - }, - } - if len(runner.invocations) != 1 || !reflect.DeepEqual(runner.invocations[0], want) { - t.Fatalf("invocations = %#v, want %#v", runner.invocations, want) - } -} - type fakeTrackerPorts struct { resolution beadsapp.TrackerResolution ledger beadsapp.LedgerResolution @@ -90,6 +62,33 @@ type fakeHygiene struct { cluster *beadsapp.ClusterReport } +type fakeScenario struct { + extraction beadsapp.ScenarioExtraction + validation beadsapp.ScenarioValidation + applied int +} + +func (fake *fakeScenario) Available() bool { return true } +func (fake *fakeScenario) PrepareScenarios(string, bool) (beadsapp.ScenarioExtraction, error) { + return fake.extraction, nil +} +func (fake *fakeScenario) ApplyScenarios(beadsapp.ScenarioExtraction) error { + fake.applied++ + return nil +} +func (fake *fakeScenario) ValidateScenarios(string) (beadsapp.ScenarioValidation, error) { + return fake.validation, nil +} + +type fakeAcceptance struct { + results []beadsapp.AcceptanceResult + nonPass bool +} + +func (fake fakeAcceptance) VerifyAcceptance([]string) ([]beadsapp.AcceptanceResult, bool, error) { + return fake.results, fake.nonPass, nil +} + func (fake fakeHygiene) Audit(bool) (*beadsapp.AuditReport, error) { return fake.audit, nil } func (fake fakeHygiene) Cluster(bool) (*beadsapp.ClusterReport, error) { return fake.cluster, nil } @@ -159,7 +158,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("directory JSON", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, ports, nil, nil, nil, nil, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"dir", "--require", "--json"}) if err := root.Execute(); err != nil { @@ -172,7 +171,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("tracker text", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, ports, nil, nil, nil, nil, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"tracker"}) if err := root.Execute(); err != nil { @@ -185,7 +184,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("exec help precedes resolution", func(t *testing.T) { ports.executed = false - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, ports, nil, nil, nil, nil, nil, nil, nil, nil).Command() root.SetArgs([]string{"exec", "--help"}) if err := root.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -198,7 +197,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("exec preserves typed exit", func(t *testing.T) { ports.executed = false ports.execErr = &beadsapp.ExitError{Code: 7} - root := NewModule(nil, ports, ports, ports, nil, nil, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, ports, nil, nil, nil, nil, nil, nil, nil, nil).Command() root.SetArgs([]string{"exec", "close", "age-x"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -219,7 +218,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { t.Run("stale claims", func(t *testing.T) { ports.listOutput = []byte(`[{"id":"age-old","status":"in_progress","assignee":"bo","updated_at":"2026-07-11T10:00:00Z"}]`) var output bytes.Buffer - root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil, nil).Command() + root := NewModule(ports, ports, ports, ports, ports, ports, ports, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"stale-claims", "--threshold", "4", "--json"}) if err := root.Execute(); err != nil { @@ -236,7 +235,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { {ID: "age-x", Status: "in_progress", Assignee: "old", UpdatedAt: "2026-07-11T10:00:00Z"}, {ID: "age-x", Status: "in_progress", Assignee: "codex", UpdatedAt: "2026-07-11T18:00:00Z"}, } - root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil, nil).Command() + root := NewModule(ports, ports, ports, ports, ports, ports, ports, nil, nil, nil, nil).Command() root.SetArgs([]string{"resume", "age-x", "--json"}) if err := root.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -249,7 +248,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { t.Run("epic terminal exit", func(t *testing.T) { ports.readOutput = []byte("{\"id\":\"age-e\",\"status\":\"open\",\"issue_type\":\"epic\"}\n{\"id\":\"age-e.1\",\"status\":\"open\",\"issue_type\":\"task\"}\n") - root := NewModule(nil, ports, ports, ports, ports, ports, ports, ports, nil, nil).Command() + root := NewModule(ports, ports, ports, ports, ports, ports, ports, nil, nil, nil, nil).Command() root.SetArgs([]string{"epic-status", "age-e", "--terminal"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -272,7 +271,7 @@ func TestModuleOwnsKnowledgeHandlers(t *testing.T) { t.Run("verify maps stale verdict", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge, nil).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, nil, knowledge, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"verify", "age-x"}) err := root.Execute() @@ -286,7 +285,7 @@ func TestModuleOwnsKnowledgeHandlers(t *testing.T) { }) t.Run("lint maps stale verdict", func(t *testing.T) { - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge, nil).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, nil, knowledge, nil, nil, nil).Command() root.SetArgs([]string{"lint"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -297,7 +296,7 @@ func TestModuleOwnsKnowledgeHandlers(t *testing.T) { t.Run("harvest renders result", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, knowledge, nil).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, nil, knowledge, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"harvest", "age-x"}) if err := root.Execute(); err != nil { @@ -328,7 +327,7 @@ func TestModuleOwnsHygieneHandlers(t *testing.T) { t.Run("strict audit maps findings", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, hygiene).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, hygiene, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"audit", "--strict"}) err := root.Execute() @@ -343,7 +342,7 @@ func TestModuleOwnsHygieneHandlers(t *testing.T) { t.Run("cluster renders representative", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, hygiene).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, hygiene, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"cluster"}) if err := root.Execute(); err != nil { @@ -354,3 +353,55 @@ func TestModuleOwnsHygieneHandlers(t *testing.T) { } }) } + +func TestModuleOwnsScenarioAndAcceptanceHandlers(t *testing.T) { + scenarioUseCases := &fakeScenario{ + extraction: beadsapp.ScenarioExtraction{ + BeadID: "age-x", + Scenarios: []scenarios.Scenario{{Name: "works", Given: "state", When: "action", Then: "result"}}, + }, + validation: beadsapp.ScenarioValidation{BeadID: "age-x", Valid: true, Scenarios: []scenarios.Scenario{{Name: "works"}}}, + } + + t.Run("write prompts before update and decline is safe", func(t *testing.T) { + scenarioUseCases.applied = 0 + var prompt bytes.Buffer + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, scenarioUseCases, nil).Command() + root.SetErr(&prompt) + root.SetIn(strings.NewReader("n\n")) + root.SetArgs([]string{"scenarios", "extract", "age-x", "--write"}) + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if scenarioUseCases.applied != 0 || !bytes.Contains(prompt.Bytes(), []byte("Proceed? [y/N]")) { + t.Fatalf("applied=%d prompt=%q", scenarioUseCases.applied, prompt.String()) + } + }) + + t.Run("write confirms update", func(t *testing.T) { + scenarioUseCases.applied = 0 + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, scenarioUseCases, nil).Command() + root.SetIn(strings.NewReader("yes\n")) + root.SetArgs([]string{"scenarios", "extract", "age-x", "--write"}) + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if scenarioUseCases.applied != 1 { + t.Fatalf("applied = %d, want 1", scenarioUseCases.applied) + } + }) + + t.Run("strict acceptance maps non-pass", func(t *testing.T) { + acceptance := fakeAcceptance{ + results: []beadsapp.AcceptanceResult{{BeadID: "age-x", IssueType: "feature", Verdict: beadsapp.AcceptanceFail, Missing: []string{"TDD signal"}}}, + nonPass: true, + } + root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, acceptance).Command() + root.SetArgs([]string{"verify-acceptance", "age-x", "--strict"}) + err := root.Execute() + var exitError *beadsapp.ExitError + if !errors.As(err, &exitError) || exitError.ExitCode() != 1 { + t.Fatalf("error = %v, want ExitError(1)", err) + } + }) +} From 6e039f21ee67b6233d779dd9ab3a384bb43c4bfc Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:58:19 -0400 Subject: [PATCH 023/108] refactor(beads): move recovery orchestration behind use cases (age-nw28h.2.1) --- cli/cmd/ao/beads_command_legacy_test.go | 22 ++- cli/cmd/ao/beads_composition.go | 13 ++ cli/cmd/ao/beads_json_compat.go | 12 ++ cli/cmd/ao/beads_module.go | 6 +- cli/cmd/ao/beads_support.go | 43 ------ cli/internal/beads/recovery_service.go | 158 +++++++++++++++++++++ cli/internal/commands/beads/module.go | 119 +++------------- cli/internal/commands/beads/module_test.go | 66 ++++++--- 8 files changed, 281 insertions(+), 158 deletions(-) create mode 100644 cli/cmd/ao/beads_composition.go create mode 100644 cli/cmd/ao/beads_json_compat.go delete mode 100644 cli/cmd/ao/beads_support.go create mode 100644 cli/internal/beads/recovery_service.go diff --git a/cli/cmd/ao/beads_command_legacy_test.go b/cli/cmd/ao/beads_command_legacy_test.go index cdaff543f..c174623c8 100644 --- a/cli/cmd/ao/beads_command_legacy_test.go +++ b/cli/cmd/ao/beads_command_legacy_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -11,6 +12,23 @@ import ( "github.com/spf13/cobra" ) +type beadsVerdictError struct{ code int } + +func (err *beadsVerdictError) Error() string { return "" } +func (err *beadsVerdictError) ExitCode() int { return err.code } + +var beadsTrackerOutput = func(args ...string) ([]byte, error) { + return currentBeadsTracker().Output(context.Background(), args...) +} +var beadsTrackerAvailable = func() bool { return currentBeadsTracker().Available() } + +func beadMinInt(left, right int) int { + if left < right { + return left + } + return right +} + type acceptBead = beadsapp.AcceptanceBead type acceptanceVerdict = beadsapp.AcceptanceVerdict @@ -147,7 +165,9 @@ func executeBeadsExec(command *cobra.Command, args []string) error { func newTestBeadsCommand(name string) *cobra.Command { tracker := currentBeadsTracker() runtime := beadsadapter.NewRuntime() - root := beadscommands.NewModule(tracker, tracker, beadsadapter.NewExecutor(tracker), tracker, tracker, runtime, runtime, nil, nil, nil, nil).Command() + directory := beadsapp.DirectoryService{Resolver: tracker, Inspector: tracker} + recovery := beadsapp.RecoveryService{StaleSource: tracker, Claims: tracker, Runtime: runtime, Resolver: tracker, Reader: runtime} + root := beadscommands.NewModule(tracker, beadsadapter.NewExecutor(tracker), directory, recovery, nil, nil, nil, nil).Command() child, _, err := root.Find([]string{name}) if err != nil || child == nil { panic("missing test beads command " + name) diff --git a/cli/cmd/ao/beads_composition.go b/cli/cmd/ao/beads_composition.go new file mode 100644 index 000000000..17b75a707 --- /dev/null +++ b/cli/cmd/ao/beads_composition.go @@ -0,0 +1,13 @@ +package main + +import ( + "os" + + beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" +) + +func currentBeadsTracker() *beadsadapter.Tracker { + return beadsadapter.NewTrackerWith(os.Getwd, os.Environ, func(name string) (string, error) { + return trackerLookPath(name) + }) +} diff --git a/cli/cmd/ao/beads_json_compat.go b/cli/cmd/ao/beads_json_compat.go new file mode 100644 index 000000000..d5ea39762 --- /dev/null +++ b/cli/cmd/ao/beads_json_compat.go @@ -0,0 +1,12 @@ +package main + +import ( + "encoding/json" + "os" +) + +func emitJSON(file *os.File, value any) error { + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} diff --git a/cli/cmd/ao/beads_module.go b/cli/cmd/ao/beads_module.go index 43f19acd5..2fc67dc36 100644 --- a/cli/cmd/ao/beads_module.go +++ b/cli/cmd/ao/beads_module.go @@ -15,9 +15,11 @@ func init() { hygiene := beadsapp.HygieneService{Repository: beadsadapter.NewHygieneRepository(tracker)} scenario := beadsapp.ScenarioService{Repository: beadsadapter.NewScenarioRepository(tracker)} acceptance := beadsapp.AcceptanceService{Repository: beadsadapter.NewAcceptanceRepository(tracker)} + directory := beadsapp.DirectoryService{Resolver: tracker, Inspector: tracker} + recovery := beadsapp.RecoveryService{StaleSource: tracker, Claims: tracker, Runtime: runtime, Resolver: tracker, Reader: runtime} module := beadscommands.NewModule( - tracker, tracker, beadsadapter.NewExecutor(tracker), tracker, tracker, - runtime, runtime, knowledge, hygiene, scenario, acceptance, + tracker, beadsadapter.NewExecutor(tracker), directory, recovery, + knowledge, hygiene, scenario, acceptance, ) command := module.Command() command.GroupID = "knowledge" diff --git a/cli/cmd/ao/beads_support.go b/cli/cmd/ao/beads_support.go deleted file mode 100644 index 44c66e654..000000000 --- a/cli/cmd/ao/beads_support.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "os" - - beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" -) - -// beadsVerdictError remains only for command families and white-box tests -// that have not yet adopted the shared command exit type. -type beadsVerdictError struct { - code int -} - -func (err *beadsVerdictError) Error() string { return "" } -func (err *beadsVerdictError) ExitCode() int { return err.code } - -var beadsTrackerOutput = func(args ...string) ([]byte, error) { - return currentBeadsTracker().Output(context.Background(), args...) -} - -var beadsTrackerAvailable = func() bool { return currentBeadsTracker().Available() } - -func currentBeadsTracker() *beadsadapter.Tracker { - return beadsadapter.NewTrackerWith(os.Getwd, os.Environ, func(name string) (string, error) { - return trackerLookPath(name) - }) -} - -func beadMinInt(left, right int) int { - if left < right { - return left - } - return right -} - -func emitJSON(file *os.File, value any) error { - encoder := json.NewEncoder(file) - encoder.SetIndent("", " ") - return encoder.Encode(value) -} diff --git a/cli/internal/beads/recovery_service.go b/cli/internal/beads/recovery_service.go new file mode 100644 index 000000000..b6e5dfe10 --- /dev/null +++ b/cli/internal/beads/recovery_service.go @@ -0,0 +1,158 @@ +package beads + +import ( + "context" + "fmt" + "path/filepath" + "time" + + "github.com/boshu2/agentops/cli/internal/epicstatus" +) + +type DirectoryResult struct { + Path string + Source string + Tracker string +} + +type DirectoryUseCases interface { + ResolveDirectory(bool) (DirectoryResult, error) +} + +type DirectoryService struct { + Resolver TrackerResolver + Inspector LedgerInspector +} + +func (service DirectoryService) ResolveDirectory(require bool) (DirectoryResult, error) { + if service.Resolver == nil || service.Inspector == nil { + return DirectoryResult{}, fmt.Errorf("beads directory ports are not configured") + } + if !service.Resolver.BeadsDirOverride() { + if resolved, err := service.Resolver.Resolve(); err == nil && resolved.Tracker == TrackerBD { + if require { + if reason := LedgerMissing(TrackerBD, service.Inspector.InspectLedger(resolved.LedgerDir)); reason != "" { + return DirectoryResult{}, fmt.Errorf("beads dir --require: %s (resolved %s for tracker bd via %s); refusing to print a path a bd write could silently fall back from", reason, resolved.LedgerDir, resolved.Source) + } + } + return DirectoryResult{Path: resolved.LedgerDir, Source: resolved.Source, Tracker: TrackerBD}, nil + } + } + resolved, err := service.Resolver.BRLedger() + if err != nil { + return DirectoryResult{}, err + } + if require { + if reason := LedgerMissing(TrackerBR, service.Inspector.InspectLedger(resolved.Path)); reason != "" { + return DirectoryResult{}, fmt.Errorf("beads dir --require: %s (resolved %s via %s); refusing to print a path a br write could silently fall back from", reason, resolved.Path, resolved.Source) + } + } + return DirectoryResult{Path: resolved.Path, Source: resolved.Source, Tracker: TrackerBR}, nil +} + +type ResumeOptions struct { + Agent string + Ledger string +} + +type ResumeResult struct { + Event TransferredEvent + PriorAgent string +} + +type RecoveryUseCases interface { + StaleClaims(context.Context, float64) ([]StaleEvent, error) + Resume(context.Context, string, ResumeOptions) (ResumeResult, error) + EpicStatus(string) (epicstatus.Result, error) +} + +type RecoveryService struct { + StaleSource StaleSource + Claims ClaimStore + Runtime ResumeRuntime + Resolver TrackerResolver + Reader LedgerReader +} + +func (service RecoveryService) StaleClaims(ctx context.Context, thresholdHours float64) ([]StaleEvent, error) { + if service.StaleSource == nil || service.Runtime == nil { + return nil, fmt.Errorf("beads stale-claims ports are not configured") + } + bounded, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + events, err := DetectStale(bounded, service.StaleSource, service.Runtime.Now(), thresholdHours) + if err != nil { + return nil, fmt.Errorf("br list: %w", err) + } + return events, nil +} + +func (service RecoveryService) Resume(ctx context.Context, beadID string, options ResumeOptions) (ResumeResult, error) { + if service.Claims == nil || service.Runtime == nil { + return ResumeResult{}, fmt.Errorf("beads resume ports are not configured") + } + bounded, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + prior, err := service.Claims.Show(bounded, beadID) + if err != nil { + return ResumeResult{}, fmt.Errorf("fetch prior state: %w", err) + } + if prior.Status != "in_progress" { + return ResumeResult{}, fmt.Errorf("bead %s is %q, not in_progress — resume only handles in_progress claims", beadID, prior.Status) + } + now := service.Runtime.Now().UTC() + agent := options.Agent + if agent == "" { + agent = service.Runtime.Actor() + } + if agent == "" { + agent = "ao-beads-resume" + } + if err := service.Claims.Claim(bounded, beadID, agent); err != nil { + return ResumeResult{}, fmt.Errorf("claim transfer: %w", err) + } + posterior, err := service.Claims.Show(bounded, beadID) + if err != nil { + posterior = StaleBeadRecord{ID: beadID, Status: "in_progress", Assignee: agent, UpdatedAt: now.Format(time.RFC3339)} + } + event := BuildTransferredEvent(beadID, agent, prior, posterior, now) + ledger, err := service.Runtime.ResolveRepoPath(options.Ledger) + if err != nil { + return ResumeResult{}, err + } + if err := service.Runtime.AppendEvent(ledger, event); err != nil { + return ResumeResult{}, fmt.Errorf("append ledger (claim already transferred): %w", err) + } + priorAgent := prior.Assignee + if priorAgent == "" { + priorAgent = "unknown" + } + return ResumeResult{Event: event, PriorAgent: priorAgent}, nil +} + +func (service RecoveryService) EpicStatus(epic string) (epicstatus.Result, error) { + if service.Resolver == nil || service.Reader == nil { + return epicstatus.Result{}, fmt.Errorf("beads epic-status ports are not configured") + } + ledger, err := service.Resolver.BRLedger() + if err != nil { + return epicstatus.Result{}, err + } + path := filepath.Join(ledger.Path, "issues.jsonl") + raw, err := service.Reader.ReadFile(path) + if err != nil { + return epicstatus.Result{}, fmt.Errorf("read ledger %s: %w", path, err) + } + records, err := ParseLedger(raw) + if err != nil { + return epicstatus.Result{}, fmt.Errorf("parse ledger: %w", err) + } + members, present := BuildMembers(epic, records) + if !present { + return epicstatus.Result{}, fmt.Errorf("epic %s not found in ledger %s", epic, ledger.Path) + } + return epicstatus.Evaluate(epic, members), nil +} + +var _ DirectoryUseCases = DirectoryService{} +var _ RecoveryUseCases = RecoveryService{} diff --git a/cli/internal/commands/beads/module.go b/cli/internal/commands/beads/module.go index 298811d7c..9cfcf8bfa 100644 --- a/cli/internal/commands/beads/module.go +++ b/cli/internal/commands/beads/module.go @@ -1,6 +1,6 @@ // Package beads owns the Cobra presentation for the beads command family. -// Behavior is delegated through Runner so tracker, filesystem, process, clock, -// and environment effects remain behind driven adapters. +// Handlers parse and render while typed application use cases own orchestration; +// tracker, filesystem, process, clock, and environment effects stay in adapters. package beads import ( @@ -9,10 +9,8 @@ import ( "encoding/json" "errors" "fmt" - "path/filepath" "sort" "strings" - "time" "github.com/spf13/cobra" @@ -42,12 +40,9 @@ type Options struct { type Module struct { resolver beadsapp.TrackerResolver - inspector beadsapp.LedgerInspector executor beadsapp.TrackerExecutor - stale beadsapp.StaleSource - claims beadsapp.ClaimStore - runtime beadsapp.ResumeRuntime - reader beadsapp.LedgerReader + directory beadsapp.DirectoryUseCases + recovery beadsapp.RecoveryUseCases knowledge beadsapp.KnowledgeUseCases hygiene beadsapp.HygieneUseCases scenario beadsapp.ScenarioUseCases @@ -56,20 +51,17 @@ type Module struct { func NewModule( resolver beadsapp.TrackerResolver, - inspector beadsapp.LedgerInspector, executor beadsapp.TrackerExecutor, - stale beadsapp.StaleSource, - claims beadsapp.ClaimStore, - runtime beadsapp.ResumeRuntime, - reader beadsapp.LedgerReader, + directory beadsapp.DirectoryUseCases, + recovery beadsapp.RecoveryUseCases, knowledge beadsapp.KnowledgeUseCases, hygiene beadsapp.HygieneUseCases, scenario beadsapp.ScenarioUseCases, acceptance beadsapp.AcceptanceUseCases, ) Module { return Module{ - resolver: resolver, inspector: inspector, executor: executor, - stale: stale, claims: claims, runtime: runtime, reader: reader, knowledge: knowledge, hygiene: hygiene, + resolver: resolver, executor: executor, directory: directory, recovery: recovery, + knowledge: knowledge, hygiene: hygiene, scenario: scenario, acceptance: acceptance, } } @@ -334,28 +326,13 @@ func (module Module) execCommand() *cobra.Command { } func (module Module) runDir(command *cobra.Command, options Options) error { - if module.resolver == nil || module.inspector == nil { - return fmt.Errorf("beads directory ports are not configured") + if module.directory == nil { + return fmt.Errorf("beads directory use cases are not configured") } - if !module.resolver.BeadsDirOverride() { - if resolved, err := module.resolver.Resolve(); err == nil && resolved.Tracker == beadsapp.TrackerBD { - if options.Require { - if reason := beadsapp.LedgerMissing(beadsapp.TrackerBD, module.inspector.InspectLedger(resolved.LedgerDir)); reason != "" { - return fmt.Errorf("beads dir --require: %s (resolved %s for tracker bd via %s); refusing to print a path a bd write could silently fall back from", reason, resolved.LedgerDir, resolved.Source) - } - } - return writeDirectory(command, options.JSON, resolved.LedgerDir, resolved.Source) - } - } - resolved, err := module.resolver.BRLedger() + resolved, err := module.directory.ResolveDirectory(options.Require) if err != nil { return err } - if options.Require { - if reason := beadsapp.LedgerMissing(beadsapp.TrackerBR, module.inspector.InspectLedger(resolved.Path)); reason != "" { - return fmt.Errorf("beads dir --require: %s (resolved %s via %s); refusing to print a path a br write could silently fall back from", reason, resolved.Path, resolved.Source) - } - } return writeDirectory(command, options.JSON, resolved.Path, resolved.Source) } @@ -643,14 +620,12 @@ func (module Module) epicStatusCommand() *cobra.Command { } func (module Module) runStale(command *cobra.Command, options Options) error { - if module.stale == nil || module.runtime == nil { - return fmt.Errorf("beads stale-claims ports are not configured") + if module.recovery == nil { + return fmt.Errorf("beads recovery use cases are not configured") } - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - events, err := beadsapp.DetectStale(ctx, module.stale, module.runtime.Now(), options.ThresholdHours) + events, err := module.recovery.StaleClaims(context.Background(), options.ThresholdHours) if err != nil { - return fmt.Errorf("br list: %w", err) + return err } if options.JSON { encoded, err := json.Marshal(events) @@ -672,81 +647,33 @@ func (module Module) runStale(command *cobra.Command, options Options) error { } func (module Module) runResume(command *cobra.Command, args []string, options Options) error { - if module.claims == nil || module.runtime == nil { - return fmt.Errorf("beads resume ports are not configured") + if module.recovery == nil { + return fmt.Errorf("beads recovery use cases are not configured") } - beadID := args[0] - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - prior, err := module.claims.Show(ctx, beadID) - if err != nil { - return fmt.Errorf("fetch prior state: %w", err) - } - if prior.Status != "in_progress" { - return fmt.Errorf("bead %s is %q, not in_progress — resume only handles in_progress claims", beadID, prior.Status) - } - now := module.runtime.Now().UTC() - agent := options.Agent - if agent == "" { - agent = module.runtime.Actor() - } - if agent == "" { - agent = "ao-beads-resume" - } - if err := module.claims.Claim(ctx, beadID, agent); err != nil { - return fmt.Errorf("claim transfer: %w", err) - } - posterior, err := module.claims.Show(ctx, beadID) - if err != nil { - posterior = beadsapp.StaleBeadRecord{ID: beadID, Status: "in_progress", Assignee: agent, UpdatedAt: now.Format(time.RFC3339)} - } - event := beadsapp.BuildTransferredEvent(beadID, agent, prior, posterior, now) - ledger, err := module.runtime.ResolveRepoPath(options.Ledger) + result, err := module.recovery.Resume(context.Background(), args[0], beadsapp.ResumeOptions{Agent: options.Agent, Ledger: options.Ledger}) if err != nil { return err } - if err := module.runtime.AppendEvent(ledger, event); err != nil { - return fmt.Errorf("append ledger (claim already transferred): %w", err) - } if options.JSON { - encoded, err := json.Marshal(event) + encoded, err := json.Marshal(result.Event) if err != nil { return fmt.Errorf("marshal transferred claim: %w", err) } _, err = fmt.Fprintln(command.OutOrStdout(), string(encoded)) return err } - priorAgent := prior.Assignee - if priorAgent == "" { - priorAgent = "unknown" - } - _, err = fmt.Fprintf(command.OutOrStdout(), "ao beads resume: %s transferred from %q to %q (prior_rev=%s, new_rev=%s)\n", beadID, priorAgent, agent, event.Transfer.PriorRevision, event.Transfer.NewRevision) + _, err = fmt.Fprintf(command.OutOrStdout(), "ao beads resume: %s transferred from %q to %q (prior_rev=%s, new_rev=%s)\n", args[0], result.PriorAgent, result.Event.NewClaimant.ID, result.Event.Transfer.PriorRevision, result.Event.Transfer.NewRevision) return err } func (module Module) runEpicStatus(command *cobra.Command, args []string, options Options) error { - if module.resolver == nil || module.reader == nil { - return fmt.Errorf("beads epic-status ports are not configured") + if module.recovery == nil { + return fmt.Errorf("beads recovery use cases are not configured") } - epic := strings.TrimSpace(args[0]) - ledger, err := module.resolver.BRLedger() + result, err := module.recovery.EpicStatus(strings.TrimSpace(args[0])) if err != nil { return err } - path := filepath.Join(ledger.Path, "issues.jsonl") - raw, err := module.reader.ReadFile(path) - if err != nil { - return fmt.Errorf("read ledger %s: %w", path, err) - } - records, err := beadsapp.ParseLedger(raw) - if err != nil { - return fmt.Errorf("parse ledger: %w", err) - } - members, present := beadsapp.BuildMembers(epic, records) - if !present { - return fmt.Errorf("epic %s not found in ledger %s", epic, ledger.Path) - } - result := epicstatus.Evaluate(epic, members) if options.JSON { encoder := json.NewEncoder(command.OutOrStdout()) encoder.SetEscapeHTML(false) diff --git a/cli/internal/commands/beads/module_test.go b/cli/internal/commands/beads/module_test.go index e09436dba..c2a166709 100644 --- a/cli/internal/commands/beads/module_test.go +++ b/cli/internal/commands/beads/module_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "errors" + "os" + "path/filepath" "reflect" "strings" "testing" @@ -14,7 +16,7 @@ import ( ) func TestModuleBuildsFreshCompleteTrees(t *testing.T) { - module := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + module := NewModule(nil, nil, nil, nil, nil, nil, nil, nil) first := module.Command() second := module.Command() if first == second { @@ -158,7 +160,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("directory JSON", func(t *testing.T) { var output bytes.Buffer - root := NewModule(ports, ports, ports, nil, nil, nil, nil, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, beadsapp.DirectoryService{Resolver: ports, Inspector: ports}, nil, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"dir", "--require", "--json"}) if err := root.Execute(); err != nil { @@ -171,7 +173,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("tracker text", func(t *testing.T) { var output bytes.Buffer - root := NewModule(ports, ports, ports, nil, nil, nil, nil, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, beadsapp.DirectoryService{Resolver: ports, Inspector: ports}, nil, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"tracker"}) if err := root.Execute(); err != nil { @@ -184,7 +186,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("exec help precedes resolution", func(t *testing.T) { ports.executed = false - root := NewModule(ports, ports, ports, nil, nil, nil, nil, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, beadsapp.DirectoryService{Resolver: ports, Inspector: ports}, nil, nil, nil, nil, nil).Command() root.SetArgs([]string{"exec", "--help"}) if err := root.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -197,7 +199,7 @@ func TestModuleOwnsDirectoryTrackerAndExecHandlers(t *testing.T) { t.Run("exec preserves typed exit", func(t *testing.T) { ports.executed = false ports.execErr = &beadsapp.ExitError{Code: 7} - root := NewModule(ports, ports, ports, nil, nil, nil, nil, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, beadsapp.DirectoryService{Resolver: ports, Inspector: ports}, nil, nil, nil, nil, nil).Command() root.SetArgs([]string{"exec", "close", "age-x"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -218,7 +220,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { t.Run("stale claims", func(t *testing.T) { ports.listOutput = []byte(`[{"id":"age-old","status":"in_progress","assignee":"bo","updated_at":"2026-07-11T10:00:00Z"}]`) var output bytes.Buffer - root := NewModule(ports, ports, ports, ports, ports, ports, ports, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, beadsapp.DirectoryService{Resolver: ports, Inspector: ports}, beadsapp.RecoveryService{StaleSource: ports, Claims: ports, Runtime: ports, Resolver: ports, Reader: ports}, nil, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"stale-claims", "--threshold", "4", "--json"}) if err := root.Execute(); err != nil { @@ -235,7 +237,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { {ID: "age-x", Status: "in_progress", Assignee: "old", UpdatedAt: "2026-07-11T10:00:00Z"}, {ID: "age-x", Status: "in_progress", Assignee: "codex", UpdatedAt: "2026-07-11T18:00:00Z"}, } - root := NewModule(ports, ports, ports, ports, ports, ports, ports, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, beadsapp.DirectoryService{Resolver: ports, Inspector: ports}, beadsapp.RecoveryService{StaleSource: ports, Claims: ports, Runtime: ports, Resolver: ports, Reader: ports}, nil, nil, nil, nil).Command() root.SetArgs([]string{"resume", "age-x", "--json"}) if err := root.Execute(); err != nil { t.Fatalf("Execute() error = %v", err) @@ -248,7 +250,7 @@ func TestModuleOwnsRecoveryHandlers(t *testing.T) { t.Run("epic terminal exit", func(t *testing.T) { ports.readOutput = []byte("{\"id\":\"age-e\",\"status\":\"open\",\"issue_type\":\"epic\"}\n{\"id\":\"age-e.1\",\"status\":\"open\",\"issue_type\":\"task\"}\n") - root := NewModule(ports, ports, ports, ports, ports, ports, ports, nil, nil, nil, nil).Command() + root := NewModule(ports, ports, beadsapp.DirectoryService{Resolver: ports, Inspector: ports}, beadsapp.RecoveryService{StaleSource: ports, Claims: ports, Runtime: ports, Resolver: ports, Reader: ports}, nil, nil, nil, nil).Command() root.SetArgs([]string{"epic-status", "age-e", "--terminal"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -271,7 +273,7 @@ func TestModuleOwnsKnowledgeHandlers(t *testing.T) { t.Run("verify maps stale verdict", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, knowledge, nil, nil, nil).Command() + root := NewModule(nil, nil, nil, nil, knowledge, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"verify", "age-x"}) err := root.Execute() @@ -285,7 +287,7 @@ func TestModuleOwnsKnowledgeHandlers(t *testing.T) { }) t.Run("lint maps stale verdict", func(t *testing.T) { - root := NewModule(nil, nil, nil, nil, nil, nil, nil, knowledge, nil, nil, nil).Command() + root := NewModule(nil, nil, nil, nil, knowledge, nil, nil, nil).Command() root.SetArgs([]string{"lint"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -296,7 +298,7 @@ func TestModuleOwnsKnowledgeHandlers(t *testing.T) { t.Run("harvest renders result", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, knowledge, nil, nil, nil).Command() + root := NewModule(nil, nil, nil, nil, knowledge, nil, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"harvest", "age-x"}) if err := root.Execute(); err != nil { @@ -327,7 +329,7 @@ func TestModuleOwnsHygieneHandlers(t *testing.T) { t.Run("strict audit maps findings", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, hygiene, nil, nil).Command() + root := NewModule(nil, nil, nil, nil, nil, hygiene, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"audit", "--strict"}) err := root.Execute() @@ -342,7 +344,7 @@ func TestModuleOwnsHygieneHandlers(t *testing.T) { t.Run("cluster renders representative", func(t *testing.T) { var output bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, hygiene, nil, nil).Command() + root := NewModule(nil, nil, nil, nil, nil, hygiene, nil, nil).Command() root.SetOut(&output) root.SetArgs([]string{"cluster"}) if err := root.Execute(); err != nil { @@ -366,7 +368,7 @@ func TestModuleOwnsScenarioAndAcceptanceHandlers(t *testing.T) { t.Run("write prompts before update and decline is safe", func(t *testing.T) { scenarioUseCases.applied = 0 var prompt bytes.Buffer - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, scenarioUseCases, nil).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, scenarioUseCases, nil).Command() root.SetErr(&prompt) root.SetIn(strings.NewReader("n\n")) root.SetArgs([]string{"scenarios", "extract", "age-x", "--write"}) @@ -380,7 +382,7 @@ func TestModuleOwnsScenarioAndAcceptanceHandlers(t *testing.T) { t.Run("write confirms update", func(t *testing.T) { scenarioUseCases.applied = 0 - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, scenarioUseCases, nil).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, scenarioUseCases, nil).Command() root.SetIn(strings.NewReader("yes\n")) root.SetArgs([]string{"scenarios", "extract", "age-x", "--write"}) if err := root.Execute(); err != nil { @@ -396,7 +398,7 @@ func TestModuleOwnsScenarioAndAcceptanceHandlers(t *testing.T) { results: []beadsapp.AcceptanceResult{{BeadID: "age-x", IssueType: "feature", Verdict: beadsapp.AcceptanceFail, Missing: []string{"TDD signal"}}}, nonPass: true, } - root := NewModule(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, acceptance).Command() + root := NewModule(nil, nil, nil, nil, nil, nil, nil, acceptance).Command() root.SetArgs([]string{"verify-acceptance", "age-x", "--strict"}) err := root.Execute() var exitError *beadsapp.ExitError @@ -405,3 +407,35 @@ func TestModuleOwnsScenarioAndAcceptanceHandlers(t *testing.T) { } }) } + +func TestLegacyProductionOwnersAreAbsent(t *testing.T) { + commandDirectory := filepath.Join("..", "..", "..", "cmd", "ao") + entries, err := os.ReadDir(commandDirectory) + if err != nil { + t.Fatal(err) + } + allowed := map[string]bool{ + "beads_composition.go": true, + "beads_exec.go": true, // yield-family compatibility delegates + "beads_json_compat.go": true, // lookup compatibility helper + "beads_module.go": true, + } + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, "beads") || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + if !allowed[name] { + t.Errorf("unexpected production beads owner remains: %s", name) + } + content, readErr := os.ReadFile(filepath.Join(commandDirectory, name)) + if readErr != nil { + t.Fatal(readErr) + } + for _, forbidden := range []string{"executeBeads", "beadsModuleRunner", "beadsTrackerOutput", "beadsTrackerAvailable", "beadsResumeAgentID", "beadsAuditJSON"} { + if strings.Contains(string(content), forbidden) { + t.Errorf("%s retains forbidden legacy declaration %q", name, forbidden) + } + } + } +} From 9f275dc8ead3f5b2965a7d77c8f83eb2acf2f375 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:03:43 -0400 Subject: [PATCH 024/108] test(beads): bind high-risk behavior to new owners (age-nw28h.2.1) --- cli/cmd/ao/beads_citation_compat.go | 35 ++++++++++ cli/cmd/ao/beads_knowledge_legacy_test.go | 16 ++--- cli/cmd/ao/citation_port_adapter.go | 19 +++--- .../adapters/beads/acceptance_test.go | 18 +++++ cli/internal/beads/acceptance_service_test.go | 22 +++++++ cli/internal/beads/hygiene.go | 9 ++- cli/internal/beads/hygiene_test.go | 65 +++++++++++++++++++ cli/internal/beads/recovery_service_test.go | 60 +++++++++++++++++ cli/internal/beads/scenario_service_test.go | 37 +++++++++++ cli/internal/commands/beads/module.go | 3 + 10 files changed, 265 insertions(+), 19 deletions(-) create mode 100644 cli/cmd/ao/beads_citation_compat.go create mode 100644 cli/internal/adapters/beads/acceptance_test.go create mode 100644 cli/internal/beads/acceptance_service_test.go create mode 100644 cli/internal/beads/hygiene_test.go create mode 100644 cli/internal/beads/recovery_service_test.go create mode 100644 cli/internal/beads/scenario_service_test.go diff --git a/cli/cmd/ao/beads_citation_compat.go b/cli/cmd/ao/beads_citation_compat.go new file mode 100644 index 000000000..79c52b9e0 --- /dev/null +++ b/cli/cmd/ao/beads_citation_compat.go @@ -0,0 +1,35 @@ +package main + +import ( + beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" + beadsapp "github.com/boshu2/agentops/cli/internal/beads" +) + +// Citation aliases keep the older BC1 CitationPort adapter source-stable while +// its effects are owned by the beads repository adapter. +type Citation = beadsapp.Citation +type CitationStatus = beadsapp.CitationStatus + +const ( + CitationFresh = beadsapp.CitationFresh + CitationStale = beadsapp.CitationStale + CitationUnknown = beadsapp.CitationUnknown +) + +func verifyFileCitation(citation *Citation, cwd string) { + verifyCitationWithRepository(citation, cwd) +} + +func verifyFunctionCitation(citation *Citation, cwd string) { + verifyCitationWithRepository(citation, cwd) +} + +func verifySymbolCitation(citation *Citation, cwd string) { + verifyCitationWithRepository(citation, cwd) +} + +func verifyCitationWithRepository(citation *Citation, cwd string) { + repository := beadsadapter.NewKnowledgeRepositoryWithWorkingDirectory(func() (string, error) { return cwd, nil }) + verified := repository.VerifyCitations([]beadsapp.Citation{*citation}) + *citation = verified[0] +} diff --git a/cli/cmd/ao/beads_knowledge_legacy_test.go b/cli/cmd/ao/beads_knowledge_legacy_test.go index 4953e9cdb..a787c9cf5 100644 --- a/cli/cmd/ao/beads_knowledge_legacy_test.go +++ b/cli/cmd/ao/beads_knowledge_legacy_test.go @@ -53,16 +53,16 @@ var ( // CitationStatus is the three-valued verdict for a single citation extracted // from a bead description. -type CitationStatus = beadsapp.CitationStatus +type LegacyCitationStatus = beadsapp.CitationStatus const ( - CitationFresh = beadsapp.CitationFresh - CitationStale = beadsapp.CitationStale - CitationUnknown = beadsapp.CitationUnknown + LegacyCitationFresh = beadsapp.CitationFresh + LegacyCitationStale = beadsapp.CitationStale + LegacyCitationUnknown = beadsapp.CitationUnknown ) // Citation is a single verifiable reference pulled from a bead description. -type Citation = beadsapp.Citation +type LegacyCitation = beadsapp.Citation // VerifyReport is the structured result of `ao beads verify`. type VerifyReport = beadsapp.VerifyReport @@ -187,7 +187,7 @@ func verifyCitationInPlace(c *Citation, cwd string) { } } -func verifyFileCitation(c *Citation, cwd string) { +func legacyVerifyFileCitation(c *Citation, cwd string) { // Strip optional :line suffix for the stat check. path := c.Raw if idx := strings.LastIndex(path, ":"); idx >= 0 { @@ -276,7 +276,7 @@ func findFilesByBasename(cwd, name string) []string { return matches } -func verifyFunctionCitation(c *Citation, cwd string) { +func legacyVerifyFunctionCitation(c *Citation, cwd string) { // c.Raw is "func Name". Grep for it across cli/ and skills/. name := strings.TrimPrefix(c.Raw, "func ") matches := grepSymbol(cwd, name) @@ -292,7 +292,7 @@ func verifyFunctionCitation(c *Citation, cwd string) { } } -func verifySymbolCitation(c *Citation, cwd string) { +func legacyVerifySymbolCitation(c *Citation, cwd string) { sym := strings.Trim(c.Raw, "`") matches := grepSymbol(cwd, sym) if len(matches) == 0 { diff --git a/cli/cmd/ao/citation_port_adapter.go b/cli/cmd/ao/citation_port_adapter.go index 856484b2b..c1c081651 100644 --- a/cli/cmd/ao/citation_port_adapter.go +++ b/cli/cmd/ao/citation_port_adapter.go @@ -6,8 +6,6 @@ import ( "fmt" "strings" - beadsadapter "github.com/boshu2/agentops/cli/internal/adapters/beads" - beadsapp "github.com/boshu2/agentops/cli/internal/beads" "github.com/boshu2/agentops/cli/internal/ports" ) @@ -45,17 +43,20 @@ func (a *productionCitationAdapter) Verify(ctx context.Context, req ports.Citati Reason: "empty Raw", }, nil } + c := Citation{Kind: string(req.Kind), Raw: req.Raw} switch req.Kind { - case ports.CitationKindFile, ports.CitationKindFunction, ports.CitationKindSymbol: + case ports.CitationKindFile: + verifyFileCitation(&c, req.Cwd) + case ports.CitationKindFunction: + verifyFunctionCitation(&c, req.Cwd) + case ports.CitationKindSymbol: + verifySymbolCitation(&c, req.Cwd) default: return ports.CitationVerdict{ Status: ports.CitationStatusUnknown, Reason: fmt.Sprintf("unknown citation kind %q (expected file|function|symbol)", req.Kind), }, nil } - repository := beadsadapter.NewKnowledgeRepositoryWithWorkingDirectory(func() (string, error) { return req.Cwd, nil }) - verified := repository.VerifyCitations([]beadsapp.Citation{{Kind: string(req.Kind), Raw: req.Raw}}) - c := verified[0] return ports.CitationVerdict{ Status: translateCitationStatus(c.Status), Reason: c.Reason, @@ -66,11 +67,11 @@ func (a *productionCitationAdapter) Verify(ctx context.Context, req ports.Citati // translateCitationStatus maps the existing CitationStatus values to // the port's CitationStatusResult. The string values are identical, // but the types are distinct — this conversion is the boundary. -func translateCitationStatus(s beadsapp.CitationStatus) ports.CitationStatusResult { +func translateCitationStatus(s CitationStatus) ports.CitationStatusResult { switch s { - case beadsapp.CitationFresh: + case CitationFresh: return ports.CitationStatusFresh - case beadsapp.CitationStale: + case CitationStale: return ports.CitationStatusStale default: return ports.CitationStatusUnknown diff --git a/cli/internal/adapters/beads/acceptance_test.go b/cli/internal/adapters/beads/acceptance_test.go new file mode 100644 index 000000000..13c12ec9e --- /dev/null +++ b/cli/internal/adapters/beads/acceptance_test.go @@ -0,0 +1,18 @@ +package beads + +import ( + "strings" + "testing" +) + +func TestAcceptanceRepositoryRejectsBDSelection(t *testing.T) { + tracker := NewTrackerWith( + func() (string, error) { return t.TempDir(), nil }, + func() []string { return []string{"AGENTOPS_TRACKER=bd"} }, + func(name string) (string, error) { return "/fake/" + name, nil }, + ) + _, err := NewAcceptanceRepository(tracker).ShowAcceptance([]string{"age-x"}) + if err == nil || !strings.Contains(err.Error(), "requires the BR acceptance wire") { + t.Fatalf("error = %v", err) + } +} diff --git a/cli/internal/beads/acceptance_service_test.go b/cli/internal/beads/acceptance_service_test.go new file mode 100644 index 000000000..e96e83ddf --- /dev/null +++ b/cli/internal/beads/acceptance_service_test.go @@ -0,0 +1,22 @@ +package beads + +import ( + "strings" + "testing" +) + +type acceptanceRepositoryFunc func([]string) ([]byte, error) + +func (function acceptanceRepositoryFunc) ShowAcceptance(ids []string) ([]byte, error) { + return function(ids) +} + +func TestAcceptanceServiceRejectsPartialTrackerResponse(t *testing.T) { + service := AcceptanceService{Repository: acceptanceRepositoryFunc(func([]string) ([]byte, error) { + return []byte(`[{"id":"age-a","issue_type":"task","description":"## Acceptance\ncomplete"}]`), nil + })} + _, _, err := service.VerifyAcceptance([]string{"age-a", "age-missing"}) + if err == nil || !strings.Contains(err.Error(), "age-missing") { + t.Fatalf("error = %v", err) + } +} diff --git a/cli/internal/beads/hygiene.go b/cli/internal/beads/hygiene.go index 2f28e67ea..319a5f554 100644 --- a/cli/internal/beads/hygiene.go +++ b/cli/internal/beads/hygiene.go @@ -92,6 +92,7 @@ type AuditReport struct { Summary AuditSummary `json:"summary"` BDAvailable bool `json:"bd_available"` Error string `json:"error,omitempty"` + Warnings []string `json:"-"` } func (report *AuditReport) FlaggedCount() int { @@ -156,7 +157,9 @@ func (service HygieneService) Audit(autoClose bool) (*AuditReport, error) { if evidence := GrepCommitsForID(commits, record.ID); evidence != "" { report.LikelyFixed = append(report.LikelyFixed, AuditFinding{ID: record.ID, Title: record.DisplayTitle(), Reason: "commit_match", Evidence: evidence}) if autoClose { - _ = service.Repository.Close(record.ID, "Auto-closed by ao beads audit: commit evidence found: "+evidence) + if closeErr := service.Repository.Close(record.ID, "Auto-closed by ao beads audit: commit evidence found: "+evidence); closeErr != nil { + report.Warnings = append(report.Warnings, closeErr.Error()) + } } continue } @@ -165,7 +168,9 @@ func (service HygieneService) Audit(autoClose bool) (*AuditReport, error) { if evidence := FileChangesSinceCommits(commits, record.CreatedAt, paths); evidence != "" { report.LikelyFixed = append(report.LikelyFixed, AuditFinding{ID: record.ID, Title: record.DisplayTitle(), Reason: "file_modified_since_creation", Evidence: evidence}) if autoClose { - _ = service.Repository.Close(record.ID, "Auto-closed by ao beads audit: mentioned files modified since creation.") + if closeErr := service.Repository.Close(record.ID, "Auto-closed by ao beads audit: mentioned files modified since creation."); closeErr != nil { + report.Warnings = append(report.Warnings, closeErr.Error()) + } } continue } diff --git a/cli/internal/beads/hygiene_test.go b/cli/internal/beads/hygiene_test.go new file mode 100644 index 000000000..04be9d430 --- /dev/null +++ b/cli/internal/beads/hygiene_test.go @@ -0,0 +1,65 @@ +package beads + +import ( + "errors" + "strings" + "testing" + "time" +) + +type fakeHygieneRepository struct { + list map[string][]BeadRecord + shown map[string]BeadRecord + commits []AuditCommit + closeErr error + closed []string + reparented []string +} + +func (fake *fakeHygieneRepository) Available() bool { return true } +func (fake *fakeHygieneRepository) List(status string) ([]BeadRecord, error) { + return fake.list[status], nil +} +func (fake *fakeHygieneRepository) Show(id string) (BeadRecord, error) { return fake.shown[id], nil } +func (fake *fakeHygieneRepository) Commits() []AuditCommit { return fake.commits } +func (fake *fakeHygieneRepository) PatternExists(string) bool { return true } +func (fake *fakeHygieneRepository) Close(id, _ string) error { + fake.closed = append(fake.closed, id) + return fake.closeErr +} +func (fake *fakeHygieneRepository) Reparent(id, parent string) error { + fake.reparented = append(fake.reparented, id+"->"+parent) + return nil +} + +func TestHygieneServicePreservesMutationEvidence(t *testing.T) { + repository := &fakeHygieneRepository{ + list: map[string][]BeadRecord{"open": {{ID: "age-x", Title: "fixed"}}}, + commits: []AuditCommit{{ShortSHA: "abc123", Subject: "finish age-x", CommitAt: time.Now()}}, + closeErr: errors.New("tracker refused close"), + } + report, err := (HygieneService{Repository: repository}).Audit(true) + if err != nil { + t.Fatal(err) + } + if len(repository.closed) != 1 || len(report.Warnings) != 1 || !strings.Contains(report.Warnings[0], "tracker refused close") { + t.Fatalf("closed=%v warnings=%v", repository.closed, report.Warnings) + } +} + +func TestHygieneServiceClustersAndReparents(t *testing.T) { + repository := &fakeHygieneRepository{ + list: map[string][]BeadRecord{"open": { + {ID: "age-e", Title: "tracker cleanup", IssueType: "epic", Labels: []string{"tracker"}}, + {ID: "age-x", Title: "tracker cleanup", Labels: []string{"tracker"}}, + }}, + shown: map[string]BeadRecord{}, + } + report, err := (HygieneService{Repository: repository}).Cluster(true) + if err != nil { + t.Fatal(err) + } + if len(report.Clusters) != 1 || report.Clusters[0].Representative != "age-e" || len(repository.reparented) != 1 { + t.Fatalf("report=%+v reparented=%v", report, repository.reparented) + } +} diff --git a/cli/internal/beads/recovery_service_test.go b/cli/internal/beads/recovery_service_test.go new file mode 100644 index 000000000..dbba61142 --- /dev/null +++ b/cli/internal/beads/recovery_service_test.go @@ -0,0 +1,60 @@ +package beads + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" +) + +type fakeRecoveryPorts struct { + shown []StaleBeadRecord + showIndex int + calls []string + appendErr error +} + +func (fake *fakeRecoveryPorts) ListInProgress(context.Context) ([]byte, error) { + return []byte("[]"), nil +} +func (fake *fakeRecoveryPorts) Show(_ context.Context, id string) (StaleBeadRecord, error) { + fake.calls = append(fake.calls, "show:"+id) + record := fake.shown[fake.showIndex] + fake.showIndex++ + return record, nil +} +func (fake *fakeRecoveryPorts) Claim(_ context.Context, id, agent string) error { + fake.calls = append(fake.calls, "claim:"+id+":"+agent) + return nil +} +func (fake *fakeRecoveryPorts) Now() time.Time { return time.Date(2026, 7, 11, 18, 0, 0, 0, time.UTC) } +func (fake *fakeRecoveryPorts) Actor() string { return "codex" } +func (fake *fakeRecoveryPorts) ResolveRepoPath(path string) (string, error) { + fake.calls = append(fake.calls, "resolve:"+path) + return "/repo/" + path, nil +} +func (fake *fakeRecoveryPorts) AppendEvent(path string, _ any) error { + fake.calls = append(fake.calls, "append:"+path) + return fake.appendErr +} + +func TestRecoveryServiceReportsLedgerFailureAfterClaim(t *testing.T) { + ports := &fakeRecoveryPorts{ + shown: []StaleBeadRecord{ + {ID: "age-x", Status: "in_progress", Assignee: "old", UpdatedAt: "2026-07-11T10:00:00Z"}, + {ID: "age-x", Status: "in_progress", Assignee: "codex", UpdatedAt: "2026-07-11T18:00:00Z"}, + }, + appendErr: errors.New("disk full"), + } + service := RecoveryService{Claims: ports, Runtime: ports} + _, err := service.Resume(context.Background(), "age-x", ResumeOptions{Ledger: "ledger.jsonl"}) + if err == nil || !strings.Contains(err.Error(), "claim already transferred") { + t.Fatalf("error = %v", err) + } + want := []string{"show:age-x", "claim:age-x:codex", "show:age-x", "resolve:ledger.jsonl", "append:/repo/ledger.jsonl"} + if !reflect.DeepEqual(ports.calls, want) { + t.Fatalf("calls = %v, want %v", ports.calls, want) + } +} diff --git a/cli/internal/beads/scenario_service_test.go b/cli/internal/beads/scenario_service_test.go new file mode 100644 index 000000000..c773b166c --- /dev/null +++ b/cli/internal/beads/scenario_service_test.go @@ -0,0 +1,37 @@ +package beads + +import ( + "testing" +) + +type fakeScenarioRepository struct { + bead FetchedBead + description string +} + +func (fake *fakeScenarioRepository) Available() bool { return true } +func (fake *fakeScenarioRepository) FetchScenarioBead(string) (FetchedBead, error) { + return fake.bead, nil +} +func (fake *fakeScenarioRepository) UpdateDescription(_ string, description string) error { + fake.description = description + return nil +} + +func TestScenarioServicePreservesDescriptionOnApply(t *testing.T) { + repository := &fakeScenarioRepository{bead: FetchedBead{ + Acceptance: "Given a state\nWhen an action\nThen a result", + Description: "original context", + }} + service := ScenarioService{Repository: repository} + extraction, err := service.PrepareScenarios("age-x", false) + if err != nil { + t.Fatal(err) + } + if err := service.ApplyScenarios(extraction); err != nil { + t.Fatal(err) + } + if repository.description[:16] != "original context" || len(repository.description) <= len("original context") { + t.Fatalf("description = %q", repository.description) + } +} diff --git a/cli/internal/commands/beads/module.go b/cli/internal/commands/beads/module.go index 9cfcf8bfa..29cee3186 100644 --- a/cli/internal/commands/beads/module.go +++ b/cli/internal/commands/beads/module.go @@ -194,6 +194,9 @@ func (module Module) runAudit(command *cobra.Command, options Options) error { if err != nil { return err } + for _, warning := range report.Warnings { + fmt.Fprintf(command.ErrOrStderr(), "WARN: %s\n", warning) + } if !report.BDAvailable { if options.JSON { return encodeJSON(command, report) From 4863195d07079bb4f995a84686b2c572afe070fb Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:07:20 -0400 Subject: [PATCH 025/108] test(beads): repair independent focused proof (age-nw28h.2.1) --- cli/internal/beads/scenario_service_test.go | 5 +++-- cli/internal/commands/beads/module_test.go | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cli/internal/beads/scenario_service_test.go b/cli/internal/beads/scenario_service_test.go index c773b166c..4df007342 100644 --- a/cli/internal/beads/scenario_service_test.go +++ b/cli/internal/beads/scenario_service_test.go @@ -1,6 +1,7 @@ package beads import ( + "strings" "testing" ) @@ -20,7 +21,7 @@ func (fake *fakeScenarioRepository) UpdateDescription(_ string, description stri func TestScenarioServicePreservesDescriptionOnApply(t *testing.T) { repository := &fakeScenarioRepository{bead: FetchedBead{ - Acceptance: "Given a state\nWhen an action\nThen a result", + Acceptance: "Given a state when an action then a result", Description: "original context", }} service := ScenarioService{Repository: repository} @@ -31,7 +32,7 @@ func TestScenarioServicePreservesDescriptionOnApply(t *testing.T) { if err := service.ApplyScenarios(extraction); err != nil { t.Fatal(err) } - if repository.description[:16] != "original context" || len(repository.description) <= len("original context") { + if !strings.HasPrefix(repository.description, "original context") || len(repository.description) <= len("original context") { t.Fatalf("description = %q", repository.description) } } diff --git a/cli/internal/commands/beads/module_test.go b/cli/internal/commands/beads/module_test.go index c2a166709..ec6a3811e 100644 --- a/cli/internal/commands/beads/module_test.go +++ b/cli/internal/commands/beads/module_test.go @@ -415,10 +415,11 @@ func TestLegacyProductionOwnersAreAbsent(t *testing.T) { t.Fatal(err) } allowed := map[string]bool{ - "beads_composition.go": true, - "beads_exec.go": true, // yield-family compatibility delegates - "beads_json_compat.go": true, // lookup compatibility helper - "beads_module.go": true, + "beads_composition.go": true, + "beads_citation_compat.go": true, // frozen citation-family adapter bridge + "beads_exec.go": true, // yield-family compatibility delegates + "beads_json_compat.go": true, // lookup compatibility helper + "beads_module.go": true, } for _, entry := range entries { name := entry.Name() From 2b0b7434722510f1f885e7739a58abb7b6527df8 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:12:18 -0400 Subject: [PATCH 026/108] chore(cli): seal beads migration lineage (age-nw28h.2.1) --- .../compatibility-baseline/families/beads/lineage.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/testdata/compatibility-baseline/families/beads/lineage.json b/cli/testdata/compatibility-baseline/families/beads/lineage.json index 9708d5197..4db7e2c49 100644 --- a/cli/testdata/compatibility-baseline/families/beads/lineage.json +++ b/cli/testdata/compatibility-baseline/families/beads/lineage.json @@ -6,5 +6,6 @@ "fixture_sha256": "fa3f72bc3b0a325d5b662a2b56ad0ccc92ac6b669871061be99ef34974b67473", "ownership_sha256": "c4baf423279f90d923030854688bc547a3170afcc093a31f0dc72c00720adccb", "scope_sha256": "55272fba3517bec8c3a80503fad1f30a551a1f76e8731e73617c8063da21215a", - "migration_state": "migrating" + "migration_state": "migrated", + "accepted_sha": "4863195d07079bb4f995a84686b2c572afe070fb" } From 0d26ea557c73d50069d6da1e492fd14cc32a790b Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:10:35 -0400 Subject: [PATCH 027/108] chore(provenance): bind pawl CONFIRMED verdict for age-nw28h.2.1 #trivial --- docs/provenance/ledger.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/provenance/ledger.jsonl b/docs/provenance/ledger.jsonl index 61798c5bb..a75f1c112 100644 --- a/docs/provenance/ledger.jsonl +++ b/docs/provenance/ledger.jsonl @@ -516,3 +516,4 @@ {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.1.4@3d8f8ae","from_type":"verdict","to_id":"3d8f8ae57b35bd0722c144fe94a03e7b2ade34bc","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.1.4 disposition=CONFIRMED","bead_id":"age-nw28h.1.4","trust_tier":"inferred","ts":"2026-07-11T16:17:01Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/P3-rebase-verdict-3d8f8ae57.txt","prev_hash":"a3185237dffeb96759eec39c069982b61da42592c2edbf595b7f31aafdc76d2a","payload_hash":"c408e98ef2a5838c9e7a53826e64b8c592d851093249be93c647bb52d70ed5bf","hash":"3e2dc04605c17f4f96e094e6ba0486cee1e424594748c646a853b13726df7dc0"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-h433.38@c9c428a","from_type":"verdict","to_id":"c9c428aadd56696d76407d24a2de99b843ecef0c","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-h433.38 disposition=CONFIRMED","bead_id":"age-h433.38","trust_tier":"inferred","ts":"2026-07-11T17:13:41Z","reviewer_family":"gemini","rounds":1,"duration_s":637,"tokens_est":177,"evidence_path":"/Users/bo/dev/agentops/.agents/pawl-evidence/age-h433.38-pawl-review.txt","prev_hash":"3e2dc04605c17f4f96e094e6ba0486cee1e424594748c646a853b13726df7dc0","payload_hash":"caf985741efe5bddebbf17b9a849aff82f4d7f84cec5836e600b2a7b9acc1025","hash":"24e841e8569aa9375c46d26cf6f6940e2a765d6e0d28ce7e0dd86240481dbae5"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.1.5@e76a715","from_type":"verdict","to_id":"e76a715b00ad560df06fbd00da9783f3bcef7ba6","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.1.5 disposition=CONFIRMED","bead_id":"age-nw28h.1.5","trust_tier":"inferred","ts":"2026-07-11T17:26:53Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/P4A-rebase-verdict-e76a715b0.txt","prev_hash":"24e841e8569aa9375c46d26cf6f6940e2a765d6e0d28ce7e0dd86240481dbae5","payload_hash":"3df68eadc14f39362ab69431db77b379cc829298967e0d76ede07b7b3b34e0e9","hash":"5ce3f9f10999351cb3ce663d1dab51fbb418291ebf9e6ad6c507acc8550a7bbe"} +{"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.1@4863195","from_type":"verdict","to_id":"4863195d07079bb4f995a84686b2c572afe070fb","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.1 disposition=CONFIRMED","bead_id":"age-nw28h.2.1","trust_tier":"inferred","ts":"2026-07-11T19:10:35Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-beads-independent-verdict-4863195d0.txt","prev_hash":"5ce3f9f10999351cb3ce663d1dab51fbb418291ebf9e6ad6c507acc8550a7bbe","payload_hash":"ff4b1163efdb38e333b7ce4358ae7dd9a56a86d0645d481750c3a1ac0a860acb","hash":"0dbb491f180b24cbd6fd602bf8812d14382184c08370a714c7e4905f3730ca0e"} From 48a6bc36351adab3a45d33a634abdad882cac98e Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:43:13 -0400 Subject: [PATCH 028/108] docs(cli): refresh commands after beads module (age-nw28h.2.1) --- cli/docs/COMMANDS.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/cli/docs/COMMANDS.md b/cli/docs/COMMANDS.md index d92f3f135..2c8953e84 100644 --- a/cli/docs/COMMANDS.md +++ b/cli/docs/COMMANDS.md @@ -1407,7 +1407,7 @@ ao beads [command] #### `ao beads audit` -Audits open and in-progress beads for backlog hygiene issues. +Audit open beads for likely-fixed, stale, or consolidatable work ``` ao beads audit [flags] @@ -1424,7 +1424,7 @@ ao beads audit [flags] #### `ao beads cluster` -Analyzes open beads for domain overlap and suggests consolidation groups. +Suggest consolidation clusters for overlapping open beads ``` ao beads cluster [flags] @@ -1440,7 +1440,7 @@ ao beads cluster [flags] #### `ao beads dir` -Print the BEADS_DIR path AgentOps will use for br subprocesses. +Print the resolved live br ledger directory ``` ao beads dir [flags] @@ -1456,7 +1456,7 @@ ao beads dir [flags] #### `ao beads epic-status` -Emit a deterministic "is this epic/wave actually done" verdict, replacing +Deterministic group-terminality verdict for an epic/wave ``` ao beads epic-status [flags] @@ -1472,7 +1472,7 @@ ao beads epic-status [flags] #### `ao beads exec` -Forward a bead command verbatim to the resolved beads tracker (bd or br), +Run a bead CRUD command against whichever tracker (bd or br) this environment uses ``` ao beads exec [args...] [flags] @@ -1480,7 +1480,7 @@ ao beads exec [args...] [flags] #### `ao beads harvest` -Reads a closed bead via 'bd show ' and writes its closure reason +Materialize a closed bead's reason as a structured learning file ``` ao beads harvest [flags] @@ -1496,7 +1496,7 @@ ao beads harvest [flags] #### `ao beads lint` -Runs 'ao beads verify' on every bead matching a status filter and +Batch-verify every open bead (or filtered set) against HEAD ``` ao beads lint [flags] @@ -1512,7 +1512,7 @@ ao beads lint [flags] #### `ao beads resume` -Transfers a stale claim via 'br update --claim', then appends a +Atomically transfer an in_progress claim from a stale agent to this one ``` ao beads resume [flags] @@ -1529,7 +1529,7 @@ ao beads resume [flags] #### `ao beads scenarios` -Turn a bead's free-text acceptance criteria into structured Gherkin +[DEPRECATED — use 'ao beads verify-acceptance'] Convert bead acceptance criteria into Gherkin scenarios ``` ao beads scenarios [command] @@ -1537,7 +1537,7 @@ ao beads scenarios [command] ##### `ao beads scenarios extract` -Read a bead's acceptance criteria via 'bd show --json', convert the +Print a candidate Gherkin '## Scenarios' block from a bead's acceptance (dry-run) ``` ao beads scenarios extract [flags] @@ -1554,7 +1554,7 @@ ao beads scenarios extract [flags] ##### `ao beads scenarios validate` -Read a bead via 'bd show --json' and validate its authored +Check that a bead's authored '## Scenarios' block is well-formed Gherkin ``` ao beads scenarios validate [flags] @@ -1569,7 +1569,7 @@ ao beads scenarios validate [flags] #### `ao beads stale-claims` -Lists in_progress beads whose claim activity is older than --threshold. +List in_progress beads whose claim looks stale ``` ao beads stale-claims [flags] @@ -1585,7 +1585,7 @@ ao beads stale-claims [flags] #### `ao beads tracker` -Detect which beads tracker AgentOps will drive here and how it was +Print the resolved beads tracker (bd or br) for this environment ``` ao beads tracker [flags] @@ -1600,7 +1600,7 @@ ao beads tracker [flags] #### `ao beads verify` -Reads a bead description via 'bd show ' and checks every file +Detect stale citations in a bead description (files, functions, symbols) ``` ao beads verify [flags] @@ -1616,7 +1616,7 @@ ao beads verify [flags] #### `ao beads verify-acceptance` -Read beads via br (never the retired bd) and check each bead carries the +Assert each bead carries the acceptance contract for its type (br-native) ``` ao beads verify-acceptance ... [flags] From 958e746a952c6f6049e09b41f4b5d1a517445c9e Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:20:24 -0400 Subject: [PATCH 029/108] test(cli): freeze capabilities compatibility contract (age-nw28h.2.2) --- .../families/capabilities/case.json | 21 +++++++++++ .../families/capabilities/ownership.json | 35 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/capabilities/case.json create mode 100644 cli/testdata/compatibility-baseline/families/capabilities/ownership.json diff --git a/cli/testdata/compatibility-baseline/families/capabilities/case.json b/cli/testdata/compatibility-baseline/families/capabilities/case.json new file mode 100644 index 000000000..80457dd56 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/capabilities/case.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "family": "capabilities", + "checks": [ + { + "id": "capabilities-black-box", + "command": "tmp=$(mktemp -d /tmp/ao-capabilities.XXXXXX); trap 'rm -r \"$tmp\"' EXIT; (cd cli && go build -o \"$tmp/ao\" ./cmd/ao); \"$tmp/ao\" capabilities --help >\"$tmp/help\"; grep -q 'Print the machine-readable contract for the whole ao CLI as JSON' \"$tmp/help\"; \"$tmp/ao\" capabilities --json >\"$tmp/first.json\"; \"$tmp/ao\" capabilities --json >\"$tmp/second.json\"; cmp -s \"$tmp/first.json\" \"$tmp/second.json\"; jq -e '.schema_version == \"1.1\" and .tool == \"ao\" and .contract_version == \"1.1\" and .platform.os != \"\" and .platform.arch != \"\" and any(.commands[]; .path == \"ao capabilities\")' \"$tmp/first.json\" >/dev/null; set +e; \"$tmp/ao\" capabilities unexpected >\"$tmp/rejected.out\" 2>\"$tmp/rejected.err\"; code=$?; set -e; test \"$code\" -eq 1; test ! -s \"$tmp/rejected.out\"; grep -qx 'Error: unknown command \"unexpected\" for \"ao capabilities\"' \"$tmp/rejected.err\"" + } + ], + "dimensions": { + "path": {"status": "evidence", "evidence": ["capabilities-black-box"]}, + "aliases": {"status": "not_applicable", "reason": "The ao capabilities command declares no aliases."}, + "accepted_args": {"status": "evidence", "evidence": ["capabilities-black-box"]}, + "rejected_args": {"status": "evidence", "evidence": ["capabilities-black-box"]}, + "stdout": {"status": "evidence", "evidence": ["capabilities-black-box"]}, + "stderr": {"status": "evidence", "evidence": ["capabilities-black-box"]}, + "exit_classes": {"status": "evidence", "evidence": ["capabilities-black-box"]}, + "tracker_selection": {"status": "not_applicable", "reason": "Capabilities projects the compiled CLI surface and does not select a tracker."}, + "ordered_effects": {"status": "not_applicable", "reason": "Capabilities is a deterministic read-side projection with no ordered external effects."} + } +} diff --git a/cli/testdata/compatibility-baseline/families/capabilities/ownership.json b/cli/testdata/compatibility-baseline/families/capabilities/ownership.json new file mode 100644 index 000000000..57b74c49e --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/capabilities/ownership.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "family": "capabilities", + "profiles": { + "default": "present", + "flywheel": "present", + "legacy": "present", + "combined": "present" + }, + "legacy_symbols": [ + "capabilitiesContractVersion", + "capabilitiesDoc", + "capPlatform", + "capFlag", + "capCommand", + "capCommandGroup", + "capabilitiesExitCodes", + "capabilitiesCommandExitCodes", + "capabilitiesEnvVars", + "capabilitiesRobotSurfaces", + "capabilitiesCmd", + "buildCapabilitiesDoc", + "runCapabilities" + ], + "live_owner": "cli/internal/commands/capabilities", + "allowed_paths": [ + "cli/cmd/ao/capabilities*.go", + "cli/cmd/ao/group_json.go", + "cli/internal/adapters/capabilities/**", + "cli/internal/capabilities/**", + "cli/internal/clicontract/**", + "cli/internal/commands/capabilities/**", + "cli/testdata/compatibility-baseline/families/capabilities/lineage.json" + ] +} From 29cc681bb52be3187c2dd379edfc33307f05afe4 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:20:40 -0400 Subject: [PATCH 030/108] test(cli): bind capabilities migration lineage (age-nw28h.2.2) --- .../families/capabilities/lineage.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/capabilities/lineage.json diff --git a/cli/testdata/compatibility-baseline/families/capabilities/lineage.json b/cli/testdata/compatibility-baseline/families/capabilities/lineage.json new file mode 100644 index 000000000..01f47c517 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/capabilities/lineage.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "family": "capabilities", + "old_implementation_sha": "48a6bc36351adab3a45d33a634abdad882cac98e", + "freeze_sha": "958e746a952c6f6049e09b41f4b5d1a517445c9e", + "fixture_sha256": "9788d6c4206e5d01e7609724bb7dec88946ebcd67302e53fca1962b7f0db5303", + "ownership_sha256": "7d9491418c90a931611d81e5e5308920db6f8a1fc3a90c4d684e5f961a85cf6f", + "scope_sha256": "62cd117cf63f443d513a84761f4c99dffcede9034ef5204b48a3c39c21e8a4e1", + "migration_state": "migrating" +} From bd171a4776a303a8c7ad867047f70967cdee8b0e Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:31:09 -0400 Subject: [PATCH 031/108] refactor(cli): migrate capabilities command module (age-nw28h.2.2) --- cli/cmd/ao/capabilities.go | 219 ++---------------- cli/cmd/ao/capabilities_oracle_test.go | 12 +- cli/cmd/ao/capabilities_test.go | 47 ++-- cli/cmd/ao/group_json.go | 19 +- cli/internal/adapters/capabilities/cobra.go | 74 ++++++ cli/internal/capabilities/document.go | 142 ++++++++++++ cli/internal/capabilities/document_test.go | 47 ++++ cli/internal/commands/capabilities/module.go | 70 ++++++ .../commands/capabilities/module_test.go | 73 ++++++ 9 files changed, 472 insertions(+), 231 deletions(-) create mode 100644 cli/internal/adapters/capabilities/cobra.go create mode 100644 cli/internal/capabilities/document.go create mode 100644 cli/internal/capabilities/document_test.go create mode 100644 cli/internal/commands/capabilities/module.go create mode 100644 cli/internal/commands/capabilities/module_test.go diff --git a/cli/cmd/ao/capabilities.go b/cli/cmd/ao/capabilities.go index c437ffacf..c6ecc372d 100644 --- a/cli/cmd/ao/capabilities.go +++ b/cli/cmd/ao/capabilities.go @@ -2,212 +2,23 @@ package main import ( - "encoding/json" - "runtime" - "sort" - + capabilitiesadapter "github.com/boshu2/agentops/cli/internal/adapters/capabilities" + capabilitiesapp "github.com/boshu2/agentops/cli/internal/capabilities" "github.com/boshu2/agentops/cli/internal/clicontract" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "gopkg.in/yaml.v3" + capabilitiescommands "github.com/boshu2/agentops/cli/internal/commands/capabilities" ) -// capabilitiesContractVersion is the version of the top-level capabilities -// contract. Bump it on any breaking change to the JSON shape so agents can -// pin against a known schema. -const capabilitiesContractVersion = "1.1" - -// capabilitiesDoc is the machine-readable contract for the whole `ao` CLI, -// emitted by `ao capabilities`. It lets an agent read the command surface, -// flag conventions, and exit-code dictionary straight from the tool instead -// of relying on an out-of-band doc lookup. -type capabilitiesDoc struct { - SchemaVersion string `json:"schema_version"` - Tool string `json:"tool"` - ToolVersion string `json:"tool_version"` - ContractVersion string `json:"contract_version"` - Platform capPlatform `json:"platform"` - GlobalFlags []capFlag `json:"global_flags"` - OutputFormats []string `json:"output_formats"` - ExitCodes map[string]string `json:"exit_codes"` - // CommandExitCodes documents the per-command typed exit codes beyond the - // global {0,1,2} — the codes an agent must interpret to branch on a verdict - // (e.g. pawl REFUTED=3, plan-pawl BLOCKED=4, governor HARDEN=3). Only - // default-build commands are listed; legacy/flywheel-tagged commands (tick, - // corpus-scan) are omitted because they are absent from the shipping binary. - CommandExitCodes map[string]map[string]string `json:"command_exit_codes"` - EnvVars map[string]string `json:"env_vars"` - RobotSurfaces map[string]string `json:"robot_surfaces"` - CommandGroups []capCommandGroup `json:"command_groups"` - Commands []clicontract.Command `json:"commands"` -} - -type capPlatform struct { - OS string `json:"os"` - Arch string `json:"arch"` -} - -type capFlag struct { - Name string `json:"name"` - Shorthand string `json:"shorthand,omitempty"` - Description string `json:"description"` -} - -type capCommand struct { - Name string `json:"name"` - Short string `json:"short"` -} - -type capCommandGroup struct { - ID string `json:"id"` - Title string `json:"title"` - Commands []capCommand `json:"commands"` -} - -// capabilitiesExitCodes documents the exit codes an agent can rely on across -// the CLI. Diagnostic commands (doctor, goals) layer an extended dictionary on -// top — see robot_surfaces for where to read it. -var capabilitiesExitCodes = map[string]string{ - "0": "success", - "1": "error: usage error, runtime failure, or — for diagnostic commands — findings present", - "2": "diagnostic: partial result or bead claimed (command-specific; see that command's --help)", -} - -// capabilitiesCommandExitCodes documents per-command typed exit codes beyond the -// global {0,1,2} dictionary — the codes `Execute()` propagates from typed errors -// so a shell/agent can branch on the verdict without parsing stdout (recon -// 2026-07-02 audit A7). Values are kept in lockstep with the code's own exit-code -// constants by TestCapabilitiesCommandExitCodesMatchConstants. Only default-build -// commands appear: `tick` (//go:build legacy) and `corpus-scan` (//go:build -// flywheel) are deliberately omitted — documenting codes for commands absent from -// the shipping binary would be the exact contract inaccuracy this fix closes. -var capabilitiesCommandExitCodes = map[string]map[string]string{ - "pawl review": { - "0": "CONFIRMED — cross-family verdict written", - "1": "hard error", - "2": "usage error", - "3": "REFUTED — auto-redo", - "4": "--converge advisory-only (no lineage written)", - }, - "plan-pawl decide": { - "0": "PASS — the door opens", - "2": "usage error", - "3": "REDO — auto-redo loop (no human)", - "4": "BLOCKED — a circuit breaker tripped (andon)", - "5": "DEGRADED — transient panel lane loss below quorum; retry the panel", - }, - "governor budget": { - "0": "ship — within error budget", - "3": "HARDEN — burn-rate over tolerance; stop the line", - }, -} - -// capabilitiesEnvVars documents the environment variables the CLI honors. -var capabilitiesEnvVars = map[string]string{ - "AGENTOPS_CONFIG": "path to the config file (overridden by --config)", - "NO_COLOR": "disable ANSI styling on all output", - "AO_DOCTOR_LOG_LEVEL": "trace|debug|info|warn|error — verbosity of the doctor surface", -} - -// capabilitiesRobotSurfaces points an agent at every machine-readable surface. -var capabilitiesRobotSurfaces = map[string]string{ - "capabilities": "ao capabilities — this contract", - "robot_docs": "ao robot-docs — paste-ready agent handbook (Markdown)", - "doctor_triage": "ao doctor --robot-triage — mega-command health triage JSON", - "doctor_capabilities": "ao doctor capabilities — extended doctor contract + exit codes", - "json_everywhere": "append --json (or -o json) to any read-side command for structured output", -} - -var capabilitiesCmd = &cobra.Command{ - Use: "capabilities", - Short: "Print the machine-readable CLI contract (JSON)", - Long: `Print the machine-readable contract for the whole ao CLI as JSON. - -This is the first command an agent should run to discover the command -surface, flag conventions, exit-code dictionary, and every other -machine-readable surface — no external documentation lookup required. - -Output is always JSON; it is stable across patch versions (pinned by -contract_version).`, - RunE: runCapabilities, -} - func init() { - capabilitiesCmd.GroupID = "core" - rootCmd.AddCommand(capabilitiesCmd) -} - -// buildCapabilitiesDoc assembles the capabilities contract by walking the -// live command tree, so it never drifts from the registered commands. -func buildCapabilitiesDoc() capabilitiesDoc { - groups := map[string]*capCommandGroup{} - var order []string - for _, g := range rootCmd.Groups() { - groups[g.ID] = &capCommandGroup{ID: g.ID, Title: g.Title} - order = append(order, g.ID) - } - const ungrouped = "" - groups[ungrouped] = &capCommandGroup{ID: ungrouped, Title: "Additional Commands:"} - order = append(order, ungrouped) - - for _, c := range rootCmd.Commands() { - if c.Hidden || c.Name() == "help" || c.Name() == "completion" { - continue - } - g, ok := groups[c.GroupID] - if !ok { - g = groups[ungrouped] - } - g.Commands = append(g.Commands, capCommand{Name: c.Name(), Short: c.Short}) - } - - var commandGroups []capCommandGroup - for _, id := range order { - g := groups[id] - if len(g.Commands) == 0 { - continue - } - sort.Slice(g.Commands, func(i, j int) bool { return g.Commands[i].Name < g.Commands[j].Name }) - commandGroups = append(commandGroups, *g) - } - - var globalFlags []capFlag - rootCmd.PersistentFlags().VisitAll(func(f *pflag.Flag) { - globalFlags = append(globalFlags, capFlag{ - Name: f.Name, - Shorthand: f.Shorthand, - Description: f.Usage, - }) - }) - sort.Slice(globalFlags, func(i, j int) bool { return globalFlags[i].Name < globalFlags[j].Name }) - - return capabilitiesDoc{ - SchemaVersion: capabilitiesContractVersion, - Tool: "ao", - ToolVersion: version, - ContractVersion: capabilitiesContractVersion, - Platform: capPlatform{OS: runtime.GOOS, Arch: runtime.GOARCH}, - GlobalFlags: globalFlags, - OutputFormats: []string{"table", "json", "yaml"}, - ExitCodes: capabilitiesExitCodes, - CommandExitCodes: capabilitiesCommandExitCodes, - EnvVars: capabilitiesEnvVars, - RobotSurfaces: capabilitiesRobotSurfaces, - CommandGroups: commandGroups, - Commands: clicontract.Inspect(rootCmd, capabilitiesCommandExitCodes), - } -} - -func runCapabilities(cmd *cobra.Command, _ []string) error { - if GetOutput() == "yaml" { - data, err := yaml.Marshal(buildCapabilitiesDoc()) - if err != nil { - return err - } - _, err = cmd.OutOrStdout().Write(data) - return err - } - enc := json.NewEncoder(cmd.OutOrStdout()) - enc.SetIndent("", " ") - return enc.Encode(buildCapabilitiesDoc()) + service := capabilitiesapp.NewService( + version, + capabilitiesadapter.NewCobraSurface(rootCmd), + capabilitiesadapter.RuntimePlatform{}, + ) + module := capabilitiescommands.NewModule(service, GetOutput) + command := module.Command() + command.GroupID = "core" + if err := clicontract.Attach(command, module.Contract()); err != nil { + panic(err) + } + rootCmd.AddCommand(command) } diff --git a/cli/cmd/ao/capabilities_oracle_test.go b/cli/cmd/ao/capabilities_oracle_test.go index 2f0bef514..396fdb9e1 100644 --- a/cli/cmd/ao/capabilities_oracle_test.go +++ b/cli/cmd/ao/capabilities_oracle_test.go @@ -42,7 +42,17 @@ func TestCapabilitiesOracleRecursivelyMatchesPublicTree(t *testing.T) { } func TestCapabilitiesExitOracleIncludesDefinedCodeFive(t *testing.T) { - if got := capabilitiesCommandExitCodes["plan-pawl decide"]["5"]; got == "" { + out, err := executeCommand("capabilities") + if err != nil { + t.Fatal(err) + } + var wire struct { + CommandExitCodes map[string]map[string]string `json:"command_exit_codes"` + } + if err := json.Unmarshal([]byte(out), &wire); err != nil { + t.Fatal(err) + } + if got := wire.CommandExitCodes["plan-pawl decide"]["5"]; got == "" { t.Fatal("plan-pawl decide defines exit 5 (DEGRADED) but capabilities omits it") } } diff --git a/cli/cmd/ao/capabilities_test.go b/cli/cmd/ao/capabilities_test.go index 627795ba3..9525c81fd 100644 --- a/cli/cmd/ao/capabilities_test.go +++ b/cli/cmd/ao/capabilities_test.go @@ -6,22 +6,17 @@ import ( "strconv" "strings" "testing" + + capabilitiesapp "github.com/boshu2/agentops/cli/internal/capabilities" ) func TestCapabilities_EmitsValidJSON(t *testing.T) { - out, err := executeCommand("capabilities") - if err != nil { - t.Fatalf("ao capabilities returned error: %v", err) - } - var doc capabilitiesDoc - if jerr := json.Unmarshal([]byte(out), &doc); jerr != nil { - t.Fatalf("ao capabilities output is not valid JSON: %v\noutput: %s", jerr, out) - } + doc := readCapabilitiesDocument(t) if doc.Tool != "ao" { t.Errorf("tool = %q, want %q", doc.Tool, "ao") } - if doc.ContractVersion != capabilitiesContractVersion { - t.Errorf("contract_version = %q, want %q", doc.ContractVersion, capabilitiesContractVersion) + if doc.ContractVersion != capabilitiesapp.ContractVersion { + t.Errorf("contract_version = %q, want %q", doc.ContractVersion, capabilitiesapp.ContractVersion) } if len(doc.CommandGroups) == 0 { t.Error("command_groups is empty; expected the live command tree") @@ -39,7 +34,7 @@ func TestCapabilities_EmitsValidJSON(t *testing.T) { // interpret (pawl/plan-pawl/governor emit 3/4), and does NOT leak legacy or // flywheel commands absent from the shipping binary. func TestCapabilities_CommandExitCodesPublished(t *testing.T) { - doc := buildCapabilitiesDoc() + doc := readCapabilitiesDocument(t) if len(doc.CommandExitCodes) == 0 { t.Fatal("command_exit_codes is empty; an agent cannot interpret codes 3-4") } @@ -59,7 +54,8 @@ func TestCapabilities_CommandExitCodesPublished(t *testing.T) { // codes in lockstep with the code's own exit-code constants, so a constant change // cannot silently drift the contract (the exact drift class audit A7 closes). func TestCapabilitiesCommandExitCodesMatchConstants(t *testing.T) { - planPawl := capabilitiesCommandExitCodes["plan-pawl decide"] + doc := readCapabilitiesDocument(t) + planPawl := doc.CommandExitCodes["plan-pawl decide"] for code, name := range map[int]string{ planPawlExitPass: "PASS", planPawlExitUsage: "usage", @@ -70,11 +66,11 @@ func TestCapabilitiesCommandExitCodesMatchConstants(t *testing.T) { t.Errorf("plan-pawl decide: exit %d (%s) is a defined const but missing from capabilities", code, name) } } - if _, ok := capabilitiesCommandExitCodes["governor budget"][strconv.Itoa(hardenExitCode)]; !ok { + if _, ok := doc.CommandExitCodes["governor budget"][strconv.Itoa(hardenExitCode)]; !ok { t.Errorf("governor budget: hardenExitCode=%d missing from capabilities", hardenExitCode) } // pawl review codes come from scripts/pawl-review.sh (the pawl.go:29 contract): 0/1/2/3/4. - pawl := capabilitiesCommandExitCodes["pawl review"] + pawl := doc.CommandExitCodes["pawl review"] for _, code := range []string{"0", "1", "2", "3", "4"} { if pawl[code] == "" { t.Errorf("pawl review: documented contract code %s missing from capabilities", code) @@ -83,7 +79,7 @@ func TestCapabilitiesCommandExitCodesMatchConstants(t *testing.T) { } func TestCapabilities_ListsRegisteredCommands(t *testing.T) { - doc := buildCapabilitiesDoc() + doc := readCapabilitiesDocument(t) seen := map[string]bool{} for _, g := range doc.CommandGroups { for _, c := range g.Commands { @@ -99,7 +95,7 @@ func TestCapabilities_ListsRegisteredCommands(t *testing.T) { } func TestCapabilities_GlobalFlagsIncludeJSON(t *testing.T) { - doc := buildCapabilitiesDoc() + doc := readCapabilitiesDocument(t) found := false for _, f := range doc.GlobalFlags { if f.Name == "json" { @@ -112,17 +108,30 @@ func TestCapabilities_GlobalFlagsIncludeJSON(t *testing.T) { } func TestCapabilities_RegisteredOnRoot(t *testing.T) { - if capabilitiesCmd.GroupID != "core" { - t.Errorf("capabilitiesCmd.GroupID = %q, want core", capabilitiesCmd.GroupID) - } for _, c := range rootCmd.Commands() { if c.Name() == "capabilities" { + if c.GroupID != "core" { + t.Errorf("capabilities GroupID = %q, want core", c.GroupID) + } return } } t.Error("capabilities command not registered on rootCmd") } +func readCapabilitiesDocument(t *testing.T) capabilitiesapp.Document { + t.Helper() + out, err := executeCommand("capabilities") + if err != nil { + t.Fatalf("ao capabilities returned error: %v", err) + } + var document capabilitiesapp.Document + if err := json.Unmarshal([]byte(out), &document); err != nil { + t.Fatalf("ao capabilities output is not valid JSON: %v\noutput: %s", err, out) + } + return document +} + func TestRobotDocs_ContainsContractSections(t *testing.T) { out, err := executeCommand("robot-docs") if err != nil { diff --git a/cli/cmd/ao/group_json.go b/cli/cmd/ao/group_json.go index 5d860044f..14c2d10b9 100644 --- a/cli/cmd/ao/group_json.go +++ b/cli/cmd/ao/group_json.go @@ -13,11 +13,16 @@ import ( // Without this, cobra prints human help text to a caller that asked for a // machine-readable structure. type groupCommandListing struct { - Command string `json:"command"` - Description string `json:"description"` - IsGroup bool `json:"is_group"` - Subcommands []capCommand `json:"subcommands"` - Hint string `json:"hint"` + Command string `json:"command"` + Description string `json:"description"` + IsGroup bool `json:"is_group"` + Subcommands []groupCommand `json:"subcommands"` + Hint string `json:"hint"` +} + +type groupCommand struct { + Name string `json:"name"` + Short string `json:"short"` } // maybeEmitGroupJSON reports whether cmd is a non-runnable parent command @@ -32,12 +37,12 @@ func maybeEmitGroupJSON(cmd *cobra.Command) bool { if !jsonFlag && output != "json" { return false } - var subs []capCommand + var subs []groupCommand for _, c := range cmd.Commands() { if c.Hidden || c.Name() == "help" { continue } - subs = append(subs, capCommand{Name: c.Name(), Short: c.Short}) + subs = append(subs, groupCommand{Name: c.Name(), Short: c.Short}) } sort.Slice(subs, func(i, j int) bool { return subs[i].Name < subs[j].Name }) listing := groupCommandListing{ diff --git a/cli/internal/adapters/capabilities/cobra.go b/cli/internal/adapters/capabilities/cobra.go new file mode 100644 index 000000000..1a3a5f445 --- /dev/null +++ b/cli/internal/adapters/capabilities/cobra.go @@ -0,0 +1,74 @@ +// Package capabilities adapts the Cobra command tree and Go runtime to the +// capabilities application's dependency-neutral ports. +package capabilities + +import ( + "runtime" + "sort" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + capabilitiesapp "github.com/boshu2/agentops/cli/internal/capabilities" + "github.com/boshu2/agentops/cli/internal/clicontract" +) + +type CobraSurface struct { + root *cobra.Command +} + +func NewCobraSurface(root *cobra.Command) CobraSurface { + return CobraSurface{root: root} +} + +func (surface CobraSurface) Snapshot(commandExitCodes map[string]map[string]string) capabilitiesapp.Snapshot { + groups := map[string]*capabilitiesapp.CommandGroup{} + var order []string + for _, group := range surface.root.Groups() { + groups[group.ID] = &capabilitiesapp.CommandGroup{ID: group.ID, Title: group.Title} + order = append(order, group.ID) + } + const ungrouped = "" + groups[ungrouped] = &capabilitiesapp.CommandGroup{ID: ungrouped, Title: "Additional Commands:"} + order = append(order, ungrouped) + + for _, command := range surface.root.Commands() { + if command.Hidden || command.Name() == "help" || command.Name() == "completion" { + continue + } + group, ok := groups[command.GroupID] + if !ok { + group = groups[ungrouped] + } + group.Commands = append(group.Commands, capabilitiesapp.Command{Name: command.Name(), Short: command.Short}) + } + + var commandGroups []capabilitiesapp.CommandGroup + for _, id := range order { + group := groups[id] + if len(group.Commands) == 0 { + continue + } + sort.Slice(group.Commands, func(i, j int) bool { return group.Commands[i].Name < group.Commands[j].Name }) + commandGroups = append(commandGroups, *group) + } + + var globalFlags []capabilitiesapp.Flag + surface.root.PersistentFlags().VisitAll(func(flag *pflag.Flag) { + globalFlags = append(globalFlags, capabilitiesapp.Flag{ + Name: flag.Name, Shorthand: flag.Shorthand, Description: flag.Usage, + }) + }) + sort.Slice(globalFlags, func(i, j int) bool { return globalFlags[i].Name < globalFlags[j].Name }) + + return capabilitiesapp.Snapshot{ + GlobalFlags: globalFlags, CommandGroups: commandGroups, + Commands: clicontract.Inspect(surface.root, commandExitCodes), + } +} + +type RuntimePlatform struct{} + +func (RuntimePlatform) Platform() capabilitiesapp.Platform { + return capabilitiesapp.Platform{OS: runtime.GOOS, Arch: runtime.GOARCH} +} diff --git a/cli/internal/capabilities/document.go b/cli/internal/capabilities/document.go new file mode 100644 index 000000000..8517ffba8 --- /dev/null +++ b/cli/internal/capabilities/document.go @@ -0,0 +1,142 @@ +// Package capabilities owns the application policy for the machine-readable +// CLI contract. It is independent of Cobra and the host runtime. +package capabilities + +import "github.com/boshu2/agentops/cli/internal/clicontract" + +const ContractVersion = "1.1" + +type Document struct { + SchemaVersion string `json:"schema_version"` + Tool string `json:"tool"` + ToolVersion string `json:"tool_version"` + ContractVersion string `json:"contract_version"` + Platform Platform `json:"platform"` + GlobalFlags []Flag `json:"global_flags"` + OutputFormats []string `json:"output_formats"` + ExitCodes map[string]string `json:"exit_codes"` + CommandExitCodes map[string]map[string]string `json:"command_exit_codes"` + EnvVars map[string]string `json:"env_vars"` + RobotSurfaces map[string]string `json:"robot_surfaces"` + CommandGroups []CommandGroup `json:"command_groups"` + Commands []clicontract.Command `json:"commands"` +} + +type Platform struct { + OS string `json:"os"` + Arch string `json:"arch"` +} + +type Flag struct { + Name string `json:"name"` + Shorthand string `json:"shorthand,omitempty"` + Description string `json:"description"` +} + +type Command struct { + Name string `json:"name"` + Short string `json:"short"` +} + +type CommandGroup struct { + ID string `json:"id"` + Title string `json:"title"` + Commands []Command `json:"commands"` +} + +type Snapshot struct { + GlobalFlags []Flag + CommandGroups []CommandGroup + Commands []clicontract.Command +} + +type SurfaceReader interface { + Snapshot(commandExitCodes map[string]map[string]string) Snapshot +} + +type PlatformReader interface { + Platform() Platform +} + +type Builder interface { + Build() Document +} + +type Service struct { + toolVersion string + surface SurfaceReader + platform PlatformReader +} + +func NewService(toolVersion string, surface SurfaceReader, platform PlatformReader) Service { + return Service{toolVersion: toolVersion, surface: surface, platform: platform} +} + +func (service Service) Build() Document { + commandExits := commandExitCodes() + snapshot := service.surface.Snapshot(commandExits) + return Document{ + SchemaVersion: ContractVersion, + Tool: "ao", + ToolVersion: service.toolVersion, + ContractVersion: ContractVersion, + Platform: service.platform.Platform(), + GlobalFlags: snapshot.GlobalFlags, + OutputFormats: []string{"table", "json", "yaml"}, + ExitCodes: exitCodes(), + CommandExitCodes: commandExits, + EnvVars: envVars(), + RobotSurfaces: robotSurfaces(), + CommandGroups: snapshot.CommandGroups, + Commands: snapshot.Commands, + } +} + +func exitCodes() map[string]string { + return map[string]string{ + "0": "success", + "1": "error: usage error, runtime failure, or — for diagnostic commands — findings present", + "2": "diagnostic: partial result or bead claimed (command-specific; see that command's --help)", + } +} + +func commandExitCodes() map[string]map[string]string { + return map[string]map[string]string{ + "pawl review": { + "0": "CONFIRMED — cross-family verdict written", + "1": "hard error", + "2": "usage error", + "3": "REFUTED — auto-redo", + "4": "--converge advisory-only (no lineage written)", + }, + "plan-pawl decide": { + "0": "PASS — the door opens", + "2": "usage error", + "3": "REDO — auto-redo loop (no human)", + "4": "BLOCKED — a circuit breaker tripped (andon)", + "5": "DEGRADED — transient panel lane loss below quorum; retry the panel", + }, + "governor budget": { + "0": "ship — within error budget", + "3": "HARDEN — burn-rate over tolerance; stop the line", + }, + } +} + +func envVars() map[string]string { + return map[string]string{ + "AGENTOPS_CONFIG": "path to the config file (overridden by --config)", + "NO_COLOR": "disable ANSI styling on all output", + "AO_DOCTOR_LOG_LEVEL": "trace|debug|info|warn|error — verbosity of the doctor surface", + } +} + +func robotSurfaces() map[string]string { + return map[string]string{ + "capabilities": "ao capabilities — this contract", + "robot_docs": "ao robot-docs — paste-ready agent handbook (Markdown)", + "doctor_triage": "ao doctor --robot-triage — mega-command health triage JSON", + "doctor_capabilities": "ao doctor capabilities — extended doctor contract + exit codes", + "json_everywhere": "append --json (or -o json) to any read-side command for structured output", + } +} diff --git a/cli/internal/capabilities/document_test.go b/cli/internal/capabilities/document_test.go new file mode 100644 index 000000000..5810d2a61 --- /dev/null +++ b/cli/internal/capabilities/document_test.go @@ -0,0 +1,47 @@ +package capabilities + +import ( + "testing" + + "github.com/boshu2/agentops/cli/internal/clicontract" +) + +type recordingSurface struct { + exits map[string]map[string]string +} + +func (surface *recordingSurface) Snapshot(exits map[string]map[string]string) Snapshot { + surface.exits = exits + return Snapshot{ + GlobalFlags: []Flag{{Name: "json", Description: "structured output"}}, + CommandGroups: []CommandGroup{{ID: "core", Title: "Core", Commands: []Command{{Name: "capabilities", Short: "contract"}}}}, + Commands: []clicontract.Command{{ID: "ao.capabilities", Path: "ao capabilities"}}, + } +} + +type fixedPlatform struct{} + +func (fixedPlatform) Platform() Platform { return Platform{OS: "test-os", Arch: "test-arch"} } + +func TestServiceBuildsCompleteContractFromPorts(t *testing.T) { + surface := &recordingSurface{} + document := NewService("v-test", surface, fixedPlatform{}).Build() + if document.SchemaVersion != ContractVersion || document.ContractVersion != ContractVersion { + t.Fatalf("contract versions = %q/%q, want %q", document.SchemaVersion, document.ContractVersion, ContractVersion) + } + if document.Tool != "ao" || document.ToolVersion != "v-test" { + t.Fatalf("tool identity = %q/%q", document.Tool, document.ToolVersion) + } + if document.Platform.OS != "test-os" || document.Platform.Arch != "test-arch" { + t.Fatalf("platform = %+v", document.Platform) + } + if len(document.GlobalFlags) != 1 || len(document.CommandGroups) != 1 || len(document.Commands) != 1 { + t.Fatalf("surface projection incomplete: %+v", document) + } + if surface.exits["plan-pawl decide"]["5"] == "" || document.CommandExitCodes["pawl review"]["3"] == "" { + t.Fatal("typed command exit codes did not cross the surface port") + } + if len(document.OutputFormats) != 3 || document.ExitCodes["0"] != "success" { + t.Fatal("stable output and exit dictionaries are incomplete") + } +} diff --git a/cli/internal/commands/capabilities/module.go b/cli/internal/commands/capabilities/module.go new file mode 100644 index 000000000..dd1d177cf --- /dev/null +++ b/cli/internal/commands/capabilities/module.go @@ -0,0 +1,70 @@ +// Package capabilities owns the Cobra presentation for the capabilities +// command. The handler delegates document construction and only renders it. +package capabilities + +import ( + "encoding/json" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + + capabilitiesapp "github.com/boshu2/agentops/cli/internal/capabilities" + "github.com/boshu2/agentops/cli/internal/clicontract" +) + +type Module struct { + builder capabilitiesapp.Builder + output func() string +} + +func NewModule(builder capabilitiesapp.Builder, output func() string) Module { + return Module{builder: builder, output: output} +} + +func (Module) Contract() clicontract.CommandContract { + return clicontract.CommandContract{ + ID: "ao.capabilities", + Profiles: clicontract.ProfileDefault | + clicontract.ProfileFlywheel | + clicontract.ProfileLegacy | + clicontract.ProfileCombined, + Args: clicontract.ArgsPolicy{Name: "none", Validate: cobra.NoArgs}, + Output: clicontract.OutputStructured, + Effects: clicontract.EffectPure, + ExitClasses: map[int]clicontract.ExitClass{0: clicontract.ExitSuccess, 1: clicontract.ExitFailure}, + } +} + +func (module Module) Command() *cobra.Command { + command := &cobra.Command{ + Use: "capabilities", + Short: "Print the machine-readable CLI contract (JSON)", + Long: `Print the machine-readable contract for the whole ao CLI as JSON. + +This is the first command an agent should run to discover the command +surface, flag conventions, exit-code dictionary, and every other +machine-readable surface — no external documentation lookup required. + +Output is always JSON; it is stable across patch versions (pinned by +contract_version).`, + } + command.RunE = func(command *cobra.Command, _ []string) error { + return module.render(command) + } + return command +} + +func (module Module) render(command *cobra.Command) error { + document := module.builder.Build() + if module.output != nil && module.output() == "yaml" { + data, err := yaml.Marshal(document) + if err != nil { + return err + } + _, err = command.OutOrStdout().Write(data) + return err + } + encoder := json.NewEncoder(command.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(document) +} diff --git a/cli/internal/commands/capabilities/module_test.go b/cli/internal/commands/capabilities/module_test.go new file mode 100644 index 000000000..626f857fd --- /dev/null +++ b/cli/internal/commands/capabilities/module_test.go @@ -0,0 +1,73 @@ +package capabilities + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + capabilitiesapp "github.com/boshu2/agentops/cli/internal/capabilities" + "github.com/boshu2/agentops/cli/internal/clicontract" +) + +type recordingBuilder struct { + calls int +} + +func (builder *recordingBuilder) Build() capabilitiesapp.Document { + builder.calls++ + return capabilitiesapp.Document{ + SchemaVersion: capabilitiesapp.ContractVersion, + Tool: "ao", + Platform: capabilitiesapp.Platform{OS: "test-os", Arch: "test-arch"}, + } +} + +func TestCommandDelegatesAndRendersJSON(t *testing.T) { + builder := &recordingBuilder{} + module := NewModule(builder, func() string { return "json" }) + command := module.Command() + var output bytes.Buffer + command.SetOut(&output) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if builder.calls != 1 { + t.Fatalf("Build calls = %d, want 1", builder.calls) + } + var document capabilitiesapp.Document + if err := json.Unmarshal(output.Bytes(), &document); err != nil { + t.Fatalf("JSON output: %v\n%s", err, output.String()) + } + if document.Platform.OS != "test-os" { + t.Fatalf("document = %+v", document) + } +} + +func TestCommandDelegatesAndRendersYAML(t *testing.T) { + builder := &recordingBuilder{} + command := NewModule(builder, func() string { return "yaml" }).Command() + var output bytes.Buffer + command.SetOut(&output) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if builder.calls != 1 || !strings.Contains(output.String(), "schemaversion: \"1.1\"") { + t.Fatalf("YAML output = %q, calls=%d", output.String(), builder.calls) + } +} + +func TestModuleOwnsFreshCommandsAndExplicitContract(t *testing.T) { + module := NewModule(&recordingBuilder{}, nil) + if module.Command() == module.Command() { + t.Fatal("Command returned shared Cobra state") + } + contract := module.Contract() + if err := clicontract.ValidateContract(contract); err != nil { + t.Fatal(err) + } + wantProfiles := clicontract.ProfileDefault | clicontract.ProfileFlywheel | clicontract.ProfileLegacy | clicontract.ProfileCombined + if contract.Profiles != wantProfiles || contract.Effects != clicontract.EffectPure || contract.Output != clicontract.OutputStructured { + t.Fatalf("contract = %+v", contract) + } +} From a77eea71cfb3df8b6054bdb718b33bef7438c41c Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:50:06 -0400 Subject: [PATCH 032/108] chore(cli): seal capabilities migration lineage (age-nw28h.2.2) --- .../compatibility-baseline/families/capabilities/lineage.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/testdata/compatibility-baseline/families/capabilities/lineage.json b/cli/testdata/compatibility-baseline/families/capabilities/lineage.json index 01f47c517..1c499964f 100644 --- a/cli/testdata/compatibility-baseline/families/capabilities/lineage.json +++ b/cli/testdata/compatibility-baseline/families/capabilities/lineage.json @@ -6,5 +6,6 @@ "fixture_sha256": "9788d6c4206e5d01e7609724bb7dec88946ebcd67302e53fca1962b7f0db5303", "ownership_sha256": "7d9491418c90a931611d81e5e5308920db6f8a1fc3a90c4d684e5f961a85cf6f", "scope_sha256": "62cd117cf63f443d513a84761f4c99dffcede9034ef5204b48a3c39c21e8a4e1", - "migration_state": "migrating" + "migration_state": "migrated", + "accepted_sha": "bd171a4776a303a8c7ad867047f70967cdee8b0e" } From d256688e8ea3d54682e36aa4f325205680420f25 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:50:19 -0400 Subject: [PATCH 033/108] chore(provenance): bind pawl CONFIRMED verdict for age-nw28h.2.2 #trivial --- docs/provenance/ledger.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/provenance/ledger.jsonl b/docs/provenance/ledger.jsonl index a75f1c112..7f98612dd 100644 --- a/docs/provenance/ledger.jsonl +++ b/docs/provenance/ledger.jsonl @@ -517,3 +517,4 @@ {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-h433.38@c9c428a","from_type":"verdict","to_id":"c9c428aadd56696d76407d24a2de99b843ecef0c","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-h433.38 disposition=CONFIRMED","bead_id":"age-h433.38","trust_tier":"inferred","ts":"2026-07-11T17:13:41Z","reviewer_family":"gemini","rounds":1,"duration_s":637,"tokens_est":177,"evidence_path":"/Users/bo/dev/agentops/.agents/pawl-evidence/age-h433.38-pawl-review.txt","prev_hash":"3e2dc04605c17f4f96e094e6ba0486cee1e424594748c646a853b13726df7dc0","payload_hash":"caf985741efe5bddebbf17b9a849aff82f4d7f84cec5836e600b2a7b9acc1025","hash":"24e841e8569aa9375c46d26cf6f6940e2a765d6e0d28ce7e0dd86240481dbae5"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.1.5@e76a715","from_type":"verdict","to_id":"e76a715b00ad560df06fbd00da9783f3bcef7ba6","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.1.5 disposition=CONFIRMED","bead_id":"age-nw28h.1.5","trust_tier":"inferred","ts":"2026-07-11T17:26:53Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/P4A-rebase-verdict-e76a715b0.txt","prev_hash":"24e841e8569aa9375c46d26cf6f6940e2a765d6e0d28ce7e0dd86240481dbae5","payload_hash":"3df68eadc14f39362ab69431db77b379cc829298967e0d76ede07b7b3b34e0e9","hash":"5ce3f9f10999351cb3ce663d1dab51fbb418291ebf9e6ad6c507acc8550a7bbe"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.1@4863195","from_type":"verdict","to_id":"4863195d07079bb4f995a84686b2c572afe070fb","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.1 disposition=CONFIRMED","bead_id":"age-nw28h.2.1","trust_tier":"inferred","ts":"2026-07-11T19:10:35Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-beads-independent-verdict-4863195d0.txt","prev_hash":"5ce3f9f10999351cb3ce663d1dab51fbb418291ebf9e6ad6c507acc8550a7bbe","payload_hash":"ff4b1163efdb38e333b7ce4358ae7dd9a56a86d0645d481750c3a1ac0a860acb","hash":"0dbb491f180b24cbd6fd602bf8812d14382184c08370a714c7e4905f3730ca0e"} +{"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.2@bd171a4","from_type":"verdict","to_id":"bd171a4776a303a8c7ad867047f70967cdee8b0e","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.2 disposition=CONFIRMED","bead_id":"age-nw28h.2.2","trust_tier":"inferred","ts":"2026-07-11T19:50:19Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-capabilities-independent-verdict-bd171a477.txt","prev_hash":"0dbb491f180b24cbd6fd602bf8812d14382184c08370a714c7e4905f3730ca0e","payload_hash":"2ed1b654481b2037e5e1b8cb439cafe29bb16cf6319181b5e42bc549eb1a9e79","hash":"9739edafbb15e8a5d7e0ac15be295093bb14ebf60ffc13f92d46db35942e5aca"} From d63c679bd50945eb0eac3bae322b22adc0553c95 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:02:26 -0400 Subject: [PATCH 034/108] fix(claim): normalize check args and ledger root (age-nw28h.2.3) --- cli/cmd/ao/claim_cmd.go | 3 ++- cli/cmd/ao/claim_cmd_test.go | 39 ++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/cli/cmd/ao/claim_cmd.go b/cli/cmd/ao/claim_cmd.go index b64d3846c..1835a8fac 100644 --- a/cli/cmd/ao/claim_cmd.go +++ b/cli/cmd/ao/claim_cmd.go @@ -58,6 +58,7 @@ var claimListCmd = &cobra.Command{ var claimCheckCmd = &cobra.Command{ Use: "check --changed [--base ]", Short: "Report proof cards for changed public claims", + Args: cobra.NoArgs, Long: `Report read-only proof cards for changed public claim markers. The checker compares the current branch/worktree to a base ref, finds changed @@ -278,7 +279,7 @@ func validateEvidenceLevelString(s string) error { } func claimBindingsPath() (string, error) { - cwd, err := resolveProjectDir() + cwd, err := repoRootOrCwd() if err != nil { return "", err } diff --git a/cli/cmd/ao/claim_cmd_test.go b/cli/cmd/ao/claim_cmd_test.go index 9ba83fd4e..55945303b 100644 --- a/cli/cmd/ao/claim_cmd_test.go +++ b/cli/cmd/ao/claim_cmd_test.go @@ -6,6 +6,8 @@ import ( "context" "encoding/json" "errors" + "os" + "path/filepath" "strings" "testing" @@ -170,6 +172,43 @@ func TestClaimCheckCommandRegistered(t *testing.T) { } } +func TestClaimCheckAcceptsNoPositionals(t *testing.T) { + if claimCheckCmd.Args == nil { + t.Fatal("claim check must declare an explicit positional-argument policy") + } + if err := claimCheckCmd.Args(claimCheckCmd, nil); err != nil { + t.Fatalf("claim check rejected zero positional arguments: %v", err) + } + if err := claimCheckCmd.Args(claimCheckCmd, []string{"unexpected"}); err == nil { + t.Fatal("claim check accepted an unexpected positional argument") + } +} + +func TestClaimBindingsPathUsesRepositoryRoot(t *testing.T) { + repo := t.TempDir() + initMinimalGitRepo(t, repo) + subdir := filepath.Join(repo, "nested") + if err := os.Mkdir(subdir, 0o755); err != nil { + t.Fatal(err) + } + old := testProjectDir + testProjectDir = subdir + t.Cleanup(func() { testProjectDir = old }) + + path, err := claimBindingsPath() + if err != nil { + t.Fatal(err) + } + root, err := resolveRepoRoot(subdir) + if err != nil { + t.Fatal(err) + } + want := filepath.Join(root, ".agents", "findings", "evidence-bindings.jsonl") + if path != want { + t.Fatalf("claimBindingsPath() = %q, want repo-rooted %q", path, want) + } +} + func TestClaimCheckJSONOutput(t *testing.T) { var got claimproof.Options stub := func(_ context.Context, opts claimproof.Options) (claimproof.Report, error) { From c6aa8b5b4c7bff71fac4512dbcb3acda44b63e66 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:55:42 -0400 Subject: [PATCH 035/108] test(cli): freeze claim compatibility contract (age-nw28h.2.3) --- .../families/claim/case.json | 21 ++++++++ .../families/claim/ownership.json | 48 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/claim/case.json create mode 100644 cli/testdata/compatibility-baseline/families/claim/ownership.json diff --git a/cli/testdata/compatibility-baseline/families/claim/case.json b/cli/testdata/compatibility-baseline/families/claim/case.json new file mode 100644 index 000000000..d99b9da8f --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/claim/case.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "family": "claim", + "checks": [ + { + "id": "claim-black-box", + "command": "tmp=$(mktemp -d /tmp/ao-claim.XXXXXX); trap 'rm -r \"$tmp\"' EXIT; (cd cli && go build -o \"$tmp/ao\" ./cmd/ao); mkdir -p \"$tmp/repo/_beads\" \"$tmp/repo/sub\" \"$tmp/bin\"; git -C \"$tmp/repo\" init -q; printf '%s\\n' '#!/usr/bin/env bash' 'printf \"%s\\n\" \"$*\" >\"$TRACE_FILE\"' 'printf \"claimed-output\\n\"' >\"$tmp/bin/br\"; chmod +x \"$tmp/bin/br\"; (cd \"$tmp/repo\" && PATH=\"$tmp/bin:$PATH\" BEADS_DIR=\"$tmp/repo/_beads\" TRACE_FILE=\"$tmp/trace\" \"$tmp/ao\" claim age-1 >\"$tmp/claim.out\"); grep -qx 'update age-1 --claim' \"$tmp/trace\"; grep -qx 'claimed-output' \"$tmp/claim.out\"; (cd \"$tmp/repo\" && \"$tmp/ao\" claim >\"$tmp/help\" && \"$tmp/ao\" claim check --help >\"$tmp/check-help\"); (cd \"$tmp/repo/sub\" && \"$tmp/ao\" claim bind --claim AOP-X --path p.md --level PG2 >\"$tmp/bind.out\"); (cd \"$tmp/repo\" && \"$tmp/ao\" claim list >\"$tmp/list.out\"); grep -q 'Claim a BR bead for harness-neutral AgentOps loops' \"$tmp/help\"; grep -q 'Report read-only proof cards for changed public claim markers' \"$tmp/check-help\"; grep -qx 'bound claim=\"AOP-X\" path=\"p.md\" level=PG2' \"$tmp/bind.out\"; jq -e '.Claim == \"AOP-X\" and .Path == \"p.md\" and .Level == \"PG2\"' \"$tmp/list.out\" >/dev/null; set +e; (cd \"$tmp/repo\" && \"$tmp/ao\" claim check --changed >\"$tmp/check.out\" 2>\"$tmp/check.err\"); check_code=$?; \"$tmp/ao\" claim one two >\"$tmp/rejected.out\" 2>\"$tmp/rejected.err\"; code=$?; set -e; test \"$check_code\" -eq 1; grep -q 'Error: claim check: claim proof: read claim registry:' \"$tmp/check.err\"; test \"$code\" -eq 1; test ! -s \"$tmp/rejected.out\"; grep -qx 'Error: accepts between 0 and 1 arg(s), received 2' \"$tmp/rejected.err\"" + } + ], + "dimensions": { + "path": {"status": "evidence", "evidence": ["claim-black-box"]}, + "aliases": {"status": "not_applicable", "reason": "The ao claim command family declares no aliases."}, + "accepted_args": {"status": "evidence", "evidence": ["claim-black-box"]}, + "rejected_args": {"status": "evidence", "evidence": ["claim-black-box"]}, + "stdout": {"status": "evidence", "evidence": ["claim-black-box"]}, + "stderr": {"status": "evidence", "evidence": ["claim-black-box"]}, + "exit_classes": {"status": "evidence", "evidence": ["claim-black-box"]}, + "tracker_selection": {"status": "evidence", "evidence": ["claim-black-box"]}, + "ordered_effects": {"status": "evidence", "evidence": ["claim-black-box"]} + } +} diff --git a/cli/testdata/compatibility-baseline/families/claim/ownership.json b/cli/testdata/compatibility-baseline/families/claim/ownership.json new file mode 100644 index 000000000..c945abf96 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/claim/ownership.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "family": "claim", + "profiles": { + "default": "present", + "flywheel": "present", + "legacy": "present", + "combined": "present" + }, + "legacy_symbols": [ + "claimCmd", + "claimBindCmd", + "claimListCmd", + "claimCheckCmd", + "claimOptions", + "claimCheckOptions", + "runClaim", + "runClaimBind", + "runClaimList", + "runClaimCheck", + "claimBindRun", + "claimListRun", + "claimCheckRun", + "writeClaimCheckHuman", + "validateEvidenceLevelString", + "claimBindingsPath", + "claimBindViaPort", + "claimListViaPort", + "productionClaimEvidence", + "newProductionClaimEvidence", + "productionPromoteEvidenceLevel", + "productionEvidenceLevelOrd", + "productionClaimEvidenceBinder", + "newProductionClaimEvidenceBinder", + "evidenceBindingRecord", + "evidenceLevelRank", + "anchorsEqual" + ], + "live_owner": "cli/internal/commands/claim", + "allowed_paths": [ + "cli/cmd/ao/claim*.go", + "cli/internal/adapters/claim/**", + "cli/internal/claim/**", + "cli/internal/claimproof/**", + "cli/internal/commands/claim/**", + "cli/testdata/compatibility-baseline/families/claim/lineage.json" + ] +} From 25150074295d7559fc78cfc7e8a82e47eb327c0e Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:55:57 -0400 Subject: [PATCH 036/108] test(cli): bind claim migration lineage (age-nw28h.2.3) --- .../compatibility-baseline/families/claim/lineage.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/claim/lineage.json diff --git a/cli/testdata/compatibility-baseline/families/claim/lineage.json b/cli/testdata/compatibility-baseline/families/claim/lineage.json new file mode 100644 index 000000000..e6b813baf --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/claim/lineage.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "family": "claim", + "old_implementation_sha": "d63c679bd50945eb0eac3bae322b22adc0553c95", + "freeze_sha": "c6aa8b5b4c7bff71fac4512dbcb3acda44b63e66", + "fixture_sha256": "834134b9e48db67016306dc26cc33c096c38478b57b601da052bbcd576847271", + "ownership_sha256": "7b75df0636c4f3af37babf3abcefa82ea030be9437369bd4b164883e270f243a", + "scope_sha256": "abf4f818b3e99612470df5e2b3dd61ad4220dcba9bf3b234bfd3468d16e197ca", + "migration_state": "migrating" +} From 66abdd2e126e5ecb81a0c70c6c222304ce5b5a21 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:14:32 -0400 Subject: [PATCH 037/108] refactor(cli): migrate claim command module (age-nw28h.2.3) --- cli/cmd/ao/claim_cmd.go | 315 ----------------- cli/cmd/ao/claim_cmd_test.go | 326 ------------------ cli/cmd/ao/claim_evidence_adapter.go | 106 ------ cli/cmd/ao/claim_evidence_adapter_test.go | 154 --------- cli/cmd/ao/claim_evidence_binder_adapter.go | 208 ----------- .../ao/claim_evidence_binder_adapter_test.go | 225 ------------ cli/cmd/ao/claim_module.go | 27 ++ cli/cmd/ao/claim_module_test.go | 25 ++ cli/internal/adapters/claim/proof.go | 49 +++ cli/internal/adapters/claim/store.go | 193 +++++++++++ cli/internal/adapters/claim/store_test.go | 56 +++ cli/internal/adapters/claim/tracker.go | 55 +++ cli/internal/adapters/claim/tracker_test.go | 51 +++ cli/internal/claim/service.go | 105 ++++++ cli/internal/claim/service_test.go | 79 +++++ cli/internal/claimproof/check.go | 84 ++--- cli/internal/claimproof/check_test.go | 33 +- cli/internal/commands/claim/module.go | 202 +++++++++++ cli/internal/commands/claim/module_test.go | 94 +++++ 19 files changed, 992 insertions(+), 1395 deletions(-) delete mode 100644 cli/cmd/ao/claim_cmd.go delete mode 100644 cli/cmd/ao/claim_cmd_test.go delete mode 100644 cli/cmd/ao/claim_evidence_adapter.go delete mode 100644 cli/cmd/ao/claim_evidence_adapter_test.go delete mode 100644 cli/cmd/ao/claim_evidence_binder_adapter.go delete mode 100644 cli/cmd/ao/claim_evidence_binder_adapter_test.go create mode 100644 cli/cmd/ao/claim_module.go create mode 100644 cli/cmd/ao/claim_module_test.go create mode 100644 cli/internal/adapters/claim/proof.go create mode 100644 cli/internal/adapters/claim/store.go create mode 100644 cli/internal/adapters/claim/store_test.go create mode 100644 cli/internal/adapters/claim/tracker.go create mode 100644 cli/internal/adapters/claim/tracker_test.go create mode 100644 cli/internal/claim/service.go create mode 100644 cli/internal/claim/service_test.go create mode 100644 cli/internal/commands/claim/module.go create mode 100644 cli/internal/commands/claim/module_test.go diff --git a/cli/cmd/ao/claim_cmd.go b/cli/cmd/ao/claim_cmd.go deleted file mode 100644 index 1835a8fac..000000000 --- a/cli/cmd/ao/claim_cmd.go +++ /dev/null @@ -1,315 +0,0 @@ -// practices: [hexagonal-architecture, ddd-bounded-context] -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "github.com/spf13/cobra" - - "github.com/boshu2/agentops/cli/internal/claimproof" - "github.com/boshu2/agentops/cli/internal/ports" -) - -// claimCmd is the parent for BC2 ClaimEvidenceBinderPort CLI -// surfaces. Built using the cycle-147 cli-wiring template. -var claimCmd = &cobra.Command{ - Use: "claim [id]", - Short: "Claim a BR bead or manage claim-evidence bindings", - Long: `Claim a BR bead for harness-neutral AgentOps loops, or bind/list claim evidence via the typed BC2 ClaimEvidenceBinderPort. - -Examples: - ao claim cp-123 - ao claim bind --claim AOP-CLAIM-X --path .agents/findings/x.md --level PG2 - ao claim list`, - RunE: runClaim, -} - -var claimBindCmd = &cobra.Command{ - Use: "bind --claim --path [--level PG1|PG2|PG3|PG4] [--anchor ...] [--author-id --judge-id ]", - Short: "Bind a claim to an evidence file at a promotion level", - Long: `Append (or upgrade) a claim→evidence binding via the typed BC2 -ClaimEvidenceBinderPort (productionClaimEvidenceBinder, cycle 116). - -The binder is append-only on disk; List replays the file and folds -to the latest per (Claim, Path). The Level can only go UP — attempting -to downgrade (e.g., PG3 → PG1) returns an error. - -Examples: - ao claim bind --claim AOP-CLAIM-X --path .agents/findings/x.md --level PG2 - ao claim bind --claim AOP-CLAIM-Y --path p.md --level PG4 --anchor L10 --anchor L20 - ao claim bind --claim AOP-CLAIM-Z --path verdict.md --level PG4 --author-id worker-1 --judge-id verifier-2`, - RunE: runClaimBind, -} - -var claimListCmd = &cobra.Command{ - Use: "list", - Short: "List recorded claim→evidence bindings (most-recent first)", - Long: `Emit all known claim→evidence bindings via the typed BC2 ClaimEvidenceBinderPort. Output is line-delimited JSON.`, - RunE: runClaimList, -} - -var claimCheckCmd = &cobra.Command{ - Use: "check --changed [--base ]", - Short: "Report proof cards for changed public claims", - Args: cobra.NoArgs, - Long: `Report read-only proof cards for changed public claim markers. - -The checker compares the current branch/worktree to a base ref, finds changed -files containing agentops:claim markers, then reports each claim's registry tier, -evidence citation status, verdict, and next action. It does not bind evidence, -promote tiers, mutate beads, or edit the claim registry. - -Examples: - ao claim check --changed - ao claim check --changed --base origin/main --json`, - RunE: runClaimCheck, -} - -type claimOptions struct { - claim string - path string - level string - anchors []string - authorID string - judgeID string - writer io.Writer - bindFn func(ctx context.Context, opts claimOptions) error - listFn func(ctx context.Context, opts claimOptions) ([]ports.EvidenceBinding, error) -} - -type claimCheckOptions struct { - repoRoot string - base string - changedOnly bool - jsonOutput bool - writer io.Writer - checkFn func(context.Context, claimproof.Options) (claimproof.Report, error) -} - -func init() { - claimCmd.GroupID = "core" - rootCmd.AddCommand(claimCmd) - - claimBindCmd.Flags().String("claim", "", "claim ID (required, e.g. AOP-CLAIM-X)") - claimBindCmd.Flags().String("path", "", "evidence file path (required, relative to repo root)") - claimBindCmd.Flags().String("level", "PG1", "promotion level: PG1|PG2|PG3|PG4") - claimBindCmd.Flags().StringArray("anchor", nil, "optional in-file anchors (repeatable)") - claimBindCmd.Flags().String("author-id", "", "artifact author identity for reviewer separation checks") - claimBindCmd.Flags().String("judge-id", "", "judge/verifier identity for reviewer separation checks") - _ = claimBindCmd.MarkFlagRequired("claim") - _ = claimBindCmd.MarkFlagRequired("path") - claimCmd.AddCommand(claimBindCmd) - - claimCmd.AddCommand(claimListCmd) - - claimCheckCmd.Flags().Bool("changed", false, "check claim markers in files changed against --base plus worktree changes") - claimCheckCmd.Flags().String("base", "origin/main", "base ref for --changed comparison") - claimCmd.AddCommand(claimCheckCmd) -} - -func runClaim(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - return cmd.Help() - } - return tickTrackerPassthrough(newTickRuntime(cmd), "update", args[0], "--claim") -} - -func runClaimBind(cmd *cobra.Command, _ []string) error { - claim, _ := cmd.Flags().GetString("claim") - path, _ := cmd.Flags().GetString("path") - level, _ := cmd.Flags().GetString("level") - anchors, _ := cmd.Flags().GetStringArray("anchor") - authorID, _ := cmd.Flags().GetString("author-id") - judgeID, _ := cmd.Flags().GetString("judge-id") - return claimBindRun(cmd.Context(), claimOptions{ - claim: claim, - path: path, - level: level, - anchors: anchors, - authorID: authorID, - judgeID: judgeID, - writer: cmd.OutOrStdout(), - }) -} - -func runClaimList(cmd *cobra.Command, _ []string) error { - return claimListRun(cmd.Context(), claimOptions{ - writer: cmd.OutOrStdout(), - }) -} - -func runClaimCheck(cmd *cobra.Command, _ []string) error { - changedOnly, _ := cmd.Flags().GetBool("changed") - base, _ := cmd.Flags().GetString("base") - repoRoot, err := repoRootOrCwd() - if err != nil { - return err - } - return claimCheckRun(cmd.Context(), claimCheckOptions{ - repoRoot: repoRoot, - base: base, - changedOnly: changedOnly, - jsonOutput: GetOutput() == "json", - writer: cmd.OutOrStdout(), - }) -} - -func claimBindRun(ctx context.Context, opts claimOptions) error { - if opts.claim == "" { - return errors.New("claim bind: --claim required\n Example: ao claim bind --claim AOP-CLAIM-1 --path internal/foo.go") - } - if opts.path == "" { - return errors.New("claim bind: --path required (evidence file, relative to repo root)\n Example: ao claim bind --claim AOP-CLAIM-1 --path internal/foo.go") - } - if err := validateEvidenceLevelString(opts.level); err != nil { - return fmt.Errorf("claim bind: %w", err) - } - fn := opts.bindFn - if fn == nil { - fn = claimBindViaPort - } - if err := fn(ctx, opts); err != nil { - return fmt.Errorf("claim bind: %w", err) - } - if opts.writer == nil { - opts.writer = os.Stdout - } - fmt.Fprintf(opts.writer, "bound claim=%q path=%q level=%s\n", opts.claim, opts.path, opts.level) - return nil -} - -func claimListRun(ctx context.Context, opts claimOptions) error { - if opts.writer == nil { - opts.writer = os.Stdout - } - fn := opts.listFn - if fn == nil { - fn = claimListViaPort - } - bindings, err := fn(ctx, opts) - if err != nil { - return fmt.Errorf("claim list: %w", err) - } - enc := json.NewEncoder(opts.writer) - for _, b := range bindings { - if err := enc.Encode(b); err != nil { - return fmt.Errorf("claim list encode: %w", err) - } - } - return nil -} - -func claimCheckRun(ctx context.Context, opts claimCheckOptions) error { - if !opts.changedOnly { - return errors.New("claim check: --changed is required for this read-only MVP") - } - if opts.writer == nil { - opts.writer = os.Stdout - } - fn := opts.checkFn - if fn == nil { - fn = claimproof.Check - } - report, err := fn(ctx, claimproof.Options{ - RepoRoot: opts.repoRoot, - Base: opts.base, - ChangedOnly: opts.changedOnly, - }) - if err != nil { - return fmt.Errorf("claim check: %w", err) - } - if opts.jsonOutput { - enc := json.NewEncoder(opts.writer) - enc.SetIndent("", " ") - return enc.Encode(report) - } - writeClaimCheckHuman(opts.writer, report) - return nil -} - -func writeClaimCheckHuman(w io.Writer, report claimproof.Report) { - fmt.Fprintf(w, "claim check: %d changed claim(s) across %d surface(s) (base=%s)\n", - report.Summary.Claims, report.Summary.ChangedSurfaces, report.Summary.Base) - if len(report.Cards) == 0 { - fmt.Fprintln(w, "No changed claim markers found.") - return - } - for _, card := range report.Cards { - fmt.Fprintf(w, "\n%s\n", card.ClaimID) - fmt.Fprintf(w, " surface: %s\n", card.Surface) - fmt.Fprintf(w, " tier: %s\n", card.Tier) - if len(card.CiteAllowed) > 0 { - fmt.Fprintf(w, " citation_ok: %t (allowed: %s)\n", card.CitationOK, strings.Join(card.CiteAllowed, ", ")) - } - fmt.Fprintf(w, " verdict: %s\n", card.Verdict) - if card.EvalBinding != "" { - fmt.Fprintf(w, " eval_binding: %s\n", card.EvalBinding) - } - if len(card.Evidence) == 0 { - fmt.Fprintln(w, " evidence: none") - } else { - fmt.Fprintln(w, " evidence:") - for _, evidence := range card.Evidence { - if evidence.Reason == "" { - fmt.Fprintf(w, " - %s [%s]\n", evidence.Path, evidence.Status) - } else { - fmt.Fprintf(w, " - %s [%s] %s\n", evidence.Path, evidence.Status, evidence.Reason) - } - } - } - fmt.Fprintf(w, " next: %s\n", card.NextAction) - } -} - -// validateEvidenceLevelString accepts PG1, PG2, PG3, PG4 (or empty -// which becomes None at the port boundary). -func validateEvidenceLevelString(s string) error { - switch strings.ToUpper(s) { - case "", "PG1", "PG2", "PG3", "PG4": - return nil - } - return fmt.Errorf("invalid --level %q (want PG1|PG2|PG3|PG4)", s) -} - -func claimBindingsPath() (string, error) { - cwd, err := repoRootOrCwd() - if err != nil { - return "", err - } - return filepath.Join(cwd, ".agents", "findings", "evidence-bindings.jsonl"), nil -} - -func claimBindViaPort(ctx context.Context, opts claimOptions) error { - path, err := claimBindingsPath() - if err != nil { - return err - } - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return fmt.Errorf("mkdir: %w", err) - } - b := newProductionClaimEvidenceBinder(path) - return b.Bind(ctx, ports.EvidenceBinding{ - Claim: ports.ClaimID(opts.claim), - Path: opts.path, - Level: ports.EvidenceLevel(strings.ToUpper(opts.level)), - Anchors: opts.anchors, - AuthorID: opts.authorID, - JudgeID: opts.judgeID, - }) -} - -func claimListViaPort(ctx context.Context, opts claimOptions) ([]ports.EvidenceBinding, error) { - path, err := claimBindingsPath() - if err != nil { - return nil, err - } - b := newProductionClaimEvidenceBinder(path) - return b.List(ctx) -} diff --git a/cli/cmd/ao/claim_cmd_test.go b/cli/cmd/ao/claim_cmd_test.go deleted file mode 100644 index 55945303b..000000000 --- a/cli/cmd/ao/claim_cmd_test.go +++ /dev/null @@ -1,326 +0,0 @@ -// practices: [tdd] -package main - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/boshu2/agentops/cli/internal/claimproof" - "github.com/boshu2/agentops/cli/internal/ports" -) - -func TestClaimBind_EmptyClaimRejected(t *testing.T) { - err := claimBindRun(context.Background(), claimOptions{path: "p", level: "PG2"}) - if err == nil { - t.Fatal("expected error on empty claim") - } -} - -func TestClaimBind_EmptyPathRejected(t *testing.T) { - err := claimBindRun(context.Background(), claimOptions{claim: "X", level: "PG2"}) - if err == nil { - t.Fatal("expected error on empty path") - } -} - -func TestClaimBind_InvalidLevelRejected(t *testing.T) { - err := claimBindRun(context.Background(), claimOptions{claim: "X", path: "p", level: "PG99"}) - if err == nil { - t.Fatal("expected error on bogus level") - } - if !strings.Contains(err.Error(), "invalid --level") { - t.Fatalf("error not informative: %v", err) - } -} - -func TestClaimBind_StubCalledWithBinding(t *testing.T) { - called := false - var gotOpts claimOptions - stub := func(_ context.Context, opts claimOptions) error { - called = true - gotOpts = opts - return nil - } - var buf bytes.Buffer - err := claimBindRun(context.Background(), claimOptions{ - claim: "AOP-CLAIM-X", - path: ".agents/findings/x.md", - level: "PG3", - anchors: []string{"L10", "L20"}, - authorID: "agent-1", - judgeID: "agent-2", - writer: &buf, - bindFn: stub, - }) - if err != nil { - t.Fatal(err) - } - if !called { - t.Fatal("bindFn not invoked") - } - if gotOpts.claim != "AOP-CLAIM-X" || gotOpts.level != "PG3" { - t.Fatalf("opts mismatch: %+v", gotOpts) - } - if gotOpts.authorID != "agent-1" || gotOpts.judgeID != "agent-2" { - t.Fatalf("reviewer metadata mismatch: %+v", gotOpts) - } - if !strings.Contains(buf.String(), `level=PG3`) { - t.Fatalf("confirmation missing level: %q", buf.String()) - } -} - -func TestClaimBind_DefaultLevelIsPG1(t *testing.T) { - // Note: level default is set by cobra; in the test path we pass it explicitly. - // This checks the validator accepts PG1 default value. - stub := func(_ context.Context, _ claimOptions) error { return nil } - err := claimBindRun(context.Background(), claimOptions{ - claim: "X", - path: "p", - level: "PG1", - bindFn: stub, - }) - if err != nil { - t.Fatal(err) - } -} - -func TestClaimBind_StubErrorWrapped(t *testing.T) { - stub := func(_ context.Context, _ claimOptions) error { - return errors.New("downgrade rejected") - } - err := claimBindRun(context.Background(), claimOptions{ - claim: "X", - path: "p", - level: "PG1", - bindFn: stub, - }) - if err == nil { - t.Fatal("expected error") - } - if !strings.Contains(err.Error(), "claim bind:") { - t.Fatalf("error not wrapped: %v", err) - } -} - -func TestClaimList_StubReturnsBindings(t *testing.T) { - stub := func(_ context.Context, _ claimOptions) ([]ports.EvidenceBinding, error) { - return []ports.EvidenceBinding{ - {Claim: "AOP-A", Path: "a.md", Level: ports.EvidenceLevelPG2}, - {Claim: "AOP-B", Path: "b.md", Level: ports.EvidenceLevelPG4}, - }, nil - } - var buf bytes.Buffer - err := claimListRun(context.Background(), claimOptions{ - writer: &buf, - listFn: stub, - }) - if err != nil { - t.Fatal(err) - } - lines := strings.Split(strings.TrimSpace(buf.String()), "\n") - if len(lines) != 2 { - t.Fatalf("len = %d, want 2", len(lines)) - } - if !strings.Contains(lines[0], `"Level":"PG2"`) || !strings.Contains(lines[1], `"Level":"PG4"`) { - t.Fatalf("levels missing: %s", buf.String()) - } -} - -func TestClaimList_EmptyBindings(t *testing.T) { - stub := func(_ context.Context, _ claimOptions) ([]ports.EvidenceBinding, error) { - return []ports.EvidenceBinding{}, nil - } - var buf bytes.Buffer - err := claimListRun(context.Background(), claimOptions{ - writer: &buf, - listFn: stub, - }) - if err != nil { - t.Fatal(err) - } - if buf.Len() != 0 { - t.Fatalf("empty should be 0 bytes, got %q", buf.String()) - } -} - -func TestClaimCheckRequiresChanged(t *testing.T) { - err := claimCheckRun(context.Background(), claimCheckOptions{}) - if err == nil { - t.Fatal("expected --changed requirement") - } - if !strings.Contains(err.Error(), "--changed") { - t.Fatalf("error not helpful: %v", err) - } -} - -func TestClaimCheckCommandRegistered(t *testing.T) { - out, err := executeCommand("claim", "check", "--help") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(out, "Report read-only proof cards for changed public claim markers.") { - t.Fatalf("help missing claim check summary:\n%s", out) - } - if !strings.Contains(out, "--changed") || !strings.Contains(out, "--base") { - t.Fatalf("help missing expected flags:\n%s", out) - } -} - -func TestClaimCheckAcceptsNoPositionals(t *testing.T) { - if claimCheckCmd.Args == nil { - t.Fatal("claim check must declare an explicit positional-argument policy") - } - if err := claimCheckCmd.Args(claimCheckCmd, nil); err != nil { - t.Fatalf("claim check rejected zero positional arguments: %v", err) - } - if err := claimCheckCmd.Args(claimCheckCmd, []string{"unexpected"}); err == nil { - t.Fatal("claim check accepted an unexpected positional argument") - } -} - -func TestClaimBindingsPathUsesRepositoryRoot(t *testing.T) { - repo := t.TempDir() - initMinimalGitRepo(t, repo) - subdir := filepath.Join(repo, "nested") - if err := os.Mkdir(subdir, 0o755); err != nil { - t.Fatal(err) - } - old := testProjectDir - testProjectDir = subdir - t.Cleanup(func() { testProjectDir = old }) - - path, err := claimBindingsPath() - if err != nil { - t.Fatal(err) - } - root, err := resolveRepoRoot(subdir) - if err != nil { - t.Fatal(err) - } - want := filepath.Join(root, ".agents", "findings", "evidence-bindings.jsonl") - if path != want { - t.Fatalf("claimBindingsPath() = %q, want repo-rooted %q", path, want) - } -} - -func TestClaimCheckJSONOutput(t *testing.T) { - var got claimproof.Options - stub := func(_ context.Context, opts claimproof.Options) (claimproof.Report, error) { - got = opts - return claimproof.Report{ - Summary: claimproof.Summary{ - Mode: "changed", - Base: "origin/main", - ChangedSurfaces: 1, - Claims: 1, - Verdicts: map[string]int{"supported": 1}, - }, - Cards: []claimproof.Card{{ - ClaimID: "AOP-CLAIM-X", - Surface: "PRODUCT.md", - Tier: "PILOT", - RegistryFound: true, - Evidence: []claimproof.Evidence{{ - Path: "docs/evidence/x.md", - Status: "tracked", - }}, - Verdict: "supported", - NextAction: "run validation", - }}, - }, nil - } - - var buf bytes.Buffer - err := claimCheckRun(context.Background(), claimCheckOptions{ - repoRoot: "/repo", - base: "origin/main", - changedOnly: true, - jsonOutput: true, - writer: &buf, - checkFn: stub, - }) - if err != nil { - t.Fatal(err) - } - if got.RepoRoot != "/repo" || got.Base != "origin/main" || !got.ChangedOnly { - t.Fatalf("opts not forwarded: %+v", got) - } - var report claimproof.Report - if err := json.Unmarshal(buf.Bytes(), &report); err != nil { - t.Fatalf("invalid JSON: %v\n%s", err, buf.String()) - } - if len(report.Cards) != 1 || report.Cards[0].ClaimID != "AOP-CLAIM-X" { - t.Fatalf("wrong report: %+v", report) - } -} - -func TestClaimCheckHumanNoop(t *testing.T) { - stub := func(_ context.Context, _ claimproof.Options) (claimproof.Report, error) { - return claimproof.Report{ - Summary: claimproof.Summary{ - Mode: "changed", - Base: "origin/main", - Verdicts: map[string]int{}, - }, - }, nil - } - - var buf bytes.Buffer - err := claimCheckRun(context.Background(), claimCheckOptions{ - base: "origin/main", - changedOnly: true, - writer: &buf, - checkFn: stub, - }) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(buf.String(), "No changed claim markers found.") { - t.Fatalf("missing no-op output: %q", buf.String()) - } -} - -func TestClaimCheckHumanShowsCitationPolicy(t *testing.T) { - stub := func(_ context.Context, _ claimproof.Options) (claimproof.Report, error) { - return claimproof.Report{ - Summary: claimproof.Summary{ - Mode: "changed", - Base: "origin/main", - ChangedSurfaces: 1, - Claims: 1, - Verdicts: map[string]int{"citation_ceiling": 1}, - }, - Cards: []claimproof.Card{{ - ClaimID: "AOP-CLAIM-X", - Surface: "PRODUCT.md", - Tier: "UNPROVEN", - CitationOK: false, - CiteAllowed: []string{"docs/comparisons/**", "docs/wiki-for-agents.md"}, - RegistryFound: true, - Verdict: "citation_ceiling", - NextAction: "promote the claim tier", - }}, - }, nil - } - - var buf bytes.Buffer - err := claimCheckRun(context.Background(), claimCheckOptions{ - base: "origin/main", - changedOnly: true, - writer: &buf, - checkFn: stub, - }) - if err != nil { - t.Fatal(err) - } - out := buf.String() - if !strings.Contains(out, "citation_ok: false") || !strings.Contains(out, "docs/comparisons/**") { - t.Fatalf("missing citation policy in human output: %q", out) - } -} diff --git a/cli/cmd/ao/claim_evidence_adapter.go b/cli/cmd/ao/claim_evidence_adapter.go deleted file mode 100644 index dd0828284..000000000 --- a/cli/cmd/ao/claim_evidence_adapter.go +++ /dev/null @@ -1,106 +0,0 @@ -// practices: [hexagonal-architecture, ddd-bounded-context] -package main - -import ( - "context" - "errors" - - "github.com/boshu2/agentops/cli/internal/ports" -) - -// productionClaimEvidence satisfies ports.ClaimEvidencePort by -// composing any GateRunnerPort (typically the cycle 115 -// productionGateRunner) with the same promotion policy as the -// in-memory adapter (cycle 141 promoteEvidenceLevel helper). -// -// The production composer wraps a real gate-runner so calling code -// can `Derive` a claim→evidence binding from a real gate invocation -// without re-implementing the policy. -type productionClaimEvidence struct { - gateRunner ports.GateRunnerPort -} - -func newProductionClaimEvidence(gateRunner ports.GateRunnerPort) *productionClaimEvidence { - return &productionClaimEvidence{gateRunner: gateRunner} -} - -// Derive runs the gate via the wrapped GateRunner and applies the -// same promotion policy as the in-memory adapter. This wrapper -// exists so production callers get the policy without recreating -// it; the policy lives in the in-memory adapter (test surface) and -// here (production surface) — kept in sync via the contract enumerated -// in cli/internal/ports/claim_evidence.go and the cycle-141 -// inmemory_claim_evidence_test.go suite. -func (a *productionClaimEvidence) Derive(ctx context.Context, req ports.ClaimEvidenceRequest, existingLevel, targetLevel ports.EvidenceLevel) (ports.ClaimEvidenceResult, error) { - if err := ctx.Err(); err != nil { - return ports.ClaimEvidenceResult{}, err - } - if req.Claim == "" { - return ports.ClaimEvidenceResult{}, errors.New("productionClaimEvidence: Claim required") - } - if req.EvidenceFile == "" { - return ports.ClaimEvidenceResult{}, errors.New("productionClaimEvidence: EvidenceFile required") - } - if a.gateRunner == nil { - return ports.ClaimEvidenceResult{}, errors.New("productionClaimEvidence: GateRunner required") - } - - verdict, err := a.gateRunner.Run(ctx, ports.GateRunRequest{Name: req.Gate}) - if err != nil { - return ports.ClaimEvidenceResult{}, err - } - - newLevel := productionPromoteEvidenceLevel(verdict.Status, existingLevel, targetLevel) - return ports.ClaimEvidenceResult{ - Binding: ports.EvidenceBinding{ - Claim: req.Claim, - Path: req.EvidenceFile, - Level: newLevel, - AuthorID: req.AuthorID, - JudgeID: req.JudgeID, - }, - Verdict: verdict, - }, nil -} - -// productionPromoteEvidenceLevel mirrors the in-memory adapter's -// promotion policy. Kept as a separate function (not imported from -// internal/ports) so the production-side stays self-contained — the -// in-memory adapter's helper is package-private. Both implementations -// must stay in sync; the cycle-141 test suite is the spec. -func productionPromoteEvidenceLevel(status ports.GateStatus, existing, target ports.EvidenceLevel) ports.EvidenceLevel { - var candidate ports.EvidenceLevel - switch status { - case ports.GateStatusPass: - candidate = target - if candidate == ports.EvidenceLevelNone { - candidate = ports.EvidenceLevelPG2 - } - case ports.GateStatusWarn: - candidate = ports.EvidenceLevelPG1 - default: - // FAIL, SKIP, UNKNOWN — no promotion - return existing - } - if productionEvidenceLevelOrd(candidate) > productionEvidenceLevelOrd(existing) { - return candidate - } - return existing -} - -func productionEvidenceLevelOrd(l ports.EvidenceLevel) int { - switch l { - case ports.EvidenceLevelPG1: - return 1 - case ports.EvidenceLevelPG2: - return 2 - case ports.EvidenceLevelPG3: - return 3 - case ports.EvidenceLevelPG4: - return 4 - } - return 0 -} - -// Compile-time assertion. -var _ ports.ClaimEvidencePort = (*productionClaimEvidence)(nil) diff --git a/cli/cmd/ao/claim_evidence_adapter_test.go b/cli/cmd/ao/claim_evidence_adapter_test.go deleted file mode 100644 index fae877fe3..000000000 --- a/cli/cmd/ao/claim_evidence_adapter_test.go +++ /dev/null @@ -1,154 +0,0 @@ -// practices: [hexagonal-architecture, tdd] -package main - -import ( - "context" - "errors" - "testing" - - "github.com/boshu2/agentops/cli/internal/ports" -) - -func newPCEFixture(verdicts map[ports.GateName]ports.GateVerdict) *productionClaimEvidence { - return newProductionClaimEvidence(ports.NewInMemoryGateRunner(verdicts)) -} - -func TestProductionClaimEvidence_PassDefaultsToPG2(t *testing.T) { - c := newPCEFixture(map[ports.GateName]ports.GateVerdict{ - "g": {Status: ports.GateStatusPass, Reason: "ok"}, - }) - res, err := c.Derive(context.Background(), ports.ClaimEvidenceRequest{ - Claim: "X", EvidenceFile: "p", Gate: "g", - }, ports.EvidenceLevelNone, ports.EvidenceLevelNone) - if err != nil { - t.Fatal(err) - } - if res.Binding.Level != ports.EvidenceLevelPG2 { - t.Fatalf("Level = %s, want PG2", res.Binding.Level) - } -} - -func TestProductionClaimEvidence_TargetLevelHonored(t *testing.T) { - c := newPCEFixture(map[ports.GateName]ports.GateVerdict{ - "g": {Status: ports.GateStatusPass, Reason: "ok"}, - }) - res, _ := c.Derive(context.Background(), ports.ClaimEvidenceRequest{ - Claim: "X", EvidenceFile: "p", Gate: "g", - }, ports.EvidenceLevelNone, ports.EvidenceLevelPG4) - if res.Binding.Level != ports.EvidenceLevelPG4 { - t.Fatalf("Level = %s, want PG4", res.Binding.Level) - } -} - -func TestProductionClaimEvidence_BindingCarriesReviewerMetadata(t *testing.T) { - c := newPCEFixture(map[ports.GateName]ports.GateVerdict{ - "g": {Status: ports.GateStatusPass, Reason: "ok"}, - }) - res, _ := c.Derive(context.Background(), ports.ClaimEvidenceRequest{ - Claim: "X", - EvidenceFile: "p", - Gate: "g", - AuthorID: "agent-1", - JudgeID: "agent-2", - }, ports.EvidenceLevelNone, ports.EvidenceLevelPG4) - if res.Binding.AuthorID != "agent-1" || res.Binding.JudgeID != "agent-2" { - t.Fatalf("reviewer metadata not carried: %+v", res.Binding) - } -} - -func TestProductionClaimEvidence_WarnToPG1(t *testing.T) { - c := newPCEFixture(map[ports.GateName]ports.GateVerdict{ - "g": {Status: ports.GateStatusWarn, Reason: "advisory"}, - }) - res, _ := c.Derive(context.Background(), ports.ClaimEvidenceRequest{ - Claim: "X", EvidenceFile: "p", Gate: "g", - }, ports.EvidenceLevelNone, ports.EvidenceLevelNone) - if res.Binding.Level != ports.EvidenceLevelPG1 { - t.Fatalf("Level = %s, want PG1", res.Binding.Level) - } -} - -func TestProductionClaimEvidence_FailKeepsExisting(t *testing.T) { - c := newPCEFixture(map[ports.GateName]ports.GateVerdict{ - "g": {Status: ports.GateStatusFail, Reason: "broken"}, - }) - res, _ := c.Derive(context.Background(), ports.ClaimEvidenceRequest{ - Claim: "X", EvidenceFile: "p", Gate: "g", - }, ports.EvidenceLevelPG3, ports.EvidenceLevelNone) - if res.Binding.Level != ports.EvidenceLevelPG3 { - t.Fatalf("Level = %s, want PG3 (existing)", res.Binding.Level) - } -} - -func TestProductionClaimEvidence_NoDowngradeOnPass(t *testing.T) { - c := newPCEFixture(map[ports.GateName]ports.GateVerdict{ - "g": {Status: ports.GateStatusPass, Reason: "ok"}, - }) - res, _ := c.Derive(context.Background(), ports.ClaimEvidenceRequest{ - Claim: "X", EvidenceFile: "p", Gate: "g", - }, ports.EvidenceLevelPG3, ports.EvidenceLevelNone) - if res.Binding.Level != ports.EvidenceLevelPG3 { - t.Fatalf("Level = %s, want PG3 (no downgrade)", res.Binding.Level) - } -} - -func TestProductionClaimEvidence_EmptyInputsRejected(t *testing.T) { - c := newPCEFixture(nil) - if _, err := c.Derive(context.Background(), ports.ClaimEvidenceRequest{EvidenceFile: "p", Gate: "g"}, ports.EvidenceLevelNone, ports.EvidenceLevelNone); err == nil { - t.Fatal("empty Claim should error") - } - if _, err := c.Derive(context.Background(), ports.ClaimEvidenceRequest{Claim: "X", Gate: "g"}, ports.EvidenceLevelNone, ports.EvidenceLevelNone); err == nil { - t.Fatal("empty EvidenceFile should error") - } -} - -func TestProductionClaimEvidence_NilGateRunnerErrors(t *testing.T) { - c := newProductionClaimEvidence(nil) - _, err := c.Derive(context.Background(), ports.ClaimEvidenceRequest{ - Claim: "X", EvidenceFile: "p", Gate: "g", - }, ports.EvidenceLevelNone, ports.EvidenceLevelNone) - if err == nil { - t.Fatal("nil GateRunner should error") - } -} - -func TestProductionClaimEvidence_PolicyMatchesInMemoryAdapter(t *testing.T) { - // Cross-check: the production policy must yield identical results - // to the in-memory adapter for the same inputs. This is the - // "spec" guarantee from the cycle-141 doc — both adapters apply - // the same policy. - cases := []struct { - status ports.GateStatus - existing ports.EvidenceLevel - target ports.EvidenceLevel - want ports.EvidenceLevel - }{ - {ports.GateStatusPass, ports.EvidenceLevelNone, ports.EvidenceLevelNone, ports.EvidenceLevelPG2}, - {ports.GateStatusPass, ports.EvidenceLevelNone, ports.EvidenceLevelPG4, ports.EvidenceLevelPG4}, - {ports.GateStatusPass, ports.EvidenceLevelPG3, ports.EvidenceLevelNone, ports.EvidenceLevelPG3}, - {ports.GateStatusWarn, ports.EvidenceLevelNone, ports.EvidenceLevelNone, ports.EvidenceLevelPG1}, - {ports.GateStatusWarn, ports.EvidenceLevelPG2, ports.EvidenceLevelNone, ports.EvidenceLevelPG2}, - {ports.GateStatusFail, ports.EvidenceLevelPG2, ports.EvidenceLevelNone, ports.EvidenceLevelPG2}, - {ports.GateStatusSkip, ports.EvidenceLevelPG3, ports.EvidenceLevelNone, ports.EvidenceLevelPG3}, - {ports.GateStatusUnknown, ports.EvidenceLevelNone, ports.EvidenceLevelNone, ports.EvidenceLevelNone}, - } - for _, tc := range cases { - got := productionPromoteEvidenceLevel(tc.status, tc.existing, tc.target) - if got != tc.want { - t.Fatalf("policy(%s, existing=%s, target=%s) = %s, want %s", - tc.status, tc.existing, tc.target, got, tc.want) - } - } -} - -func TestProductionClaimEvidence_HonorsContextCancellation(t *testing.T) { - c := newPCEFixture(nil) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - _, err := c.Derive(ctx, ports.ClaimEvidenceRequest{ - Claim: "X", EvidenceFile: "p", Gate: "g", - }, ports.EvidenceLevelNone, ports.EvidenceLevelNone) - if !errors.Is(err, context.Canceled) { - t.Fatalf("error = %v, want context.Canceled", err) - } -} diff --git a/cli/cmd/ao/claim_evidence_binder_adapter.go b/cli/cmd/ao/claim_evidence_binder_adapter.go deleted file mode 100644 index 38f2cf0f3..000000000 --- a/cli/cmd/ao/claim_evidence_binder_adapter.go +++ /dev/null @@ -1,208 +0,0 @@ -// practices: [hexagonal-architecture, ddd-bounded-context] -package main - -import ( - "bufio" - "context" - "encoding/json" - "errors" - "fmt" - "os" - "sync" - - "github.com/boshu2/agentops/cli/internal/ports" -) - -// productionClaimEvidenceBinder satisfies ports.ClaimEvidenceBinderPort -// by appending EvidenceBinding records to a JSONL file (typically -// .agents/findings/evidence-bindings.jsonl). -// -// Semantics: -// - Append-only on disk. List replays the file and folds it into the -// current view (latest binding per Claim+Path wins; level can only -// go up). -// - Bind enforces idempotency + upgrade-only by reading the current -// file once under lock, checking the existing binding for the same -// (Claim, Path) pair, rejecting downgrades, and skipping the write -// when the level is unchanged AND Anchors are identical. -// - Empty Claim or empty Path is a structural-rejection error per -// port contract. -// - List returns bindings most-recently-bound first (replayed file -// reversed). -// -// Shape: same JSONL pattern proven across cycles 108-110. -type productionClaimEvidenceBinder struct { - mu sync.Mutex - path string -} - -func newProductionClaimEvidenceBinder(path string) *productionClaimEvidenceBinder { - return &productionClaimEvidenceBinder{path: path} -} - -// Bind appends a binding after the upgrade-only check. -func (b *productionClaimEvidenceBinder) Bind(ctx context.Context, binding ports.EvidenceBinding) error { - if err := ctx.Err(); err != nil { - return err - } - if binding.Claim == "" { - return errors.New("productionClaimEvidenceBinder: Claim required") - } - if binding.Path == "" { - return errors.New("productionClaimEvidenceBinder: Path required") - } - if err := ports.ValidateEvidenceBindingReviewers(binding); err != nil { - return fmt.Errorf("productionClaimEvidenceBinder: %w", err) - } - if b.path == "" { - return errors.New("productionClaimEvidenceBinder: file path required") - } - - b.mu.Lock() - defer b.mu.Unlock() - - existing, err := b.scanLatestFor(binding.Claim, binding.Path) - if err != nil { - return err - } - if existing != nil { - if evidenceLevelRank(binding.Level) < evidenceLevelRank(existing.Level) { - return fmt.Errorf("productionClaimEvidenceBinder: refusing downgrade %s → %s for claim %q path %q", - existing.Level, binding.Level, binding.Claim, binding.Path) - } - if binding.Level == existing.Level && anchorsEqual(binding.Anchors, existing.Anchors) { - return nil // pure idempotent no-op - } - } - - payload, err := json.Marshal(evidenceBindingRecord{ - Claim: string(binding.Claim), - Path: binding.Path, - Level: string(binding.Level), - Anchors: binding.Anchors, - AuthorID: binding.AuthorID, - JudgeID: binding.JudgeID, - }) - if err != nil { - return fmt.Errorf("productionClaimEvidenceBinder marshal: %w", err) - } - f, err := os.OpenFile(b.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - return fmt.Errorf("productionClaimEvidenceBinder open %q: %w", b.path, err) - } - defer func() { _ = f.Close() }() - if _, err := f.Write(append(payload, '\n')); err != nil { - return fmt.Errorf("productionClaimEvidenceBinder write: %w", err) - } - return nil -} - -// List returns all bindings, most-recent first. -func (b *productionClaimEvidenceBinder) List(ctx context.Context) ([]ports.EvidenceBinding, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - b.mu.Lock() - defer b.mu.Unlock() - all, err := b.scanAllLocked() - if err != nil { - return nil, err - } - out := make([]ports.EvidenceBinding, 0, len(all)) - for i := len(all) - 1; i >= 0; i-- { - out = append(out, all[i]) - } - return out, nil -} - -// scanLatestFor walks the file and returns the most recent binding -// for the given (claim, path), or nil if none. Assumes caller holds -// b.mu. -func (b *productionClaimEvidenceBinder) scanLatestFor(claim ports.ClaimID, path string) (*ports.EvidenceBinding, error) { - all, err := b.scanAllLocked() - if err != nil { - return nil, err - } - var found *ports.EvidenceBinding - for i := range all { - if all[i].Claim == claim && all[i].Path == path { - cp := all[i] - found = &cp - } - } - return found, nil -} - -// scanAllLocked reads the file in order. Missing file → empty slice. -func (b *productionClaimEvidenceBinder) scanAllLocked() ([]ports.EvidenceBinding, error) { - out := make([]ports.EvidenceBinding, 0) - f, err := os.Open(b.path) - if err != nil { - if os.IsNotExist(err) { - return out, nil - } - return nil, fmt.Errorf("productionClaimEvidenceBinder open %q: %w", b.path, err) - } - defer func() { _ = f.Close() }() - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 64*1024), 1024*1024) - for scanner.Scan() { - var rec evidenceBindingRecord - if err := json.Unmarshal(scanner.Bytes(), &rec); err != nil { - continue - } - out = append(out, ports.EvidenceBinding{ - Claim: ports.ClaimID(rec.Claim), - Path: rec.Path, - Level: ports.EvidenceLevel(rec.Level), - Anchors: rec.Anchors, - AuthorID: rec.AuthorID, - JudgeID: rec.JudgeID, - }) - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("productionClaimEvidenceBinder scan: %w", err) - } - return out, nil -} - -// evidenceBindingRecord is the on-disk shape. -type evidenceBindingRecord struct { - Claim string `json:"claim"` - Path string `json:"path"` - Level string `json:"level,omitempty"` - Anchors []string `json:"anchors,omitempty"` - AuthorID string `json:"author_id,omitempty"` - JudgeID string `json:"judge_id,omitempty"` -} - -// evidenceLevelRank gives EvidenceLevel an integer ordering for the -// upgrade-only check. None < PG1 < PG2 < PG3 < PG4. -func evidenceLevelRank(l ports.EvidenceLevel) int { - switch l { - case ports.EvidenceLevelPG1: - return 1 - case ports.EvidenceLevelPG2: - return 2 - case ports.EvidenceLevelPG3: - return 3 - case ports.EvidenceLevelPG4: - return 4 - } - return 0 -} - -func anchorsEqual(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} - -// Compile-time assertion: productionClaimEvidenceBinder satisfies the port. -var _ ports.ClaimEvidenceBinderPort = (*productionClaimEvidenceBinder)(nil) diff --git a/cli/cmd/ao/claim_evidence_binder_adapter_test.go b/cli/cmd/ao/claim_evidence_binder_adapter_test.go deleted file mode 100644 index 8af9d1481..000000000 --- a/cli/cmd/ao/claim_evidence_binder_adapter_test.go +++ /dev/null @@ -1,225 +0,0 @@ -// practices: [hexagonal-architecture, tdd] -package main - -import ( - "context" - "errors" - "path/filepath" - "strings" - "testing" - - "github.com/boshu2/agentops/cli/internal/ports" -) - -// Sibling pattern: cycle 115 gate_runner_adapter_test.go. - -func newTempBinder(t *testing.T) *productionClaimEvidenceBinder { - t.Helper() - path := filepath.Join(t.TempDir(), "evidence-bindings.jsonl") - return newProductionClaimEvidenceBinder(path) -} - -func TestProductionClaimEvidenceBinder_BindCreatesEntry(t *testing.T) { - b := newTempBinder(t) - err := b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: "AOP-CLAIM-X", - Path: ".agents/findings/x.md", - Level: ports.EvidenceLevelPG2, - }) - if err != nil { - t.Fatal(err) - } - list, _ := b.List(context.Background()) - if len(list) != 1 { - t.Fatalf("len = %d, want 1", len(list)) - } - if list[0].Claim != "AOP-CLAIM-X" || list[0].Level != ports.EvidenceLevelPG2 { - t.Fatalf("got %+v", list[0]) - } -} - -func TestProductionClaimEvidenceBinder_UpgradeAllowed(t *testing.T) { - b := newTempBinder(t) - _ = b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: "AOP-CLAIM-X", - Path: "p", - Level: ports.EvidenceLevelPG1, - }) - err := b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: "AOP-CLAIM-X", - Path: "p", - Level: ports.EvidenceLevelPG3, - AuthorID: "agent-1", - JudgeID: "agent-2", - }) - if err != nil { - t.Fatalf("PG1 → PG3 upgrade rejected: %v", err) - } - list, _ := b.List(context.Background()) - if list[0].Level != ports.EvidenceLevelPG3 { - t.Fatalf("Level = %s, want PG3 (upgrade not visible)", list[0].Level) - } -} - -func TestProductionClaimEvidenceBinder_DowngradeRejected(t *testing.T) { - b := newTempBinder(t) - _ = b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: "AOP-CLAIM-X", - Path: "p", - Level: ports.EvidenceLevelPG3, - AuthorID: "agent-1", - JudgeID: "agent-2", - }) - err := b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: "AOP-CLAIM-X", Path: "p", Level: ports.EvidenceLevelPG1, - }) - if err == nil { - t.Fatal("PG3 → PG1 downgrade should error, got nil") - } -} - -func TestProductionClaimEvidenceBinder_IdempotentSameLevel(t *testing.T) { - b := newTempBinder(t) - req := ports.EvidenceBinding{Claim: "X", Path: "p", Level: ports.EvidenceLevelPG2} - _ = b.Bind(context.Background(), req) - _ = b.Bind(context.Background(), req) // second call — should NOT append - list, _ := b.List(context.Background()) - if len(list) != 1 { - t.Fatalf("len = %d, want 1 (idempotent no-op)", len(list)) - } -} - -func TestProductionClaimEvidenceBinder_DifferentAnchorsCausesAppend(t *testing.T) { - b := newTempBinder(t) - _ = b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: "X", Path: "p", Level: ports.EvidenceLevelPG2, - Anchors: []string{"L10"}, - }) - _ = b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: "X", Path: "p", Level: ports.EvidenceLevelPG2, - Anchors: []string{"L20"}, - }) - list, _ := b.List(context.Background()) - if list[0].Anchors[0] != "L20" { - t.Fatalf("Anchors[0] = %v, want L20", list[0].Anchors) - } -} - -func TestProductionClaimEvidenceBinder_ListMostRecentFirst(t *testing.T) { - b := newTempBinder(t) - for _, claim := range []ports.ClaimID{"A", "B", "C"} { - _ = b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: claim, Path: "p", Level: ports.EvidenceLevelPG1, - }) - } - list, _ := b.List(context.Background()) - if list[0].Claim != "C" || list[2].Claim != "A" { - t.Fatalf("order wrong: %v %v %v", list[0].Claim, list[1].Claim, list[2].Claim) - } -} - -func TestProductionClaimEvidenceBinder_EmptyClaimRejected(t *testing.T) { - b := newTempBinder(t) - err := b.Bind(context.Background(), ports.EvidenceBinding{Path: "p", Level: ports.EvidenceLevelPG1}) - if err == nil { - t.Fatal("expected empty-claim rejection, got nil") - } -} - -func TestProductionClaimEvidenceBinder_EmptyPathRejected(t *testing.T) { - b := newTempBinder(t) - err := b.Bind(context.Background(), ports.EvidenceBinding{Claim: "X", Level: ports.EvidenceLevelPG1}) - if err == nil { - t.Fatal("expected empty-path rejection, got nil") - } -} - -func TestProductionClaimEvidenceBinder_HighAssuranceWithoutReviewersRejected(t *testing.T) { - b := newTempBinder(t) - err := b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: "X", - Path: "p", - Level: ports.EvidenceLevelPG3, - }) - if err == nil { - t.Fatal("expected PG3 reviewer metadata rejection, got nil") - } - if !strings.Contains(err.Error(), "author_id and judge_id") { - t.Fatalf("error = %v, want author/judge guidance", err) - } -} - -func TestProductionClaimEvidenceBinder_SelfGradeRejected(t *testing.T) { - b := newTempBinder(t) - err := b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: "X", - Path: "p", - Level: ports.EvidenceLevelPG4, - AuthorID: "agent-1", - JudgeID: "agent-1", - }) - if err == nil { - t.Fatal("expected self-grade rejection, got nil") - } - if !strings.Contains(err.Error(), "author_id and judge_id") { - t.Fatalf("error = %v, want author/judge guidance", err) - } -} - -func TestProductionClaimEvidenceBinder_PreservesReviewerMetadata(t *testing.T) { - b := newTempBinder(t) - if err := b.Bind(context.Background(), ports.EvidenceBinding{ - Claim: "X", - Path: "p", - Level: ports.EvidenceLevelPG4, - AuthorID: "agent-1", - JudgeID: "agent-2", - }); err != nil { - t.Fatalf("distinct author/judge rejected: %v", err) - } - list, err := b.List(context.Background()) - if err != nil { - t.Fatal(err) - } - if list[0].AuthorID != "agent-1" || list[0].JudgeID != "agent-2" { - t.Fatalf("reviewer metadata not preserved: %+v", list[0]) - } -} - -func TestProductionClaimEvidenceBinder_EmptyFilePathErrors(t *testing.T) { - b := newProductionClaimEvidenceBinder("") - err := b.Bind(context.Background(), ports.EvidenceBinding{Claim: "X", Path: "p"}) - if err == nil { - t.Fatal("expected error on empty file path, got nil") - } -} - -func TestProductionClaimEvidenceBinder_HonorsContextCancellation(t *testing.T) { - b := newTempBinder(t) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - if err := b.Bind(ctx, ports.EvidenceBinding{Claim: "X", Path: "p"}); err == nil { - t.Fatal("Bind: expected cancellation, got nil") - } else if !errors.Is(err, context.Canceled) { - t.Fatalf("Bind error = %v, want context.Canceled", err) - } - if _, err := b.List(ctx); err == nil { - t.Fatal("List: expected cancellation, got nil") - } else if !errors.Is(err, context.Canceled) { - t.Fatalf("List error = %v, want context.Canceled", err) - } -} - -func TestProductionClaimEvidenceBinder_ListMissingFileEmpty(t *testing.T) { - b := newProductionClaimEvidenceBinder(filepath.Join(t.TempDir(), "missing.jsonl")) - list, err := b.List(context.Background()) - if err != nil { - t.Fatal(err) - } - if list == nil { - t.Fatal("missing-file List should be non-nil empty") - } - if len(list) != 0 { - t.Fatalf("len = %d, want 0", len(list)) - } -} diff --git a/cli/cmd/ao/claim_module.go b/cli/cmd/ao/claim_module.go new file mode 100644 index 000000000..d4bd9ff1f --- /dev/null +++ b/cli/cmd/ao/claim_module.go @@ -0,0 +1,27 @@ +// practices: [hexagonal-architecture, ddd-bounded-context] +package main + +import ( + claimadapter "github.com/boshu2/agentops/cli/internal/adapters/claim" + claimapp "github.com/boshu2/agentops/cli/internal/claim" + "github.com/boshu2/agentops/cli/internal/clicontract" + claimcommands "github.com/boshu2/agentops/cli/internal/commands/claim" +) + +func init() { + tracker := currentBeadsTracker() + service := claimapp.NewService( + claimadapter.NewTracker(tracker, func(code int, message string) error { + return &tickExitError{code: code, msg: message} + }), + claimadapter.NewEvidenceStore(repoRootOrCwd), + claimadapter.NewProof(repoRootOrCwd), + ) + module := claimcommands.NewModule(service, GetOutput) + command := module.Command() + command.GroupID = "core" + if err := clicontract.Attach(command, module.Contract()); err != nil { + panic(err) + } + rootCmd.AddCommand(command) +} diff --git a/cli/cmd/ao/claim_module_test.go b/cli/cmd/ao/claim_module_test.go new file mode 100644 index 000000000..738b9d7ba --- /dev/null +++ b/cli/cmd/ao/claim_module_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "strings" + "testing" +) + +func TestClaimModuleRegisteredWithCorrectedCheckArgs(t *testing.T) { + out, err := executeCommand("claim", "check", "--help") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "Report read-only proof cards for changed public claim markers.") { + t.Fatalf("claim check help = %q", out) + } + for _, command := range rootCmd.Commands() { + if command.Name() == "claim" { + if command.GroupID != "core" { + t.Fatalf("claim group = %q", command.GroupID) + } + return + } + } + t.Fatal("claim command not registered") +} diff --git a/cli/internal/adapters/claim/proof.go b/cli/internal/adapters/claim/proof.go new file mode 100644 index 000000000..3747b8434 --- /dev/null +++ b/cli/internal/adapters/claim/proof.go @@ -0,0 +1,49 @@ +package claim + +import ( + "context" + "errors" + "os" + "os/exec" + + "github.com/boshu2/agentops/cli/internal/claimproof" +) + +type Proof struct { + repoRoot func() (string, error) +} + +func NewProof(repoRoot func() (string, error)) Proof { + return Proof{repoRoot: repoRoot} +} + +func (proof Proof) Check(ctx context.Context, base string, changedOnly bool) (claimproof.Report, error) { + if proof.repoRoot == nil { + return claimproof.Report{}, errors.New("claim proof root resolver is not configured") + } + root, err := proof.repoRoot() + if err != nil { + return claimproof.Report{}, err + } + return claimproof.Check(ctx, claimproof.Options{ + RepoRoot: root, Base: base, ChangedOnly: changedOnly, Workspace: LocalWorkspace{}, + }) +} + +type LocalWorkspace struct{} + +func (LocalWorkspace) WorkingDirectory() (string, error) { return os.Getwd() } +func (LocalWorkspace) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } +func (LocalWorkspace) Stat(path string) error { + _, err := os.Stat(path) + return err +} +func (LocalWorkspace) IsNotExist(err error) bool { return os.IsNotExist(err) } +func (LocalWorkspace) Git(ctx context.Context, repoRoot string, args ...string) (string, error) { + command := exec.CommandContext(ctx, "git", args...) + command.Dir = repoRoot + output, err := command.Output() + return string(output), err +} + +var _ claimproof.Workspace = LocalWorkspace{} diff --git a/cli/internal/adapters/claim/store.go b/cli/internal/adapters/claim/store.go new file mode 100644 index 000000000..26ab1bb77 --- /dev/null +++ b/cli/internal/adapters/claim/store.go @@ -0,0 +1,193 @@ +package claim + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/boshu2/agentops/cli/internal/ports" +) + +type EvidenceStore struct { + mu sync.Mutex + directory func() (string, error) +} + +func NewEvidenceStore(directory func() (string, error)) *EvidenceStore { + return &EvidenceStore{directory: directory} +} + +func (store *EvidenceStore) Bind(ctx context.Context, binding ports.EvidenceBinding) error { + if err := ctx.Err(); err != nil { + return err + } + if binding.Claim == "" { + return errors.New("productionClaimEvidenceBinder: Claim required") + } + if binding.Path == "" { + return errors.New("productionClaimEvidenceBinder: Path required") + } + if err := ports.ValidateEvidenceBindingReviewers(binding); err != nil { + return fmt.Errorf("productionClaimEvidenceBinder: %w", err) + } + path, err := store.path() + if err != nil { + return err + } + + store.mu.Lock() + defer store.mu.Unlock() + existing, err := store.scanLatest(path, binding.Claim, binding.Path) + if err != nil { + return err + } + if existing != nil { + if levelRank(binding.Level) < levelRank(existing.Level) { + return fmt.Errorf("productionClaimEvidenceBinder: refusing downgrade %s → %s for claim %q path %q", existing.Level, binding.Level, binding.Claim, binding.Path) + } + if binding.Level == existing.Level && sameAnchors(binding.Anchors, existing.Anchors) { + return nil + } + } + payload, err := json.Marshal(bindingRecord{ + Claim: string(binding.Claim), Path: binding.Path, Level: string(binding.Level), + Anchors: binding.Anchors, AuthorID: binding.AuthorID, JudgeID: binding.JudgeID, + }) + if err != nil { + return fmt.Errorf("productionClaimEvidenceBinder marshal: %w", err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("mkdir: %w", err) + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return fmt.Errorf("productionClaimEvidenceBinder open %q: %w", path, err) + } + defer func() { _ = file.Close() }() + if _, err := file.Write(append(payload, '\n')); err != nil { + return fmt.Errorf("productionClaimEvidenceBinder write: %w", err) + } + return nil +} + +func (store *EvidenceStore) List(ctx context.Context) ([]ports.EvidenceBinding, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + path, err := store.path() + if err != nil { + return nil, err + } + store.mu.Lock() + defer store.mu.Unlock() + all, err := store.scan(path) + if err != nil { + return nil, err + } + out := make([]ports.EvidenceBinding, 0, len(all)) + for index := len(all) - 1; index >= 0; index-- { + out = append(out, all[index]) + } + return out, nil +} + +func (store *EvidenceStore) path() (string, error) { + if store == nil || store.directory == nil { + return "", errors.New("productionClaimEvidenceBinder: file path required") + } + directory, err := store.directory() + if err != nil { + return "", err + } + if directory == "" { + return "", errors.New("productionClaimEvidenceBinder: file path required") + } + return filepath.Join(directory, ".agents", "findings", "evidence-bindings.jsonl"), nil +} + +func (store *EvidenceStore) scanLatest(path string, claim ports.ClaimID, evidencePath string) (*ports.EvidenceBinding, error) { + all, err := store.scan(path) + if err != nil { + return nil, err + } + var found *ports.EvidenceBinding + for index := range all { + if all[index].Claim == claim && all[index].Path == evidencePath { + copyOf := all[index] + found = ©Of + } + } + return found, nil +} + +func (store *EvidenceStore) scan(path string) ([]ports.EvidenceBinding, error) { + out := make([]ports.EvidenceBinding, 0) + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return out, nil + } + return nil, fmt.Errorf("productionClaimEvidenceBinder open %q: %w", path, err) + } + defer func() { _ = file.Close() }() + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 64*1024), 1024*1024) + for scanner.Scan() { + var record bindingRecord + if err := json.Unmarshal(scanner.Bytes(), &record); err != nil { + continue + } + out = append(out, ports.EvidenceBinding{ + Claim: ports.ClaimID(record.Claim), Path: record.Path, + Level: ports.EvidenceLevel(record.Level), Anchors: record.Anchors, + AuthorID: record.AuthorID, JudgeID: record.JudgeID, + }) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("productionClaimEvidenceBinder scan: %w", err) + } + return out, nil +} + +type bindingRecord struct { + Claim string `json:"claim"` + Path string `json:"path"` + Level string `json:"level,omitempty"` + Anchors []string `json:"anchors,omitempty"` + AuthorID string `json:"author_id,omitempty"` + JudgeID string `json:"judge_id,omitempty"` +} + +func levelRank(level ports.EvidenceLevel) int { + switch level { + case ports.EvidenceLevelPG1: + return 1 + case ports.EvidenceLevelPG2: + return 2 + case ports.EvidenceLevelPG3: + return 3 + case ports.EvidenceLevelPG4: + return 4 + default: + return 0 + } +} + +func sameAnchors(left, right []string) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} + +var _ ports.ClaimEvidenceBinderPort = (*EvidenceStore)(nil) diff --git a/cli/internal/adapters/claim/store_test.go b/cli/internal/adapters/claim/store_test.go new file mode 100644 index 000000000..644e694d2 --- /dev/null +++ b/cli/internal/adapters/claim/store_test.go @@ -0,0 +1,56 @@ +package claim + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/boshu2/agentops/cli/internal/ports" +) + +func TestEvidenceStoreHistoryIdempotenceAndDowngrade(t *testing.T) { + root := t.TempDir() + store := NewEvidenceStore(func() (string, error) { return root, nil }) + ctx := context.Background() + pg2 := ports.EvidenceBinding{Claim: "AOP-X", Path: "p.md", Level: ports.EvidenceLevelPG2} + if err := store.Bind(ctx, pg2); err != nil { + t.Fatal(err) + } + if err := store.Bind(ctx, pg2); err != nil { + t.Fatal(err) + } + pg3 := ports.EvidenceBinding{Claim: "AOP-X", Path: "p.md", Level: ports.EvidenceLevelPG3, AuthorID: "author", JudgeID: "judge"} + if err := store.Bind(ctx, pg3); err != nil { + t.Fatal(err) + } + if err := store.Bind(ctx, pg2); err == nil { + t.Fatal("downgrade succeeded") + } + bindings, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + if len(bindings) != 2 || bindings[0].Level != ports.EvidenceLevelPG3 || bindings[1].Level != ports.EvidenceLevelPG2 { + t.Fatalf("physical newest-first history = %+v", bindings) + } +} + +func TestEvidenceStoreSkipsMalformedRowsAndUsesRepoRoot(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, ".agents", "findings", "evidence-bindings.jsonl") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("not-json\n{\"claim\":\"A\",\"path\":\"p\",\"level\":\"PG1\"}\n"), 0o644); err != nil { + t.Fatal(err) + } + store := NewEvidenceStore(func() (string, error) { return root, nil }) + bindings, err := store.List(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(bindings) != 1 || bindings[0].Claim != "A" { + t.Fatalf("bindings = %+v", bindings) + } +} diff --git a/cli/internal/adapters/claim/tracker.go b/cli/internal/adapters/claim/tracker.go new file mode 100644 index 000000000..a5e43c739 --- /dev/null +++ b/cli/internal/adapters/claim/tracker.go @@ -0,0 +1,55 @@ +// Package claim contains driven adapters for the claim command family. +package claim + +import ( + "context" + "errors" + "os/exec" + "strings" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" + claimapp "github.com/boshu2/agentops/cli/internal/claim" +) + +type ExitErrorFactory func(code int, message string) error + +type Resolver interface { + Resolve() (beadsapp.TrackerResolution, error) +} + +type Tracker struct { + resolver Resolver + exitError ExitErrorFactory +} + +func NewTracker(resolver Resolver, exitError ExitErrorFactory) Tracker { + return Tracker{resolver: resolver, exitError: exitError} +} + +func (tracker Tracker) Claim(ctx context.Context, id string, streams claimapp.Streams) error { + if tracker.resolver == nil || tracker.exitError == nil { + return errors.New("claim tracker adapter is not configured") + } + resolution, err := tracker.resolver.Resolve() + if err != nil { + return err + } + command := exec.CommandContext(ctx, resolution.Binary, "update", id, "--claim") // #nosec G204 -- tracker resolver constrains the binary to br|bd. + command.Dir = resolution.WorkDir + command.Env = append([]string(nil), resolution.ChildEnv...) + output, runErr := command.CombinedOutput() + if len(output) > 0 && streams.Stdout != nil { + _, _ = streams.Stdout.Write(output) + } + if runErr == nil { + return nil + } + code := 127 + var processExit *exec.ExitError + if errors.As(runErr, &processExit) { + code = processExit.ExitCode() + } + return tracker.exitError(code, strings.TrimSpace(runErr.Error())) +} + +var _ claimapp.TrackerClaimer = Tracker{} diff --git a/cli/internal/adapters/claim/tracker_test.go b/cli/internal/adapters/claim/tracker_test.go new file mode 100644 index 000000000..7ea13dd8f --- /dev/null +++ b/cli/internal/adapters/claim/tracker_test.go @@ -0,0 +1,51 @@ +package claim + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + beadsapp "github.com/boshu2/agentops/cli/internal/beads" + claimapp "github.com/boshu2/agentops/cli/internal/claim" +) + +type fixedResolver struct{ resolution beadsapp.TrackerResolution } + +func (resolver fixedResolver) Resolve() (beadsapp.TrackerResolution, error) { + return resolver.resolution, nil +} + +type exitMarker struct { + code int + message string +} + +func (marker *exitMarker) Error() string { return marker.message } + +func TestTrackerPreservesMergedOutputArgsAndChildExit(t *testing.T) { + root := t.TempDir() + script := filepath.Join(root, "br") + content := "#!/bin/sh\nprintf 'args:%s\\n' \"$*\"\nprintf 'child-stderr\\n' >&2\nexit 7\n" + if err := os.WriteFile(script, []byte(content), 0o755); err != nil { + t.Fatal(err) + } + var captured *exitMarker + tracker := NewTracker(fixedResolver{beadsapp.TrackerResolution{ + Tracker: beadsapp.TrackerBR, Binary: script, WorkDir: root, ChildEnv: os.Environ(), + }}, func(code int, message string) error { + captured = &exitMarker{code: code, message: message} + return captured + }) + var output bytes.Buffer + err := tracker.Claim(context.Background(), "age-1", claimapp.Streams{Stdout: &output}) + if !errors.Is(err, captured) || captured.code != 7 || captured.message != "exit status 7" { + t.Fatalf("error=%v captured=%+v", err, captured) + } + if got := output.String(); !strings.Contains(got, "args:update age-1 --claim") || !strings.Contains(got, "child-stderr") { + t.Fatalf("merged child output = %q", got) + } +} diff --git a/cli/internal/claim/service.go b/cli/internal/claim/service.go new file mode 100644 index 000000000..20bd9d1e3 --- /dev/null +++ b/cli/internal/claim/service.go @@ -0,0 +1,105 @@ +// Package claim owns application policy for claiming tracker work, binding +// evidence, and checking changed public claims. +package claim + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + + "github.com/boshu2/agentops/cli/internal/claimproof" + "github.com/boshu2/agentops/cli/internal/ports" +) + +type Streams struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +type TrackerClaimer interface { + Claim(context.Context, string, Streams) error +} + +type ProofChecker interface { + Check(context.Context, string, bool) (claimproof.Report, error) +} + +type BindRequest struct { + Claim string + Path string + Level string + Anchors []string + AuthorID string + JudgeID string +} + +type Service struct { + tracker TrackerClaimer + binder ports.ClaimEvidenceBinderPort + checker ProofChecker +} + +func NewService(tracker TrackerClaimer, binder ports.ClaimEvidenceBinderPort, checker ProofChecker) Service { + return Service{tracker: tracker, binder: binder, checker: checker} +} + +func (service Service) Claim(ctx context.Context, id string, streams Streams) error { + if service.tracker == nil { + return errors.New("claim tracker is not configured") + } + return service.tracker.Claim(ctx, id, streams) +} + +func (service Service) Bind(ctx context.Context, request BindRequest) error { + if request.Claim == "" { + return errors.New("claim bind: --claim required\n Example: ao claim bind --claim AOP-CLAIM-1 --path internal/foo.go") + } + if request.Path == "" { + return errors.New("claim bind: --path required (evidence file, relative to repo root)\n Example: ao claim bind --claim AOP-CLAIM-1 --path internal/foo.go") + } + level := strings.ToUpper(request.Level) + switch level { + case "", "PG1", "PG2", "PG3", "PG4": + default: + return fmt.Errorf("claim bind: invalid --level %q (want PG1|PG2|PG3|PG4)", request.Level) + } + if service.binder == nil { + return errors.New("claim bind: evidence binder is not configured") + } + if err := service.binder.Bind(ctx, ports.EvidenceBinding{ + Claim: ports.ClaimID(request.Claim), Path: request.Path, + Level: ports.EvidenceLevel(level), Anchors: request.Anchors, + AuthorID: request.AuthorID, JudgeID: request.JudgeID, + }); err != nil { + return fmt.Errorf("claim bind: %w", err) + } + return nil +} + +func (service Service) List(ctx context.Context) ([]ports.EvidenceBinding, error) { + if service.binder == nil { + return nil, errors.New("claim list: evidence binder is not configured") + } + bindings, err := service.binder.List(ctx) + if err != nil { + return nil, fmt.Errorf("claim list: %w", err) + } + return bindings, nil +} + +func (service Service) Check(ctx context.Context, base string, changedOnly bool) (claimproof.Report, error) { + if !changedOnly { + return claimproof.Report{}, errors.New("claim check: --changed is required for this read-only MVP") + } + if service.checker == nil { + return claimproof.Report{}, errors.New("claim check: proof checker is not configured") + } + report, err := service.checker.Check(ctx, base, changedOnly) + if err != nil { + return claimproof.Report{}, fmt.Errorf("claim check: %w", err) + } + return report, nil +} diff --git a/cli/internal/claim/service_test.go b/cli/internal/claim/service_test.go new file mode 100644 index 000000000..0c1168142 --- /dev/null +++ b/cli/internal/claim/service_test.go @@ -0,0 +1,79 @@ +package claim + +import ( + "context" + "errors" + "testing" + + "github.com/boshu2/agentops/cli/internal/claimproof" + "github.com/boshu2/agentops/cli/internal/ports" +) + +type fakeTracker struct{ id string } + +func (tracker *fakeTracker) Claim(_ context.Context, id string, _ Streams) error { + tracker.id = id + return nil +} + +type fakeBinder struct { + binding ports.EvidenceBinding + list []ports.EvidenceBinding + err error +} + +func (binder *fakeBinder) Bind(_ context.Context, binding ports.EvidenceBinding) error { + binder.binding = binding + return binder.err +} +func (binder *fakeBinder) List(context.Context) ([]ports.EvidenceBinding, error) { + return binder.list, binder.err +} + +type fakeChecker struct{ changed bool } + +func (checker *fakeChecker) Check(_ context.Context, base string, changed bool) (claimproof.Report, error) { + checker.changed = changed + return claimproof.Report{Summary: claimproof.Summary{Base: base}}, nil +} + +func TestServiceDelegatesClaimAndNormalizesBinding(t *testing.T) { + tracker, binder, checker := &fakeTracker{}, &fakeBinder{}, &fakeChecker{} + service := NewService(tracker, binder, checker) + if err := service.Claim(context.Background(), "age-1", Streams{}); err != nil { + t.Fatal(err) + } + if tracker.id != "age-1" { + t.Fatalf("claimed %q", tracker.id) + } + request := BindRequest{Claim: "AOP-X", Path: "p.md", Level: "pg2", Anchors: []string{"L1"}} + if err := service.Bind(context.Background(), request); err != nil { + t.Fatal(err) + } + if binder.binding.Level != ports.EvidenceLevelPG2 || binder.binding.Claim != "AOP-X" { + t.Fatalf("binding = %+v", binder.binding) + } + if _, err := service.Check(context.Background(), "base", true); err != nil || !checker.changed { + t.Fatalf("check err=%v changed=%t", err, checker.changed) + } +} + +func TestServiceRejectsInvalidRequestsBeforeEffects(t *testing.T) { + binder := &fakeBinder{err: errors.New("must not run")} + service := NewService(&fakeTracker{}, binder, &fakeChecker{}) + for _, request := range []BindRequest{ + {Path: "p", Level: "PG1"}, + {Claim: "A", Level: "PG1"}, + {Claim: "A", Path: "p", Level: "PG9"}, + } { + if err := service.Bind(context.Background(), request); err == nil { + t.Fatalf("Bind(%+v) succeeded", request) + } + } + if binder.binding.Claim != "" { + t.Fatalf("binder ran: %+v", binder.binding) + } + if _, err := service.Check(context.Background(), "base", false); err == nil { + t.Fatal("Check without --changed succeeded") + } +} diff --git a/cli/internal/claimproof/check.go b/cli/internal/claimproof/check.go index c4ec40612..d86ec97d6 100644 --- a/cli/internal/claimproof/check.go +++ b/cli/internal/claimproof/check.go @@ -4,12 +4,9 @@ package claimproof import ( - "bufio" "context" "errors" "fmt" - "os" - "os/exec" "path/filepath" "regexp" "sort" @@ -24,15 +21,20 @@ const defaultBase = "origin/main" var markerRe = regexp.MustCompile(`agentops:claim:(AOP-CLAIM-[A-Z0-9-]+)`) -// GitRunner runs a git subcommand in repoRoot and returns stdout. -type GitRunner func(ctx context.Context, repoRoot string, args ...string) (string, error) +type Workspace interface { + WorkingDirectory() (string, error) + ReadFile(string) ([]byte, error) + Stat(string) error + IsNotExist(error) bool + Git(context.Context, string, ...string) (string, error) +} // Options configure a claim proof-card check. type Options struct { RepoRoot string Base string ChangedOnly bool - RunGit GitRunner + Workspace Workspace } // Report is the structured output for ao claim check. @@ -97,8 +99,11 @@ type markerHit struct { // Check returns proof cards for changed claim markers. func Check(ctx context.Context, opts Options) (Report, error) { + if opts.Workspace == nil { + return Report{}, errors.New("claim proof: workspace is required") + } if opts.RepoRoot == "" { - cwd, err := os.Getwd() + cwd, err := opts.Workspace.WorkingDirectory() if err != nil { return Report{}, fmt.Errorf("claim proof: get working directory: %w", err) } @@ -112,7 +117,7 @@ func Check(ctx context.Context, opts Options) (Report, error) { return Report{}, errors.New("claim proof: only --changed mode is implemented") } - reg, err := loadRegistry(opts.RepoRoot) + reg, err := loadRegistry(opts.Workspace, opts.RepoRoot) if err != nil { return Report{}, err } @@ -121,7 +126,7 @@ func Check(ctx context.Context, opts Options) (Report, error) { if err != nil { return Report{}, err } - hits, err := markerHits(opts.RepoRoot, changed) + hits, err := markerHits(opts.Workspace, opts.RepoRoot, changed) if err != nil { return Report{}, err } @@ -147,9 +152,9 @@ func Check(ctx context.Context, opts Options) (Report, error) { return report, nil } -func loadRegistry(repoRoot string) (registryFile, error) { +func loadRegistry(workspace Workspace, repoRoot string) (registryFile, error) { path := filepath.Join(repoRoot, "docs", "contracts", "claim-registry.yaml") - data, err := os.ReadFile(path) + data, err := workspace.ReadFile(path) if err != nil { return registryFile{}, fmt.Errorf("claim proof: read claim registry: %w", err) } @@ -164,18 +169,13 @@ func loadRegistry(repoRoot string) (registryFile, error) { } func changedPaths(ctx context.Context, opts Options) ([]string, error) { - run := opts.RunGit - if run == nil { - run = realGit - } - var out []string for _, args := range [][]string{ {"diff", "--name-only", opts.Base + "...HEAD"}, {"diff", "--name-only", "HEAD"}, {"ls-files", "--others", "--exclude-standard"}, } { - s, err := run(ctx, opts.RepoRoot, args...) + s, err := opts.Workspace.Git(ctx, opts.RepoRoot, args...) if err != nil { return nil, fmt.Errorf("claim proof: git %s: %w", strings.Join(args, " "), err) } @@ -184,7 +184,7 @@ func changedPaths(ctx context.Context, opts Options) ([]string, error) { return dedupe(out), nil } -func markerHits(repoRoot string, paths []string) ([]markerHit, error) { +func markerHits(workspace Workspace, repoRoot string, paths []string) ([]markerHit, error) { seen := map[string]bool{} var hits []markerHit for _, rel := range paths { @@ -192,9 +192,9 @@ func markerHits(repoRoot string, paths []string) ([]markerHit, error) { if rel == "" || shouldSkipPath(rel) || !looksLikeClaimHost(rel) { continue } - ids, err := extractClaimIDs(filepath.Join(repoRoot, filepath.FromSlash(rel))) + ids, err := extractClaimIDs(workspace, filepath.Join(repoRoot, filepath.FromSlash(rel))) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if workspace.IsNotExist(err) { continue } return nil, err @@ -235,28 +235,22 @@ func looksLikeClaimHost(path string) bool { } } -func extractClaimIDs(path string) ([]string, error) { - f, err := os.Open(path) +func extractClaimIDs(workspace Workspace, path string) ([]string, error) { + data, err := workspace.ReadFile(path) if err != nil { return nil, err } - // Read-only handle: a Close error cannot lose data, so it is safe to drop. - defer func() { _ = f.Close() }() - seen := map[string]bool{} var ids []string - scanner := bufio.NewScanner(f) - for scanner.Scan() { - for _, match := range markerRe.FindAllStringSubmatch(scanner.Text(), -1) { - id := match[1] - if seen[id] { - continue - } - seen[id] = true - ids = append(ids, id) + for _, match := range markerRe.FindAllStringSubmatch(string(data), -1) { + id := match[1] + if seen[id] { + continue } + seen[id] = true + ids = append(ids, id) } - return ids, scanner.Err() + return ids, nil } func buildCard(ctx context.Context, opts Options, hit markerHit, row registryRow, found bool, tiers map[string]registryTier) Card { @@ -297,8 +291,8 @@ func classifyEvidence(ctx context.Context, opts Options, path string) Evidence { return e } abs := filepath.Join(opts.RepoRoot, filepath.FromSlash(e.Path)) - if _, err := os.Stat(abs); err != nil { - if errors.Is(err, os.ErrNotExist) { + if err := opts.Workspace.Stat(abs); err != nil { + if opts.Workspace.IsNotExist(err) { e.Status = "missing" e.Reason = "registry evidence path does not exist in the working tree" return e @@ -317,11 +311,7 @@ func classifyEvidence(ctx context.Context, opts Options, path string) Evidence { } func isTracked(ctx context.Context, opts Options, path string) bool { - run := opts.RunGit - if run == nil { - run = realGit - } - _, err := run(ctx, opts.RepoRoot, "ls-files", "--error-unmatch", "--", path) + _, err := opts.Workspace.Git(ctx, opts.RepoRoot, "ls-files", "--error-unmatch", "--", path) return err == nil } @@ -382,13 +372,3 @@ func dedupe(in []string) []string { } return out } - -func realGit(ctx context.Context, repoRoot string, args ...string) (string, error) { - cmd := exec.CommandContext(ctx, "git", args...) - cmd.Dir = repoRoot - out, err := cmd.Output() - if err != nil { - return "", err - } - return string(out), nil -} diff --git a/cli/internal/claimproof/check_test.go b/cli/internal/claimproof/check_test.go index 7eb1c7bb3..ee86e98b8 100644 --- a/cli/internal/claimproof/check_test.go +++ b/cli/internal/claimproof/check_test.go @@ -30,12 +30,12 @@ claims: RepoRoot: root, Base: "origin/main", ChangedOnly: true, - RunGit: fakeGit(map[string]fakeGitResult{ + Workspace: testWorkspace{run: fakeGit(map[string]fakeGitResult{ "diff --name-only origin/main...HEAD": {out: "PRODUCT.md\n"}, "diff --name-only HEAD": {}, "ls-files --others --exclude-standard": {}, "ls-files --error-unmatch -- docs/evidence/value.md": {out: "docs/evidence/value.md\n"}, - }), + })}, }) if err != nil { t.Fatal(err) @@ -81,11 +81,11 @@ claims: report, err := Check(context.Background(), Options{ RepoRoot: root, ChangedOnly: true, - RunGit: fakeGit(map[string]fakeGitResult{ + Workspace: testWorkspace{run: fakeGit(map[string]fakeGitResult{ "diff --name-only origin/main...HEAD": {out: "README.md\n"}, "diff --name-only HEAD": {}, "ls-files --others --exclude-standard": {}, - }), + })}, }) if err != nil { t.Fatal(err) @@ -126,12 +126,12 @@ claims: RepoRoot: root, Base: "origin/main", ChangedOnly: true, - RunGit: fakeGit(map[string]fakeGitResult{ + Workspace: testWorkspace{run: fakeGit(map[string]fakeGitResult{ "diff --name-only origin/main...HEAD": {out: "PRODUCT.md\n"}, "diff --name-only HEAD": {}, "ls-files --others --exclude-standard": {}, "ls-files --error-unmatch -- docs/evidence/value.md": {out: "docs/evidence/value.md\n"}, - }), + })}, }) if err != nil { t.Fatal(err) @@ -155,11 +155,11 @@ claims: {} report, err := Check(context.Background(), Options{ RepoRoot: root, ChangedOnly: true, - RunGit: fakeGit(map[string]fakeGitResult{ + Workspace: testWorkspace{run: fakeGit(map[string]fakeGitResult{ "diff --name-only origin/main...HEAD": {out: "cli/cmd/ao/foo.go\n"}, "diff --name-only HEAD": {}, "ls-files --others --exclude-standard": {}, - }), + })}, }) if err != nil { t.Fatal(err) @@ -177,7 +177,22 @@ type fakeGitResult struct { err error } -func fakeGit(results map[string]fakeGitResult) GitRunner { +type testGitRunner func(context.Context, string, ...string) (string, error) + +type testWorkspace struct{ run testGitRunner } + +func (testWorkspace) WorkingDirectory() (string, error) { return os.Getwd() } +func (testWorkspace) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } +func (testWorkspace) Stat(path string) error { + _, err := os.Stat(path) + return err +} +func (testWorkspace) IsNotExist(err error) bool { return os.IsNotExist(err) } +func (workspace testWorkspace) Git(ctx context.Context, root string, args ...string) (string, error) { + return workspace.run(ctx, root, args...) +} + +func fakeGit(results map[string]fakeGitResult) testGitRunner { return func(_ context.Context, _ string, args ...string) (string, error) { key := strings.Join(args, " ") if result, ok := results[key]; ok { diff --git a/cli/internal/commands/claim/module.go b/cli/internal/commands/claim/module.go new file mode 100644 index 000000000..c494ddb43 --- /dev/null +++ b/cli/internal/commands/claim/module.go @@ -0,0 +1,202 @@ +// Package claim owns the Cobra presentation for the claim command family. +// Handlers parse, delegate, and render; application policy and effects live +// behind injected ports. +package claim + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/spf13/cobra" + + claimapp "github.com/boshu2/agentops/cli/internal/claim" + "github.com/boshu2/agentops/cli/internal/claimproof" + "github.com/boshu2/agentops/cli/internal/clicontract" + "github.com/boshu2/agentops/cli/internal/ports" +) + +type UseCases interface { + Claim(context.Context, string, claimapp.Streams) error + Bind(context.Context, claimapp.BindRequest) error + List(context.Context) ([]ports.EvidenceBinding, error) + Check(context.Context, string, bool) (claimproof.Report, error) +} + +type Module struct { + useCases UseCases + output func() string +} + +func NewModule(useCases UseCases, output func() string) Module { + return Module{useCases: useCases, output: output} +} + +func (Module) Contract() clicontract.CommandContract { + return clicontract.CommandContract{ + ID: "ao.claim", + Profiles: clicontract.ProfileDefault | + clicontract.ProfileFlywheel | + clicontract.ProfileLegacy | + clicontract.ProfileCombined, + Args: clicontract.ArgsPolicy{Name: "range", Validate: cobra.RangeArgs(0, 1)}, + Output: clicontract.OutputNone, + Effects: clicontract.EffectFilesystem | clicontract.EffectProcess | + clicontract.EffectTracker | clicontract.EffectEnvironment, + ExitClasses: map[int]clicontract.ExitClass{0: clicontract.ExitSuccess, 1: clicontract.ExitFailure}, + } +} + +func (module Module) Command() *cobra.Command { + root := &cobra.Command{ + Use: "claim [id]", + Short: "Claim a BR bead or manage claim-evidence bindings", + Args: cobra.RangeArgs(0, 1), + Long: `Claim a BR bead for harness-neutral AgentOps loops, or bind/list claim evidence via the typed BC2 ClaimEvidenceBinderPort. + +Examples: + ao claim cp-123 + ao claim bind --claim AOP-CLAIM-X --path .agents/findings/x.md --level PG2 + ao claim list`, + } + root.RunE = func(command *cobra.Command, args []string) error { + if len(args) != 1 { + return command.Help() + } + return module.useCases.Claim(command.Context(), args[0], claimapp.Streams{ + Stdin: command.InOrStdin(), Stdout: command.OutOrStdout(), Stderr: command.ErrOrStderr(), + }) + } + root.AddCommand(module.bindCommand(), module.listCommand(), module.checkCommand()) + return root +} + +func (module Module) bindCommand() *cobra.Command { + var request claimapp.BindRequest + command := &cobra.Command{ + Use: "bind --claim --path [--level PG1|PG2|PG3|PG4] [--anchor ...] [--author-id --judge-id ]", + Short: "Bind a claim to an evidence file at a promotion level", + Long: `Append (or upgrade) a claim→evidence binding via the typed BC2 +ClaimEvidenceBinderPort (productionClaimEvidenceBinder, cycle 116). + +The binder is append-only on disk; List replays the file and folds +to the latest per (Claim, Path). The Level can only go UP — attempting +to downgrade (e.g., PG3 → PG1) returns an error. + +Examples: + ao claim bind --claim AOP-CLAIM-X --path .agents/findings/x.md --level PG2 + ao claim bind --claim AOP-CLAIM-Y --path p.md --level PG4 --anchor L10 --anchor L20 + ao claim bind --claim AOP-CLAIM-Z --path verdict.md --level PG4 --author-id worker-1 --judge-id verifier-2`, + Args: cobra.NoArgs, + } + command.Flags().StringVar(&request.Claim, "claim", "", "claim ID (required, e.g. AOP-CLAIM-X)") + command.Flags().StringVar(&request.Path, "path", "", "evidence file path (required, relative to repo root)") + command.Flags().StringVar(&request.Level, "level", "PG1", "promotion level: PG1|PG2|PG3|PG4") + command.Flags().StringArrayVar(&request.Anchors, "anchor", nil, "optional in-file anchors (repeatable)") + command.Flags().StringVar(&request.AuthorID, "author-id", "", "artifact author identity for reviewer separation checks") + command.Flags().StringVar(&request.JudgeID, "judge-id", "", "judge/verifier identity for reviewer separation checks") + _ = command.MarkFlagRequired("claim") + _ = command.MarkFlagRequired("path") + command.RunE = func(command *cobra.Command, _ []string) error { + if err := module.useCases.Bind(command.Context(), request); err != nil { + return err + } + _, err := fmt.Fprintf(command.OutOrStdout(), "bound claim=%q path=%q level=%s\n", request.Claim, request.Path, request.Level) + return err + } + return command +} + +func (module Module) listCommand() *cobra.Command { + command := &cobra.Command{ + Use: "list", + Short: "List recorded claim→evidence bindings (most-recent first)", + Long: `Emit all known claim→evidence bindings via the typed BC2 ClaimEvidenceBinderPort. Output is line-delimited JSON.`, + Args: cobra.NoArgs, + } + command.RunE = func(command *cobra.Command, _ []string) error { + bindings, err := module.useCases.List(command.Context()) + if err != nil { + return err + } + encoder := json.NewEncoder(command.OutOrStdout()) + for _, binding := range bindings { + if err := encoder.Encode(binding); err != nil { + return fmt.Errorf("claim list encode: %w", err) + } + } + return nil + } + return command +} + +func (module Module) checkCommand() *cobra.Command { + var changedOnly bool + var base string + command := &cobra.Command{ + Use: "check --changed [--base ]", + Short: "Report proof cards for changed public claims", + Long: `Report read-only proof cards for changed public claim markers. + +The checker compares the current branch/worktree to a base ref, finds changed +files containing agentops:claim markers, then reports each claim's registry tier, +evidence citation status, verdict, and next action. It does not bind evidence, +promote tiers, mutate beads, or edit the claim registry. + +Examples: + ao claim check --changed + ao claim check --changed --base origin/main --json`, + Args: cobra.NoArgs, + } + command.Flags().BoolVar(&changedOnly, "changed", false, "check claim markers in files changed against --base plus worktree changes") + command.Flags().StringVar(&base, "base", "origin/main", "base ref for --changed comparison") + command.RunE = func(command *cobra.Command, _ []string) error { + report, err := module.useCases.Check(command.Context(), base, changedOnly) + if err != nil { + return err + } + if module.output != nil && module.output() == "json" { + encoder := json.NewEncoder(command.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(report) + } + writeHuman(command, report) + return nil + } + return command +} + +func writeHuman(command *cobra.Command, report claimproof.Report) { + writer := command.OutOrStdout() + fmt.Fprintf(writer, "claim check: %d changed claim(s) across %d surface(s) (base=%s)\n", report.Summary.Claims, report.Summary.ChangedSurfaces, report.Summary.Base) + if len(report.Cards) == 0 { + fmt.Fprintln(writer, "No changed claim markers found.") + return + } + for _, card := range report.Cards { + fmt.Fprintf(writer, "\n%s\n", card.ClaimID) + fmt.Fprintf(writer, " surface: %s\n", card.Surface) + fmt.Fprintf(writer, " tier: %s\n", card.Tier) + if len(card.CiteAllowed) > 0 { + fmt.Fprintf(writer, " citation_ok: %t (allowed: %s)\n", card.CitationOK, strings.Join(card.CiteAllowed, ", ")) + } + fmt.Fprintf(writer, " verdict: %s\n", card.Verdict) + if card.EvalBinding != "" { + fmt.Fprintf(writer, " eval_binding: %s\n", card.EvalBinding) + } + if len(card.Evidence) == 0 { + fmt.Fprintln(writer, " evidence: none") + } else { + fmt.Fprintln(writer, " evidence:") + for _, evidence := range card.Evidence { + if evidence.Reason == "" { + fmt.Fprintf(writer, " - %s [%s]\n", evidence.Path, evidence.Status) + } else { + fmt.Fprintf(writer, " - %s [%s] %s\n", evidence.Path, evidence.Status, evidence.Reason) + } + } + } + fmt.Fprintf(writer, " next: %s\n", card.NextAction) + } +} diff --git a/cli/internal/commands/claim/module_test.go b/cli/internal/commands/claim/module_test.go new file mode 100644 index 000000000..40679f60b --- /dev/null +++ b/cli/internal/commands/claim/module_test.go @@ -0,0 +1,94 @@ +package claim + +import ( + "bytes" + "context" + "encoding/json" + "testing" + + claimapp "github.com/boshu2/agentops/cli/internal/claim" + "github.com/boshu2/agentops/cli/internal/claimproof" + "github.com/boshu2/agentops/cli/internal/clicontract" + "github.com/boshu2/agentops/cli/internal/ports" +) + +type fakeUseCases struct { + claimed string + bound claimapp.BindRequest + list []ports.EvidenceBinding + report claimproof.Report +} + +func (fake *fakeUseCases) Claim(_ context.Context, id string, _ claimapp.Streams) error { + fake.claimed = id + return nil +} +func (fake *fakeUseCases) Bind(_ context.Context, request claimapp.BindRequest) error { + fake.bound = request + return nil +} +func (fake *fakeUseCases) List(context.Context) ([]ports.EvidenceBinding, error) { + return fake.list, nil +} +func (fake *fakeUseCases) Check(context.Context, string, bool) (claimproof.Report, error) { + return fake.report, nil +} + +func TestModuleBuildsFreshTreeAndDelegates(t *testing.T) { + fake := &fakeUseCases{} + module := NewModule(fake, func() string { return "json" }) + if module.Command() == module.Command() { + t.Fatal("Command returned shared Cobra state") + } + command := module.Command() + command.SetArgs([]string{"age-1"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if fake.claimed != "age-1" { + t.Fatalf("claimed %q", fake.claimed) + } + if err := clicontract.ValidateContract(module.Contract()); err != nil { + t.Fatal(err) + } +} + +func TestBindListAndCheckRenderThroughUseCases(t *testing.T) { + fake := &fakeUseCases{ + list: []ports.EvidenceBinding{{Claim: "A", Path: "p", Level: ports.EvidenceLevelPG2}}, + report: claimproof.Report{Summary: claimproof.Summary{Base: "base", Verdicts: map[string]int{}}}, + } + module := NewModule(fake, func() string { return "json" }) + bind := module.Command() + bind.SetArgs([]string{"bind", "--claim", "A", "--path", "p", "--level", "pg2"}) + if err := bind.Execute(); err != nil { + t.Fatal(err) + } + if fake.bound.Level != "pg2" { + t.Fatalf("bound = %+v", fake.bound) + } + + list := module.Command() + var listOut bytes.Buffer + list.SetOut(&listOut) + list.SetArgs([]string{"list"}) + if err := list.Execute(); err != nil { + t.Fatal(err) + } + var binding ports.EvidenceBinding + if err := json.Unmarshal(listOut.Bytes(), &binding); err != nil || binding.Claim != "A" { + t.Fatalf("list JSON err=%v binding=%+v", err, binding) + } + + check := module.Command() + var checkOut bytes.Buffer + check.SetOut(&checkOut) + check.SetArgs([]string{"check", "--changed", "--base", "base"}) + if err := check.Execute(); err != nil { + t.Fatal(err) + } + var report claimproof.Report + if err := json.Unmarshal(checkOut.Bytes(), &report); err != nil || report.Summary.Base != "base" { + t.Fatalf("check JSON err=%v report=%+v", err, report) + } +} From 9b1206a1e3934fe55ae71761a4017be64ae19f41 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:15:54 -0400 Subject: [PATCH 038/108] test(claim): preserve bind command surface coverage (age-nw28h.2.3) --- cli/cmd/ao/claim_module_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cli/cmd/ao/claim_module_test.go b/cli/cmd/ao/claim_module_test.go index 738b9d7ba..a50a09f96 100644 --- a/cli/cmd/ao/claim_module_test.go +++ b/cli/cmd/ao/claim_module_test.go @@ -23,3 +23,16 @@ func TestClaimModuleRegisteredWithCorrectedCheckArgs(t *testing.T) { } t.Fatal("claim command not registered") } + +func TestClaimBindCommandDelegatesThroughModule(t *testing.T) { + old := testProjectDir + testProjectDir = t.TempDir() + t.Cleanup(func() { testProjectDir = old }) + out, err := executeCommand("claim", "bind", "--claim", "AOP-X", "--path", "p.md", "--level", "PG2") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, `bound claim="AOP-X" path="p.md" level=PG2`) { + t.Fatalf("claim bind output = %q", out) + } +} From 2172c77f05692169948f3b034970c90d64a366c1 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:24:19 -0400 Subject: [PATCH 039/108] fix(claim): preserve tracker runtime compatibility (age-nw28h.2.3) --- cli/cmd/ao/claim_module.go | 3 +- cli/internal/adapters/claim/tracker.go | 49 +++++++---- cli/internal/adapters/claim/tracker_test.go | 91 +++++++++++++++++---- 3 files changed, 109 insertions(+), 34 deletions(-) diff --git a/cli/cmd/ao/claim_module.go b/cli/cmd/ao/claim_module.go index d4bd9ff1f..ad290649c 100644 --- a/cli/cmd/ao/claim_module.go +++ b/cli/cmd/ao/claim_module.go @@ -9,9 +9,8 @@ import ( ) func init() { - tracker := currentBeadsTracker() service := claimapp.NewService( - claimadapter.NewTracker(tracker, func(code int, message string) error { + claimadapter.NewTracker(func(code int, message string) error { return &tickExitError{code: code, msg: message} }), claimadapter.NewEvidenceStore(repoRootOrCwd), diff --git a/cli/internal/adapters/claim/tracker.go b/cli/internal/adapters/claim/tracker.go index a5e43c739..408eed618 100644 --- a/cli/internal/adapters/claim/tracker.go +++ b/cli/internal/adapters/claim/tracker.go @@ -4,39 +4,60 @@ package claim import ( "context" "errors" + "os" "os/exec" "strings" - beadsapp "github.com/boshu2/agentops/cli/internal/beads" claimapp "github.com/boshu2/agentops/cli/internal/claim" + "github.com/boshu2/agentops/cli/internal/trackerresolve" ) type ExitErrorFactory func(code int, message string) error -type Resolver interface { - Resolve() (beadsapp.TrackerResolution, error) -} - type Tracker struct { - resolver Resolver - exitError ExitErrorFactory + workingDirectory func() (string, error) + environment func() []string + lookPath trackerresolve.LookPath + exitError ExitErrorFactory } -func NewTracker(resolver Resolver, exitError ExitErrorFactory) Tracker { - return Tracker{resolver: resolver, exitError: exitError} +func NewTracker(exitError ExitErrorFactory) Tracker { + return NewTrackerWith(os.Getwd, os.Environ, exec.LookPath, exitError) +} + +func NewTrackerWith( + workingDirectory func() (string, error), + environment func() []string, + lookPath trackerresolve.LookPath, + exitError ExitErrorFactory, +) Tracker { + return Tracker{ + workingDirectory: workingDirectory, environment: environment, + lookPath: lookPath, exitError: exitError, + } } func (tracker Tracker) Claim(ctx context.Context, id string, streams claimapp.Streams) error { - if tracker.resolver == nil || tracker.exitError == nil { + if tracker.workingDirectory == nil || tracker.environment == nil || tracker.lookPath == nil || tracker.exitError == nil { return errors.New("claim tracker adapter is not configured") } - resolution, err := tracker.resolver.Resolve() + cwd, err := tracker.workingDirectory() if err != nil { - return err + return tracker.exitError(127, strings.TrimSpace(err.Error())) + } + environment := append([]string(nil), tracker.environment()...) + resolution, err := trackerresolve.ResolveWithLookPath(cwd, environment, tracker.lookPath) + if err != nil { + return tracker.exitError(127, strings.TrimSpace(err.Error())) } command := exec.CommandContext(ctx, resolution.Binary, "update", id, "--claim") // #nosec G204 -- tracker resolver constrains the binary to br|bd. - command.Dir = resolution.WorkDir - command.Env = append([]string(nil), resolution.ChildEnv...) + command.Dir = cwd + if resolution.Tracker == trackerresolve.BR { + if _, present := trackerresolve.BeadsDirValue(environment); !present { + environment = append(environment, "BEADS_DIR="+resolution.LedgerDir) + } + } + command.Env = environment output, runErr := command.CombinedOutput() if len(output) > 0 && streams.Stdout != nil { _, _ = streams.Stdout.Write(output) diff --git a/cli/internal/adapters/claim/tracker_test.go b/cli/internal/adapters/claim/tracker_test.go index 7ea13dd8f..b121a18b8 100644 --- a/cli/internal/adapters/claim/tracker_test.go +++ b/cli/internal/adapters/claim/tracker_test.go @@ -4,21 +4,15 @@ import ( "bytes" "context" "errors" + "fmt" "os" "path/filepath" "strings" "testing" - beadsapp "github.com/boshu2/agentops/cli/internal/beads" claimapp "github.com/boshu2/agentops/cli/internal/claim" ) -type fixedResolver struct{ resolution beadsapp.TrackerResolution } - -func (resolver fixedResolver) Resolve() (beadsapp.TrackerResolution, error) { - return resolver.resolution, nil -} - type exitMarker struct { code int message string @@ -26,20 +20,24 @@ type exitMarker struct { func (marker *exitMarker) Error() string { return marker.message } +func captureExit(target **exitMarker) ExitErrorFactory { + return func(code int, message string) error { + *target = &exitMarker{code: code, message: message} + return *target + } +} + func TestTrackerPreservesMergedOutputArgsAndChildExit(t *testing.T) { root := t.TempDir() - script := filepath.Join(root, "br") - content := "#!/bin/sh\nprintf 'args:%s\\n' \"$*\"\nprintf 'child-stderr\\n' >&2\nexit 7\n" - if err := os.WriteFile(script, []byte(content), 0o755); err != nil { - t.Fatal(err) - } + script := writeTrackerScript(t, root, "br", "printf 'args:%s\\n' \"$*\"\nprintf 'child-stderr\\n' >&2\nexit 7\n") + environment := []string{"AGENTOPS_TRACKER=br", "PATH=" + root} var captured *exitMarker - tracker := NewTracker(fixedResolver{beadsapp.TrackerResolution{ - Tracker: beadsapp.TrackerBR, Binary: script, WorkDir: root, ChildEnv: os.Environ(), - }}, func(code int, message string) error { - captured = &exitMarker{code: code, message: message} - return captured - }) + tracker := NewTrackerWith( + func() (string, error) { return root, nil }, + func() []string { return environment }, + lookPathFor("br", script), + captureExit(&captured), + ) var output bytes.Buffer err := tracker.Claim(context.Background(), "age-1", claimapp.Streams{Stdout: &output}) if !errors.Is(err, captured) || captured.code != 7 || captured.message != "exit status 7" { @@ -49,3 +47,60 @@ func TestTrackerPreservesMergedOutputArgsAndChildExit(t *testing.T) { t.Fatalf("merged child output = %q", got) } } + +func TestTrackerPreservesBDCallerDirectoryAndForeignBeadsDir(t *testing.T) { + root := t.TempDir() + subdir := filepath.Join(root, "sub") + if err := os.Mkdir(subdir, 0o755); err != nil { + t.Fatal(err) + } + script := writeTrackerScript(t, root, "bd", "pwd\nprintf 'beads=%s\\n' \"${BEADS_DIR-unset}\"\nprintf 'args:%s\\n' \"$*\"\n") + environment := []string{"AGENTOPS_TRACKER=bd", "BEADS_DIR=/foreign", "PATH=" + root} + tracker := NewTrackerWith( + func() (string, error) { return subdir, nil }, + func() []string { return environment }, + lookPathFor("bd", script), + func(code int, message string) error { return fmt.Errorf("exit %d: %s", code, message) }, + ) + var output bytes.Buffer + if err := tracker.Claim(context.Background(), "age-2", claimapp.Streams{Stdout: &output}); err != nil { + t.Fatal(err) + } + got := output.String() + if !strings.Contains(got, subdir) || !strings.Contains(got, "beads=/foreign") || !strings.Contains(got, "args:update age-2 --claim") { + t.Fatalf("BD execution context = %q", got) + } +} + +func TestTrackerResolutionFailureMapsTo127(t *testing.T) { + root := t.TempDir() + var captured *exitMarker + tracker := NewTrackerWith( + func() (string, error) { return root, nil }, + func() []string { return []string{"PATH=/missing"} }, + func(string) (string, error) { return "", errors.New("not found") }, + captureExit(&captured), + ) + err := tracker.Claim(context.Background(), "age-x", claimapp.Streams{}) + if !errors.Is(err, captured) || captured.code != 127 || !strings.Contains(captured.message, "cannot resolve a beads tracker") { + t.Fatalf("resolution error=%v captured=%+v", err, captured) + } +} + +func writeTrackerScript(t *testing.T, directory, name, body string) string { + t.Helper() + path := filepath.Join(directory, name) + if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body), 0o755); err != nil { + t.Fatal(err) + } + return path +} + +func lookPathFor(name, path string) func(string) (string, error) { + return func(candidate string) (string, error) { + if candidate == name { + return path, nil + } + return "", errors.New("not found") + } +} From ceae886fe4d40b7ca768c5a69db4b7a04d654677 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:30:21 -0400 Subject: [PATCH 040/108] chore(cli): seal claim migration lineage (age-nw28h.2.3) --- .../compatibility-baseline/families/claim/lineage.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/testdata/compatibility-baseline/families/claim/lineage.json b/cli/testdata/compatibility-baseline/families/claim/lineage.json index e6b813baf..924aa71b1 100644 --- a/cli/testdata/compatibility-baseline/families/claim/lineage.json +++ b/cli/testdata/compatibility-baseline/families/claim/lineage.json @@ -6,5 +6,6 @@ "fixture_sha256": "834134b9e48db67016306dc26cc33c096c38478b57b601da052bbcd576847271", "ownership_sha256": "7b75df0636c4f3af37babf3abcefa82ea030be9437369bd4b164883e270f243a", "scope_sha256": "abf4f818b3e99612470df5e2b3dd61ad4220dcba9bf3b234bfd3468d16e197ca", - "migration_state": "migrating" + "migration_state": "migrated", + "accepted_sha": "2172c77f05692169948f3b034970c90d64a366c1" } From afc09ed3ed469f77ea61b8f99cdd060f0c0d5435 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:30:36 -0400 Subject: [PATCH 041/108] chore(provenance): bind pawl CONFIRMED verdict for age-nw28h.2.3 #trivial --- docs/provenance/ledger.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/provenance/ledger.jsonl b/docs/provenance/ledger.jsonl index 7f98612dd..7dda064fa 100644 --- a/docs/provenance/ledger.jsonl +++ b/docs/provenance/ledger.jsonl @@ -518,3 +518,4 @@ {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.1.5@e76a715","from_type":"verdict","to_id":"e76a715b00ad560df06fbd00da9783f3bcef7ba6","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.1.5 disposition=CONFIRMED","bead_id":"age-nw28h.1.5","trust_tier":"inferred","ts":"2026-07-11T17:26:53Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/P4A-rebase-verdict-e76a715b0.txt","prev_hash":"24e841e8569aa9375c46d26cf6f6940e2a765d6e0d28ce7e0dd86240481dbae5","payload_hash":"3df68eadc14f39362ab69431db77b379cc829298967e0d76ede07b7b3b34e0e9","hash":"5ce3f9f10999351cb3ce663d1dab51fbb418291ebf9e6ad6c507acc8550a7bbe"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.1@4863195","from_type":"verdict","to_id":"4863195d07079bb4f995a84686b2c572afe070fb","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.1 disposition=CONFIRMED","bead_id":"age-nw28h.2.1","trust_tier":"inferred","ts":"2026-07-11T19:10:35Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-beads-independent-verdict-4863195d0.txt","prev_hash":"5ce3f9f10999351cb3ce663d1dab51fbb418291ebf9e6ad6c507acc8550a7bbe","payload_hash":"ff4b1163efdb38e333b7ce4358ae7dd9a56a86d0645d481750c3a1ac0a860acb","hash":"0dbb491f180b24cbd6fd602bf8812d14382184c08370a714c7e4905f3730ca0e"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.2@bd171a4","from_type":"verdict","to_id":"bd171a4776a303a8c7ad867047f70967cdee8b0e","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.2 disposition=CONFIRMED","bead_id":"age-nw28h.2.2","trust_tier":"inferred","ts":"2026-07-11T19:50:19Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-capabilities-independent-verdict-bd171a477.txt","prev_hash":"0dbb491f180b24cbd6fd602bf8812d14382184c08370a714c7e4905f3730ca0e","payload_hash":"2ed1b654481b2037e5e1b8cb439cafe29bb16cf6319181b5e42bc549eb1a9e79","hash":"9739edafbb15e8a5d7e0ac15be295093bb14ebf60ffc13f92d46db35942e5aca"} +{"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.3@2172c77","from_type":"verdict","to_id":"2172c77f05692169948f3b034970c90d64a366c1","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.3 disposition=CONFIRMED","bead_id":"age-nw28h.2.3","trust_tier":"inferred","ts":"2026-07-11T20:30:36Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-claim-independent-verdict-2172c77f0.txt","prev_hash":"9739edafbb15e8a5d7e0ac15be295093bb14ebf60ffc13f92d46db35942e5aca","payload_hash":"7d01c886ce25d14d82b1cabbf379bc4e777f083328ee4695ca8344979eae9855","hash":"be2caecd5f64c47b68e12dd6a220eacac168450f3c6c0160f4428f4a917e92d0"} From 85adb9922a0506edfcc4c1c22e12786401bdc8ca Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:22:15 -0400 Subject: [PATCH 042/108] fix(close): make persistence retries monotonic (age-nw28h.2.4) --- cli/cmd/ao/tick.go | 150 +++++++++++++----- cli/cmd/ao/tick_test.go | 338 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 411 insertions(+), 77 deletions(-) diff --git a/cli/cmd/ao/tick.go b/cli/cmd/ao/tick.go index 67f432d54..c056e4b66 100644 --- a/cli/cmd/ao/tick.go +++ b/cli/cmd/ao/tick.go @@ -91,7 +91,21 @@ func (rt tickRuntime) runTracker(args ...string) ([]byte, int, error) { if err != nil { return nil, 127, err } - return rt.run(resolution.Binary, args...) + return rt.runResolvedTracker(resolution, args...) +} + +func (rt tickRuntime) runResolvedTracker(resolution trackerResolution, args ...string) ([]byte, int, error) { + command := exec.Command(resolution.Binary, args...) // #nosec G204 -- resolution constrains the executable to br|bd. + command.Dir = resolution.WorkDir + command.Env = append([]string(nil), resolution.ChildEnv...) + out, err := command.CombinedOutput() + if err == nil { + return out, 0, nil + } + if exitErr, ok := err.(*exec.ExitError); ok { + return out, exitErr.ExitCode(), err + } + return out, 127, err } var verdictGateCmd = &cobra.Command{ @@ -347,33 +361,34 @@ func tickClose(rt tickRuntime, id, msg, evidence string, paths []string) error { } func tickCloseResolved(rt tickRuntime, resolution trackerResolution, id, msg, evidence string, paths []string) error { - evFirst := tickFirstEvidenceToken(evidence) - if reason := tickEvidenceRefusal(rt, evidence, evFirst); reason != "" { - fmt.Fprintf(rt.stderr, "REFUSED close %s: evidence %q %s\n", id, evidence, reason) - return &tickExitError{code: tickExitCloseRef} + if _, err := tickClosePreflight(rt, id, evidence, paths); err != nil { + return err } - ledgerDir := resolution.LedgerDir - if _, code, err := rt.run(resolution.Binary, "close", id, "--reason", "evidence: "+evidence); err != nil || code != 0 { + if _, code, err := rt.runResolvedTracker(resolution, "close", id, "--reason", "evidence: "+evidence); err != nil || code != 0 { fmt.Fprintf(rt.stderr, "FAILED close %s: %s close failed or skipped; no files staged\n", id, resolution.Tracker) return &tickExitError{code: tickExitCloseFail} } + return tickPersistClosedResolved(rt, resolution, id, msg, paths, false) +} +func tickPersistClosedResolved(rt tickRuntime, resolution trackerResolution, id, msg string, paths []string, alreadyClosed bool) error { + ledgerDir := resolution.LedgerDir closedRef := "none" if resolution.Tracker == trackerBR { before := tickGitRevParseInDir(rt, ledgerDir) if before == "" { before = "none" } - if _, code, syncErr := rt.run(resolution.Binary, "sync", "--flush-only"); syncErr != nil || code != 0 { - _, _, _ = rt.run(resolution.Binary, "sync") + if _, code, syncErr := rt.runResolvedTracker(resolution, "sync", "--flush-only"); syncErr != nil || code != 0 { + if _, fallbackCode, fallbackErr := rt.runResolvedTracker(resolution, "sync"); fallbackErr != nil || fallbackCode != 0 { + return tickClosePersistenceError("tracker sync failed", fallbackCode) + } } issuesPath := filepath.Join(ledgerDir, "issues.jsonl") metadataPath := filepath.Join(ledgerDir, "metadata.json") if !tickLedgerShowsClosed(issuesPath, id) { - _, _, _ = rt.run(resolution.Binary, "update", id, "--status", "open") - _, _, _ = rt.run(resolution.Binary, "sync", "--flush-only") - fmt.Fprintf(rt.stderr, "FAILED close %s: %s ledger does not show a closed bead after close; bead reopened\n", id, resolution.Tracker) + fmt.Fprintf(rt.stderr, "FAILED close %s: %s ledger does not prove the bead is closed\n", id, resolution.Tracker) return &tickExitError{code: tickExitCloseFail} } @@ -382,7 +397,7 @@ func tickCloseResolved(rt tickRuntime, resolution trackerResolution, id, msg, ev ledgerStage = append(ledgerStage, "metadata.json") } if _, code, addErr := rt.run("git", ledgerStage...); addErr != nil || code != 0 { - return &tickExitError{code: code, msg: "ledger git add failed"} + return tickClosePersistenceError("ledger git add failed", code) } ledgerCommitOut, code, commitErr := rt.run("git", "-C", ledgerDir, "commit", "-q", "-m", msg) ledgerCommitNoop := false @@ -390,7 +405,7 @@ func tickCloseResolved(rt tickRuntime, resolution trackerResolution, id, msg, ev if tickGitCommitNothingToCommit(ledgerCommitOut) { ledgerCommitNoop = true } else { - return &tickExitError{code: code, msg: "ledger git commit failed"} + return tickClosePersistenceError("ledger git commit failed", code) } } after := tickGitRevParseInDir(rt, ledgerDir) @@ -398,31 +413,29 @@ func tickCloseResolved(rt tickRuntime, resolution trackerResolution, id, msg, ev after = "none" } if before == after && !ledgerCommitNoop { - _, _, _ = rt.run(resolution.Binary, "update", id, "--status", "open") - fmt.Fprintf(rt.stderr, "FAILED close %s: ledger git commit did not land; bead reopened\n", id) - return &tickExitError{code: tickExitNoCommit} + return tickClosePersistenceError("ledger git commit did not land", tickExitNoCommit) } closedRef = after } else { // bd persists into its Dolt-backed store itself. Never inspect, stage, or // commit the unrelated BR JSONL ledger; verify through the selected bd // backend instead. - if !tickTrackerShowsClosed(rt, resolution, id) { - _, _, _ = rt.run(resolution.Binary, "update", id, "--status", "open") - fmt.Fprintf(rt.stderr, "FAILED close %s: %s does not report the issue closed; issue reopened\n", id, resolution.Tracker) + closed, err := tickTrackerClosedState(rt, resolution, id) + if err != nil || !closed { + fmt.Fprintf(rt.stderr, "FAILED close %s: %s does not prove the issue is closed\n", id, resolution.Tracker) return &tickExitError{code: tickExitCloseFail} } } - stage := tickPublicStagePaths(rt, ledgerDir, evFirst, paths) + stage := tickPublicStagePaths(rt, ledgerDir, "", paths) if len(stage) > 0 { args := append([]string{"add", "--"}, stage...) if _, code, err := rt.run("git", args...); err != nil || code != 0 { - return &tickExitError{code: code, msg: "git add failed"} + return tickClosePersistenceError("public git add failed", code) } out, code, err := rt.run("git", "commit", "-q", "-m", msg) if (err != nil || code != 0) && !tickGitCommitNothingToCommit(out) { - return &tickExitError{code: code, msg: "git commit failed"} + return tickClosePersistenceError("public git commit failed", code) } } if resolution.Tracker == trackerBD { @@ -431,7 +444,11 @@ func tickCloseResolved(rt tickRuntime, resolution trackerResolution, id, msg, ev closedRef = "none" } } - fmt.Fprintf(rt.stdout, "closed %s @ %s\n", id, tickShortSHA(closedRef)) + verb := "closed" + if alreadyClosed { + verb = "already closed" + } + fmt.Fprintf(rt.stdout, "%s %s @ %s\n", verb, id, tickShortSHA(closedRef)) return nil } @@ -440,34 +457,75 @@ func tickClosePort(rt tickRuntime, id, msg, evidence string, paths []string) err if err != nil { return err } - if tickTrackerShowsClosed(rt, resolution, id) { - after := tickGitRevParse(rt) - if after == "" { - after = "none" - } - fmt.Fprintf(rt.stdout, "already closed %s @ %s\n", id, tickShortSHA(after)) - return nil + if _, err := tickClosePreflight(rt, id, evidence, paths); err != nil { + return err + } + closed, err := tickTrackerClosedState(rt, resolution, id) + if err != nil { + fmt.Fprintf(rt.stderr, "FAILED close %s: %s status cannot be determined\n", id, resolution.Tracker) + return &tickExitError{code: tickExitCloseFail} + } + if closed { + return tickPersistClosedResolved(rt, resolution, id, msg, paths, true) } return tickCloseResolved(rt, resolution, id, msg, evidence, paths) } func tickTrackerShowsClosed(rt tickRuntime, resolution trackerResolution, id string) bool { + closed, err := tickTrackerClosedState(rt, resolution, id) + return err == nil && closed +} + +func tickTrackerClosedState(rt tickRuntime, resolution trackerResolution, id string) (bool, error) { if resolution.Tracker == trackerBR { - return tickLedgerShowsClosed(filepath.Join(resolution.LedgerDir, "issues.jsonl"), id) + return tickLedgerClosedState(filepath.Join(resolution.LedgerDir, "issues.jsonl"), id) } queries := [][]string{{"show", id, "--json"}, {"list", "--all", "--json"}} + var querySucceeded bool for _, args := range queries { - out, code, err := rt.run(resolution.Binary, args...) + out, code, err := rt.runResolvedTracker(resolution, args...) if err != nil || code != 0 { continue } + querySucceeded = true for _, item := range tickParseBeads(out) { - if item.ID == id && item.Status == "closed" { - return true + if item.ID == id { + return item.Status == "closed", nil } } } - return false + if querySucceeded { + return false, fmt.Errorf("issue %s not found", id) + } + return false, fmt.Errorf("tracker status query failed") +} + +func tickClosePreflight(rt tickRuntime, id, evidence string, paths []string) (string, error) { + evFirst := tickFirstEvidenceToken(evidence) + if reason := tickEvidenceRefusal(rt, evidence, evFirst); reason != "" { + fmt.Fprintf(rt.stderr, "REFUSED close %s: evidence %q %s\n", id, evidence, reason) + return "", &tickExitError{code: tickExitCloseRef} + } + for _, path := range paths { + if tickPathExists(rt.workDir, path) || tickGitTracksPath(rt, path) { + continue + } + fmt.Fprintf(rt.stderr, "REFUSED close %s: path %q resolves to no real or tracked artifact\n", id, path) + return "", &tickExitError{code: tickExitCloseRef} + } + return evFirst, nil +} + +func tickGitTracksPath(rt tickRuntime, path string) bool { + if path == "" { + return false + } + _, code, err := rt.run("git", "ls-files", "--error-unmatch", "--", path) + return err == nil && code == 0 +} + +func tickClosePersistenceError(message string, childCode int) error { + return &tickExitError{code: tickExitNoCommit, msg: fmt.Sprintf("%s (child exit %d)", message, childCode)} } // tickEvidenceRefusal applies the close-evidence gate and returns a non-empty @@ -736,10 +794,8 @@ func tickGitCommitNothingToCommit(out []byte) bool { } func tickPublicStagePaths(rt tickRuntime, ledgerDir, evFirst string, paths []string) []string { + _ = evFirst // Evidence is validation-only; callers explicitly select public paths. stage := []string{} - if tickPathExists(rt.workDir, evFirst) { - stage = tickAppendPublicStagePath(stage, rt.workDir, ledgerDir, evFirst) - } for _, path := range paths { stage = tickAppendPublicStagePath(stage, rt.workDir, ledgerDir, path) } @@ -772,19 +828,27 @@ func tickShortSHA(sha string) string { } func tickLedgerShowsClosed(path, id string) bool { + closed, err := tickLedgerClosedState(path, id) + return err == nil && closed +} + +func tickLedgerClosedState(path, id string) (bool, error) { f, err := os.Open(path) if err != nil { - return false + return false, err } defer func() { _ = f.Close() }() scanner := bufio.NewScanner(f) for scanner.Scan() { var item tickBead - if json.Unmarshal(scanner.Bytes(), &item) == nil && item.ID == id && item.Status == "closed" { - return true + if json.Unmarshal(scanner.Bytes(), &item) == nil && item.ID == id { + return item.Status == "closed", nil } } - return false + if err := scanner.Err(); err != nil { + return false, err + } + return false, fmt.Errorf("issue %s not found", id) } func tickReadVerdict(rt tickRuntime, path string) (string, error) { diff --git a/cli/cmd/ao/tick_test.go b/cli/cmd/ao/tick_test.go index e31ad0a44..718b2cff9 100644 --- a/cli/cmd/ao/tick_test.go +++ b/cli/cmd/ao/tick_test.go @@ -105,41 +105,84 @@ func tickTestLedgerLine(id, status string) string { return `{"_type": "issue", "id": "` + id + `", "title": "Node 2", "status": "` + status + `", "priority": 2, "issue_type": "task", "created_at": "2026-05-08T12:09:00Z", "updated_at": "2026-05-30T12:57:05Z", "closed_at": "2026-05-08T12:09:11Z", "close_reason": "Closed", "dependency_count": 0, "dependent_count": 1, "comment_count": 0}` + "\n" } -func TestTickClosePortAlreadyClosedIsIdempotent(t *testing.T) { - tests := []struct { - name string - ledgerDir string - }{ - {name: "br workspace _beads", ledgerDir: "_beads"}, +func TestTickClosePortAlreadyClosedCompletesPendingPersistence(t *testing.T) { + t.Setenv("BEADS_DIR", "") + t.Setenv("GIT_AUTHOR_NAME", "tick-test") + t.Setenv("GIT_AUTHOR_EMAIL", "tick@test") + t.Setenv("GIT_COMMITTER_NAME", "tick-test") + t.Setenv("GIT_COMMITTER_EMAIL", "tick@test") + dir := t.TempDir() + ledger := filepath.Join(dir, "_beads") + if err := os.MkdirAll(filepath.Join(dir, "docs"), 0o755); err != nil { + t.Fatal(err) } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Setenv("BEADS_DIR", "") - dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, tc.ledgerDir), 0o755); err != nil { - t.Fatal(err) - } - body := tickTestLedgerLine("cp-done", "closed") - if err := os.WriteFile(filepath.Join(dir, tc.ledgerDir, "issues.jsonl"), []byte(body), 0o644); err != nil { - t.Fatal(err) - } - var stdout bytes.Buffer - rt := tickRuntime{workDir: dir, stdout: &stdout, stderr: &bytes.Buffer{}} - if err := tickClosePort(rt, "cp-done", "msg", "missing-evidence.md", nil); err != nil { - t.Fatalf("tickClosePort() unexpected error: %v", err) - } - if got := stdout.String(); got != "already closed cp-done @ none\n" { - t.Fatalf("tickClosePort() stdout = %q", got) - } - }) + if err := os.MkdirAll(ledger, 0o755); err != nil { + t.Fatal(err) + } + runGit := func(workDir string, args ...string) string { + t.Helper() + command := exec.Command("git", args...) + command.Dir = workDir + out, err := command.CombinedOutput() + if err != nil { + t.Fatalf("git %v in %s: %v\n%s", args, workDir, err, out) + } + return strings.TrimSpace(string(out)) + } + runGit(dir, "init", "-q") + if err := os.WriteFile(filepath.Join(dir, "proof.md"), []byte("proof\n"), 0o644); err != nil { + t.Fatal(err) + } + runGit(dir, "add", "proof.md") + runGit(dir, "commit", "-q", "-m", "seed public") + publicBefore := runGit(dir, "rev-parse", "HEAD") + + runGit(ledger, "init", "-q") + if err := os.WriteFile(filepath.Join(ledger, "issues.jsonl"), []byte(tickTestLedgerLine("cp-done", "open")), 0o644); err != nil { + t.Fatal(err) + } + runGit(ledger, "add", "issues.jsonl") + runGit(ledger, "commit", "-q", "-m", "seed ledger") + if err := os.WriteFile(filepath.Join(ledger, "issues.jsonl"), []byte(tickTestLedgerLine("cp-done", "closed")), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "docs", "close.md"), []byte("pending public persistence\n"), 0o644); err != nil { + t.Fatal(err) + } + + fakebin := filepath.Join(dir, "fakebin") + if err := os.MkdirAll(fakebin, 0o755); err != nil { + t.Fatal(err) + } + fakeBR := "#!/usr/bin/env bash\ncase \"${1:-}\" in sync) exit 0 ;; *) echo \"unexpected br call: $*\" >&2; exit 43 ;; esac\n" + if err := os.WriteFile(filepath.Join(fakebin, "br"), []byte(fakeBR), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fakebin+string(os.PathListSeparator)+os.Getenv("PATH")) + + var stdout, stderr bytes.Buffer + rt := tickRuntime{workDir: dir, stdout: &stdout, stderr: &stderr} + if err := tickClosePort(rt, "cp-done", "finish close", "proof.md", []string{"docs/close.md"}); err != nil { + t.Fatalf("tickClosePort() error: %v (stderr=%q)", err, stderr.String()) + } + ledgerAfter := runGit(ledger, "rev-parse", "HEAD") + if ledgerAfter == "" || ledgerAfter == publicBefore { + t.Fatalf("ledger close was not committed: ledger=%q public-before=%q", ledgerAfter, publicBefore) + } + publicAfter := runGit(dir, "rev-parse", "HEAD") + if publicAfter == publicBefore { + t.Fatal("pending public paths were not committed on retry") + } + if got, want := stdout.String(), "already closed cp-done @ "+tickShortSHA(ledgerAfter)+"\n"; got != want { + t.Fatalf("tickClosePort() stdout = %q, want %q", got, want) } } func TestTickClosePortBDDoesNotTrustOrCommitBRLedger(t *testing.T) { t.Setenv("AGENTOPS_TRACKER", "bd") - t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_DIR", "/foreign/br-ledger") dir := t.TempDir() - for _, sub := range []string{"_beads", ".beads", "fakebin"} { + for _, sub := range []string{"_beads", ".beads", "fakebin", "nested"} { if err := os.MkdirAll(filepath.Join(dir, sub), 0o755); err != nil { t.Fatal(err) } @@ -172,7 +215,7 @@ func TestTickClosePortBDDoesNotTrustOrCommitBRLedger(t *testing.T) { t.Fatal(err) } fakeBD := `#!/usr/bin/env bash -printf '%s\n' "$*" >> "${TICK_TEST_BD_LOG:?}" +printf 'pwd=%s beads=%s args=%s\n' "$PWD" "${BEADS_DIR-unset}" "$*" >> "${TICK_TEST_BD_LOG:?}" case "${1:-}" in show) status="$(tr -d '\n' < "${TICK_TEST_BD_STATE:?}")" @@ -195,8 +238,8 @@ esac t.Setenv("TICK_TEST_BD_LOG", logFile) var stdout, stderr bytes.Buffer - rt := tickRuntime{workDir: dir, stdout: &stdout, stderr: &stderr} - if err := tickClosePort(rt, "agentops-123", "close bd issue", "proof.md", nil); err != nil { + rt := tickRuntime{workDir: filepath.Join(dir, "nested"), stdout: &stdout, stderr: &stderr} + if err := tickClosePort(rt, "agentops-123", "close bd issue", filepath.Join(dir, "proof.md"), nil); err != nil { t.Fatalf("tickClosePort() error: %v (stderr=%q)", err, stderr.String()) } logBody, err := os.ReadFile(logFile) @@ -204,9 +247,18 @@ esac t.Fatalf("bd was never invoked: %v", err) } logText := string(logBody) - if !strings.Contains(logText, "close agentops-123 --reason evidence: proof.md") { + resolvedDir, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(logText, "close agentops-123 --reason evidence: "+filepath.Join(dir, "proof.md")) { t.Fatalf("BD close missing; calls:\n%s", logText) } + for _, line := range strings.Split(strings.TrimSpace(logText), "\n") { + if !strings.Contains(line, "pwd="+resolvedDir+" ") || !strings.Contains(line, "beads=unset ") { + t.Fatalf("BD child did not use canonical repo root with BEADS_DIR stripped: %q", line) + } + } if strings.Contains(logText, "sync") { t.Fatalf("BR-only sync leaked into BD close; calls:\n%s", logText) } @@ -219,6 +271,224 @@ esac } } +func TestTickPublicStagePathsSkipsGitignoredRuntimeCorpusEvidence(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, ".agents", "evidence"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(".agents/\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".agents", "evidence", "close.md"), []byte("runtime proof\n"), 0o644); err != nil { + t.Fatal(err) + } + command := exec.Command("git", "init", "-q") + command.Dir = dir + if out, err := command.CombinedOutput(); err != nil { + t.Fatalf("git init: %v\n%s", err, out) + } + rt := tickRuntime{workDir: dir, stdout: &bytes.Buffer{}, stderr: &bytes.Buffer{}} + got := tickPublicStagePaths(rt, filepath.Join(dir, "_beads"), ".agents/evidence/close.md", []string{"docs/result.md"}) + want := []string{"docs/result.md"} + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("tickPublicStagePaths() = %q, want %q", got, want) + } +} + +func TestTickClosePortPreflightsExplicitPathsBeforeTrackerMutation(t *testing.T) { + for _, status := range []string{"open", "closed"} { + t.Run(status, func(t *testing.T) { + t.Setenv("BEADS_DIR", "") + dir := t.TempDir() + ledger := filepath.Join(dir, "_beads") + fakebin := filepath.Join(dir, "fakebin") + if err := os.MkdirAll(ledger, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(fakebin, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(ledger, "issues.jsonl"), []byte(tickTestLedgerLine("cp-path", status)), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "proof.md"), []byte("proof\n"), 0o644); err != nil { + t.Fatal(err) + } + logFile := filepath.Join(dir, "br.log") + fakeBR := "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >> \"${TICK_TEST_BR_LOG:?}\"\nexit 0\n" + if err := os.WriteFile(filepath.Join(fakebin, "br"), []byte(fakeBR), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fakebin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("TICK_TEST_BR_LOG", logFile) + var stdout, stderr bytes.Buffer + rt := tickRuntime{workDir: dir, stdout: &stdout, stderr: &stderr} + err := tickClosePort(rt, "cp-path", "close", "proof.md", []string{"missing-result.md"}) + if code := tickExitCode(err); code != tickExitCloseRef { + t.Fatalf("exit = %d, want %d (err=%v stderr=%q)", code, tickExitCloseRef, err, stderr.String()) + } + if _, err := os.Stat(logFile); !os.IsNotExist(err) { + t.Fatalf("tracker was invoked before preflight completed: err=%v", err) + } + if stdout.Len() != 0 { + t.Fatalf("success output emitted on refusal: %q", stdout.String()) + } + }) + } +} + +func TestTickClosePortRetriesForwardAfterPublicPersistenceFailure(t *testing.T) { + t.Setenv("BEADS_DIR", "") + t.Setenv("GIT_AUTHOR_NAME", "tick-test") + t.Setenv("GIT_AUTHOR_EMAIL", "tick@test") + t.Setenv("GIT_COMMITTER_NAME", "tick-test") + t.Setenv("GIT_COMMITTER_EMAIL", "tick@test") + dir := t.TempDir() + ledger := filepath.Join(dir, "_beads") + fakebin := filepath.Join(dir, "fakebin") + for _, path := range []string{ledger, fakebin, filepath.Join(dir, "docs")} { + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + } + realGit, err := exec.LookPath("git") + if err != nil { + t.Fatal(err) + } + runGit := func(workDir string, args ...string) string { + t.Helper() + command := exec.Command(realGit, args...) + command.Dir = workDir + out, err := command.CombinedOutput() + if err != nil { + t.Fatalf("git %v in %s: %v\n%s", args, workDir, err, out) + } + return strings.TrimSpace(string(out)) + } + runGit(dir, "init", "-q") + if err := os.WriteFile(filepath.Join(dir, "proof.md"), []byte("proof\n"), 0o644); err != nil { + t.Fatal(err) + } + runGit(dir, "add", "proof.md") + runGit(dir, "commit", "-q", "-m", "seed public") + if err := os.WriteFile(filepath.Join(dir, "docs", "result.md"), []byte("result\n"), 0o644); err != nil { + t.Fatal(err) + } + runGit(ledger, "init", "-q") + if err := os.WriteFile(filepath.Join(ledger, "issues.jsonl"), []byte(tickTestLedgerLine("cp-retry", "open")), 0o644); err != nil { + t.Fatal(err) + } + runGit(ledger, "add", "issues.jsonl") + runGit(ledger, "commit", "-q", "-m", "seed ledger") + + brLog := filepath.Join(dir, "br.log") + failOnce := filepath.Join(dir, "fail-public-add-once") + if err := os.WriteFile(failOnce, []byte("1\n"), 0o644); err != nil { + t.Fatal(err) + } + fakeBR := `#!/usr/bin/env bash +printf '%s\n' "$*" >> "${TICK_TEST_BR_LOG:?}" +case "${1:-}" in + close) sed 's/"status": "open"/"status": "closed"/' "${TICK_TEST_LEDGER:?}/issues.jsonl" > "${TICK_TEST_LEDGER:?}/issues.tmp" && mv "${TICK_TEST_LEDGER:?}/issues.tmp" "${TICK_TEST_LEDGER:?}/issues.jsonl" ;; + sync) ;; + *) echo "unexpected br call: $*" >&2; exit 43 ;; +esac +` + fakeGit := `#!/usr/bin/env bash +if [ "${1:-}" = "add" ] && [ -f "${TICK_TEST_FAIL_ONCE:?}" ]; then + rm -f "${TICK_TEST_FAIL_ONCE:?}" + exit 77 +fi +exec "${TICK_TEST_REAL_GIT:?}" "$@" +` + if err := os.WriteFile(filepath.Join(fakebin, "br"), []byte(fakeBR), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fakebin, "git"), []byte(fakeGit), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fakebin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("TICK_TEST_BR_LOG", brLog) + t.Setenv("TICK_TEST_LEDGER", ledger) + t.Setenv("TICK_TEST_FAIL_ONCE", failOnce) + t.Setenv("TICK_TEST_REAL_GIT", realGit) + + var firstOut, firstErr bytes.Buffer + firstRT := tickRuntime{workDir: dir, stdout: &firstOut, stderr: &firstErr} + err = tickClosePort(firstRT, "cp-retry", "finish close", "proof.md", []string{"docs/result.md"}) + if code := tickExitCode(err); code != tickExitNoCommit { + t.Fatalf("first exit = %d, want %d (err=%v stderr=%q)", code, tickExitNoCommit, err, firstErr.String()) + } + if firstOut.Len() != 0 { + t.Fatalf("success output emitted on persistence failure: %q", firstOut.String()) + } + if !tickLedgerShowsClosed(filepath.Join(ledger, "issues.jsonl"), "cp-retry") { + t.Fatal("persistence failure reopened a proven-closed BR bead") + } + + var retryOut, retryErr bytes.Buffer + retryRT := tickRuntime{workDir: dir, stdout: &retryOut, stderr: &retryErr} + if err := tickClosePort(retryRT, "cp-retry", "finish close", "proof.md", []string{"docs/result.md"}); err != nil { + t.Fatalf("retry error: %v (stderr=%q)", err, retryErr.String()) + } + logBody, err := os.ReadFile(brLog) + if err != nil { + t.Fatal(err) + } + closeCalls := 0 + for _, line := range strings.Split(strings.TrimSpace(string(logBody)), "\n") { + if strings.HasPrefix(line, "close ") { + closeCalls++ + } + if strings.HasPrefix(line, "update ") { + t.Fatalf("retry-forward state machine attempted rollback: %q", line) + } + } + if closeCalls != 1 { + t.Fatalf("tracker close calls = %d, want exactly 1 across failure+retry\n%s", closeCalls, logBody) + } + ledgerHead := runGit(ledger, "rev-parse", "HEAD") + if got, want := retryOut.String(), "already closed cp-retry @ "+tickShortSHA(ledgerHead)+"\n"; got != want { + t.Fatalf("retry stdout = %q, want %q", got, want) + } +} + +func TestTickClosePortTrackerStatusFailureIsNotTreatedAsOpen(t *testing.T) { + t.Setenv("AGENTOPS_TRACKER", "bd") + t.Setenv("BEADS_DIR", "/foreign/br-ledger") + dir := t.TempDir() + fakebin := filepath.Join(dir, "fakebin") + if err := os.MkdirAll(fakebin, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "proof.md"), []byte("proof\n"), 0o644); err != nil { + t.Fatal(err) + } + logFile := filepath.Join(dir, "bd.log") + fakeBD := "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >> \"${TICK_TEST_BD_LOG:?}\"\nexit 77\n" + if err := os.WriteFile(filepath.Join(fakebin, "bd"), []byte(fakeBD), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fakebin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("TICK_TEST_BD_LOG", logFile) + var stdout, stderr bytes.Buffer + rt := tickRuntime{workDir: dir, stdout: &stdout, stderr: &stderr} + err := tickClosePort(rt, "agentops-unknown", "close", "proof.md", nil) + if code := tickExitCode(err); code != tickExitCloseFail { + t.Fatalf("exit = %d, want %d (err=%v stderr=%q)", code, tickExitCloseFail, err, stderr.String()) + } + logBody, err := os.ReadFile(logFile) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(logBody), "close ") { + t.Fatalf("unknown status was treated as open:\n%s", logBody) + } + if stdout.Len() != 0 { + t.Fatalf("success output emitted on status failure: %q", stdout.String()) + } +} + func TestTickLedgerDirResolution(t *testing.T) { tests := []struct { name string @@ -463,8 +733,8 @@ esac strings.Contains(line, "issues.jsonl") || strings.Contains(line, "metadata.json") { t.Fatalf("public git add staged ledger path: %q", line) } - if line != "add -- evidence/proof.md docs/close.md" { - t.Fatalf("public git add = %q, want evidence and caller path only", line) + if line != "add -- docs/close.md" { + t.Fatalf("public git add = %q, want caller path only", line) } } } From 3d7c4029ac34abc5db3ccf1cfbad457ef61a322a Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:24:53 -0400 Subject: [PATCH 043/108] test(cli): freeze close compatibility contract (age-nw28h.2.4) --- .../families/close/case.json | 25 ++++++++++ .../families/close/ownership.json | 46 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/close/case.json create mode 100644 cli/testdata/compatibility-baseline/families/close/ownership.json diff --git a/cli/testdata/compatibility-baseline/families/close/case.json b/cli/testdata/compatibility-baseline/families/close/case.json new file mode 100644 index 000000000..d2708f58b --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/close/case.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "family": "close", + "checks": [ + { + "id": "close-cli-surface", + "command": "tmp=$(mktemp -d /tmp/ao-close-surface.XXXXXX); trap 'rm -rf \"$tmp\"' EXIT; (cd cli && go build -o \"$tmp/ao\" ./cmd/ao && go build -tags legacy -o \"$tmp/ao-legacy\" ./cmd/ao); \"$tmp/ao\" close --help >\"$tmp/help\"; \"$tmp/ao-legacy\" tick close --help >\"$tmp/legacy-help\"; grep -qx 'Usage:' \"$tmp/help\"; grep -Fq 'ao close [paths...]' \"$tmp/help\"; grep -Fq 'ao tick close [paths...]' \"$tmp/legacy-help\"; set +e; \"$tmp/ao\" close only-two args >\"$tmp/out\" 2>\"$tmp/err\"; code=$?; set -e; test \"$code\" -eq 1; test ! -s \"$tmp/out\"; grep -qx 'Error: requires at least 3 arg(s), only received 2' \"$tmp/err\"" + }, + { + "id": "close-transaction-semantics", + "command": "cd cli && go test ./cmd/ao -run '^(TestTickClosePortAlreadyClosedCompletesPendingPersistence|TestTickClosePortBDDoesNotTrustOrCommitBRLedger|TestTickPublicStagePathsSkipsGitignoredRuntimeCorpusEvidence|TestTickClosePortPreflightsExplicitPathsBeforeTrackerMutation|TestTickClosePortRetriesForwardAfterPublicPersistenceFailure|TestTickClosePortTrackerStatusFailureIsNotTreatedAsOpen|TestTickClose_PersistsLedgerInOwnRepoNotPublic|TestTickCloseLinkedWorktreeUsesCanonicalBeadsLedger)$' -count=1" + } + ], + "dimensions": { + "path": {"status": "evidence", "evidence": ["close-cli-surface"]}, + "aliases": {"status": "evidence", "evidence": ["close-cli-surface"]}, + "accepted_args": {"status": "evidence", "evidence": ["close-cli-surface", "close-transaction-semantics"]}, + "rejected_args": {"status": "evidence", "evidence": ["close-cli-surface", "close-transaction-semantics"]}, + "stdout": {"status": "evidence", "evidence": ["close-transaction-semantics"]}, + "stderr": {"status": "evidence", "evidence": ["close-cli-surface", "close-transaction-semantics"]}, + "exit_classes": {"status": "evidence", "evidence": ["close-cli-surface", "close-transaction-semantics"]}, + "tracker_selection": {"status": "evidence", "evidence": ["close-transaction-semantics"]}, + "ordered_effects": {"status": "evidence", "evidence": ["close-transaction-semantics"]} + } +} diff --git a/cli/testdata/compatibility-baseline/families/close/ownership.json b/cli/testdata/compatibility-baseline/families/close/ownership.json new file mode 100644 index 000000000..be45e1d88 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/close/ownership.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "family": "close", + "profiles": { + "default": "present", + "flywheel": "present", + "legacy": "present", + "combined": "present" + }, + "legacy_symbols": [ + "closeCmd", + "tickClose", + "tickCloseResolved", + "tickPersistClosedResolved", + "tickClosePort", + "tickTrackerShowsClosed", + "tickTrackerClosedState", + "tickClosePreflight", + "tickGitTracksPath", + "tickClosePersistenceError", + "tickEvidenceRefusal", + "tickHasGitHead", + "tickRepoRelEvidence", + "tickEvidenceCommittedRel", + "tickEvidenceRuntimeCorpus", + "tickFirstEvidenceToken", + "tickGitRevParseInDir", + "tickGitCommitNothingToCommit", + "tickPublicStagePaths", + "tickAppendPublicStagePath", + "tickIsPrivateLedgerPath", + "tickLedgerShowsClosed", + "tickLedgerClosedState" + ], + "live_owner": "cli/internal/commands/close", + "allowed_paths": [ + "cli/cmd/ao/close*.go", + "cli/cmd/ao/tick.go", + "cli/cmd/ao/tick_test.go", + "cli/cmd/ao/tick_cmd_legacy.go", + "cli/internal/adapters/close/**", + "cli/internal/close/**", + "cli/internal/commands/close/**", + "cli/testdata/compatibility-baseline/families/close/lineage.json" + ] +} From f0a06c2f160ac8f2ee2ccf428adf92ba31e2b05c Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:25:33 -0400 Subject: [PATCH 044/108] chore(cli): bind close migration lineage (age-nw28h.2.4) --- .../compatibility-baseline/families/close/lineage.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/close/lineage.json diff --git a/cli/testdata/compatibility-baseline/families/close/lineage.json b/cli/testdata/compatibility-baseline/families/close/lineage.json new file mode 100644 index 000000000..ed25a67b6 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/close/lineage.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "family": "close", + "old_implementation_sha": "85adb9922a0506edfcc4c1c22e12786401bdc8ca", + "freeze_sha": "3d7c4029ac34abc5db3ccf1cfbad457ef61a322a", + "fixture_sha256": "20482e48578cab821b243f2500a82a4212b3541dfdfea69ef69734d3775dc196", + "ownership_sha256": "3f9e48bc80357bb5ab9453427a423ffde64fbd387081591def6e33e6bbc853e0", + "scope_sha256": "3577ffbc9a8679d61367bfe13a159c9939050c5bd2cf5333e078f85c2211ec0a", + "migration_state": "migrating" +} From be19dc5be4f64121b5776cab0c056c3ac193e07f Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:45:08 -0400 Subject: [PATCH 045/108] refactor(close): extract explicit command module (age-nw28h.2.4) --- cli/cmd/ao/close_module.go | 23 + cli/cmd/ao/close_smoke.go | 57 +++ cli/cmd/ao/close_test_shims_test.go | 96 ++++ cli/cmd/ao/tick.go | 534 +-------------------- cli/cmd/ao/tick_cmd_legacy.go | 9 +- cli/internal/adapters/close/repository.go | 260 ++++++++++ cli/internal/adapters/close/runtime.go | 26 + cli/internal/adapters/close/tracker.go | 143 ++++++ cli/internal/close/service.go | 199 ++++++++ cli/internal/close/service_test.go | 150 ++++++ cli/internal/commands/close/module.go | 93 ++++ cli/internal/commands/close/module_test.go | 71 +++ 12 files changed, 1121 insertions(+), 540 deletions(-) create mode 100644 cli/cmd/ao/close_module.go create mode 100644 cli/cmd/ao/close_smoke.go create mode 100644 cli/cmd/ao/close_test_shims_test.go create mode 100644 cli/internal/adapters/close/repository.go create mode 100644 cli/internal/adapters/close/runtime.go create mode 100644 cli/internal/adapters/close/tracker.go create mode 100644 cli/internal/close/service.go create mode 100644 cli/internal/close/service_test.go create mode 100644 cli/internal/commands/close/module.go create mode 100644 cli/internal/commands/close/module_test.go diff --git a/cli/cmd/ao/close_module.go b/cli/cmd/ao/close_module.go new file mode 100644 index 000000000..8bb900b79 --- /dev/null +++ b/cli/cmd/ao/close_module.go @@ -0,0 +1,23 @@ +// practices: [hexagonal-architecture, ddd-bounded-context] +package main + +import ( + closeadapter "github.com/boshu2/agentops/cli/internal/adapters/close" + "github.com/boshu2/agentops/cli/internal/clicontract" + closeapp "github.com/boshu2/agentops/cli/internal/close" + closecommands "github.com/boshu2/agentops/cli/internal/commands/close" +) + +func newCloseService(runtime closeapp.Runtime) *closeapp.Service { + return closeapp.NewService(runtime, closeadapter.Tracker{}, closeadapter.Repository{}) +} + +var closeModule = closecommands.NewModule(newCloseService(closeadapter.SystemRuntime{})) + +func init() { + command := closeModule.Command() + if err := clicontract.Attach(command, closeModule.Contract()); err != nil { + panic(err) + } + rootCmd.AddCommand(command) +} diff --git a/cli/cmd/ao/close_smoke.go b/cli/cmd/ao/close_smoke.go new file mode 100644 index 000000000..a40949501 --- /dev/null +++ b/cli/cmd/ao/close_smoke.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "os" + "path/filepath" + + closeadapter "github.com/boshu2/agentops/cli/internal/adapters/close" + closeapp "github.com/boshu2/agentops/cli/internal/close" +) + +func runCloseSmokeFailure(rt tickRuntime, tmp string) bool { + fakebin := filepath.Join(tmp, "fakebin") + _ = os.MkdirAll(fakebin, 0o755) + fakeBR := `#!/usr/bin/env bash +if [ "${1:-}" = "close" ]; then echo "stubbed br close failure" >&2; exit 42; fi +if [ "${1:-}" = "sync" ] || [ "${1:-}" = "update" ]; then exit 0; fi +echo "unexpected br call: $*" >&2; exit 43 +` + fakeGitLog := filepath.Join(tmp, "fake-git.log") + fakeGit := `#!/usr/bin/env bash +case "${1:-}" in + rev-parse) echo fakehead ;; + add) : ;; + commit) printf 'commit\n' >> "${TICK_SMOKE_FAKE_GIT_LOG:?}" ;; + *) : ;; +esac +` + _ = os.WriteFile(filepath.Join(fakebin, "br"), []byte(fakeBR), 0o755) + _ = os.WriteFile(filepath.Join(fakebin, "git"), []byte(fakeGit), 0o755) + evidence := filepath.Join(tmp, "close-evidence.md") + _ = os.WriteFile(evidence, []byte("evidence"), 0o644) + env := append(os.Environ(), rt.env...) + env = append(env, + "PATH="+fakebin+string(os.PathListSeparator)+envValue(env, "PATH"), + "TICK_SMOKE_FAKE_GIT_LOG="+fakeGitLog, + ) + service := newCloseService(closeadapter.StaticRuntime{WorkDir: rt.workDir, Env: env}) + _, err := service.Execute(context.Background(), closeapp.Request{ + ID: "cp-smoke", Message: "smoke close should not commit", Evidence: evidence, Mode: closeapp.ModeStrict, + }) + if err == nil { + return false + } + info, statErr := os.Stat(fakeGitLog) + return os.IsNotExist(statErr) || (statErr == nil && info.Size() == 0) +} + +func envValue(env []string, key string) string { + prefix := key + "=" + for index := len(env) - 1; index >= 0; index-- { + if len(env[index]) >= len(prefix) && env[index][:len(prefix)] == prefix { + return env[index][len(prefix):] + } + } + return "" +} diff --git a/cli/cmd/ao/close_test_shims_test.go b/cli/cmd/ao/close_test_shims_test.go new file mode 100644 index 000000000..d9871af93 --- /dev/null +++ b/cli/cmd/ao/close_test_shims_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + closeadapter "github.com/boshu2/agentops/cli/internal/adapters/close" + closeapp "github.com/boshu2/agentops/cli/internal/close" + "github.com/boshu2/agentops/cli/internal/trackerresolve" +) + +const ( + tickExitCloseRef = closeapp.ExitRefused + tickExitNoCommit = closeapp.ExitPersistence + tickExitCloseFail = closeapp.ExitTracker +) + +func tickClosePort(rt tickRuntime, id, message, evidence string, paths []string) error { + return runCloseTestShim(rt, closeapp.Request{ + ID: id, Message: message, Evidence: evidence, Paths: paths, Mode: closeapp.ModeEnsure, + }) +} + +func tickClose(rt tickRuntime, id, message, evidence string, paths []string) error { + return runCloseTestShim(rt, closeapp.Request{ + ID: id, Message: message, Evidence: evidence, Paths: paths, Mode: closeapp.ModeStrict, + }) +} + +func runCloseTestShim(rt tickRuntime, request closeapp.Request) error { + env := append(os.Environ(), rt.env...) + service := newCloseService(closeadapter.StaticRuntime{WorkDir: rt.workDir, Env: env}) + result, err := service.Execute(context.Background(), request) + if err != nil { + if failure, ok := err.(*closeapp.Failure); ok && failure.Message != "" { + fmt.Fprintln(testWriter(rt.stderr), failure.Message) + } + return err + } + verb := "closed" + if result.AlreadyClosed { + verb = "already closed" + } + fmt.Fprintf(testWriter(rt.stdout), "%s %s @ %s\n", verb, result.ID, closeapp.ShortRef(result.Ref)) + return nil +} + +func testWriter(writer io.Writer) io.Writer { + if writer == nil { + return io.Discard + } + return writer +} + +func tickEvidenceRefusal(rt tickRuntime, _ string, first string) string { + return closeadapter.EvidenceRefusal(context.Background(), closeapp.Snapshot{ + WorkDir: rt.workDir, Env: append(os.Environ(), rt.env...), + }, first) +} + +func tickFirstEvidenceToken(evidence string) string { + evidence = strings.TrimSpace(evidence) + if index := strings.IndexByte(evidence, ' '); index >= 0 { + return evidence[:index] + } + return evidence +} + +func tickPublicStagePaths(rt tickRuntime, ledgerDir, _ string, paths []string) []string { + return closeadapter.PublicStagePaths(rt.workDir, ledgerDir, paths) +} + +func tickLedgerShowsClosed(path, id string) bool { + closed, err := (closeadapter.Repository{}).LedgerStatus(context.Background(), closeapp.Resolution{ + LedgerDir: filepath.Dir(path), + }, id) + return err == nil && closed +} + +func tickShortSHA(ref string) string { return closeapp.ShortRef(ref) } + +func tickLedgerDir(rt tickRuntime) string { + return trackerresolve.ResolveLedger(rt.workDir, append(os.Environ(), rt.env...), trackerresolve.BR).Path +} + +func tickStagePath(root, path string) string { + relative, err := filepath.Rel(root, path) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return path + } + return relative +} diff --git a/cli/cmd/ao/tick.go b/cli/cmd/ao/tick.go index c056e4b66..ed4a8403c 100644 --- a/cli/cmd/ao/tick.go +++ b/cli/cmd/ao/tick.go @@ -19,12 +19,9 @@ import ( const ( tickExitUsage = 1 - tickExitCloseRef = 3 - tickExitNoCommit = 4 tickExitUnverified = 5 tickExitCouncil = 6 tickExitDisagree = 8 - tickExitCloseFail = 10 ) type tickExitError struct { @@ -163,22 +160,12 @@ var readyCmd = &cobra.Command{ }, } -var closeCmd = &cobra.Command{ - Use: "close [paths...]", - Short: "Close a bead and persist the explicit ledger/evidence paths", - Args: cobra.MinimumNArgs(3), - RunE: func(cmd *cobra.Command, args []string) error { - return tickClosePort(newTickRuntime(cmd), args[0], args[1], args[2], args[3:]) - }, -} - func init() { // The `tick` command + its subcommands are archived behind //go:build legacy // (tick_cmd_legacy.go). These sibling top-level commands are spine — they use // the untagged tick engine below and ship in the default binary. rootCmd.AddCommand( readyCmd, - closeCmd, verdictGateCmd, councilGateCmd, installGuardsCmd, @@ -352,383 +339,6 @@ func tickAnyOpenOrInProgress(list []tickBead) bool { return false } -func tickClose(rt tickRuntime, id, msg, evidence string, paths []string) error { - resolution, err := resolveTracker(rt.workDir, append(os.Environ(), rt.env...)) - if err != nil { - return err - } - return tickCloseResolved(rt, resolution, id, msg, evidence, paths) -} - -func tickCloseResolved(rt tickRuntime, resolution trackerResolution, id, msg, evidence string, paths []string) error { - if _, err := tickClosePreflight(rt, id, evidence, paths); err != nil { - return err - } - - if _, code, err := rt.runResolvedTracker(resolution, "close", id, "--reason", "evidence: "+evidence); err != nil || code != 0 { - fmt.Fprintf(rt.stderr, "FAILED close %s: %s close failed or skipped; no files staged\n", id, resolution.Tracker) - return &tickExitError{code: tickExitCloseFail} - } - return tickPersistClosedResolved(rt, resolution, id, msg, paths, false) -} - -func tickPersistClosedResolved(rt tickRuntime, resolution trackerResolution, id, msg string, paths []string, alreadyClosed bool) error { - ledgerDir := resolution.LedgerDir - closedRef := "none" - if resolution.Tracker == trackerBR { - before := tickGitRevParseInDir(rt, ledgerDir) - if before == "" { - before = "none" - } - if _, code, syncErr := rt.runResolvedTracker(resolution, "sync", "--flush-only"); syncErr != nil || code != 0 { - if _, fallbackCode, fallbackErr := rt.runResolvedTracker(resolution, "sync"); fallbackErr != nil || fallbackCode != 0 { - return tickClosePersistenceError("tracker sync failed", fallbackCode) - } - } - issuesPath := filepath.Join(ledgerDir, "issues.jsonl") - metadataPath := filepath.Join(ledgerDir, "metadata.json") - if !tickLedgerShowsClosed(issuesPath, id) { - fmt.Fprintf(rt.stderr, "FAILED close %s: %s ledger does not prove the bead is closed\n", id, resolution.Tracker) - return &tickExitError{code: tickExitCloseFail} - } - - ledgerStage := []string{"-C", ledgerDir, "add", "--", "issues.jsonl"} - if tickPathExists(rt.workDir, metadataPath) { - ledgerStage = append(ledgerStage, "metadata.json") - } - if _, code, addErr := rt.run("git", ledgerStage...); addErr != nil || code != 0 { - return tickClosePersistenceError("ledger git add failed", code) - } - ledgerCommitOut, code, commitErr := rt.run("git", "-C", ledgerDir, "commit", "-q", "-m", msg) - ledgerCommitNoop := false - if commitErr != nil || code != 0 { - if tickGitCommitNothingToCommit(ledgerCommitOut) { - ledgerCommitNoop = true - } else { - return tickClosePersistenceError("ledger git commit failed", code) - } - } - after := tickGitRevParseInDir(rt, ledgerDir) - if after == "" { - after = "none" - } - if before == after && !ledgerCommitNoop { - return tickClosePersistenceError("ledger git commit did not land", tickExitNoCommit) - } - closedRef = after - } else { - // bd persists into its Dolt-backed store itself. Never inspect, stage, or - // commit the unrelated BR JSONL ledger; verify through the selected bd - // backend instead. - closed, err := tickTrackerClosedState(rt, resolution, id) - if err != nil || !closed { - fmt.Fprintf(rt.stderr, "FAILED close %s: %s does not prove the issue is closed\n", id, resolution.Tracker) - return &tickExitError{code: tickExitCloseFail} - } - } - - stage := tickPublicStagePaths(rt, ledgerDir, "", paths) - if len(stage) > 0 { - args := append([]string{"add", "--"}, stage...) - if _, code, err := rt.run("git", args...); err != nil || code != 0 { - return tickClosePersistenceError("public git add failed", code) - } - out, code, err := rt.run("git", "commit", "-q", "-m", msg) - if (err != nil || code != 0) && !tickGitCommitNothingToCommit(out) { - return tickClosePersistenceError("public git commit failed", code) - } - } - if resolution.Tracker == trackerBD { - closedRef = tickGitRevParse(rt) - if closedRef == "" { - closedRef = "none" - } - } - verb := "closed" - if alreadyClosed { - verb = "already closed" - } - fmt.Fprintf(rt.stdout, "%s %s @ %s\n", verb, id, tickShortSHA(closedRef)) - return nil -} - -func tickClosePort(rt tickRuntime, id, msg, evidence string, paths []string) error { - resolution, err := resolveTracker(rt.workDir, append(os.Environ(), rt.env...)) - if err != nil { - return err - } - if _, err := tickClosePreflight(rt, id, evidence, paths); err != nil { - return err - } - closed, err := tickTrackerClosedState(rt, resolution, id) - if err != nil { - fmt.Fprintf(rt.stderr, "FAILED close %s: %s status cannot be determined\n", id, resolution.Tracker) - return &tickExitError{code: tickExitCloseFail} - } - if closed { - return tickPersistClosedResolved(rt, resolution, id, msg, paths, true) - } - return tickCloseResolved(rt, resolution, id, msg, evidence, paths) -} - -func tickTrackerShowsClosed(rt tickRuntime, resolution trackerResolution, id string) bool { - closed, err := tickTrackerClosedState(rt, resolution, id) - return err == nil && closed -} - -func tickTrackerClosedState(rt tickRuntime, resolution trackerResolution, id string) (bool, error) { - if resolution.Tracker == trackerBR { - return tickLedgerClosedState(filepath.Join(resolution.LedgerDir, "issues.jsonl"), id) - } - queries := [][]string{{"show", id, "--json"}, {"list", "--all", "--json"}} - var querySucceeded bool - for _, args := range queries { - out, code, err := rt.runResolvedTracker(resolution, args...) - if err != nil || code != 0 { - continue - } - querySucceeded = true - for _, item := range tickParseBeads(out) { - if item.ID == id { - return item.Status == "closed", nil - } - } - } - if querySucceeded { - return false, fmt.Errorf("issue %s not found", id) - } - return false, fmt.Errorf("tracker status query failed") -} - -func tickClosePreflight(rt tickRuntime, id, evidence string, paths []string) (string, error) { - evFirst := tickFirstEvidenceToken(evidence) - if reason := tickEvidenceRefusal(rt, evidence, evFirst); reason != "" { - fmt.Fprintf(rt.stderr, "REFUSED close %s: evidence %q %s\n", id, evidence, reason) - return "", &tickExitError{code: tickExitCloseRef} - } - for _, path := range paths { - if tickPathExists(rt.workDir, path) || tickGitTracksPath(rt, path) { - continue - } - fmt.Fprintf(rt.stderr, "REFUSED close %s: path %q resolves to no real or tracked artifact\n", id, path) - return "", &tickExitError{code: tickExitCloseRef} - } - return evFirst, nil -} - -func tickGitTracksPath(rt tickRuntime, path string) bool { - if path == "" { - return false - } - _, code, err := rt.run("git", "ls-files", "--error-unmatch", "--", path) - return err == nil && code == 0 -} - -func tickClosePersistenceError(message string, childCode int) error { - return &tickExitError{code: tickExitNoCommit, msg: fmt.Sprintf("%s (child exit %d)", message, childCode)} -} - -// tickEvidenceRefusal applies the close-evidence gate and returns a non-empty -// reason if the close must be refused, or "" if it may proceed. -// -// Durable git-blob binding (age-l7yh): a repo-internal evidence path must be a -// committed git blob in HEAD (an ancestor of the close commit tickClose is about -// to create). Evidence that is present in the working tree but not in history -// looks durable yet isn't — refuse it. The gate uses git itself for every -// classification (repo-root relativization, cat-file, ls-tree, diff, -// check-ignore) rather than string heuristics on the cited text, so it cannot be -// bypassed by an absolute path inside the repo, a subdirectory-relative path, a -// crafted substring, a committed directory (tree), a dirty tracked file, a -// committed symlink, or a symlink/component whose target escapes the repo. -// -// Only two classes are allowed: -// - a committed, clean, regular-file blob in HEAD (the durable ideal); -// - the gitignored AgentOps runtime corpus (.agents/**), confirmed via -// git check-ignore — ephemeral by design (admitted state under .ao/ is -// tracked and so passes the committed-blob check directly). -// -// Everything else is refused, INCLUDING evidence that resolves outside the repo: -// it cannot be a durable in-repo git blob, and exempting outside paths was the -// fail-open that crafted symlinks exploited. -// -// No Agent-Mail lookup is involved, so Agent-Mail availability never blocks a -// close. This strengthens "no verdict = not done" without replacing main's -// ContextID-independence verdict model. Where there is no git history to bind to -// (not a repo, before the first commit) or repo placement is undeterminable -// (a stubbed harness), the binding falls back to plain existence so a legit -// close is not blocked. -func tickEvidenceRefusal(rt tickRuntime, evidence, evFirst string) string { - _ = evidence // classification is by the cited token + git, not the raw text - if evFirst == "" { - return "resolves to no real artifact" - } - // Resolve once: tickResolvePath returns evFirst unchanged when it is already - // absolute, else joins it under rt.workDir. os.Stat on that absolute path is - // the existence check — an absolute evidence path is never falsely treated as - // missing by being re-joined under workDir. - abs := tickResolvePath(rt.workDir, evFirst) - if _, err := os.Stat(abs); err != nil { - return "resolves to no real artifact" - } - if !tickHasGitHead(rt) { - return "" // no history to bind to — existence is the only available check - } - root, rel, ok := tickRepoRelEvidence(rt, evFirst) - if !ok { - return "" // repo placement undeterminable (e.g. a stubbed test harness) - } - if rel == ".." || strings.HasPrefix(rel, "../") { - // Resolves outside the repo (including via a symlink whose target escapes - // it). Such evidence cannot be a durable in-repo git blob, so refuse it - // rather than exempt it — exempting outside paths was the fail-open that - // let crafted symlinks bypass the binding. - return "resolves outside the repository; durable evidence must be a committed file tracked in this repo" - } - if tickEvidenceCommittedRel(rt, root, rel) { - return "" // durable: a committed regular-file blob, ancestor of the close commit - } - if tickEvidenceRuntimeCorpus(rt, root, rel) { - return "" // gitignored .agents/** runtime corpus — ephemeral by design - } - return "is present but not a committed git blob in history (durable-evidence binding); commit the evidence before closing" -} - -// tickHasGitHead reports whether rt.workDir is inside a git repo with at least -// one commit (a resolvable HEAD). Used to gate the durable-evidence binding so -// it never fires where there is no history to bind to. -func tickHasGitHead(rt tickRuntime) bool { - _, code, err := rt.run("git", "rev-parse", "--verify", "-q", "HEAD") - return err == nil && code == 0 -} - -// tickRepoRelEvidence maps the cited evidence token to the repo top-level plus a -// clean repo-root-relative slash path. The bool reports only whether repo -// placement could be DETERMINED (a real git top-level was resolved) — not -// whether the path is inside; the returned rel may begin with "../" when the -// path escapes the repo, and the caller refuses that. Relativizing against the -// repo top-level — rather than trusting the caller's spelling — is what makes the -// binding immune to absolute-inside-repo and subdirectory bypasses. -// -// The parent directory is symlink-resolved (so a symlinked prefix like macOS -// /var -> /private/var, or any symlinked component that escapes the repo, -// resolves to its real location and is then correctly judged inside or outside) -// while the leaf name is kept as-is. A leaf or component symlink that escapes the -// repo yields a "../" rel and is refused by the caller — escaping is never -// exempted. -func tickRepoRelEvidence(rt tickRuntime, evFirst string) (root, rel string, ok bool) { - out, code, err := rt.run("git", "rev-parse", "--show-toplevel") - if err != nil || code != 0 { - return "", "", false - } - root = strings.TrimSpace(string(out)) - // A real git top-level is an absolute path; anything else (e.g. a stubbed - // test harness echoing a token) means placement is undeterminable. - if !filepath.IsAbs(root) { - return "", "", false - } - if resolved, e := filepath.EvalSymlinks(root); e == nil { - root = resolved - } - var lexAbs string - if filepath.IsAbs(evFirst) { - // Resolve the parent directory (so a symlinked prefix like /var -> - // /private/var matches the resolved root) but keep the leaf name as-is, so - // a leaf symlink is not followed out of the repo. - clean := filepath.Clean(evFirst) - parent := filepath.Dir(clean) - if resolved, e := filepath.EvalSymlinks(parent); e == nil { - parent = resolved - } - lexAbs = filepath.Join(parent, filepath.Base(clean)) - } else { - base := rt.workDir - if resolved, e := filepath.EvalSymlinks(base); e == nil { - base = resolved - } - lexAbs = filepath.Clean(filepath.Join(base, evFirst)) - } - r, err := filepath.Rel(root, lexAbs) - if err != nil { - return root, "", false // cannot relativize (e.g. different volume) — undeterminable - } - // Determinable: r is the repo-relative path. It may begin with "../" when the - // path (or a resolved symlink target) escapes the repo; the caller refuses - // that rather than exempting it. - return root, filepath.ToSlash(r), true -} - -// tickEvidenceCommittedRel reports whether the repo-root-relative path rel is -// durable evidence: a committed FILE in HEAD whose working-tree content matches -// HEAD exactly. Two checks: -// -// - `git cat-file -t HEAD:` must report type "blob" — `-e` alone also -// succeeds for a tree, which would let a close cite a whole committed -// directory (e.g. docs/) instead of a specific durable artifact. rel is -// root-relative, which is what the HEAD: form expects. -// - `git -C root diff --quiet HEAD -- ` must report no difference — a -// committed path with uncommitted or staged content changes is NOT durably -// in history, so the cited working-tree artifact must equal the committed -// blob. diff runs with -C root because its pathspec is cwd-relative whereas -// rel is root-relative. -// -// A clean committed regular-file blob is durable evidence reachable from (an -// ancestor of) the close commit. A committed SYMLINK is also a blob, but its -// bytes are just the target path — the pointed-at content is not in history — so -// it is rejected via its tree mode (120000). -func tickEvidenceCommittedRel(rt tickRuntime, root, rel string) bool { - if rel == "" { - return false - } - out, code, err := rt.run("git", "cat-file", "-t", "HEAD:"+rel) - if err != nil || code != 0 || strings.TrimSpace(string(out)) != "blob" { - return false - } - // Reject a committed symlink: ls-tree reports its mode as 120000. Require a - // regular file (100644/100755) so the durable bytes ARE the evidence, not a - // pointer to an untracked, possibly-external, mutable target. - lt, ltcode, lterr := rt.run("git", "-C", root, "ls-tree", "HEAD", "--", rel) - if lterr != nil || ltcode != 0 { - return false - } - if mode := strings.TrimSpace(string(lt)); !strings.HasPrefix(mode, "100") { - return false - } - _, dcode, derr := rt.run("git", "-C", root, "diff", "--quiet", "HEAD", "--", rel) - return derr == nil && dcode == 0 -} - -// tickEvidenceRuntimeCorpus reports whether the repo-root-relative path rel is -// in the AgentOps runtime corpus (.agents/**) AND is actually gitignored. Both -// conditions are required: scoping to .agents/** keeps other gitignored paths -// (build artifacts, logs, temp files) from serving as durable evidence, and -// confirming via git check-ignore honors the documented exemption being for the -// *gitignored* corpus (a force-added, tracked .agents path is not exempt — it is -// committed and so passes the blob check anyway). check-ignore runs with -C root -// so the root-relative rel resolves correctly regardless of rt.workDir. -func tickEvidenceRuntimeCorpus(rt tickRuntime, root, rel string) bool { - if rel != ".agents" && !strings.HasPrefix(rel, ".agents/") { - return false - } - _, code, err := rt.run("git", "-C", root, "check-ignore", "-q", "--", rel) - return err == nil && code == 0 -} - -func tickFirstEvidenceToken(evidence string) string { - ev := strings.TrimSpace(evidence) - if i := strings.IndexByte(ev, ' '); i >= 0 { - return ev[:i] - } - return ev -} - -func tickPathExists(root, path string) bool { - if path == "" { - return false - } - _, err := os.Stat(tickResolvePath(root, path)) - return err == nil -} - func tickResolvePath(root, path string) string { if filepath.IsAbs(path) { return path @@ -736,40 +346,6 @@ func tickResolvePath(root, path string) string { return filepath.Join(root, path) } -// tickLedgerDir resolves the tracker ledger directory used for close -// verification and staging. Resolution order matches what spawned `br` -// child processes see: explicit BEADS_DIR (rt.env overrides inherited process -// env), then the canonical git-common-dir `_beads` ledger for linked worktrees. -func tickLedgerDir(rt tickRuntime) string { - if dir := tickEnvValue(rt, "BEADS_DIR"); dir != "" { - return tickResolvePath(rt.workDir, dir) - } - return resolveBeadsDir(rt.workDir, append(os.Environ(), rt.env...)).Path -} - -// tickEnvValue mirrors the environment a tickRuntime child process receives: -// rt.env entries are appended after os.Environ() in rt.run, so the last -// rt.env match wins over the inherited process value. -func tickEnvValue(rt tickRuntime, key string) string { - prefix := key + "=" - for i := len(rt.env) - 1; i >= 0; i-- { - if strings.HasPrefix(rt.env[i], prefix) { - return strings.TrimPrefix(rt.env[i], prefix) - } - } - return os.Getenv(key) -} - -// tickStagePath renders a ledger path for git add: repo-relative when the -// path sits under the work dir, absolute otherwise. -func tickStagePath(root, path string) string { - rel, err := filepath.Rel(root, path) - if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return path - } - return rel -} - func tickGitRevParse(rt tickRuntime) string { out, code, err := rt.run("git", "rev-parse", "HEAD") if err != nil || code != 0 { @@ -778,79 +354,6 @@ func tickGitRevParse(rt tickRuntime) string { return strings.TrimSpace(string(out)) } -func tickGitRevParseInDir(rt tickRuntime, dir string) string { - out, code, err := rt.run("git", "-C", dir, "rev-parse", "HEAD") - if err != nil || code != 0 { - return "" - } - return strings.TrimSpace(string(out)) -} - -func tickGitCommitNothingToCommit(out []byte) bool { - text := strings.ToLower(string(out)) - return strings.Contains(text, "nothing to commit") || - strings.Contains(text, "nothing added to commit") || - strings.Contains(text, "no changes added to commit") -} - -func tickPublicStagePaths(rt tickRuntime, ledgerDir, evFirst string, paths []string) []string { - _ = evFirst // Evidence is validation-only; callers explicitly select public paths. - stage := []string{} - for _, path := range paths { - stage = tickAppendPublicStagePath(stage, rt.workDir, ledgerDir, path) - } - return stage -} - -func tickAppendPublicStagePath(stage []string, root, ledgerDir, path string) []string { - if path == "" || tickIsPrivateLedgerPath(root, ledgerDir, path) { - return stage - } - return append(stage, path) -} - -func tickIsPrivateLedgerPath(root, ledgerDir, path string) bool { - resolved := filepath.Clean(tickResolvePath(root, path)) - ledger := filepath.Clean(tickResolvePath(root, ledgerDir)) - if rel, err := filepath.Rel(ledger, resolved); err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return true - } - clean := filepath.Clean(path) - return clean == "_beads" || strings.HasPrefix(clean, "_beads"+string(filepath.Separator)) || - clean == ".beads" || strings.HasPrefix(clean, ".beads"+string(filepath.Separator)) -} - -func tickShortSHA(sha string) string { - if len(sha) <= 7 { - return sha - } - return sha[:7] -} - -func tickLedgerShowsClosed(path, id string) bool { - closed, err := tickLedgerClosedState(path, id) - return err == nil && closed -} - -func tickLedgerClosedState(path, id string) (bool, error) { - f, err := os.Open(path) - if err != nil { - return false, err - } - defer func() { _ = f.Close() }() - scanner := bufio.NewScanner(f) - for scanner.Scan() { - var item tickBead - if json.Unmarshal(scanner.Bytes(), &item) == nil && item.ID == id { - return item.Status == "closed", nil - } - } - if err := scanner.Err(); err != nil { - return false, err - } - return false, fmt.Errorf("issue %s not found", id) -} - func tickReadVerdict(rt tickRuntime, path string) (string, error) { var b []byte var err error @@ -1223,7 +726,7 @@ func tickSmoke(rt tickRuntime) error { } else { failLine("4 chaos bare-verdict accepted") } - if tickSmokeCloseFailure(rt, tmp) { + if runCloseSmokeFailure(rt, tmp) { passLine("5 close aborts before git commit when br close fails") } else { failLine("5 close failure path committed") @@ -1254,45 +757,12 @@ func tickExitCode(err error) int { if err == nil { return 0 } - if e, ok := err.(*tickExitError); ok { + if e, ok := err.(interface{ ExitCode() int }); ok { return e.ExitCode() } return 1 } -func tickSmokeCloseFailure(rt tickRuntime, tmp string) bool { - fakebin := filepath.Join(tmp, "fakebin") - _ = os.MkdirAll(fakebin, 0o755) - fakeBR := `#!/usr/bin/env bash -if [ "${1:-}" = "close" ]; then echo "stubbed br close failure" >&2; exit 42; fi -if [ "${1:-}" = "sync" ] || [ "${1:-}" = "update" ]; then exit 0; fi -echo "unexpected br call: $*" >&2; exit 43 -` - fakeGitLog := filepath.Join(tmp, "fake-git.log") - fakeGit := `#!/usr/bin/env bash -case "${1:-}" in - rev-parse) echo fakehead ;; - add) : ;; - commit) printf 'commit\n' >> "${TICK_SMOKE_FAKE_GIT_LOG:?}" ;; - *) : ;; -esac -` - _ = os.WriteFile(filepath.Join(fakebin, "br"), []byte(fakeBR), 0o755) - _ = os.WriteFile(filepath.Join(fakebin, "git"), []byte(fakeGit), 0o755) - evidence := filepath.Join(tmp, "close-evidence.md") - _ = os.WriteFile(evidence, []byte("evidence"), 0o644) - smokeRT := rt - smokeRT.stdout = io.Discard - smokeRT.stderr = io.Discard - smokeRT.env = []string{ - "PATH=" + fakebin + string(os.PathListSeparator) + os.Getenv("PATH"), - "TICK_SMOKE_FAKE_GIT_LOG=" + fakeGitLog, - } - code := tickExitCode(tickClose(smokeRT, "cp-smoke", "smoke close should not commit", evidence, nil)) - info, err := os.Stat(fakeGitLog) - return code != 0 && (os.IsNotExist(err) || (err == nil && info.Size() == 0)) -} - func tickNoClaudePrintOnScriptSurfaces(rt tickRuntime) bool { out, code, err := rt.run("git", "ls-files", "--", "*.sh", ".githooks/*", "bin/*") if err != nil || code != 0 { diff --git a/cli/cmd/ao/tick_cmd_legacy.go b/cli/cmd/ao/tick_cmd_legacy.go index f318c94d6..f17c04646 100644 --- a/cli/cmd/ao/tick_cmd_legacy.go +++ b/cli/cmd/ao/tick_cmd_legacy.go @@ -70,14 +70,7 @@ var tickReopenCmd = &cobra.Command{ }, } -var tickCloseCmd = &cobra.Command{ - Use: "close [paths...]", - Short: "Close a bead and persist the explicit ledger/evidence paths", - Args: cobra.MinimumNArgs(3), - RunE: func(cmd *cobra.Command, args []string) error { - return tickClose(newTickRuntime(cmd), args[0], args[1], args[2], args[3:]) - }, -} +var tickCloseCmd = closeModule.LegacyCommand() var tickVerdictGateCmd = &cobra.Command{ Use: "verdict-gate ", diff --git a/cli/internal/adapters/close/repository.go b/cli/internal/adapters/close/repository.go new file mode 100644 index 000000000..4f47f7227 --- /dev/null +++ b/cli/internal/adapters/close/repository.go @@ -0,0 +1,260 @@ +package close + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + closeapp "github.com/boshu2/agentops/cli/internal/close" +) + +type Repository struct{} + +func (Repository) Preflight(ctx context.Context, snapshot closeapp.Snapshot, resolution closeapp.Resolution, evidence string, paths []string) error { + first := firstEvidenceToken(evidence) + if reason := EvidenceRefusal(ctx, snapshot, first); reason != "" { + return fmt.Errorf("evidence %q %s", evidence, reason) + } + for _, path := range paths { + if pathExists(snapshot.WorkDir, path) || gitTracksPath(ctx, snapshot, path) { + continue + } + return fmt.Errorf("path %q resolves to no real or tracked artifact", path) + } + return nil +} + +func (Repository) LedgerStatus(_ context.Context, resolution closeapp.Resolution, id string) (bool, error) { + file, err := os.Open(filepath.Join(resolution.LedgerDir, "issues.jsonl")) + if err != nil { + return false, err + } + defer func() { _ = file.Close() }() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + var item trackerRecord + if json.Unmarshal(scanner.Bytes(), &item) == nil && item.ID == id { + return item.Status == "closed", nil + } + } + if err := scanner.Err(); err != nil { + return false, err + } + return false, fmt.Errorf("issue %s not found", id) +} + +func (Repository) CommitLedger(ctx context.Context, snapshot closeapp.Snapshot, resolution closeapp.Resolution, message string) (string, error) { + before := gitHead(ctx, snapshot, resolution.LedgerDir) + stage := []string{"-C", resolution.LedgerDir, "add", "--", "issues.jsonl"} + if pathExists(snapshot.WorkDir, filepath.Join(resolution.LedgerDir, "metadata.json")) { + stage = append(stage, "metadata.json") + } + if out, code, err := runGit(ctx, snapshot, stage...); err != nil || code != 0 { + return "", effectError("ledger git add", code, out, err) + } + out, code, err := runGit(ctx, snapshot, "-C", resolution.LedgerDir, "commit", "-q", "-m", message) + noop := false + if err != nil || code != 0 { + if gitCommitNothingToCommit(out) { + noop = true + } else { + return "", effectError("ledger git commit", code, out, err) + } + } + after := gitHead(ctx, snapshot, resolution.LedgerDir) + if before == after && !noop { + return "", fmt.Errorf("ledger git commit did not land") + } + return noneIfEmpty(after), nil +} + +func (Repository) CommitPublic(ctx context.Context, snapshot closeapp.Snapshot, resolution closeapp.Resolution, message string, paths []string) (string, error) { + stage := PublicStagePaths(snapshot.WorkDir, resolution.LedgerDir, paths) + if len(stage) > 0 { + args := append([]string{"add", "--"}, stage...) + if out, code, err := runGit(ctx, snapshot, args...); err != nil || code != 0 { + return "", effectError("public git add", code, out, err) + } + out, code, err := runGit(ctx, snapshot, "commit", "-q", "-m", message) + if (err != nil || code != 0) && !gitCommitNothingToCommit(out) { + return "", effectError("public git commit", code, out, err) + } + } + return noneIfEmpty(gitHead(ctx, snapshot, snapshot.WorkDir)), nil +} + +func PublicStagePaths(root, ledgerDir string, paths []string) []string { + stage := make([]string, 0, len(paths)) + for _, path := range paths { + if path == "" || isPrivateLedgerPath(root, ledgerDir, path) { + continue + } + stage = append(stage, path) + } + return stage +} + +func EvidenceRefusal(ctx context.Context, snapshot closeapp.Snapshot, first string) string { + if first == "" { + return "resolves to no real artifact" + } + abs := resolvePath(snapshot.WorkDir, first) + if _, err := os.Stat(abs); err != nil { + return "resolves to no real artifact" + } + if !hasGitHead(ctx, snapshot) { + return "" + } + root, rel, ok := repoRelativeEvidence(ctx, snapshot, first) + if !ok { + return "" + } + if rel == ".." || strings.HasPrefix(rel, "../") { + return "resolves outside the repository; durable evidence must be a committed file tracked in this repo" + } + if evidenceCommitted(ctx, snapshot, root, rel) || evidenceRuntimeCorpus(ctx, snapshot, root, rel) { + return "" + } + return "is present but not a committed git blob in history (durable-evidence binding); commit the evidence before closing" +} + +func firstEvidenceToken(evidence string) string { + evidence = strings.TrimSpace(evidence) + if index := strings.IndexByte(evidence, ' '); index >= 0 { + return evidence[:index] + } + return evidence +} + +func pathExists(root, path string) bool { + if path == "" { + return false + } + _, err := os.Stat(resolvePath(root, path)) + return err == nil +} + +func resolvePath(root, path string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(root, path) +} + +func gitTracksPath(ctx context.Context, snapshot closeapp.Snapshot, path string) bool { + if path == "" { + return false + } + _, code, err := runGit(ctx, snapshot, "ls-files", "--error-unmatch", "--", path) + return err == nil && code == 0 +} + +func hasGitHead(ctx context.Context, snapshot closeapp.Snapshot) bool { + _, code, err := runGit(ctx, snapshot, "rev-parse", "--verify", "-q", "HEAD") + return err == nil && code == 0 +} + +func repoRelativeEvidence(ctx context.Context, snapshot closeapp.Snapshot, first string) (string, string, bool) { + out, code, err := runGit(ctx, snapshot, "rev-parse", "--show-toplevel") + if err != nil || code != 0 { + return "", "", false + } + root := strings.TrimSpace(string(out)) + if !filepath.IsAbs(root) { + return "", "", false + } + if resolved, resolveErr := filepath.EvalSymlinks(root); resolveErr == nil { + root = resolved + } + var absolute string + if filepath.IsAbs(first) { + clean := filepath.Clean(first) + parent := filepath.Dir(clean) + if resolved, resolveErr := filepath.EvalSymlinks(parent); resolveErr == nil { + parent = resolved + } + absolute = filepath.Join(parent, filepath.Base(clean)) + } else { + base := snapshot.WorkDir + if resolved, resolveErr := filepath.EvalSymlinks(base); resolveErr == nil { + base = resolved + } + absolute = filepath.Clean(filepath.Join(base, first)) + } + relative, err := filepath.Rel(root, absolute) + if err != nil { + return root, "", false + } + return root, filepath.ToSlash(relative), true +} + +func evidenceCommitted(ctx context.Context, snapshot closeapp.Snapshot, root, relative string) bool { + if relative == "" { + return false + } + out, code, err := runGit(ctx, snapshot, "cat-file", "-t", "HEAD:"+relative) + if err != nil || code != 0 || strings.TrimSpace(string(out)) != "blob" { + return false + } + tree, treeCode, treeErr := runGit(ctx, snapshot, "-C", root, "ls-tree", "HEAD", "--", relative) + if treeErr != nil || treeCode != 0 || !strings.HasPrefix(strings.TrimSpace(string(tree)), "100") { + return false + } + _, diffCode, diffErr := runGit(ctx, snapshot, "-C", root, "diff", "--quiet", "HEAD", "--", relative) + return diffErr == nil && diffCode == 0 +} + +func evidenceRuntimeCorpus(ctx context.Context, snapshot closeapp.Snapshot, root, relative string) bool { + if relative != ".agents" && !strings.HasPrefix(relative, ".agents/") { + return false + } + _, code, err := runGit(ctx, snapshot, "-C", root, "check-ignore", "-q", "--", relative) + return err == nil && code == 0 +} + +func isPrivateLedgerPath(root, ledgerDir, path string) bool { + resolved := filepath.Clean(resolvePath(root, path)) + ledger := filepath.Clean(resolvePath(root, ledgerDir)) + if relative, err := filepath.Rel(ledger, resolved); err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return true + } + clean := filepath.Clean(path) + return clean == "_beads" || strings.HasPrefix(clean, "_beads"+string(filepath.Separator)) || + clean == ".beads" || strings.HasPrefix(clean, ".beads"+string(filepath.Separator)) +} + +func gitHead(ctx context.Context, snapshot closeapp.Snapshot, directory string) string { + out, code, err := runGit(ctx, snapshot, "-C", directory, "rev-parse", "HEAD") + if err != nil || code != 0 { + return "" + } + return strings.TrimSpace(string(out)) +} + +func gitCommitNothingToCommit(out []byte) bool { + text := strings.ToLower(string(out)) + return strings.Contains(text, "nothing to commit") || strings.Contains(text, "nothing added to commit") || strings.Contains(text, "no changes added to commit") +} + +func runGit(ctx context.Context, snapshot closeapp.Snapshot, args ...string) ([]byte, int, error) { + binary, err := lookPath(snapshot.Env, "git") + if err != nil { + return nil, 127, err + } + command := exec.CommandContext(ctx, binary, args...) // #nosec G204 -- binary is resolved from the caller's explicit PATH. + command.Dir = snapshot.WorkDir + command.Env = append([]string(nil), snapshot.Env...) + return combined(command) +} + +func noneIfEmpty(value string) string { + if value == "" { + return "none" + } + return value +} diff --git a/cli/internal/adapters/close/runtime.go b/cli/internal/adapters/close/runtime.go new file mode 100644 index 000000000..1031bfbbc --- /dev/null +++ b/cli/internal/adapters/close/runtime.go @@ -0,0 +1,26 @@ +package close + +import ( + "os" + + closeapp "github.com/boshu2/agentops/cli/internal/close" +) + +type SystemRuntime struct{} + +func (SystemRuntime) Snapshot() (closeapp.Snapshot, error) { + cwd, err := os.Getwd() + if err != nil { + return closeapp.Snapshot{}, err + } + return closeapp.Snapshot{WorkDir: cwd, Env: os.Environ()}, nil +} + +type StaticRuntime struct { + WorkDir string + Env []string +} + +func (runtime StaticRuntime) Snapshot() (closeapp.Snapshot, error) { + return closeapp.Snapshot{WorkDir: runtime.WorkDir, Env: append([]string(nil), runtime.Env...)}, nil +} diff --git a/cli/internal/adapters/close/tracker.go b/cli/internal/adapters/close/tracker.go new file mode 100644 index 000000000..8661502f2 --- /dev/null +++ b/cli/internal/adapters/close/tracker.go @@ -0,0 +1,143 @@ +package close + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + closeapp "github.com/boshu2/agentops/cli/internal/close" + "github.com/boshu2/agentops/cli/internal/trackerresolve" +) + +type Tracker struct{} + +type trackerRecord struct { + ID string `json:"id"` + Status string `json:"status"` +} + +func (Tracker) Resolve(_ context.Context, snapshot closeapp.Snapshot) (closeapp.Resolution, error) { + resolved, err := trackerresolve.ResolveWithLookPath(snapshot.WorkDir, snapshot.Env, func(name string) (string, error) { + return lookPath(snapshot.Env, name) + }) + if err != nil { + return closeapp.Resolution{}, err + } + return closeapp.Resolution{ + Backend: resolved.Tracker, Binary: resolved.Binary, LedgerDir: resolved.LedgerDir, + RepoRoot: resolved.RepoRoot, WorkDir: resolved.WorkDir, ChildEnv: append([]string(nil), resolved.ChildEnv...), + }, nil +} + +func (Tracker) Status(ctx context.Context, resolution closeapp.Resolution, id string) (bool, error) { + queries := [][]string{{"show", id, "--json"}, {"list", "--all", "--json"}} + querySucceeded := false + for _, args := range queries { + out, code, err := runTracker(ctx, resolution, args...) + if err != nil || code != 0 { + continue + } + querySucceeded = true + for _, item := range parseRecords(out) { + if item.ID == id { + return item.Status == "closed", nil + } + } + } + if querySucceeded { + return false, fmt.Errorf("issue %s not found", id) + } + return false, fmt.Errorf("tracker status query failed") +} + +func (Tracker) Close(ctx context.Context, resolution closeapp.Resolution, id, reason string) error { + out, code, err := runTracker(ctx, resolution, "close", id, "--reason", reason) + if err != nil || code != 0 { + return effectError("tracker close", code, out, err) + } + return nil +} + +func (Tracker) Sync(ctx context.Context, resolution closeapp.Resolution) error { + out, code, err := runTracker(ctx, resolution, "sync", "--flush-only") + if err == nil && code == 0 { + return nil + } + out, code, err = runTracker(ctx, resolution, "sync") + if err != nil || code != 0 { + return effectError("tracker sync", code, out, err) + } + return nil +} + +func runTracker(ctx context.Context, resolution closeapp.Resolution, args ...string) ([]byte, int, error) { + command := exec.CommandContext(ctx, resolution.Binary, args...) // #nosec G204 -- tracker resolution constrains the binary to br|bd. + command.Dir = resolution.WorkDir + command.Env = append([]string(nil), resolution.ChildEnv...) + return combined(command) +} + +func parseRecords(out []byte) []trackerRecord { + var records []trackerRecord + if json.Unmarshal(out, &records) == nil { + return records + } + var wrapped struct { + Issues []trackerRecord `json:"issues"` + } + if json.Unmarshal(out, &wrapped) == nil { + return wrapped.Issues + } + return nil +} + +func lookPath(env []string, name string) (string, error) { + if strings.ContainsRune(name, filepath.Separator) { + return name, nil + } + pathValue := envLast(env, "PATH") + for _, directory := range filepath.SplitList(pathValue) { + candidate := filepath.Join(directory, name) + info, err := os.Stat(candidate) + if err == nil && !info.IsDir() && info.Mode()&0o111 != 0 { + return candidate, nil + } + } + return "", exec.ErrNotFound +} + +func envLast(env []string, key string) string { + prefix := key + "=" + for index := len(env) - 1; index >= 0; index-- { + if strings.HasPrefix(env[index], prefix) { + return strings.TrimPrefix(env[index], prefix) + } + } + return "" +} + +func combined(command *exec.Cmd) ([]byte, int, error) { + out, err := command.CombinedOutput() + if err == nil { + return out, 0, nil + } + if exitErr, ok := err.(*exec.ExitError); ok { + return out, exitErr.ExitCode(), err + } + return out, 127, err +} + +func effectError(operation string, code int, out []byte, cause error) error { + detail := strings.TrimSpace(string(out)) + if detail == "" && cause != nil { + detail = cause.Error() + } + if detail == "" { + detail = "unknown failure" + } + return fmt.Errorf("%s (child exit %d): %s", operation, code, detail) +} diff --git a/cli/internal/close/service.go b/cli/internal/close/service.go new file mode 100644 index 000000000..6b099520e --- /dev/null +++ b/cli/internal/close/service.go @@ -0,0 +1,199 @@ +// Package close owns the close transaction policy independently of Cobra and +// concrete tracker, filesystem, and git effects. +package close + +import ( + "context" + "fmt" +) + +const ( + BackendBR = "br" + BackendBD = "bd" + + ExitRefused = 3 + ExitPersistence = 4 + ExitTracker = 10 +) + +type Mode uint8 + +const ( + ModeEnsure Mode = iota + ModeStrict +) + +type Snapshot struct { + WorkDir string + Env []string +} + +type Resolution struct { + Backend string + Binary string + LedgerDir string + RepoRoot string + WorkDir string + ChildEnv []string +} + +type Request struct { + ID string + Message string + Evidence string + Paths []string + Mode Mode +} + +type Result struct { + ID string + Ref string + AlreadyClosed bool +} + +type Runtime interface { + Snapshot() (Snapshot, error) +} + +type Tracker interface { + Resolve(context.Context, Snapshot) (Resolution, error) + Status(context.Context, Resolution, string) (bool, error) + Close(context.Context, Resolution, string, string) error + Sync(context.Context, Resolution) error +} + +type Repository interface { + Preflight(context.Context, Snapshot, Resolution, string, []string) error + LedgerStatus(context.Context, Resolution, string) (bool, error) + CommitLedger(context.Context, Snapshot, Resolution, string) (string, error) + CommitPublic(context.Context, Snapshot, Resolution, string, []string) (string, error) +} + +type Failure struct { + Code int + Message string + Cause error +} + +func (failure *Failure) Error() string { + if failure == nil { + return "" + } + if failure.Message != "" { + return failure.Message + } + if failure.Cause != nil { + return failure.Cause.Error() + } + return "close failed" +} + +func (failure *Failure) Unwrap() error { return failure.Cause } +func (failure *Failure) ExitCode() int { return failure.Code } + +type Service struct { + runtime Runtime + tracker Tracker + repository Repository +} + +func NewService(runtime Runtime, tracker Tracker, repository Repository) *Service { + return &Service{runtime: runtime, tracker: tracker, repository: repository} +} + +func (service *Service) Execute(ctx context.Context, request Request) (Result, error) { + snapshot, err := service.runtime.Snapshot() + if err != nil { + return Result{}, &Failure{Code: 1, Message: err.Error(), Cause: err} + } + resolution, err := service.tracker.Resolve(ctx, snapshot) + if err != nil { + return Result{}, &Failure{Code: 1, Message: err.Error(), Cause: err} + } + if err := service.repository.Preflight(ctx, snapshot, resolution, request.Evidence, request.Paths); err != nil { + return Result{}, &Failure{ + Code: ExitRefused, Message: fmt.Sprintf("REFUSED close %s: %s", request.ID, err), Cause: err, + } + } + + alreadyClosed := false + if request.Mode == ModeEnsure { + closed, statusErr := service.closed(ctx, resolution, request.ID) + if statusErr != nil { + return Result{}, &Failure{ + Code: ExitTracker, Message: fmt.Sprintf("FAILED close %s: %s status cannot be determined", request.ID, resolution.Backend), Cause: statusErr, + } + } + alreadyClosed = closed + if !closed { + if err := service.close(ctx, resolution, request); err != nil { + return Result{}, err + } + } + } else if err := service.close(ctx, resolution, request); err != nil { + return Result{}, err + } + + ref := "none" + if resolution.Backend == BackendBR { + if err := service.tracker.Sync(ctx, resolution); err != nil { + return Result{}, persistenceFailure("tracker sync failed", err) + } + closed, statusErr := service.repository.LedgerStatus(ctx, resolution, request.ID) + if statusErr != nil || !closed { + return Result{}, &Failure{ + Code: ExitTracker, Message: fmt.Sprintf("FAILED close %s: %s ledger does not prove the bead is closed", request.ID, resolution.Backend), Cause: statusErr, + } + } + ref, err = service.repository.CommitLedger(ctx, snapshot, resolution, request.Message) + if err != nil { + return Result{}, persistenceFailure("ledger persistence failed", err) + } + } else { + closed, statusErr := service.tracker.Status(ctx, resolution, request.ID) + if statusErr != nil || !closed { + return Result{}, &Failure{ + Code: ExitTracker, Message: fmt.Sprintf("FAILED close %s: %s does not prove the issue is closed", request.ID, resolution.Backend), Cause: statusErr, + } + } + } + + publicRef, err := service.repository.CommitPublic(ctx, snapshot, resolution, request.Message, request.Paths) + if err != nil { + return Result{}, persistenceFailure("public persistence failed", err) + } + if resolution.Backend == BackendBD { + ref = publicRef + } + if ref == "" { + ref = "none" + } + return Result{ID: request.ID, Ref: ref, AlreadyClosed: alreadyClosed}, nil +} + +func (service *Service) closed(ctx context.Context, resolution Resolution, id string) (bool, error) { + if resolution.Backend == BackendBR { + return service.repository.LedgerStatus(ctx, resolution, id) + } + return service.tracker.Status(ctx, resolution, id) +} + +func (service *Service) close(ctx context.Context, resolution Resolution, request Request) error { + if err := service.tracker.Close(ctx, resolution, request.ID, "evidence: "+request.Evidence); err != nil { + return &Failure{ + Code: ExitTracker, Message: fmt.Sprintf("FAILED close %s: %s close failed or skipped; no files staged", request.ID, resolution.Backend), Cause: err, + } + } + return nil +} + +func persistenceFailure(message string, cause error) error { + return &Failure{Code: ExitPersistence, Message: message + ": " + cause.Error(), Cause: cause} +} + +func ShortRef(ref string) string { + if len(ref) <= 7 { + return ref + } + return ref[:7] +} diff --git a/cli/internal/close/service_test.go b/cli/internal/close/service_test.go new file mode 100644 index 000000000..b9e707880 --- /dev/null +++ b/cli/internal/close/service_test.go @@ -0,0 +1,150 @@ +package close + +import ( + "context" + "errors" + "reflect" + "testing" +) + +type fakeRuntime struct{ snapshot Snapshot } + +func (runtime fakeRuntime) Snapshot() (Snapshot, error) { return runtime.snapshot, nil } + +type fakeTracker struct { + resolution Resolution + closed bool + statusErr error + syncErr error + events *[]string +} + +func (tracker *fakeTracker) Resolve(context.Context, Snapshot) (Resolution, error) { + *tracker.events = append(*tracker.events, "resolve") + return tracker.resolution, nil +} + +func (tracker *fakeTracker) Status(context.Context, Resolution, string) (bool, error) { + *tracker.events = append(*tracker.events, "tracker-status") + return tracker.closed, tracker.statusErr +} + +func (tracker *fakeTracker) Close(context.Context, Resolution, string, string) error { + *tracker.events = append(*tracker.events, "close") + tracker.closed = true + return nil +} + +func (tracker *fakeTracker) Sync(context.Context, Resolution) error { + *tracker.events = append(*tracker.events, "sync") + return tracker.syncErr +} + +type fakeRepository struct { + closed bool + ledgerStatuses []bool + preflightErr error + commitLedgerErr error + commitPublicErr error + events *[]string +} + +func (repository *fakeRepository) Preflight(context.Context, Snapshot, Resolution, string, []string) error { + *repository.events = append(*repository.events, "preflight") + return repository.preflightErr +} + +func (repository *fakeRepository) LedgerStatus(context.Context, Resolution, string) (bool, error) { + *repository.events = append(*repository.events, "ledger-status") + if len(repository.ledgerStatuses) > 0 { + closed := repository.ledgerStatuses[0] + repository.ledgerStatuses = repository.ledgerStatuses[1:] + return closed, nil + } + return repository.closed, nil +} + +func (repository *fakeRepository) CommitLedger(context.Context, Snapshot, Resolution, string) (string, error) { + *repository.events = append(*repository.events, "commit-ledger") + return "ledger-head", repository.commitLedgerErr +} + +func (repository *fakeRepository) CommitPublic(context.Context, Snapshot, Resolution, string, []string) (string, error) { + *repository.events = append(*repository.events, "commit-public") + return "public-head", repository.commitPublicErr +} + +func TestServiceBRCloseOrdersMonotonicDurabilityPhases(t *testing.T) { + events := []string{} + tracker := &fakeTracker{resolution: Resolution{Backend: BackendBR}, events: &events} + repository := &fakeRepository{ledgerStatuses: []bool{false, true}, events: &events} + service := NewService(fakeRuntime{snapshot: Snapshot{WorkDir: "/repo"}}, tracker, repository) + result, err := service.Execute(context.Background(), Request{ID: "age-1", Message: "done", Evidence: "proof", Mode: ModeEnsure}) + if err != nil { + t.Fatal(err) + } + want := []string{"resolve", "preflight", "ledger-status", "close", "sync", "ledger-status", "commit-ledger", "commit-public"} + if !reflect.DeepEqual(events, want) { + t.Fatalf("events = %v, want %v", events, want) + } + if result.Ref != "ledger-head" || result.AlreadyClosed { + t.Fatalf("result = %+v", result) + } +} + +func TestServiceBRAlreadyClosedStillEnsuresPersistence(t *testing.T) { + events := []string{} + tracker := &fakeTracker{resolution: Resolution{Backend: BackendBR}, events: &events} + repository := &fakeRepository{closed: true, events: &events} + service := NewService(fakeRuntime{}, tracker, repository) + result, err := service.Execute(context.Background(), Request{ID: "age-1", Mode: ModeEnsure}) + if err != nil { + t.Fatal(err) + } + if !result.AlreadyClosed { + t.Fatalf("result = %+v, want already closed", result) + } + for _, event := range events { + if event == "close" { + t.Fatalf("already-closed ensure repeated tracker close: %v", events) + } + } + want := []string{"resolve", "preflight", "ledger-status", "sync", "ledger-status", "commit-ledger", "commit-public"} + if !reflect.DeepEqual(events, want) { + t.Fatalf("events = %v, want %v", events, want) + } +} + +func TestServicePersistenceFailureIsStableAndNeverRollsBack(t *testing.T) { + events := []string{} + tracker := &fakeTracker{resolution: Resolution{Backend: BackendBR}, events: &events} + repository := &fakeRepository{closed: true, commitPublicErr: errors.New("disk full"), events: &events} + service := NewService(fakeRuntime{}, tracker, repository) + _, err := service.Execute(context.Background(), Request{ID: "age-1", Mode: ModeEnsure}) + var failure *Failure + if !errors.As(err, &failure) || failure.Code != ExitPersistence { + t.Fatalf("error = %#v, want persistence failure", err) + } + for _, event := range events { + if event == "reopen" || event == "close" { + t.Fatalf("failure moved tracker state backward: %v", events) + } + } +} + +func TestServiceBDUnknownStatusFailsClosed(t *testing.T) { + events := []string{} + tracker := &fakeTracker{ + resolution: Resolution{Backend: BackendBD}, statusErr: errors.New("bd unavailable"), events: &events, + } + repository := &fakeRepository{events: &events} + service := NewService(fakeRuntime{}, tracker, repository) + _, err := service.Execute(context.Background(), Request{ID: "agentops-1", Mode: ModeEnsure}) + var failure *Failure + if !errors.As(err, &failure) || failure.Code != ExitTracker { + t.Fatalf("error = %#v, want tracker failure", err) + } + if !reflect.DeepEqual(events, []string{"resolve", "preflight", "tracker-status"}) { + t.Fatalf("events = %v; unknown status must not be treated as open", events) + } +} diff --git a/cli/internal/commands/close/module.go b/cli/internal/commands/close/module.go new file mode 100644 index 000000000..a816a40f5 --- /dev/null +++ b/cli/internal/commands/close/module.go @@ -0,0 +1,93 @@ +// Package close owns Cobra presentation for the close command family. +package close + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/spf13/cobra" + + "github.com/boshu2/agentops/cli/internal/clicontract" + closeapp "github.com/boshu2/agentops/cli/internal/close" +) + +type UseCases interface { + Execute(context.Context, closeapp.Request) (closeapp.Result, error) +} + +type Module struct { + useCases UseCases +} + +func NewModule(useCases UseCases) Module { + return Module{useCases: useCases} +} + +func (Module) Contract() clicontract.CommandContract { + return clicontract.CommandContract{ + ID: "ao.close", + Profiles: clicontract.ProfileDefault | + clicontract.ProfileFlywheel | + clicontract.ProfileLegacy | + clicontract.ProfileCombined, + Args: clicontract.ArgsPolicy{Name: "minimum-3", Validate: cobra.MinimumNArgs(3)}, + Output: clicontract.OutputText, + Effects: clicontract.EffectFilesystem | clicontract.EffectProcess | clicontract.EffectTracker | clicontract.EffectEnvironment, + ExitClasses: map[int]clicontract.ExitClass{ + 0: clicontract.ExitSuccess, 1: clicontract.ExitFailure, closeapp.ExitRefused: clicontract.ExitConflict, + closeapp.ExitPersistence: clicontract.ExitPartial, closeapp.ExitTracker: clicontract.ExitFailure, + }, + } +} + +func (module Module) Command() *cobra.Command { + return module.command(closeapp.ModeEnsure) +} + +func (module Module) LegacyCommand() *cobra.Command { + return module.command(closeapp.ModeStrict) +} + +func (module Module) command(mode closeapp.Mode) *cobra.Command { + command := &cobra.Command{ + Use: "close [paths...]", + Short: "Close a bead and persist the explicit ledger/evidence paths", + Args: cobra.MinimumNArgs(3), + } + command.RunE = func(command *cobra.Command, args []string) error { + result, err := module.useCases.Execute(command.Context(), closeapp.Request{ + ID: args[0], Message: args[1], Evidence: args[2], Paths: append([]string(nil), args[3:]...), Mode: mode, + }) + if err != nil { + command.SilenceErrors = true + command.SilenceUsage = true + return renderFailure(command.ErrOrStderr(), err) + } + verb := "closed" + if result.AlreadyClosed { + verb = "already closed" + } + _, err = fmt.Fprintf(command.OutOrStdout(), "%s %s @ %s\n", verb, result.ID, closeapp.ShortRef(result.Ref)) + return err + } + return command +} + +type exitError struct{ code int } + +func (failure *exitError) Error() string { return "" } +func (failure *exitError) ExitCode() int { return failure.code } + +func renderFailure(stderr io.Writer, err error) error { + var failure *closeapp.Failure + if errors.As(err, &failure) { + if failure.Message != "" { + fmt.Fprintln(stderr, failure.Message) + } + return &exitError{code: failure.Code} + } + fmt.Fprintln(stderr, err) + return &exitError{code: 1} +} diff --git a/cli/internal/commands/close/module_test.go b/cli/internal/commands/close/module_test.go new file mode 100644 index 000000000..bca889f8e --- /dev/null +++ b/cli/internal/commands/close/module_test.go @@ -0,0 +1,71 @@ +package close + +import ( + "bytes" + "context" + "errors" + "testing" + + closeapp "github.com/boshu2/agentops/cli/internal/close" +) + +type fakeUseCases struct { + request closeapp.Request + result closeapp.Result + err error +} + +func (useCases *fakeUseCases) Execute(_ context.Context, request closeapp.Request) (closeapp.Result, error) { + useCases.request = request + return useCases.result, useCases.err +} + +func TestModuleCommandParsesDelegatesAndRenders(t *testing.T) { + useCases := &fakeUseCases{result: closeapp.Result{ID: "age-1", Ref: "123456789", AlreadyClosed: true}} + command := NewModule(useCases).Command() + var stdout, stderr bytes.Buffer + command.SetOut(&stdout) + command.SetErr(&stderr) + command.SetArgs([]string{"age-1", "finish", "proof.md", "docs/result.md"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if useCases.request.Mode != closeapp.ModeEnsure || len(useCases.request.Paths) != 1 { + t.Fatalf("request = %+v", useCases.request) + } + if got, want := stdout.String(), "already closed age-1 @ 1234567\n"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } + if stderr.Len() != 0 { + t.Fatalf("stderr = %q", stderr.String()) + } +} + +func TestModuleFailureRendersOnceAndPreservesExitClass(t *testing.T) { + useCases := &fakeUseCases{err: &closeapp.Failure{Code: closeapp.ExitPersistence, Message: "public persistence failed"}} + command := NewModule(useCases).Command() + var stdout, stderr bytes.Buffer + command.SetOut(&stdout) + command.SetErr(&stderr) + command.SetArgs([]string{"age-1", "finish", "proof.md"}) + err := command.Execute() + var exit interface{ ExitCode() int } + if !errors.As(err, &exit) || exit.ExitCode() != closeapp.ExitPersistence { + t.Fatalf("error = %#v", err) + } + if got, want := stderr.String(), "public persistence failed\n"; got != want { + t.Fatalf("stderr = %q, want %q", got, want) + } + if stdout.Len() != 0 { + t.Fatalf("stdout = %q", stdout.String()) + } +} + +func TestModuleContractPublishesStableCloseExits(t *testing.T) { + contract := (Module{}).Contract() + for _, code := range []int{0, 1, closeapp.ExitRefused, closeapp.ExitPersistence, closeapp.ExitTracker} { + if contract.ExitClasses[code] == "" { + t.Fatalf("missing exit class %d", code) + } + } +} From 0782b566af8be3b013b649f95f607b5bbebc4fc5 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:57:29 -0400 Subject: [PATCH 046/108] fix(close): resume BR persistence from tracker state (age-nw28h.2.4) --- cli/cmd/ao/tick_test.go | 6 +- .../adapters/close/retry_integration_test.go | 116 ++++++++++++++++++ cli/internal/adapters/close/tracker.go | 8 +- cli/internal/adapters/close/tracker_test.go | 10 ++ cli/internal/close/service.go | 3 - cli/internal/close/service_test.go | 47 ++++++- 6 files changed, 179 insertions(+), 11 deletions(-) create mode 100644 cli/internal/adapters/close/retry_integration_test.go create mode 100644 cli/internal/adapters/close/tracker_test.go diff --git a/cli/cmd/ao/tick_test.go b/cli/cmd/ao/tick_test.go index 718b2cff9..6d072f802 100644 --- a/cli/cmd/ao/tick_test.go +++ b/cli/cmd/ao/tick_test.go @@ -154,7 +154,7 @@ func TestTickClosePortAlreadyClosedCompletesPendingPersistence(t *testing.T) { if err := os.MkdirAll(fakebin, 0o755); err != nil { t.Fatal(err) } - fakeBR := "#!/usr/bin/env bash\ncase \"${1:-}\" in sync) exit 0 ;; *) echo \"unexpected br call: $*\" >&2; exit 43 ;; esac\n" + fakeBR := "#!/usr/bin/env bash\ncase \"${1:-}\" in show) printf '%s\\n' '{\"id\":\"cp-done\",\"status\":\"closed\"}' ;; sync) exit 0 ;; *) echo \"unexpected br call: $*\" >&2; exit 43 ;; esac\n" if err := os.WriteFile(filepath.Join(fakebin, "br"), []byte(fakeBR), 0o755); err != nil { t.Fatal(err) } @@ -389,6 +389,10 @@ func TestTickClosePortRetriesForwardAfterPublicPersistenceFailure(t *testing.T) fakeBR := `#!/usr/bin/env bash printf '%s\n' "$*" >> "${TICK_TEST_BR_LOG:?}" case "${1:-}" in + show) + if grep -q '"status": "closed"' "${TICK_TEST_LEDGER:?}/issues.jsonl"; then status=closed; else status=open; fi + printf '{"id":"%s","status":"%s"}\n' "${2:-}" "$status" + ;; close) sed 's/"status": "open"/"status": "closed"/' "${TICK_TEST_LEDGER:?}/issues.jsonl" > "${TICK_TEST_LEDGER:?}/issues.tmp" && mv "${TICK_TEST_LEDGER:?}/issues.tmp" "${TICK_TEST_LEDGER:?}/issues.jsonl" ;; sync) ;; *) echo "unexpected br call: $*" >&2; exit 43 ;; diff --git a/cli/internal/adapters/close/retry_integration_test.go b/cli/internal/adapters/close/retry_integration_test.go new file mode 100644 index 000000000..605623610 --- /dev/null +++ b/cli/internal/adapters/close/retry_integration_test.go @@ -0,0 +1,116 @@ +package close + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + closeapp "github.com/boshu2/agentops/cli/internal/close" +) + +func TestBRSyncFailureRetryUsesTrackerStateAndClosesExactlyOnce(t *testing.T) { + t.Setenv("GIT_AUTHOR_NAME", "close-test") + t.Setenv("GIT_AUTHOR_EMAIL", "close@test") + t.Setenv("GIT_COMMITTER_NAME", "close-test") + t.Setenv("GIT_COMMITTER_EMAIL", "close@test") + dir := t.TempDir() + ledger := filepath.Join(dir, "_beads") + fakebin := filepath.Join(dir, "fakebin") + for _, path := range []string{ledger, fakebin, filepath.Join(dir, "docs")} { + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + } + realGit, err := exec.LookPath("git") + if err != nil { + t.Fatal(err) + } + runGit := func(workDir string, args ...string) { + t.Helper() + command := exec.Command(realGit, args...) + command.Dir = workDir + if out, err := command.CombinedOutput(); err != nil { + t.Fatalf("git %v in %s: %v\n%s", args, workDir, err, out) + } + } + runGit(dir, "init", "-q") + if err := os.WriteFile(filepath.Join(dir, "proof.md"), []byte("proof\n"), 0o644); err != nil { + t.Fatal(err) + } + runGit(dir, "add", "proof.md") + runGit(dir, "commit", "-q", "-m", "seed public") + if err := os.WriteFile(filepath.Join(dir, "docs", "result.md"), []byte("result\n"), 0o644); err != nil { + t.Fatal(err) + } + runGit(ledger, "init", "-q") + if err := os.WriteFile(filepath.Join(ledger, "issues.jsonl"), []byte(`{"id":"age-retry","status":"open"}`+"\n"), 0o644); err != nil { + t.Fatal(err) + } + runGit(ledger, "add", "issues.jsonl") + runGit(ledger, "commit", "-q", "-m", "seed ledger") + + stateFile := filepath.Join(dir, "tracker-state") + syncCount := filepath.Join(dir, "sync-count") + closeLog := filepath.Join(dir, "close.log") + if err := os.WriteFile(stateFile, []byte("open\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(syncCount, []byte("0\n"), 0o644); err != nil { + t.Fatal(err) + } + fakeBR := `#!/usr/bin/env bash +case "${1:-}" in + show) + status="$(tr -d '\n' < "${CLOSE_TEST_STATE:?}")" + printf '{"id":"%s","status":"%s"}\n' "${2:-}" "$status" + ;; + close) + printf '%s\n' "$*" >> "${CLOSE_TEST_CLOSE_LOG:?}" + printf 'closed\n' > "${CLOSE_TEST_STATE:?}" + ;; + sync) + count="$(tr -d '\n' < "${CLOSE_TEST_SYNC_COUNT:?}")" + count=$((count + 1)) + printf '%s\n' "$count" > "${CLOSE_TEST_SYNC_COUNT:?}" + if [ "$count" -le 2 ]; then exit 77; fi + sed 's/"status":"open"/"status":"closed"/' "${BEADS_DIR:?}/issues.jsonl" > "${BEADS_DIR:?}/issues.tmp" + mv "${BEADS_DIR:?}/issues.tmp" "${BEADS_DIR:?}/issues.jsonl" + ;; + list) exit 78 ;; + *) echo "unexpected br call: $*" >&2; exit 79 ;; +esac +` + if err := os.WriteFile(filepath.Join(fakebin, "br"), []byte(fakeBR), 0o755); err != nil { + t.Fatal(err) + } + env := append(os.Environ(), + "PATH="+fakebin+string(os.PathListSeparator)+os.Getenv("PATH"), + "AGENTOPS_TRACKER=br", "BEADS_DIR="+ledger, + "CLOSE_TEST_STATE="+stateFile, "CLOSE_TEST_SYNC_COUNT="+syncCount, "CLOSE_TEST_CLOSE_LOG="+closeLog, + ) + service := closeapp.NewService(StaticRuntime{WorkDir: dir, Env: env}, Tracker{}, Repository{}) + request := closeapp.Request{ + ID: "age-retry", Message: "finish close", Evidence: "proof.md", Paths: []string{"docs/result.md"}, Mode: closeapp.ModeEnsure, + } + _, err = service.Execute(context.Background(), request) + var failure *closeapp.Failure + if !errors.As(err, &failure) || failure.Code != closeapp.ExitPersistence { + t.Fatalf("first error = %#v, want persistence failure", err) + } + if result, err := service.Execute(context.Background(), request); err != nil { + t.Fatalf("retry: %v", err) + } else if !result.AlreadyClosed { + t.Fatalf("retry result = %+v, want already closed", result) + } + logBody, err := os.ReadFile(closeLog) + if err != nil { + t.Fatal(err) + } + if got := strings.Count(strings.TrimSpace(string(logBody)), "close age-retry"); got != 1 { + t.Fatalf("close calls = %d, want 1; log=%q", got, logBody) + } +} diff --git a/cli/internal/adapters/close/tracker.go b/cli/internal/adapters/close/tracker.go index 8661502f2..ba6a377c3 100644 --- a/cli/internal/adapters/close/tracker.go +++ b/cli/internal/adapters/close/tracker.go @@ -63,11 +63,11 @@ func (Tracker) Close(ctx context.Context, resolution closeapp.Resolution, id, re } func (Tracker) Sync(ctx context.Context, resolution closeapp.Resolution) error { - out, code, err := runTracker(ctx, resolution, "sync", "--flush-only") + _, code, err := runTracker(ctx, resolution, "sync", "--flush-only") if err == nil && code == 0 { return nil } - out, code, err = runTracker(ctx, resolution, "sync") + out, code, err := runTracker(ctx, resolution, "sync") if err != nil || code != 0 { return effectError("tracker sync", code, out, err) } @@ -86,6 +86,10 @@ func parseRecords(out []byte) []trackerRecord { if json.Unmarshal(out, &records) == nil { return records } + var single trackerRecord + if json.Unmarshal(out, &single) == nil && single.ID != "" { + return []trackerRecord{single} + } var wrapped struct { Issues []trackerRecord `json:"issues"` } diff --git a/cli/internal/adapters/close/tracker_test.go b/cli/internal/adapters/close/tracker_test.go new file mode 100644 index 000000000..63c1b1551 --- /dev/null +++ b/cli/internal/adapters/close/tracker_test.go @@ -0,0 +1,10 @@ +package close + +import "testing" + +func TestParseRecordsAcceptsSingleShowObject(t *testing.T) { + records := parseRecords([]byte(`{"id":"agentops-1","status":"closed"}`)) + if len(records) != 1 || records[0].ID != "agentops-1" || records[0].Status != "closed" { + t.Fatalf("parseRecords(single object) = %+v", records) + } +} diff --git a/cli/internal/close/service.go b/cli/internal/close/service.go index 6b099520e..2e752d81b 100644 --- a/cli/internal/close/service.go +++ b/cli/internal/close/service.go @@ -172,9 +172,6 @@ func (service *Service) Execute(ctx context.Context, request Request) (Result, e } func (service *Service) closed(ctx context.Context, resolution Resolution, id string) (bool, error) { - if resolution.Backend == BackendBR { - return service.repository.LedgerStatus(ctx, resolution, id) - } return service.tracker.Status(ctx, resolution, id) } diff --git a/cli/internal/close/service_test.go b/cli/internal/close/service_test.go index b9e707880..f07aeb76f 100644 --- a/cli/internal/close/service_test.go +++ b/cli/internal/close/service_test.go @@ -16,6 +16,9 @@ type fakeTracker struct { closed bool statusErr error syncErr error + syncErrs []error + onSync func() + closeCount int events *[]string } @@ -31,12 +34,24 @@ func (tracker *fakeTracker) Status(context.Context, Resolution, string) (bool, e func (tracker *fakeTracker) Close(context.Context, Resolution, string, string) error { *tracker.events = append(*tracker.events, "close") + tracker.closeCount++ tracker.closed = true return nil } func (tracker *fakeTracker) Sync(context.Context, Resolution) error { *tracker.events = append(*tracker.events, "sync") + if len(tracker.syncErrs) > 0 { + err := tracker.syncErrs[0] + tracker.syncErrs = tracker.syncErrs[1:] + if err == nil && tracker.onSync != nil { + tracker.onSync() + } + return err + } + if tracker.syncErr == nil && tracker.onSync != nil { + tracker.onSync() + } return tracker.syncErr } @@ -77,13 +92,13 @@ func (repository *fakeRepository) CommitPublic(context.Context, Snapshot, Resolu func TestServiceBRCloseOrdersMonotonicDurabilityPhases(t *testing.T) { events := []string{} tracker := &fakeTracker{resolution: Resolution{Backend: BackendBR}, events: &events} - repository := &fakeRepository{ledgerStatuses: []bool{false, true}, events: &events} + repository := &fakeRepository{ledgerStatuses: []bool{true}, events: &events} service := NewService(fakeRuntime{snapshot: Snapshot{WorkDir: "/repo"}}, tracker, repository) result, err := service.Execute(context.Background(), Request{ID: "age-1", Message: "done", Evidence: "proof", Mode: ModeEnsure}) if err != nil { t.Fatal(err) } - want := []string{"resolve", "preflight", "ledger-status", "close", "sync", "ledger-status", "commit-ledger", "commit-public"} + want := []string{"resolve", "preflight", "tracker-status", "close", "sync", "ledger-status", "commit-ledger", "commit-public"} if !reflect.DeepEqual(events, want) { t.Fatalf("events = %v, want %v", events, want) } @@ -94,7 +109,7 @@ func TestServiceBRCloseOrdersMonotonicDurabilityPhases(t *testing.T) { func TestServiceBRAlreadyClosedStillEnsuresPersistence(t *testing.T) { events := []string{} - tracker := &fakeTracker{resolution: Resolution{Backend: BackendBR}, events: &events} + tracker := &fakeTracker{resolution: Resolution{Backend: BackendBR}, closed: true, events: &events} repository := &fakeRepository{closed: true, events: &events} service := NewService(fakeRuntime{}, tracker, repository) result, err := service.Execute(context.Background(), Request{ID: "age-1", Mode: ModeEnsure}) @@ -109,7 +124,7 @@ func TestServiceBRAlreadyClosedStillEnsuresPersistence(t *testing.T) { t.Fatalf("already-closed ensure repeated tracker close: %v", events) } } - want := []string{"resolve", "preflight", "ledger-status", "sync", "ledger-status", "commit-ledger", "commit-public"} + want := []string{"resolve", "preflight", "tracker-status", "sync", "ledger-status", "commit-ledger", "commit-public"} if !reflect.DeepEqual(events, want) { t.Fatalf("events = %v, want %v", events, want) } @@ -117,7 +132,7 @@ func TestServiceBRAlreadyClosedStillEnsuresPersistence(t *testing.T) { func TestServicePersistenceFailureIsStableAndNeverRollsBack(t *testing.T) { events := []string{} - tracker := &fakeTracker{resolution: Resolution{Backend: BackendBR}, events: &events} + tracker := &fakeTracker{resolution: Resolution{Backend: BackendBR}, closed: true, events: &events} repository := &fakeRepository{closed: true, commitPublicErr: errors.New("disk full"), events: &events} service := NewService(fakeRuntime{}, tracker, repository) _, err := service.Execute(context.Background(), Request{ID: "age-1", Mode: ModeEnsure}) @@ -148,3 +163,25 @@ func TestServiceBDUnknownStatusFailsClosed(t *testing.T) { t.Fatalf("events = %v; unknown status must not be treated as open", events) } } + +func TestServiceBRSyncFailureRetryDoesNotCloseTwice(t *testing.T) { + events := []string{} + repository := &fakeRepository{closed: false, events: &events} + tracker := &fakeTracker{ + resolution: Resolution{Backend: BackendBR}, syncErrs: []error{errors.New("flush failed"), nil}, + onSync: func() { repository.closed = true }, events: &events, + } + service := NewService(fakeRuntime{}, tracker, repository) + request := Request{ID: "age-1", Mode: ModeEnsure} + _, err := service.Execute(context.Background(), request) + var failure *Failure + if !errors.As(err, &failure) || failure.Code != ExitPersistence { + t.Fatalf("first error = %#v, want persistence failure", err) + } + if _, err := service.Execute(context.Background(), request); err != nil { + t.Fatalf("retry: %v", err) + } + if tracker.closeCount != 1 { + t.Fatalf("tracker close count = %d, want 1; events=%v", tracker.closeCount, events) + } +} From 5bda2dc7c22a885212d48d86461169873f4c3688 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:03:28 -0400 Subject: [PATCH 047/108] chore(cli): seal close migration lineage (age-nw28h.2.4) --- .../compatibility-baseline/families/close/lineage.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/testdata/compatibility-baseline/families/close/lineage.json b/cli/testdata/compatibility-baseline/families/close/lineage.json index ed25a67b6..95f446a82 100644 --- a/cli/testdata/compatibility-baseline/families/close/lineage.json +++ b/cli/testdata/compatibility-baseline/families/close/lineage.json @@ -6,5 +6,6 @@ "fixture_sha256": "20482e48578cab821b243f2500a82a4212b3541dfdfea69ef69734d3775dc196", "ownership_sha256": "3f9e48bc80357bb5ab9453427a423ffde64fbd387081591def6e33e6bbc853e0", "scope_sha256": "3577ffbc9a8679d61367bfe13a159c9939050c5bd2cf5333e078f85c2211ec0a", - "migration_state": "migrating" + "migration_state": "migrated", + "accepted_sha": "0782b566af8be3b013b649f95f607b5bbebc4fc5" } From 45c065022902f3c4a756adfab969d5b699bf478a Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:04:47 -0400 Subject: [PATCH 048/108] chore(provenance): bind pawl CONFIRMED verdict for age-nw28h.2.4 #trivial --- docs/provenance/ledger.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/provenance/ledger.jsonl b/docs/provenance/ledger.jsonl index 7dda064fa..43b7d9f95 100644 --- a/docs/provenance/ledger.jsonl +++ b/docs/provenance/ledger.jsonl @@ -519,3 +519,4 @@ {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.1@4863195","from_type":"verdict","to_id":"4863195d07079bb4f995a84686b2c572afe070fb","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.1 disposition=CONFIRMED","bead_id":"age-nw28h.2.1","trust_tier":"inferred","ts":"2026-07-11T19:10:35Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-beads-independent-verdict-4863195d0.txt","prev_hash":"5ce3f9f10999351cb3ce663d1dab51fbb418291ebf9e6ad6c507acc8550a7bbe","payload_hash":"ff4b1163efdb38e333b7ce4358ae7dd9a56a86d0645d481750c3a1ac0a860acb","hash":"0dbb491f180b24cbd6fd602bf8812d14382184c08370a714c7e4905f3730ca0e"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.2@bd171a4","from_type":"verdict","to_id":"bd171a4776a303a8c7ad867047f70967cdee8b0e","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.2 disposition=CONFIRMED","bead_id":"age-nw28h.2.2","trust_tier":"inferred","ts":"2026-07-11T19:50:19Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-capabilities-independent-verdict-bd171a477.txt","prev_hash":"0dbb491f180b24cbd6fd602bf8812d14382184c08370a714c7e4905f3730ca0e","payload_hash":"2ed1b654481b2037e5e1b8cb439cafe29bb16cf6319181b5e42bc549eb1a9e79","hash":"9739edafbb15e8a5d7e0ac15be295093bb14ebf60ffc13f92d46db35942e5aca"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.3@2172c77","from_type":"verdict","to_id":"2172c77f05692169948f3b034970c90d64a366c1","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.3 disposition=CONFIRMED","bead_id":"age-nw28h.2.3","trust_tier":"inferred","ts":"2026-07-11T20:30:36Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-claim-independent-verdict-2172c77f0.txt","prev_hash":"9739edafbb15e8a5d7e0ac15be295093bb14ebf60ffc13f92d46db35942e5aca","payload_hash":"7d01c886ce25d14d82b1cabbf379bc4e777f083328ee4695ca8344979eae9855","hash":"be2caecd5f64c47b68e12dd6a220eacac168450f3c6c0160f4428f4a917e92d0"} +{"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.4@0782b56","from_type":"verdict","to_id":"0782b566af8be3b013b649f95f607b5bbebc4fc5","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.4 disposition=CONFIRMED","bead_id":"age-nw28h.2.4","trust_tier":"inferred","ts":"2026-07-11T22:04:47Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-close-independent-verdict-0782b566a.txt","prev_hash":"be2caecd5f64c47b68e12dd6a220eacac168450f3c6c0160f4428f4a917e92d0","payload_hash":"a451e922b8fb7db3c5d35d3815169a837ec3e0b650486fdfd1c1a1c976e9d168","hash":"671a7c4580ed2345d62f32c92074f974f51455e3ea341f4eb371445a8b8f395b"} From bd3f65176e27f7bd0b20589626d7d61a26f6eb3a Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:06:41 -0400 Subject: [PATCH 049/108] fix(config): reject junk args and sort model overrides (age-nw28h.2.5) --- cli/cmd/ao/config.go | 15 ++++++++++++--- cli/cmd/ao/config_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/cli/cmd/ao/config.go b/cli/cmd/ao/config.go index e1f91130e..e652e5be7 100644 --- a/cli/cmd/ao/config.go +++ b/cli/cmd/ao/config.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "github.com/spf13/cobra" @@ -44,8 +45,9 @@ Configure in .agentops/config.yaml: crank: budget Or via environment variables: - AGENTOPS_MODEL_TIER=budget - AGENTOPS_COUNCIL_MODEL_TIER=quality`, + AGENTOPS_MODEL_TIER=budget + AGENTOPS_COUNCIL_MODEL_TIER=quality`, + Args: cobra.NoArgs, RunE: runConfigModels, } @@ -83,6 +85,7 @@ Environment variables: Examples: ao config --show # Show resolved configuration ao config --show --json # Output as JSON`, + Args: cobra.NoArgs, RunE: runConfig, } @@ -238,7 +241,13 @@ func runConfigModels(_ *cobra.Command, _ []string) error { fmt.Println() if len(cfg.Models.SkillOverrides) > 0 { fmt.Println(" Skill overrides:") - for skill, tier := range cfg.Models.SkillOverrides { + skills := make([]string, 0, len(cfg.Models.SkillOverrides)) + for skill := range cfg.Models.SkillOverrides { + skills = append(skills, skill) + } + sort.Strings(skills) + for _, skill := range skills { + tier := cfg.Models.SkillOverrides[skill] resolved := cfg.ResolveTier(skill) if tier == resolved { fmt.Printf(" %-12s → %s\n", skill, tier) diff --git a/cli/cmd/ao/config_test.go b/cli/cmd/ao/config_test.go index e93ba5d6c..63dd6a7e6 100644 --- a/cli/cmd/ao/config_test.go +++ b/cli/cmd/ao/config_test.go @@ -501,3 +501,37 @@ func TestRunConfig_ShowTable_NoEnvVars(t *testing.T) { t.Errorf("expected '(none set)' for no env vars, got: %q", stdout) } } + +func TestConfigCommandsRejectPositionalArgs(t *testing.T) { + if err := configCmd.Args(configCmd, []string{"junk"}); err == nil { + t.Fatal("config accepted an unexpected positional argument") + } + if err := configModelsCmd.Args(configModelsCmd, []string{"junk"}); err == nil { + t.Fatal("config models accepted an unexpected positional argument") + } +} + +func TestRunConfigModelsSortsSkillOverrides(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + t.Setenv("AGENTOPS_CONFIG", "") + if err := os.MkdirAll(filepath.Join(dir, ".agentops"), 0o755); err != nil { + t.Fatal(err) + } + body := "models:\n skill_overrides:\n zebra: budget\n alpha: quality\n" + if err := os.WriteFile(filepath.Join(dir, ".agentops", "config.yaml"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + oldOutput := output + output = "table" + defer func() { output = oldOutput }() + stdout, err := captureStdout(t, func() error { return runConfigModels(&cobra.Command{}, nil) }) + if err != nil { + t.Fatal(err) + } + alpha := strings.Index(stdout, "alpha") + zebra := strings.Index(stdout, "zebra") + if alpha < 0 || zebra < 0 || alpha > zebra { + t.Fatalf("skill overrides are not sorted:\n%s", stdout) + } +} From 3034cfa9bff8b86b46a36f381d06f65192bea425 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:09:09 -0400 Subject: [PATCH 050/108] test(cli): freeze config compatibility contract (age-nw28h.2.5) --- .../families/config/case.json | 25 ++++++++++++++++ .../families/config/ownership.json | 29 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/config/case.json create mode 100644 cli/testdata/compatibility-baseline/families/config/ownership.json diff --git a/cli/testdata/compatibility-baseline/families/config/case.json b/cli/testdata/compatibility-baseline/families/config/case.json new file mode 100644 index 000000000..64d4bc310 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/config/case.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "family": "config", + "checks": [ + { + "id": "config-cli-surface", + "command": "tmp=$(mktemp -d /tmp/ao-config-surface.XXXXXX); (cd cli && go build -o \"$tmp/ao\" ./cmd/ao); mkdir -p \"$tmp/home\" \"$tmp/repo\"; HOME=\"$tmp/home\" \"$tmp/ao\" config --help >\"$tmp/help\"; grep -Fq 'ao config [flags]' \"$tmp/help\"; grep -Fq 'models Show model cost tier configuration' \"$tmp/help\"; HOME=\"$tmp/home\" \"$tmp/ao\" config >\"$tmp/noargs\"; grep -Fq 'View and manage AgentOps configuration.' \"$tmp/noargs\"; HOME=\"$tmp/home\" \"$tmp/ao\" --json config --show >\"$tmp/show.json\"; jq -e '.output.value != null and .dream_report_dir.value != null' \"$tmp/show.json\" >/dev/null; HOME=\"$tmp/home\" \"$tmp/ao\" --json config models >\"$tmp/models.json\"; jq -e '.default_tier == \"balanced\" and (.tiers|length) == 3' \"$tmp/models.json\" >/dev/null; (cd \"$tmp/repo\" && HOME=\"$tmp/home\" \"$tmp/ao\" config models --set-tier quality >\"$tmp/write.out\"); grep -qx 'Set default model tier to \"quality\"' \"$tmp/write.out\"; grep -q 'default_tier: quality' \"$tmp/repo/.agentops/config.yaml\"; set +e; HOME=\"$tmp/home\" \"$tmp/ao\" config junk >\"$tmp/junk.out\" 2>\"$tmp/junk.err\"; junk=$?; HOME=\"$tmp/home\" \"$tmp/ao\" config models --set-tier premium >\"$tmp/bad.out\" 2>\"$tmp/bad.err\"; bad=$?; set -e; test \"$junk\" -eq 1; test \"$bad\" -eq 1; grep -qx 'Error: unknown command \"junk\" for \"ao config\"' \"$tmp/junk.err\"; grep -q 'Error: invalid tier \"premium\"' \"$tmp/bad.err\"" + }, + { + "id": "config-go-behavior", + "command": "cd cli && go test ./cmd/ao -run '^(TestConfigFlag_MaterializesAndIsHonored|TestRunConfig_.*|TestRunConfigModels_.*|TestConfigModels_.*|TestConfigCommandsRejectPositionalArgs|TestCobraConfigCommand|TestJSONValidity_Config)$' -count=1" + } + ], + "dimensions": { + "path": {"status": "evidence", "evidence": ["config-cli-surface"]}, + "aliases": {"status": "not_applicable", "reason": "The ao config family declares no aliases."}, + "accepted_args": {"status": "evidence", "evidence": ["config-cli-surface", "config-go-behavior"]}, + "rejected_args": {"status": "evidence", "evidence": ["config-cli-surface", "config-go-behavior"]}, + "stdout": {"status": "evidence", "evidence": ["config-cli-surface", "config-go-behavior"]}, + "stderr": {"status": "evidence", "evidence": ["config-cli-surface", "config-go-behavior"]}, + "exit_classes": {"status": "evidence", "evidence": ["config-cli-surface", "config-go-behavior"]}, + "tracker_selection": {"status": "not_applicable", "reason": "The config family does not select or execute an issue tracker."}, + "ordered_effects": {"status": "evidence", "evidence": ["config-cli-surface", "config-go-behavior"]} + } +} diff --git a/cli/testdata/compatibility-baseline/families/config/ownership.json b/cli/testdata/compatibility-baseline/families/config/ownership.json new file mode 100644 index 000000000..5b1bd28e9 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/config/ownership.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "family": "config", + "profiles": { + "default": "present", + "flywheel": "present", + "legacy": "present", + "combined": "present" + }, + "legacy_symbols": [ + "configShow", + "modelsSetTier", + "modelsSetSkill", + "configModelsWriteResult", + "configModelsCmd", + "configCmd", + "runConfig", + "runConfigModels", + "handleModelsWrite" + ], + "live_owner": "cli/internal/commands/config", + "allowed_paths": [ + "cli/cmd/ao/config*.go", + "cli/internal/adapters/config/**", + "cli/internal/commands/config/**", + "cli/internal/config/**", + "cli/testdata/compatibility-baseline/families/config/lineage.json" + ] +} From 399995049cb0ede6e88c1da76470c977eb7713a5 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:09:48 -0400 Subject: [PATCH 051/108] chore(cli): bind config migration lineage (age-nw28h.2.5) --- .../families/config/lineage.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/config/lineage.json diff --git a/cli/testdata/compatibility-baseline/families/config/lineage.json b/cli/testdata/compatibility-baseline/families/config/lineage.json new file mode 100644 index 000000000..4e4ddd6da --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/config/lineage.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "family": "config", + "old_implementation_sha": "bd3f65176e27f7bd0b20589626d7d61a26f6eb3a", + "freeze_sha": "3034cfa9bff8b86b46a36f381d06f65192bea425", + "fixture_sha256": "3c5dd5a716854332a3985b91b3beb9a12cce656747625fac79371f2e54f10ff8", + "ownership_sha256": "0414989420add10ca0f3ce0a5e491a9b5ce9dadd98b346b3ce7ae991e2af9b80", + "scope_sha256": "7ab979017b4cf7d74fabcbe09c78f83feb70e740e575565e5daea9237e0e04bb", + "migration_state": "migrating" +} From c55ebbacb90fe44bc7097b45240965c5f21abd30 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:16:52 -0400 Subject: [PATCH 052/108] refactor(config): extract explicit command module (age-nw28h.2.5) --- cli/cmd/ao/config.go | 333 -------------------- cli/cmd/ao/config_module.go | 20 ++ cli/cmd/ao/config_test_shims_test.go | 53 ++++ cli/internal/adapters/config/gateway.go | 44 +++ cli/internal/commands/config/module.go | 303 ++++++++++++++++++ cli/internal/commands/config/module_test.go | 71 +++++ cli/internal/config/command_service.go | 112 +++++++ cli/internal/config/command_service_test.go | 48 +++ 8 files changed, 651 insertions(+), 333 deletions(-) delete mode 100644 cli/cmd/ao/config.go create mode 100644 cli/cmd/ao/config_module.go create mode 100644 cli/cmd/ao/config_test_shims_test.go create mode 100644 cli/internal/adapters/config/gateway.go create mode 100644 cli/internal/commands/config/module.go create mode 100644 cli/internal/commands/config/module_test.go create mode 100644 cli/internal/config/command_service.go create mode 100644 cli/internal/config/command_service_test.go diff --git a/cli/cmd/ao/config.go b/cli/cmd/ao/config.go deleted file mode 100644 index e652e5be7..000000000 --- a/cli/cmd/ao/config.go +++ /dev/null @@ -1,333 +0,0 @@ -// practices: [twelve-factor-app, pragmatic-programmer] -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/spf13/cobra" - - "github.com/boshu2/agentops/cli/internal/config" -) - -var ( - configShow bool - modelsSetTier string - modelsSetSkill string -) - -type configModelsWriteResult struct { - Updated bool `json:"updated"` - DefaultTier string `json:"default_tier,omitempty"` - SkillOverrides map[string]string `json:"skill_overrides,omitempty"` -} - -var configModelsCmd = &cobra.Command{ - Use: "models", - Short: "Show model cost tier configuration", - Long: `Display the current model cost tier settings with sources. - -Cost tiers map to model quality levels: - quality → opus (high-stakes decisions, architecture) - balanced → sonnet (default, routine reviews) - budget → haiku (quick checks, simple tasks) - inherit → uses default tier (falls back to balanced) - -Configure in .agentops/config.yaml: - models: - default_tier: balanced - skill_overrides: - council: quality - crank: budget - -Or via environment variables: - AGENTOPS_MODEL_TIER=budget - AGENTOPS_COUNCIL_MODEL_TIER=quality`, - Args: cobra.NoArgs, - RunE: runConfigModels, -} - -var configCmd = &cobra.Command{ - Use: "config", - Short: "Manage configuration", - Long: `View and manage AgentOps configuration. - -Configuration priority (highest to lowest): - 1. Command-line flags - 2. Environment variables (AGENTOPS_*) - 3. Project config (.agentops/config.yaml) - 4. Home config (~/.agentops/config.yaml) - 5. Defaults - -Environment variables: - AGENTOPS_CONFIG - Explicit config file path (overrides default project config location) - AGENTOPS_OUTPUT - Default output format (table, json, yaml) - AGENTOPS_BASE_DIR - Data directory path - AGENTOPS_VERBOSE - Enable verbose output (true/1) - AGENTOPS_NO_SC - Disable Smart Connections (true/1) - AGENTOPS_RPI_WORKTREE_MODE - RPI worktree policy (auto|always|never) - AGENTOPS_RPI_RUNTIME / AGENTOPS_RPI_RUNTIME_MODE - RPI runtime mode (auto|direct|stream) - AGENTOPS_RPI_RUNTIME_COMMAND - Runtime command used by legacy internal RPI paths (default: claude) - AGENTOPS_RPI_AO_COMMAND - ao command used for ratchet/checkpoint calls (default: ao) - AGENTOPS_RPI_BD_COMMAND - bd command used for epic/child checks (default: bd) - AGENTOPS_RPI_TMUX_COMMAND - tmux command used for status liveness probes (default: tmux) - AGENTOPS_FLYWHEEL_AUTO_PROMOTE_THRESHOLD - Default auto-promote age threshold (e.g. 24h) - AGENTOPS_MODEL_TIER - Default model cost tier (quality/balanced/budget) - AGENTOPS_COUNCIL_MODEL_TIER - Council-specific model tier override - AGENTOPS_DREAM_REPORT_DIR - Default output directory for overnight Dream reports - AGENTOPS_DREAM_RUN_TIMEOUT - Default timeout for overnight Dream runs - AGENTOPS_DREAM_KEEP_AWAKE - Default keep-awake behavior for overnight Dream runs - -Examples: - ao config --show # Show resolved configuration - ao config --show --json # Output as JSON`, - Args: cobra.NoArgs, - RunE: runConfig, -} - -func init() { - configCmd.GroupID = "config" - rootCmd.AddCommand(configCmd) - configCmd.Flags().BoolVar(&configShow, "show", false, "Show resolved configuration with sources") - configCmd.AddCommand(configModelsCmd) - configModelsCmd.Flags().StringVar(&modelsSetTier, "set-tier", "", "Set the default model cost tier (quality, balanced, budget)") - configModelsCmd.Flags().StringVar(&modelsSetSkill, "set-skill", "", "Set a skill-specific tier override (e.g. council=quality)") - _ = configModelsCmd.RegisterFlagCompletionFunc("set-tier", staticCompletionFunc("quality", "balanced", "budget")) -} - -func runConfig(cmd *cobra.Command, args []string) error { - if !configShow { - // Show help if no flags - return cmd.Help() - } - - // Get resolved config with sources - resolved := config.Resolve(GetOutput(), "", GetVerbose()) - - if GetOutput() == "json" { - data, err := json.MarshalIndent(resolved, "", " ") - if err != nil { - return fmt.Errorf("marshal config: %w", err) - } - fmt.Println(string(data)) - return nil - } - - // Print table format - fmt.Println("AgentOps Configuration") - fmt.Println("=====================") - fmt.Println() - - fmt.Println("Config files:") - homeConfig := filepath.Join(os.Getenv("HOME"), ".agentops", "config.yaml") - if _, err := os.Stat(homeConfig); err == nil { - fmt.Printf(" ✓ Home: %s\n", homeConfig) - } else { - fmt.Printf(" ✗ Home: %s (not found)\n", homeConfig) - } - - cwd, _ := os.Getwd() - projectConfig := filepath.Join(cwd, ".agentops", "config.yaml") - if _, err := os.Stat(projectConfig); err == nil { - fmt.Printf(" ✓ Project: %s\n", projectConfig) - } else { - fmt.Printf(" ✗ Project: %s (not found)\n", projectConfig) - } - - fmt.Println() - fmt.Println("Resolved values:") - fmt.Printf(" output: %v (from %s)\n", resolved.Output.Value, resolved.Output.Source) - fmt.Printf(" base_dir: %v (from %s)\n", resolved.BaseDir.Value, resolved.BaseDir.Source) - fmt.Printf(" verbose: %v (from %s)\n", resolved.Verbose.Value, resolved.Verbose.Source) - fmt.Printf(" rpi.worktree_mode: %v (from %s)\n", resolved.RPIWorktreeMode.Value, resolved.RPIWorktreeMode.Source) - fmt.Printf(" rpi.runtime_mode: %v (from %s)\n", resolved.RPIRuntimeMode.Value, resolved.RPIRuntimeMode.Source) - fmt.Printf(" rpi.runtime_command: %v (from %s)\n", resolved.RPIRuntimeCommand.Value, resolved.RPIRuntimeCommand.Source) - fmt.Printf(" rpi.ao_command: %v (from %s)\n", resolved.RPIAOCommand.Value, resolved.RPIAOCommand.Source) - fmt.Printf(" rpi.bd_command: %v (from %s)\n", resolved.RPIBDCommand.Value, resolved.RPIBDCommand.Source) - fmt.Printf(" rpi.tmux_command: %v (from %s)\n", resolved.RPITmuxCommand.Value, resolved.RPITmuxCommand.Source) - fmt.Printf(" dream.report_dir: %v (from %s)\n", resolved.DreamReportDir.Value, resolved.DreamReportDir.Source) - fmt.Printf(" dream.run_timeout: %v (from %s)\n", resolved.DreamRunTimeout.Value, resolved.DreamRunTimeout.Source) - fmt.Printf(" dream.keep_awake: %v (from %s)\n", resolved.DreamKeepAwake.Value, resolved.DreamKeepAwake.Source) - fmt.Printf(" dream.runners: %v (from %s)\n", resolved.DreamRunners.Value, resolved.DreamRunners.Source) - fmt.Printf(" dream.scheduler: %v (from %s)\n", resolved.DreamScheduler.Value, resolved.DreamScheduler.Source) - fmt.Printf(" dream.schedule_at: %v (from %s)\n", resolved.DreamScheduleAt.Value, resolved.DreamScheduleAt.Source) - fmt.Printf(" dream.consensus: %v (from %s)\n", resolved.DreamConsensus.Value, resolved.DreamConsensus.Source) - fmt.Printf(" dream.creative: %v (from %s)\n", resolved.DreamCreativeLane.Value, resolved.DreamCreativeLane.Source) - - fmt.Println() - fmt.Println("Environment variables (if set):") - envVars := []string{ - "AGENTOPS_CONFIG", - "AGENTOPS_OUTPUT", - "AGENTOPS_BASE_DIR", - "AGENTOPS_VERBOSE", - "AGENTOPS_NO_SC", - "AGENTOPS_RPI_WORKTREE_MODE", - "AGENTOPS_RPI_RUNTIME", - "AGENTOPS_RPI_RUNTIME_MODE", - "AGENTOPS_RPI_RUNTIME_COMMAND", - "AGENTOPS_RPI_AO_COMMAND", - "AGENTOPS_RPI_BD_COMMAND", - "AGENTOPS_RPI_TMUX_COMMAND", - "AGENTOPS_FLYWHEEL_AUTO_PROMOTE_THRESHOLD", - "AGENTOPS_MODEL_TIER", - "AGENTOPS_COUNCIL_MODEL_TIER", - "AGENTOPS_DREAM_REPORT_DIR", - "AGENTOPS_DREAM_RUN_TIMEOUT", - "AGENTOPS_DREAM_KEEP_AWAKE", - } - anySet := false - for _, env := range envVars { - if v := os.Getenv(env); v != "" { - fmt.Printf(" %s=%s\n", env, v) - anySet = true - } - } - if !anySet { - fmt.Println(" (none set)") - } - - return nil -} - -func runConfigModels(_ *cobra.Command, _ []string) error { - // Handle write operations if either flag is set. - if modelsSetTier != "" || modelsSetSkill != "" { - return handleModelsWrite() - } - - cfg, err := config.Load(nil) - if err != nil { - return fmt.Errorf("load config: %w", err) - } - - if GetOutput() == "json" { - data, err := json.MarshalIndent(cfg.Models, "", " ") - if err != nil { - return fmt.Errorf("marshal models config: %w", err) - } - fmt.Println(string(data)) - return nil - } - - fmt.Println("Model Cost Tiers") - fmt.Println("================") - fmt.Println() - - fmt.Printf(" Default tier: %s\n", cfg.Models.DefaultTier) - fmt.Println() - - fmt.Println(" Available tiers:") - for _, name := range []string{"quality", "balanced", "budget"} { - tier, ok := cfg.Models.Tiers[name] - if !ok { - continue - } - marker := " " - if name == cfg.ResolveTier("") { - marker = "*" - } - codex := tier.Codex - if codex == "" { - codex = "(default)" - } - fmt.Printf(" %s %-10s claude=%-8s codex=%s\n", marker, name, tier.Claude, codex) - } - - fmt.Println() - if len(cfg.Models.SkillOverrides) > 0 { - fmt.Println(" Skill overrides:") - skills := make([]string, 0, len(cfg.Models.SkillOverrides)) - for skill := range cfg.Models.SkillOverrides { - skills = append(skills, skill) - } - sort.Strings(skills) - for _, skill := range skills { - tier := cfg.Models.SkillOverrides[skill] - resolved := cfg.ResolveTier(skill) - if tier == resolved { - fmt.Printf(" %-12s → %s\n", skill, tier) - } else { - fmt.Printf(" %-12s → %s (resolves to %s)\n", skill, tier, resolved) - } - } - } else { - fmt.Println(" Skill overrides: (none)") - } - - fmt.Println() - fmt.Println(" Environment overrides:") - modelEnvVars := []string{ - "AGENTOPS_MODEL_TIER", - "AGENTOPS_COUNCIL_MODEL_TIER", - "COUNCIL_CLAUDE_MODEL", - } - anyModelEnv := false - for _, env := range modelEnvVars { - if v := os.Getenv(env); v != "" { - fmt.Printf(" %s=%s\n", env, v) - anyModelEnv = true - } - } - if !anyModelEnv { - fmt.Println(" (none set)") - } - - return nil -} - -func handleModelsWrite() error { - saveCfg := &config.Config{} - result := configModelsWriteResult{Updated: true} - - if modelsSetTier != "" { - if modelsSetTier == "inherit" { - return fmt.Errorf("invalid tier %q for default: \"inherit\" is only valid for skill overrides", modelsSetTier) - } - if !config.ValidTiers[modelsSetTier] { - return fmt.Errorf("invalid tier %q: must be one of quality, balanced, budget", modelsSetTier) - } - saveCfg.Models.DefaultTier = modelsSetTier - result.DefaultTier = modelsSetTier - } - - if modelsSetSkill != "" { - parts := strings.SplitN(modelsSetSkill, "=", 2) - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - return fmt.Errorf("invalid --set-skill format %q: expected skill=tier (e.g. council=quality)", modelsSetSkill) - } - skill, tier := parts[0], parts[1] - if !config.ValidTiers[tier] { - return fmt.Errorf("invalid tier %q for skill %q: must be one of quality, balanced, budget, inherit", tier, skill) - } - saveCfg.Models.SkillOverrides = map[string]string{skill: tier} - result.SkillOverrides = map[string]string{skill: tier} - } - - if err := config.Save(saveCfg); err != nil { - return fmt.Errorf("saving config: %w", err) - } - - if GetOutput() == "json" { - data, err := json.MarshalIndent(result, "", " ") - if err != nil { - return fmt.Errorf("marshal models write result: %w", err) - } - fmt.Println(string(data)) - return nil - } - - if modelsSetTier != "" { - fmt.Printf("Set default model tier to %q\n", modelsSetTier) - } - if modelsSetSkill != "" { - parts := strings.SplitN(modelsSetSkill, "=", 2) - fmt.Printf("Set skill %q tier to %q\n", parts[0], parts[1]) - } - - return nil -} diff --git a/cli/cmd/ao/config_module.go b/cli/cmd/ao/config_module.go new file mode 100644 index 000000000..335ae373a --- /dev/null +++ b/cli/cmd/ao/config_module.go @@ -0,0 +1,20 @@ +// practices: [hexagonal-architecture, twelve-factor-app] +package main + +import ( + configadapter "github.com/boshu2/agentops/cli/internal/adapters/config" + "github.com/boshu2/agentops/cli/internal/clicontract" + configcommands "github.com/boshu2/agentops/cli/internal/commands/config" + configapp "github.com/boshu2/agentops/cli/internal/config" +) + +var configModule = configcommands.NewModule(configapp.NewCommandService(configadapter.Gateway{}), GetOutput, GetVerbose) +var configCommand = configModule.Command() + +func init() { + configCommand.GroupID = "config" + if err := clicontract.Attach(configCommand, configModule.Contract()); err != nil { + panic(err) + } + rootCmd.AddCommand(configCommand) +} diff --git a/cli/cmd/ao/config_test_shims_test.go b/cli/cmd/ao/config_test_shims_test.go new file mode 100644 index 000000000..ec9aebf27 --- /dev/null +++ b/cli/cmd/ao/config_test_shims_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "os" + + configapp "github.com/boshu2/agentops/cli/internal/config" + "github.com/spf13/cobra" +) + +var ( + configShow bool + modelsSetTier string + modelsSetSkill string + configCmd = configCommand + configModelsCmd = func() *cobra.Command { + command, _, err := configCommand.Find([]string{"models"}) + if err != nil { + panic(err) + } + return command + }() +) + +type configModelsWriteResult = configapp.ModelsWriteResult + +func runConfig(command *cobra.Command, args []string) error { + fresh := configModule.Command() + if configShow { + _ = fresh.Flags().Set("show", "true") + } + fresh.SetOut(os.Stdout) + return fresh.RunE(command, args) +} + +func runConfigModels(command *cobra.Command, args []string) error { + fresh := configModule.Command() + models, _, err := fresh.Find([]string{"models"}) + if err != nil { + return err + } + if modelsSetTier != "" { + _ = models.Flags().Set("set-tier", modelsSetTier) + } + if modelsSetSkill != "" { + _ = models.Flags().Set("set-skill", modelsSetSkill) + } + models.SetOut(os.Stdout) + return models.RunE(command, args) +} + +func handleModelsWrite() error { + return runConfigModels(&cobra.Command{}, nil) +} diff --git a/cli/internal/adapters/config/gateway.go b/cli/internal/adapters/config/gateway.go new file mode 100644 index 000000000..037d2d96b --- /dev/null +++ b/cli/internal/adapters/config/gateway.go @@ -0,0 +1,44 @@ +package config + +import ( + "os" + "path/filepath" + + configapp "github.com/boshu2/agentops/cli/internal/config" +) + +type Gateway struct{} + +func (Gateway) Resolve(output string, verbose bool) *configapp.ResolvedConfig { + return configapp.Resolve(output, "", verbose) +} + +func (Gateway) Files() (configapp.ConfigFiles, error) { + home := filepath.Join(os.Getenv("HOME"), ".agentops", "config.yaml") + cwd, err := os.Getwd() + if err != nil { + return configapp.ConfigFiles{}, err + } + project := filepath.Join(cwd, ".agentops", "config.yaml") + return configapp.ConfigFiles{ + HomePath: home, HomeExists: exists(home), ProjectPath: project, ProjectExists: exists(project), + }, nil +} + +func (Gateway) Environment(keys []string) map[string]string { + values := make(map[string]string) + for _, key := range keys { + if value := os.Getenv(key); value != "" { + values[key] = value + } + } + return values +} + +func (Gateway) Load() (*configapp.Config, error) { return configapp.Load(nil) } +func (Gateway) Save(value *configapp.Config) error { return configapp.Save(value) } + +func exists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/cli/internal/commands/config/module.go b/cli/internal/commands/config/module.go new file mode 100644 index 000000000..c330366bf --- /dev/null +++ b/cli/internal/commands/config/module.go @@ -0,0 +1,303 @@ +// Package config owns Cobra presentation for the config command family. +package config + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "github.com/spf13/cobra" + + "github.com/boshu2/agentops/cli/internal/clicontract" + configapp "github.com/boshu2/agentops/cli/internal/config" +) + +var showEnvironmentKeys = []string{ + "AGENTOPS_CONFIG", "AGENTOPS_OUTPUT", "AGENTOPS_BASE_DIR", "AGENTOPS_VERBOSE", "AGENTOPS_NO_SC", + "AGENTOPS_RPI_WORKTREE_MODE", "AGENTOPS_RPI_RUNTIME", "AGENTOPS_RPI_RUNTIME_MODE", + "AGENTOPS_RPI_RUNTIME_COMMAND", "AGENTOPS_RPI_AO_COMMAND", "AGENTOPS_RPI_BD_COMMAND", "AGENTOPS_RPI_TMUX_COMMAND", + "AGENTOPS_FLYWHEEL_AUTO_PROMOTE_THRESHOLD", "AGENTOPS_MODEL_TIER", "AGENTOPS_COUNCIL_MODEL_TIER", + "AGENTOPS_DREAM_REPORT_DIR", "AGENTOPS_DREAM_RUN_TIMEOUT", "AGENTOPS_DREAM_KEEP_AWAKE", +} + +var modelEnvironmentKeys = []string{"AGENTOPS_MODEL_TIER", "AGENTOPS_COUNCIL_MODEL_TIER", "COUNCIL_CLAUDE_MODEL"} + +type UseCases interface { + Show(context.Context, string, bool) (configapp.ShowResult, error) + Models(context.Context) (configapp.ModelsResult, error) + WriteModels(context.Context, configapp.ModelsWriteRequest) (configapp.ModelsWriteResult, error) +} + +type Module struct { + useCases UseCases + output func() string + verbose func() bool +} + +type options struct { + show bool + setTier string + setSkill string +} + +func NewModule(useCases UseCases, output func() string, verbose func() bool) Module { + return Module{useCases: useCases, output: output, verbose: verbose} +} + +func (Module) Contract() clicontract.CommandContract { + return clicontract.CommandContract{ + ID: "ao.config", + Profiles: clicontract.ProfileDefault | clicontract.ProfileFlywheel | clicontract.ProfileLegacy | clicontract.ProfileCombined, + Args: clicontract.ArgsPolicy{Name: "none", Validate: cobra.NoArgs}, Output: clicontract.OutputText, + Effects: clicontract.EffectFilesystem | clicontract.EffectEnvironment, + ExitClasses: map[int]clicontract.ExitClass{0: clicontract.ExitSuccess, 1: clicontract.ExitFailure}, + } +} + +func (module Module) Command() *cobra.Command { + var commandOptions options + root := &cobra.Command{ + Use: "config", Short: "Manage configuration", Args: cobra.NoArgs, + Long: configLong, + } + root.Flags().BoolVar(&commandOptions.show, "show", false, "Show resolved configuration with sources") + root.RunE = func(command *cobra.Command, _ []string) error { + if !commandOptions.show { + return command.Help() + } + result, err := module.useCases.Show(command.Context(), module.output(), module.verbose()) + if err != nil { + return err + } + return renderShow(command, module.output(), result) + } + models := module.modelsCommand(&commandOptions) + root.AddCommand(models) + return root +} + +func (module Module) modelsCommand(commandOptions *options) *cobra.Command { + command := &cobra.Command{Use: "models", Short: "Show model cost tier configuration", Long: modelsLong, Args: cobra.NoArgs} + command.Flags().StringVar(&commandOptions.setTier, "set-tier", "", "Set the default model cost tier (quality, balanced, budget)") + command.Flags().StringVar(&commandOptions.setSkill, "set-skill", "", "Set a skill-specific tier override (e.g. council=quality)") + _ = command.RegisterFlagCompletionFunc("set-tier", staticValues("quality", "balanced", "budget")) + command.RunE = func(command *cobra.Command, _ []string) error { + if commandOptions.setTier != "" || commandOptions.setSkill != "" { + result, err := module.useCases.WriteModels(command.Context(), configapp.ModelsWriteRequest{ + DefaultTier: commandOptions.setTier, Skill: commandOptions.setSkill, + }) + if err != nil { + return err + } + return renderModelsWrite(command, module.output(), commandOptions.setTier, commandOptions.setSkill, result) + } + result, err := module.useCases.Models(command.Context()) + if err != nil { + return err + } + return renderModels(command, module.output(), result) + } + return command +} + +func renderShow(command *cobra.Command, output string, result configapp.ShowResult) error { + if output == "json" { + return writeJSON(command, result.Resolved, "marshal config") + } + w := command.OutOrStdout() + fmt.Fprintln(w, "AgentOps Configuration") + fmt.Fprintln(w, "=====================") + fmt.Fprintln(w) + fmt.Fprintln(w, "Config files:") + printConfigFile(w, "Home: ", result.ConfigFiles.HomePath, result.ConfigFiles.HomeExists) + printConfigFile(w, "Project:", result.ConfigFiles.ProjectPath, result.ConfigFiles.ProjectExists) + r := result.Resolved + fmt.Fprintln(w) + fmt.Fprintln(w, "Resolved values:") + fmt.Fprintf(w, " output: %v (from %s)\n", r.Output.Value, r.Output.Source) + fmt.Fprintf(w, " base_dir: %v (from %s)\n", r.BaseDir.Value, r.BaseDir.Source) + fmt.Fprintf(w, " verbose: %v (from %s)\n", r.Verbose.Value, r.Verbose.Source) + fmt.Fprintf(w, " rpi.worktree_mode: %v (from %s)\n", r.RPIWorktreeMode.Value, r.RPIWorktreeMode.Source) + fmt.Fprintf(w, " rpi.runtime_mode: %v (from %s)\n", r.RPIRuntimeMode.Value, r.RPIRuntimeMode.Source) + fmt.Fprintf(w, " rpi.runtime_command: %v (from %s)\n", r.RPIRuntimeCommand.Value, r.RPIRuntimeCommand.Source) + fmt.Fprintf(w, " rpi.ao_command: %v (from %s)\n", r.RPIAOCommand.Value, r.RPIAOCommand.Source) + fmt.Fprintf(w, " rpi.bd_command: %v (from %s)\n", r.RPIBDCommand.Value, r.RPIBDCommand.Source) + fmt.Fprintf(w, " rpi.tmux_command: %v (from %s)\n", r.RPITmuxCommand.Value, r.RPITmuxCommand.Source) + fmt.Fprintf(w, " dream.report_dir: %v (from %s)\n", r.DreamReportDir.Value, r.DreamReportDir.Source) + fmt.Fprintf(w, " dream.run_timeout: %v (from %s)\n", r.DreamRunTimeout.Value, r.DreamRunTimeout.Source) + fmt.Fprintf(w, " dream.keep_awake: %v (from %s)\n", r.DreamKeepAwake.Value, r.DreamKeepAwake.Source) + fmt.Fprintf(w, " dream.runners: %v (from %s)\n", r.DreamRunners.Value, r.DreamRunners.Source) + fmt.Fprintf(w, " dream.scheduler: %v (from %s)\n", r.DreamScheduler.Value, r.DreamScheduler.Source) + fmt.Fprintf(w, " dream.schedule_at: %v (from %s)\n", r.DreamScheduleAt.Value, r.DreamScheduleAt.Source) + fmt.Fprintf(w, " dream.consensus: %v (from %s)\n", r.DreamConsensus.Value, r.DreamConsensus.Source) + fmt.Fprintf(w, " dream.creative: %v (from %s)\n", r.DreamCreativeLane.Value, r.DreamCreativeLane.Source) + fmt.Fprintln(w) + fmt.Fprintln(w, "Environment variables (if set):") + printEnvironment(w, showEnvironmentKeys, result.Environment, " ") + return nil +} + +func renderModels(command *cobra.Command, output string, result configapp.ModelsResult) error { + if output == "json" { + return writeJSON(command, result.Config.Models, "marshal models config") + } + w := command.OutOrStdout() + cfg := result.Config + fmt.Fprintln(w, "Model Cost Tiers") + fmt.Fprintln(w, "================") + fmt.Fprintln(w) + fmt.Fprintf(w, " Default tier: %s\n\n", cfg.Models.DefaultTier) + fmt.Fprintln(w, " Available tiers:") + for _, name := range []string{"quality", "balanced", "budget"} { + tier, ok := cfg.Models.Tiers[name] + if !ok { + continue + } + marker := " " + if name == cfg.ResolveTier("") { + marker = "*" + } + codex := tier.Codex + if codex == "" { + codex = "(default)" + } + fmt.Fprintf(w, " %s %-10s claude=%-8s codex=%s\n", marker, name, tier.Claude, codex) + } + fmt.Fprintln(w) + if len(cfg.Models.SkillOverrides) == 0 { + fmt.Fprintln(w, " Skill overrides: (none)") + } else { + fmt.Fprintln(w, " Skill overrides:") + skills := make([]string, 0, len(cfg.Models.SkillOverrides)) + for skill := range cfg.Models.SkillOverrides { + skills = append(skills, skill) + } + sort.Strings(skills) + for _, skill := range skills { + tier, resolved := cfg.Models.SkillOverrides[skill], cfg.ResolveTier(skill) + if tier == resolved { + fmt.Fprintf(w, " %-12s → %s\n", skill, tier) + } else { + fmt.Fprintf(w, " %-12s → %s (resolves to %s)\n", skill, tier, resolved) + } + } + } + fmt.Fprintln(w) + fmt.Fprintln(w, " Environment overrides:") + printEnvironment(w, modelEnvironmentKeys, result.Environment, " ") + return nil +} + +func renderModelsWrite(command *cobra.Command, output, tier, skill string, result configapp.ModelsWriteResult) error { + if output == "json" { + return writeJSON(command, result, "marshal models write result") + } + w := command.OutOrStdout() + if tier != "" { + fmt.Fprintf(w, "Set default model tier to %q\n", tier) + } + if skill != "" { + parts := splitSkill(skill) + fmt.Fprintf(w, "Set skill %q tier to %q\n", parts[0], parts[1]) + } + return nil +} + +func splitSkill(value string) []string { + for index := 0; index < len(value); index++ { + if value[index] == '=' { + return []string{value[:index], value[index+1:]} + } + } + return []string{value, ""} +} + +func writeJSON(command *cobra.Command, value any, label string) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return fmt.Errorf("%s: %w", label, err) + } + _, err = fmt.Fprintln(command.OutOrStdout(), string(data)) + return err +} + +func printConfigFile(w interface{ Write([]byte) (int, error) }, label, path string, exists bool) { + if exists { + fmt.Fprintf(w, " ✓ %s %s\n", label, path) + } else { + fmt.Fprintf(w, " ✗ %s %s (not found)\n", label, path) + } +} + +func printEnvironment(w interface{ Write([]byte) (int, error) }, keys []string, values map[string]string, indent string) { + printed := false + for _, key := range keys { + if value := values[key]; value != "" { + fmt.Fprintf(w, "%s%s=%s\n", indent, key, value) + printed = true + } + } + if !printed { + fmt.Fprintf(w, "%s(none set)\n", indent) + } +} + +func staticValues(values ...string) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + sorted := append([]string(nil), values...) + sort.Strings(sorted) + return func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return sorted, cobra.ShellCompDirectiveNoFileComp + } +} + +const configLong = `View and manage AgentOps configuration. + +Configuration priority (highest to lowest): + 1. Command-line flags + 2. Environment variables (AGENTOPS_*) + 3. Project config (.agentops/config.yaml) + 4. Home config (~/.agentops/config.yaml) + 5. Defaults + +Environment variables: + AGENTOPS_CONFIG - Explicit config file path (overrides default project config location) + AGENTOPS_OUTPUT - Default output format (table, json, yaml) + AGENTOPS_BASE_DIR - Data directory path + AGENTOPS_VERBOSE - Enable verbose output (true/1) + AGENTOPS_NO_SC - Disable Smart Connections (true/1) + AGENTOPS_RPI_WORKTREE_MODE - RPI worktree policy (auto|always|never) + AGENTOPS_RPI_RUNTIME / AGENTOPS_RPI_RUNTIME_MODE - RPI runtime mode (auto|direct|stream) + AGENTOPS_RPI_RUNTIME_COMMAND - Runtime command used by legacy internal RPI paths (default: claude) + AGENTOPS_RPI_AO_COMMAND - ao command used for ratchet/checkpoint calls (default: ao) + AGENTOPS_RPI_BD_COMMAND - bd command used for epic/child checks (default: bd) + AGENTOPS_RPI_TMUX_COMMAND - tmux command used for status liveness probes (default: tmux) + AGENTOPS_FLYWHEEL_AUTO_PROMOTE_THRESHOLD - Default auto-promote age threshold (e.g. 24h) + AGENTOPS_MODEL_TIER - Default model cost tier (quality/balanced/budget) + AGENTOPS_COUNCIL_MODEL_TIER - Council-specific model tier override + AGENTOPS_DREAM_REPORT_DIR - Default output directory for overnight Dream reports + AGENTOPS_DREAM_RUN_TIMEOUT - Default timeout for overnight Dream runs + AGENTOPS_DREAM_KEEP_AWAKE - Default keep-awake behavior for overnight Dream runs + +Examples: + ao config --show # Show resolved configuration + ao config --show --json # Output as JSON` + +const modelsLong = `Display the current model cost tier settings with sources. + +Cost tiers map to model quality levels: + quality → opus (high-stakes decisions, architecture) + balanced → sonnet (default, routine reviews) + budget → haiku (quick checks, simple tasks) + inherit → uses default tier (falls back to balanced) + +Configure in .agentops/config.yaml: + models: + default_tier: balanced + skill_overrides: + council: quality + crank: budget + +Or via environment variables: + AGENTOPS_MODEL_TIER=budget + AGENTOPS_COUNCIL_MODEL_TIER=quality` diff --git a/cli/internal/commands/config/module_test.go b/cli/internal/commands/config/module_test.go new file mode 100644 index 000000000..4111e1f16 --- /dev/null +++ b/cli/internal/commands/config/module_test.go @@ -0,0 +1,71 @@ +package config + +import ( + "bytes" + "context" + "strings" + "testing" + + configapp "github.com/boshu2/agentops/cli/internal/config" +) + +type fakeUseCases struct { + showResult configapp.ShowResult + modelsResult configapp.ModelsResult + writeRequest configapp.ModelsWriteRequest + writeResult configapp.ModelsWriteResult +} + +func (useCases *fakeUseCases) Show(context.Context, string, bool) (configapp.ShowResult, error) { + return useCases.showResult, nil +} + +func (useCases *fakeUseCases) Models(context.Context) (configapp.ModelsResult, error) { + return useCases.modelsResult, nil +} + +func (useCases *fakeUseCases) WriteModels(_ context.Context, request configapp.ModelsWriteRequest) (configapp.ModelsWriteResult, error) { + useCases.writeRequest = request + return useCases.writeResult, nil +} + +func TestModuleShowJSONUsesCommandWriter(t *testing.T) { + useCases := &fakeUseCases{showResult: configapp.ShowResult{Resolved: &configapp.ResolvedConfig{}}} + command := NewModule(useCases, func() string { return "json" }, func() bool { return false }).Command() + var stdout bytes.Buffer + command.SetOut(&stdout) + command.SetArgs([]string{"--show"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(stdout.String(), "{\n") || !strings.Contains(stdout.String(), `"output"`) { + t.Fatalf("stdout = %q", stdout.String()) + } +} + +func TestModuleModelsWriteParsesDelegatesAndRenders(t *testing.T) { + useCases := &fakeUseCases{writeResult: configapp.ModelsWriteResult{Updated: true, DefaultTier: "quality"}} + command := NewModule(useCases, func() string { return "table" }, func() bool { return false }).Command() + var stdout bytes.Buffer + command.SetOut(&stdout) + command.SetArgs([]string{"models", "--set-tier", "quality", "--set-skill", "council=budget"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if useCases.writeRequest.DefaultTier != "quality" || useCases.writeRequest.Skill != "council=budget" { + t.Fatalf("request = %+v", useCases.writeRequest) + } + if got := stdout.String(); got != "Set default model tier to \"quality\"\nSet skill \"council\" tier to \"budget\"\n" { + t.Fatalf("stdout = %q", got) + } +} + +func TestModuleContractDeclaresConfigEffects(t *testing.T) { + contract := (Module{}).Contract() + if contract.ExitClasses[0] == "" || contract.ExitClasses[1] == "" { + t.Fatalf("exit classes = %+v", contract.ExitClasses) + } + if contract.Effects == 0 { + t.Fatal("config effects are undeclared") + } +} diff --git a/cli/internal/config/command_service.go b/cli/internal/config/command_service.go new file mode 100644 index 000000000..37b8d5550 --- /dev/null +++ b/cli/internal/config/command_service.go @@ -0,0 +1,112 @@ +package config + +import ( + "context" + "fmt" + "strings" +) + +var showEnvironmentKeys = []string{ + "AGENTOPS_CONFIG", "AGENTOPS_OUTPUT", "AGENTOPS_BASE_DIR", "AGENTOPS_VERBOSE", "AGENTOPS_NO_SC", + "AGENTOPS_RPI_WORKTREE_MODE", "AGENTOPS_RPI_RUNTIME", "AGENTOPS_RPI_RUNTIME_MODE", + "AGENTOPS_RPI_RUNTIME_COMMAND", "AGENTOPS_RPI_AO_COMMAND", "AGENTOPS_RPI_BD_COMMAND", "AGENTOPS_RPI_TMUX_COMMAND", + "AGENTOPS_FLYWHEEL_AUTO_PROMOTE_THRESHOLD", "AGENTOPS_MODEL_TIER", "AGENTOPS_COUNCIL_MODEL_TIER", + "AGENTOPS_DREAM_REPORT_DIR", "AGENTOPS_DREAM_RUN_TIMEOUT", "AGENTOPS_DREAM_KEEP_AWAKE", +} + +var modelEnvironmentKeys = []string{"AGENTOPS_MODEL_TIER", "AGENTOPS_COUNCIL_MODEL_TIER", "COUNCIL_CLAUDE_MODEL"} + +type ConfigFiles struct { + HomePath string + HomeExists bool + ProjectPath string + ProjectExists bool +} + +type ShowResult struct { + Resolved *ResolvedConfig + ConfigFiles ConfigFiles + Environment map[string]string +} + +type ModelsResult struct { + Config *Config + Environment map[string]string +} + +type ModelsWriteRequest struct { + DefaultTier string + Skill string +} + +type ModelsWriteResult struct { + Updated bool `json:"updated"` + DefaultTier string `json:"default_tier,omitempty"` + SkillOverrides map[string]string `json:"skill_overrides,omitempty"` +} + +type CommandGateway interface { + Resolve(string, bool) *ResolvedConfig + Files() (ConfigFiles, error) + Environment([]string) map[string]string + Load() (*Config, error) + Save(*Config) error +} + +type CommandService struct { + gateway CommandGateway +} + +func NewCommandService(gateway CommandGateway) *CommandService { + return &CommandService{gateway: gateway} +} + +func (service *CommandService) Show(_ context.Context, output string, verbose bool) (ShowResult, error) { + files, err := service.gateway.Files() + if err != nil { + return ShowResult{}, err + } + return ShowResult{ + Resolved: service.gateway.Resolve(output, verbose), ConfigFiles: files, + Environment: service.gateway.Environment(showEnvironmentKeys), + }, nil +} + +func (service *CommandService) Models(_ context.Context) (ModelsResult, error) { + loaded, err := service.gateway.Load() + if err != nil { + return ModelsResult{}, fmt.Errorf("load config: %w", err) + } + return ModelsResult{Config: loaded, Environment: service.gateway.Environment(modelEnvironmentKeys)}, nil +} + +func (service *CommandService) WriteModels(_ context.Context, request ModelsWriteRequest) (ModelsWriteResult, error) { + save := &Config{} + result := ModelsWriteResult{Updated: true} + if request.DefaultTier != "" { + if request.DefaultTier == "inherit" { + return ModelsWriteResult{}, fmt.Errorf("invalid tier %q for default: \"inherit\" is only valid for skill overrides", request.DefaultTier) + } + if !ValidTiers[request.DefaultTier] { + return ModelsWriteResult{}, fmt.Errorf("invalid tier %q: must be one of quality, balanced, budget", request.DefaultTier) + } + save.Models.DefaultTier = request.DefaultTier + result.DefaultTier = request.DefaultTier + } + if request.Skill != "" { + parts := strings.SplitN(request.Skill, "=", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return ModelsWriteResult{}, fmt.Errorf("invalid --set-skill format %q: expected skill=tier (e.g. council=quality)", request.Skill) + } + skill, tier := parts[0], parts[1] + if !ValidTiers[tier] { + return ModelsWriteResult{}, fmt.Errorf("invalid tier %q for skill %q: must be one of quality, balanced, budget, inherit", tier, skill) + } + save.Models.SkillOverrides = map[string]string{skill: tier} + result.SkillOverrides = map[string]string{skill: tier} + } + if err := service.gateway.Save(save); err != nil { + return ModelsWriteResult{}, fmt.Errorf("saving config: %w", err) + } + return result, nil +} diff --git a/cli/internal/config/command_service_test.go b/cli/internal/config/command_service_test.go new file mode 100644 index 000000000..c07720cff --- /dev/null +++ b/cli/internal/config/command_service_test.go @@ -0,0 +1,48 @@ +package config + +import ( + "context" + "strings" + "testing" +) + +type fakeCommandGateway struct{ saved *Config } + +func (gateway *fakeCommandGateway) Resolve(string, bool) *ResolvedConfig { return &ResolvedConfig{} } +func (gateway *fakeCommandGateway) Files() (ConfigFiles, error) { return ConfigFiles{}, nil } +func (gateway *fakeCommandGateway) Environment([]string) map[string]string { + return map[string]string{} +} +func (gateway *fakeCommandGateway) Load() (*Config, error) { return Default(), nil } +func (gateway *fakeCommandGateway) Save(value *Config) error { + gateway.saved = value + return nil +} + +func TestCommandServiceWriteModelsValidatesAndBuildsPatch(t *testing.T) { + gateway := &fakeCommandGateway{} + service := NewCommandService(gateway) + result, err := service.WriteModels(context.Background(), ModelsWriteRequest{DefaultTier: "budget", Skill: "council=quality"}) + if err != nil { + t.Fatal(err) + } + if result.DefaultTier != "budget" || result.SkillOverrides["council"] != "quality" { + t.Fatalf("result = %+v", result) + } + if gateway.saved == nil || gateway.saved.Models.DefaultTier != "budget" || gateway.saved.Models.SkillOverrides["council"] != "quality" { + t.Fatalf("saved patch = %+v", gateway.saved) + } +} + +func TestCommandServiceWriteModelsRejectsInvalidInputBeforeSave(t *testing.T) { + for _, request := range []ModelsWriteRequest{{DefaultTier: "inherit"}, {DefaultTier: "premium"}, {Skill: "broken"}, {Skill: "council=premium"}} { + gateway := &fakeCommandGateway{} + _, err := NewCommandService(gateway).WriteModels(context.Background(), request) + if err == nil || !strings.Contains(err.Error(), "invalid") { + t.Fatalf("request %+v error = %v", request, err) + } + if gateway.saved != nil { + t.Fatalf("invalid request %+v reached save", request) + } + } +} From b6eee931263d255eccd310fe85edbe51b5e44b91 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:26:10 -0400 Subject: [PATCH 053/108] chore(cli): seal config migration lineage (age-nw28h.2.5) --- .../compatibility-baseline/families/config/lineage.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/testdata/compatibility-baseline/families/config/lineage.json b/cli/testdata/compatibility-baseline/families/config/lineage.json index 4e4ddd6da..7ea4f6275 100644 --- a/cli/testdata/compatibility-baseline/families/config/lineage.json +++ b/cli/testdata/compatibility-baseline/families/config/lineage.json @@ -6,5 +6,6 @@ "fixture_sha256": "3c5dd5a716854332a3985b91b3beb9a12cce656747625fac79371f2e54f10ff8", "ownership_sha256": "0414989420add10ca0f3ce0a5e491a9b5ce9dadd98b346b3ce7ae991e2af9b80", "scope_sha256": "7ab979017b4cf7d74fabcbe09c78f83feb70e740e575565e5daea9237e0e04bb", - "migration_state": "migrating" + "migration_state": "migrated", + "accepted_sha": "c55ebbacb90fe44bc7097b45240965c5f21abd30" } From 8a8efedfce8cbad39d665c44f20635a368219abc Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:26:32 -0400 Subject: [PATCH 054/108] chore(provenance): bind pawl CONFIRMED verdict for age-nw28h.2.5 #trivial --- docs/provenance/ledger.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/provenance/ledger.jsonl b/docs/provenance/ledger.jsonl index 43b7d9f95..0a9222d2b 100644 --- a/docs/provenance/ledger.jsonl +++ b/docs/provenance/ledger.jsonl @@ -520,3 +520,4 @@ {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.2@bd171a4","from_type":"verdict","to_id":"bd171a4776a303a8c7ad867047f70967cdee8b0e","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.2 disposition=CONFIRMED","bead_id":"age-nw28h.2.2","trust_tier":"inferred","ts":"2026-07-11T19:50:19Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-capabilities-independent-verdict-bd171a477.txt","prev_hash":"0dbb491f180b24cbd6fd602bf8812d14382184c08370a714c7e4905f3730ca0e","payload_hash":"2ed1b654481b2037e5e1b8cb439cafe29bb16cf6319181b5e42bc549eb1a9e79","hash":"9739edafbb15e8a5d7e0ac15be295093bb14ebf60ffc13f92d46db35942e5aca"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.3@2172c77","from_type":"verdict","to_id":"2172c77f05692169948f3b034970c90d64a366c1","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.3 disposition=CONFIRMED","bead_id":"age-nw28h.2.3","trust_tier":"inferred","ts":"2026-07-11T20:30:36Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-claim-independent-verdict-2172c77f0.txt","prev_hash":"9739edafbb15e8a5d7e0ac15be295093bb14ebf60ffc13f92d46db35942e5aca","payload_hash":"7d01c886ce25d14d82b1cabbf379bc4e777f083328ee4695ca8344979eae9855","hash":"be2caecd5f64c47b68e12dd6a220eacac168450f3c6c0160f4428f4a917e92d0"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.4@0782b56","from_type":"verdict","to_id":"0782b566af8be3b013b649f95f607b5bbebc4fc5","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.4 disposition=CONFIRMED","bead_id":"age-nw28h.2.4","trust_tier":"inferred","ts":"2026-07-11T22:04:47Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-close-independent-verdict-0782b566a.txt","prev_hash":"be2caecd5f64c47b68e12dd6a220eacac168450f3c6c0160f4428f4a917e92d0","payload_hash":"a451e922b8fb7db3c5d35d3815169a837ec3e0b650486fdfd1c1a1c976e9d168","hash":"671a7c4580ed2345d62f32c92074f974f51455e3ea341f4eb371445a8b8f395b"} +{"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.5@c55ebba","from_type":"verdict","to_id":"c55ebbacb90fe44bc7097b45240965c5f21abd30","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.5 disposition=CONFIRMED","bead_id":"age-nw28h.2.5","trust_tier":"inferred","ts":"2026-07-11T22:26:32Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-config-independent-verdict-c55ebbacb.txt","prev_hash":"671a7c4580ed2345d62f32c92074f974f51455e3ea341f4eb371445a8b8f395b","payload_hash":"65b9734effd05e76746ac5c29fad79f7eba85a0613aa220fa4669f0744507f9e","hash":"0a925b381ecc0937e340c884fc83a9f992a30a6fc49c0c4f94efc4aa500d1dc8"} From e54af35a79de81e792449b8824cef50a647e9487 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:28:49 -0400 Subject: [PATCH 055/108] test(cli): freeze council-gate compatibility contract (age-nw28h.2.6) --- .../families/council-gate/case.json | 25 +++++++++++++++++++ .../families/council-gate/ownership.json | 18 +++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/council-gate/case.json create mode 100644 cli/testdata/compatibility-baseline/families/council-gate/ownership.json diff --git a/cli/testdata/compatibility-baseline/families/council-gate/case.json b/cli/testdata/compatibility-baseline/families/council-gate/case.json new file mode 100644 index 000000000..5bde2b9b7 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/council-gate/case.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "family": "council-gate", + "checks": [ + { + "id": "council-gate-black-box", + "command": "tmp=$(mktemp -d /tmp/ao-council-gate.XXXXXX); (cd cli && go build -o \"$tmp/ao\" ./cmd/ao); printf 'author: worker\\njudge: judge-a\\njudge_program: codex\\njudge_model_family: gpt\\ncontext_id: ctx-a\\nVERDICT: PASS\\nCOMMANDS RUN:\\n go test ./...\\n' >\"$tmp/a\"; printf 'author: worker\\njudge: judge-b\\njudge_program: codex\\njudge_model_family: gpt\\ncontext_id: ctx-b\\nVERDICT: PASS\\nCOMMANDS RUN:\\n go test ./...\\n' >\"$tmp/b\"; sed 's/VERDICT: PASS/VERDICT: FAIL/' \"$tmp/b\" >\"$tmp/fail\"; sed '/COMMANDS RUN:/,$d' \"$tmp/b\" >\"$tmp/unverified\"; \"$tmp/ao\" council-gate \"$tmp/a\" \"$tmp/b\" >\"$tmp/pass.out\"; grep -qx 'COUNCIL PASS: 2/2 judges unanimous across 2 distinct contexts (1 model families)' \"$tmp/pass.out\"; set +e; \"$tmp/ao\" council-gate \"$tmp/a\" \"$tmp/fail\" >\"$tmp/mixed.out\" 2>\"$tmp/mixed.err\"; mixed=$?; \"$tmp/ao\" council-gate \"$tmp/a\" \"$tmp/unverified\" >\"$tmp/unverified.out\" 2>\"$tmp/unverified.err\"; unverified=$?; \"$tmp/ao\" council-gate \"$tmp/a\" >\"$tmp/args.out\" 2>\"$tmp/args.err\"; args=$?; set -e; test \"$mixed\" -eq 8; test \"$unverified\" -eq 6; test \"$args\" -eq 1; grep -qx 'DISAGREEMENT: 1 PASS / 1 FAIL - fail-closed; dispatch tie-break' \"$tmp/mixed.err\"; grep -q 'verdict(s) unverified' \"$tmp/unverified.err\"; grep -qx 'Error: requires at least 2 arg(s), only received 1' \"$tmp/args.err\"" + }, + { + "id": "council-gate-go-behavior", + "command": "cd cli && go test ./cmd/ao -run '^(TestTickCouncilGate.*|TestTickVerdict.*)$' -count=1" + } + ], + "dimensions": { + "path": {"status": "evidence", "evidence": ["council-gate-black-box"]}, + "aliases": {"status": "not_applicable", "reason": "The council-gate command declares no aliases."}, + "accepted_args": {"status": "evidence", "evidence": ["council-gate-black-box"]}, + "rejected_args": {"status": "evidence", "evidence": ["council-gate-black-box"]}, + "stdout": {"status": "evidence", "evidence": ["council-gate-black-box", "council-gate-go-behavior"]}, + "stderr": {"status": "evidence", "evidence": ["council-gate-black-box", "council-gate-go-behavior"]}, + "exit_classes": {"status": "evidence", "evidence": ["council-gate-black-box", "council-gate-go-behavior"]}, + "tracker_selection": {"status": "not_applicable", "reason": "Council verdict aggregation does not select an issue tracker."}, + "ordered_effects": {"status": "evidence", "evidence": ["council-gate-black-box", "council-gate-go-behavior"]} + } +} diff --git a/cli/testdata/compatibility-baseline/families/council-gate/ownership.json b/cli/testdata/compatibility-baseline/families/council-gate/ownership.json new file mode 100644 index 000000000..f007591a5 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/council-gate/ownership.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "family": "council-gate", + "profiles": {"default":"present","flywheel":"present","legacy":"present","combined":"present"}, + "legacy_symbols": [ + "councilGateCmd", "tickCouncilGate", "tickVerdictLineCap", "tickScanVerdictLines", + "tickVerdictHasCommandsRun", "tickVerdictIdentityInfo", "tickVerdictIdentity", + "tickVerdictMetadataValue", "tickNormalizeMetadataKey", "tickNormalizeIdentityValue", + "tickNormalizeModelFamily", "tickUnknownModelFamily", "tickVerdictTokenCounts" + ], + "live_owner": "cli/internal/commands/council_gate", + "allowed_paths": [ + "cli/cmd/ao/council_gate*.go", "cli/cmd/ao/tick.go", "cli/cmd/ao/tick_test.go", "cli/cmd/ao/tick_cmd_legacy.go", + "cli/cmd/ao/codex.go", "cli/cmd/ao/converge_canary.go", + "cli/internal/adapters/council_gate/**", "cli/internal/councilgate/**", "cli/internal/commands/council_gate/**", + "cli/internal/verdict/**", "cli/testdata/compatibility-baseline/families/council-gate/lineage.json" + ] +} From e847ec73b4e551b5aaa9acc648d17cec62d5772e Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:44:34 -0400 Subject: [PATCH 056/108] test(cli): bind council-gate migration lineage --- .../families/council-gate/lineage.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/council-gate/lineage.json diff --git a/cli/testdata/compatibility-baseline/families/council-gate/lineage.json b/cli/testdata/compatibility-baseline/families/council-gate/lineage.json new file mode 100644 index 000000000..ea5c30898 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/council-gate/lineage.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "family": "council-gate", + "old_implementation_sha": "8a8efedfce8cbad39d665c44f20635a368219abc", + "freeze_sha": "e54af35a79de81e792449b8824cef50a647e9487", + "fixture_sha256": "be0e78852c998cf61f76ac635fa0fa8c86c071969b0012168ee47ecc78b40247", + "ownership_sha256": "8677bfaf108ee9da2fde85527c10ca1e10167612f49ddf9254ff60155a4d3d6b", + "scope_sha256": "d0bfb45b0a3592d67d411c21645bf83c65a2dc085c4d55b17c4d58a8ac8c737b", + "migration_state": "migrating" +} From e38c499458012d159a6397e5b56af5be5ba5d3ba Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:43:03 -0400 Subject: [PATCH 057/108] refactor(council-gate): extract explicit command module --- cli/cmd/ao/codex.go | 9 +- cli/cmd/ao/converge_canary.go | 8 +- cli/cmd/ao/council_gate_legacy_test.go | 24 ++ cli/cmd/ao/council_gate_module.go | 39 +++ cli/cmd/ao/tick.go | 277 +----------------- cli/cmd/ao/tick_cmd_legacy.go | 11 +- cli/internal/adapters/council_gate/reader.go | 42 +++ .../adapters/council_gate/reader_test.go | 27 ++ cli/internal/commands/council_gate/module.go | 94 ++++++ .../commands/council_gate/module_test.go | 49 ++++ cli/internal/councilgate/service.go | 115 ++++++++ cli/internal/councilgate/service_test.go | 48 +++ cli/internal/verdict/parser.go | 170 +++++++++++ cli/internal/verdict/parser_test.go | 24 ++ 14 files changed, 655 insertions(+), 282 deletions(-) create mode 100644 cli/cmd/ao/council_gate_legacy_test.go create mode 100644 cli/cmd/ao/council_gate_module.go create mode 100644 cli/internal/adapters/council_gate/reader.go create mode 100644 cli/internal/adapters/council_gate/reader_test.go create mode 100644 cli/internal/commands/council_gate/module.go create mode 100644 cli/internal/commands/council_gate/module_test.go create mode 100644 cli/internal/councilgate/service.go create mode 100644 cli/internal/councilgate/service_test.go create mode 100644 cli/internal/verdict/parser.go create mode 100644 cli/internal/verdict/parser_test.go diff --git a/cli/cmd/ao/codex.go b/cli/cmd/ao/codex.go index 26bb850fa..d77bb2bf1 100644 --- a/cli/cmd/ao/codex.go +++ b/cli/cmd/ao/codex.go @@ -28,6 +28,7 @@ import ( cliRPI "github.com/boshu2/agentops/cli/internal/rpi" "github.com/boshu2/agentops/cli/internal/storage" "github.com/boshu2/agentops/cli/internal/types" + verdictparse "github.com/boshu2/agentops/cli/internal/verdict" ) var ( @@ -1397,12 +1398,12 @@ func codexFinalMessageVerdict(text string) codexReceiptVerdict { if status == "ERROR" { return verdict } - if !tickVerdictHasCommandsRun(text) { + if !verdictparse.HasCommandsRun(text) { verdict.Status = "ERROR" verdict.Summary = "Final verdict missing non-empty COMMANDS RUN body." return verdict } - identity, gaps := tickVerdictIdentity(text) + identity, gaps := verdictparse.Identity(text) if len(gaps) > 0 { verdict.Status = "ERROR" verdict.Summary = "Final verdict identity unproven: " + strings.Join(gaps, "; ") @@ -1470,7 +1471,7 @@ func validateCodexRunReceipt(receipt codexRunReceipt) error { gaps = append(gaps, "successful Codex run did not produce a verifiable verdict") } if status == "PASS" || status == "WARN" || status == "FAIL" { - identity := tickVerdictIdentityInfo{ + identity := verdictparse.IdentityInfo{ Author: receipt.Verdict.AuthorID, JudgeName: receipt.Verdict.JudgeName, JudgeProgram: receipt.Verdict.JudgeProgram, @@ -1485,7 +1486,7 @@ func validateCodexRunReceipt(receipt codexRunReceipt) error { if strings.TrimSpace(identity.JudgeProgram) == "" { gaps = append(gaps, "missing judge_program") } - if strings.TrimSpace(identity.JudgeModelFamily) == "" || tickUnknownModelFamily(identity.JudgeModelFamily) { + if strings.TrimSpace(identity.JudgeModelFamily) == "" || verdictparse.UnknownModelFamily(identity.JudgeModelFamily) { gaps = append(gaps, "missing or unknown judge_model_family") } if strings.TrimSpace(identity.Author) != "" && strings.TrimSpace(identity.Author) == strings.TrimSpace(identity.JudgeName) { diff --git a/cli/cmd/ao/converge_canary.go b/cli/cmd/ao/converge_canary.go index 0a24b675c..3d3d6fe8c 100644 --- a/cli/cmd/ao/converge_canary.go +++ b/cli/cmd/ao/converge_canary.go @@ -8,7 +8,11 @@ package main // planted positive (or that rejects everything) fails the canary and aborts the // run. Precedent: chaos-test / tickSmoke plant fixtures and assert reject codes. -import "fmt" +import ( + "fmt" + + verdictparse "github.com/boshu2/agentops/cli/internal/verdict" +) // convergeCanaryRejectCode is the sentinel reject code a gate returns for a // planted positive (distinct, non-zero; aligned with the tickExitCouncil family). @@ -25,7 +29,7 @@ type convergeCanaryGate func(verdictBody string) (rejected bool, code int) // a missing context_id. An independent-context verdict with a complete identity // is accepted. var convergeProductionCanaryGate convergeCanaryGate = func(verdictBody string) (bool, int) { - _, gaps := tickVerdictIdentity(verdictBody) + _, gaps := verdictparse.Identity(verdictBody) if len(gaps) > 0 { return true, convergeCanaryRejectCode } diff --git a/cli/cmd/ao/council_gate_legacy_test.go b/cli/cmd/ao/council_gate_legacy_test.go new file mode 100644 index 000000000..c8384ed37 --- /dev/null +++ b/cli/cmd/ao/council_gate_legacy_test.go @@ -0,0 +1,24 @@ +package main + +import verdictparse "github.com/boshu2/agentops/cli/internal/verdict" + +const tickVerdictLineCap = verdictparse.LineCap + +type tickVerdictIdentityInfo = verdictparse.IdentityInfo + +func tickScanVerdictLines(text string) ([]string, error) { return verdictparse.ScanLines(text) } +func tickVerdictHasCommandsRun(text string) bool { return verdictparse.HasCommandsRun(text) } +func tickVerdictIdentity(text string) (tickVerdictIdentityInfo, []string) { + return verdictparse.Identity(text) +} +func tickVerdictMetadataValue(text string, keys ...string) string { + return verdictparse.MetadataValue(text, keys...) +} +func tickNormalizeMetadataKey(key string) string { return verdictparse.NormalizeMetadataKey(key) } +func tickNormalizeIdentityValue(value string) string { + return verdictparse.NormalizeIdentityValue(value) +} +func tickNormalizeModelFamily(value string) string { return verdictparse.NormalizeModelFamily(value) } +func tickUnknownModelFamily(family string) bool { return verdictparse.UnknownModelFamily(family) } +func tickVerdictTokenCounts(text string) (int, int) { return verdictparse.TokenCounts(text) } +func tickCouncilGate(rt tickRuntime, paths []string) error { return runCouncilGate(rt, paths) } diff --git a/cli/cmd/ao/council_gate_module.go b/cli/cmd/ao/council_gate_module.go new file mode 100644 index 000000000..8765ff1a2 --- /dev/null +++ b/cli/cmd/ao/council_gate_module.go @@ -0,0 +1,39 @@ +// practices: [hexagonal-architecture, ddd-bounded-context] +package main + +import ( + counciladapter "github.com/boshu2/agentops/cli/internal/adapters/council_gate" + "github.com/boshu2/agentops/cli/internal/clicontract" + councilcommands "github.com/boshu2/agentops/cli/internal/commands/council_gate" + "github.com/boshu2/agentops/cli/internal/councilgate" +) + +func newCouncilGateModule(workDir string, requireCrossFamily bool) councilcommands.Module { + reader := counciladapter.Reader{WorkDir: workDir} + service := councilgate.NewService(reader, councilgate.Policy{RequireCrossFamily: requireCrossFamily}) + return councilcommands.NewModule(service) +} + +func systemCouncilGateModule() councilcommands.Module { + service := councilgate.NewService(counciladapter.Reader{}, councilgate.Policy{}) + return councilcommands.NewModule(service) +} + +var councilGateModule = systemCouncilGateModule() +var councilGateCommand = councilGateModule.Command() + +func runCouncilGate(rt tickRuntime, paths []string) error { + command := newCouncilGateModule(rt.workDir, rt.requireCrossFamily).Command() + command.SetIn(rt.stdin) + command.SetOut(rt.stdout) + command.SetErr(rt.stderr) + command.SetArgs(paths) + return command.ExecuteContext(rootCmd.Context()) +} + +func init() { + if err := clicontract.Attach(councilGateCommand, councilGateModule.Contract()); err != nil { + panic(err) + } + rootCmd.AddCommand(councilGateCommand) +} diff --git a/cli/cmd/ao/tick.go b/cli/cmd/ao/tick.go index ed4a8403c..b1b91481e 100644 --- a/cli/cmd/ao/tick.go +++ b/cli/cmd/ao/tick.go @@ -13,7 +13,7 @@ import ( "regexp" "strings" - "github.com/boshu2/agentops/cli/internal/liveness" + verdictparse "github.com/boshu2/agentops/cli/internal/verdict" "github.com/spf13/cobra" ) @@ -114,15 +114,6 @@ var verdictGateCmd = &cobra.Command{ }, } -var councilGateCmd = &cobra.Command{ - Use: "council-gate [...]", - Short: "Fail-closed two-plus judge verdict aggregation", - Args: cobra.MinimumNArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { - return tickCouncilGate(newTickRuntime(cmd), args) - }, -} - var installGuardsCmd = &cobra.Command{ Use: "install-guards", Short: "Install repo-local git guard hooks", @@ -167,7 +158,6 @@ func init() { rootCmd.AddCommand( readyCmd, verdictGateCmd, - councilGateCmd, installGuardsCmd, guardStatusCmd, chaosTestCmd, @@ -373,11 +363,11 @@ func tickRunVerdictGate(rt tickRuntime, path string) error { if err != nil { return err } - if !tickVerdictHasCommandsRun(text) { + if !verdictparse.HasCommandsRun(text) { fmt.Fprintln(rt.stderr, "REJECTED: verdict has no non-empty 'COMMANDS RUN' body - unverified; route to tie-break") return &tickExitError{code: tickExitUnverified} } - if _, gaps := tickVerdictIdentity(text); len(gaps) > 0 { + if _, gaps := verdictparse.Identity(text); len(gaps) > 0 { fmt.Fprintf(rt.stderr, "REJECTED: verdict identity unproven: %s\n", strings.Join(gaps, "; ")) return &tickExitError{code: tickExitUnverified} } @@ -385,253 +375,6 @@ func tickRunVerdictGate(rt tickRuntime, path string) error { return nil } -// tickVerdictLineCap is the deliberate maximum length of a single line in a -// council verdict artifact. It raises the implicit 64KB bufio.Scanner token -// limit to a bounded ceiling: any legitimate verdict line fits well under it, -// while a pathological line beyond it fails the scan closed rather than silently -// truncating the rest of the artifact. -const tickVerdictLineCap = 1 << 20 // 1 MiB - -// tickScanVerdictLines reads a verdict artifact into its lines with an explicit, -// bounded scan buffer and surfaces any scanner error — including a line longer -// than tickVerdictLineCap. Callers in the council verdict path MUST fail closed -// on a non-nil error: an artifact that cannot be fully scanned must never be -// treated as a PASS (Codex sweep F-03 — an unchecked scanner.Err() let a -// 70000-byte line hide a trailing FAIL behind a truncated scan). -func tickScanVerdictLines(text string) ([]string, error) { - scanner := bufio.NewScanner(strings.NewReader(text)) - scanner.Buffer(make([]byte, 0, 64*1024), tickVerdictLineCap) - var lines []string - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - if err := scanner.Err(); err != nil { - return nil, err - } - return lines, nil -} - -func tickVerdictHasCommandsRun(text string) bool { - lines, err := tickScanVerdictLines(text) - if err != nil { - // Fail closed: an unscannable artifact has no verifiable COMMANDS RUN body. - return false - } - inBlock := false - commandHeader := regexp.MustCompile(`(?i)commands[ _-]*run`) - for _, line := range lines { - lower := strings.ToLower(line) - if commandHeader.MatchString(line) { - inBlock = true - if i := strings.Index(line, ":"); i >= 0 && strings.TrimSpace(line[i+1:]) != "" { - return true - } - continue - } - if inBlock && regexp.MustCompile(`^\s*reasons`).MatchString(lower) { - inBlock = false - } - if inBlock && strings.TrimSpace(line) != "" { - return true - } - } - return false -} - -func tickCouncilGate(rt tickRuntime, paths []string) error { - n := len(paths) - pass, fail, unverified := 0, 0, 0 - families := map[string]bool{} - judges := map[string]bool{} - contexts := map[string]bool{} - for _, path := range paths { - text, err := tickReadVerdict(rt, path) - identity, identityGaps := tickVerdictIdentity(text) - // A verdict with any identity gap — now including a missing context_id or a - // judge context equal to the author context (self-judge) — is unverified and - // fails closed. - if err != nil || !tickVerdictHasCommandsRun(text) || len(identityGaps) > 0 { - unverified++ - continue - } - // The independence axis is the judge CONTEXT, not the judge name: a - // duplicate context is one judge regardless of how it labels itself. - // Canonicalize first so a single judge cannot forge "distinct" contexts via - // whitespace/case/unicode variants. - canonCtx := liveness.CanonicalizeContextID(identity.ContextID) - if contexts[canonCtx] { - fmt.Fprintf(rt.stderr, "FAIL-CLOSED: duplicate judge context %q does not count as an independent judge\n", identity.ContextID) - return &tickExitError{code: tickExitCouncil} - } - contexts[canonCtx] = true - // Retain the judge-name dedup as a secondary guard. - if judges[identity.JudgeName] { - fmt.Fprintf(rt.stderr, "FAIL-CLOSED: duplicate judge %q does not count as an independent judge\n", identity.JudgeName) - return &tickExitError{code: tickExitCouncil} - } - judges[identity.JudgeName] = true - p, f := tickVerdictTokenCounts(text) - switch { - case p == 1 && f == 0: - pass++ - families[identity.JudgeModelFamily] = true - case f == 1 && p == 0: - fail++ - default: - unverified++ - } - } - if unverified > 0 { - fmt.Fprintf(rt.stderr, "FAIL-CLOSED: %d/%d verdict(s) unverified (no COMMANDS RUN / identity gap)\n", unverified, n) - return &tickExitError{code: tickExitCouncil} - } - if pass == n { - // Context floor: need >=2 distinct non-author judge contexts. - if len(contexts) < 2 { - fmt.Fprintf(rt.stderr, "FAIL-CLOSED: PASS quorum has %d distinct judge context; need at least 2 independent contexts\n", len(contexts)) - return &tickExitError{code: tickExitCouncil} - } - // Optional cross-family strengthener: only gated when explicitly required. - if rt.requireCrossFamily && len(families) < 2 { - fmt.Fprintf(rt.stderr, "FAIL-CLOSED: --require-cross-family set but PASS quorum spans %d model family; need at least 2 (cross-family)\n", len(families)) - return &tickExitError{code: tickExitCouncil} - } - fmt.Fprintf(rt.stdout, "COUNCIL PASS: %d/%d judges unanimous across %d distinct contexts (%d model families)\n", pass, n, len(contexts), len(families)) - return nil - } - if fail == n { - fmt.Fprintf(rt.stderr, "FAIL-CLOSED: %d/%d judges FAIL\n", fail, n) - return &tickExitError{code: tickExitCouncil} - } - fmt.Fprintf(rt.stderr, "DISAGREEMENT: %d PASS / %d FAIL - fail-closed; dispatch tie-break\n", pass, fail) - return &tickExitError{code: tickExitDisagree} -} - -type tickVerdictIdentityInfo struct { - Author string - JudgeName string - JudgeProgram string - JudgeModelFamily string - // ContextID is the judge's session-identity axis (the independence axis): two - // distinct ContextIDs are two independent judges even on the same model. - ContextID string - // AuthorContextID is the author's context — a judge whose ContextID equals it - // is self-judging and fails closed. - AuthorContextID string -} - -func tickVerdictIdentity(text string) (tickVerdictIdentityInfo, []string) { - info := tickVerdictIdentityInfo{ - Author: tickNormalizeIdentityValue(tickVerdictMetadataValue(text, "author", "author_id", "author-id", "author id", "author_name", "author-name", "author name")), - JudgeName: tickNormalizeIdentityValue(tickVerdictMetadataValue(text, "judge", "judge_name", "judge-name", "judge name", "judge_id", "judge-id", "judge id")), - JudgeProgram: tickNormalizeIdentityValue(tickVerdictMetadataValue(text, "judge_program", "judge-program", "judge program", "program", "validator_program", "validator-program", "validator program")), - JudgeModelFamily: tickNormalizeModelFamily(tickVerdictMetadataValue(text, "judge_model_family", "judge-model-family", "judge model family", "model_family", "model-family", "model family", "family")), - ContextID: tickNormalizeIdentityValue(tickVerdictMetadataValue(text, "context_id", "context-id", "context id", "validator_session", "validator-session", "validator session")), - AuthorContextID: tickNormalizeIdentityValue(tickVerdictMetadataValue(text, "author_context_id", "author-context-id", "author context id")), - } - var gaps []string - if info.Author == "" { - gaps = append(gaps, "missing author") - } - if info.JudgeName == "" { - gaps = append(gaps, "missing judge.name") - } - if info.JudgeProgram == "" { - gaps = append(gaps, "missing judge.program") - } - if info.JudgeModelFamily == "" { - gaps = append(gaps, "missing judge.model_family") - } else if tickUnknownModelFamily(info.JudgeModelFamily) { - gaps = append(gaps, "judge.model_family is unknown") - } - if info.ContextID == "" { - gaps = append(gaps, "missing judge.context_id") - } - if info.Author != "" && info.JudgeName != "" && info.Author == info.JudgeName { - gaps = append(gaps, "judge.name equals author") - } - if info.AuthorContextID != "" && info.ContextID != "" && info.AuthorContextID == info.ContextID { - gaps = append(gaps, "judge.context_id equals author context") - } - if tickVerdictMetadataValue(text, "allow_self", "allow-self", "allow self", "self_waiver", "self-waiver", "self waiver") != "" { - gaps = append(gaps, "self-judge waiver must be external and principal-logged, not verdict-authored") - } - return info, gaps -} - -func tickVerdictMetadataValue(text string, keys ...string) string { - wanted := map[string]bool{} - for _, key := range keys { - wanted[tickNormalizeMetadataKey(key)] = true - } - lines, err := tickScanVerdictLines(text) - if err != nil { - // Fail closed: an unscannable artifact yields no trusted metadata, which - // surfaces as an identity gap upstream. - return "" - } - for _, raw := range lines { - line := strings.TrimSpace(raw) - line = strings.TrimPrefix(line, "-") - line = strings.TrimPrefix(strings.TrimSpace(line), "*") - line = strings.TrimSpace(line) - i := strings.Index(line, ":") - if i < 0 { - continue - } - key := tickNormalizeMetadataKey(line[:i]) - if !wanted[key] { - continue - } - return strings.Trim(strings.TrimSpace(line[i+1:]), "`\"'") - } - return "" -} - -func tickNormalizeMetadataKey(key string) string { - key = strings.ToLower(strings.TrimSpace(key)) - key = strings.ReplaceAll(key, "-", "_") - key = strings.ReplaceAll(key, " ", "_") - return key -} - -func tickNormalizeIdentityValue(value string) string { - return strings.TrimSpace(value) -} - -func tickNormalizeModelFamily(value string) string { - return strings.ToLower(strings.TrimSpace(value)) -} - -func tickUnknownModelFamily(family string) bool { - switch strings.ToLower(strings.TrimSpace(family)) { - case "", "unknown", "none", "n/a", "na", "unset": - return true - default: - return false - } -} - -func tickVerdictTokenCounts(text string) (pass, fail int) { - lines, err := tickScanVerdictLines(text) - if err != nil { - // Fail closed: an unscannable artifact yields no PASS token. Returning - // (0, 0) lands in the council's default branch (unverified), never a PASS. - return 0, 0 - } - passRE := regexp.MustCompile(`(?i)^\s*VERDICT:\s*PASS\b`) - failRE := regexp.MustCompile(`(?i)^\s*VERDICT:\s*FAIL\b`) - for _, line := range lines { - if passRE.MatchString(line) { - pass++ - } - if failRE.MatchString(line) { - fail++ - } - } - return pass, fail -} - func tickInstallGuards(rt tickRuntime) error { if out, code, err := rt.run("git", "config", "core.hooksPath", ".githooks"); err != nil || code != 0 { _, _ = rt.stderr.Write(out) @@ -688,8 +431,8 @@ func tickSmoke(rt tickRuntime) error { failLine("1 guard-status NOT active") } - if !tickVerdictHasCommandsRun("COMMANDS RUN:\nREASONS:\nbecause\n") && - tickVerdictHasCommandsRun("COMMANDS RUN:\n ao tick guard-status\nREASONS: ok\n") { + if !verdictparse.HasCommandsRun("COMMANDS RUN:\nREASONS:\nbecause\n") && + verdictparse.HasCommandsRun("COMMANDS RUN:\n ao tick guard-status\nREASONS: ok\n") { passLine("2 verdict-gate rejects empty / accepts cited") } else { failLine("2 verdict-gate command body matrix failed") @@ -713,15 +456,15 @@ func tickSmoke(rt tickRuntime) error { quietRT := rt quietRT.stdout = io.Discard quietRT.stderr = io.Discard - if tickExitCode(tickCouncilGate(quietRT, []string{pass1, pass2})) == 0 && - tickExitCode(tickCouncilGate(quietRT, []string{pass1, fail1})) == tickExitDisagree && - tickExitCode(tickCouncilGate(quietRT, []string{pass1, unver})) == tickExitCouncil && - tickExitCode(tickCouncilGate(quietRT, []string{pass1, contra})) == tickExitCouncil { + if tickExitCode(runCouncilGate(quietRT, []string{pass1, pass2})) == 0 && + tickExitCode(runCouncilGate(quietRT, []string{pass1, fail1})) == tickExitDisagree && + tickExitCode(runCouncilGate(quietRT, []string{pass1, unver})) == tickExitCouncil && + tickExitCode(runCouncilGate(quietRT, []string{pass1, contra})) == tickExitCouncil { passLine("3 council-gate 2PASS/mixed/unverified/contradictory") } else { failLine("3 council-gate matrix failed") } - if !tickVerdictHasCommandsRun("VERDICT: FAIL\n") { + if !verdictparse.HasCommandsRun("VERDICT: FAIL\n") { passLine("4 chaos bare-verdict rejected") } else { failLine("4 chaos bare-verdict accepted") diff --git a/cli/cmd/ao/tick_cmd_legacy.go b/cli/cmd/ao/tick_cmd_legacy.go index f17c04646..0d844eaff 100644 --- a/cli/cmd/ao/tick_cmd_legacy.go +++ b/cli/cmd/ao/tick_cmd_legacy.go @@ -81,14 +81,7 @@ var tickVerdictGateCmd = &cobra.Command{ }, } -var tickCouncilGateCmd = &cobra.Command{ - Use: "council-gate [...]", - Short: "Fail-closed two-plus judge verdict aggregation", - Args: cobra.MinimumNArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { - return tickCouncilGate(newTickRuntime(cmd), args) - }, -} +var tickCouncilGateCommand = councilGateModule.LegacyCommand() var tickInstallGuardsCmd = &cobra.Command{ Use: "install-guards", @@ -127,7 +120,7 @@ func init() { tickReopenCmd, tickCloseCmd, tickVerdictGateCmd, - tickCouncilGateCmd, + tickCouncilGateCommand, tickInstallGuardsCmd, tickGuardStatusCmd, tickSmokeCmd, diff --git a/cli/internal/adapters/council_gate/reader.go b/cli/internal/adapters/council_gate/reader.go new file mode 100644 index 000000000..4c5f23b12 --- /dev/null +++ b/cli/internal/adapters/council_gate/reader.go @@ -0,0 +1,42 @@ +// Package council_gate contains driven adapters for council verdict artifacts. +package council_gate + +import ( + "context" + "io" + "os" + "path/filepath" +) + +type Reader struct { + WorkDir string + WorkingDirectory func() (string, error) +} + +func (reader Reader) Read(ctx context.Context, path string, stdin io.Reader) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + var data []byte + var err error + if path == "-" { + data, err = io.ReadAll(stdin) + } else { + if !filepath.IsAbs(path) { + workDir := reader.WorkDir + if workDir == "" { + resolve := reader.WorkingDirectory + if resolve == nil { + resolve = os.Getwd + } + workDir, err = resolve() + if err != nil { + return "", err + } + } + path = filepath.Join(workDir, path) + } + data, err = os.ReadFile(path) + } + return string(data), err +} diff --git a/cli/internal/adapters/council_gate/reader_test.go b/cli/internal/adapters/council_gate/reader_test.go new file mode 100644 index 000000000..9d33cb548 --- /dev/null +++ b/cli/internal/adapters/council_gate/reader_test.go @@ -0,0 +1,27 @@ +package council_gate + +import ( + "context" + "io" + "os" + "path/filepath" + "testing" +) + +func TestReaderResolvesWorkingDirectoryAtReadTime(t *testing.T) { + first, second := t.TempDir(), t.TempDir() + for directory, body := range map[string]string{first: "first", second: "second"} { + if err := os.WriteFile(filepath.Join(directory, "verdict.md"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + current := first + reader := Reader{WorkingDirectory: func() (string, error) { return current, nil }} + if body, err := reader.Read(context.Background(), "verdict.md", io.Reader(nil)); err != nil || body != "first" { + t.Fatalf("first read body=%q err=%v", body, err) + } + current = second + if body, err := reader.Read(context.Background(), "verdict.md", io.Reader(nil)); err != nil || body != "second" { + t.Fatalf("second read body=%q err=%v", body, err) + } +} diff --git a/cli/internal/commands/council_gate/module.go b/cli/internal/commands/council_gate/module.go new file mode 100644 index 000000000..16970cd1c --- /dev/null +++ b/cli/internal/commands/council_gate/module.go @@ -0,0 +1,94 @@ +// Package council_gate owns Cobra presentation for council-gate. +package council_gate + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/boshu2/agentops/cli/internal/clicontract" + "github.com/boshu2/agentops/cli/internal/councilgate" +) + +type UseCases interface { + Evaluate(context.Context, councilgate.Request) councilgate.Result +} + +type Module struct { + useCases UseCases +} + +func NewModule(useCases UseCases) Module { return Module{useCases: useCases} } + +func (Module) Contract() clicontract.CommandContract { + return clicontract.CommandContract{ + ID: "ao.council-gate", + Profiles: clicontract.ProfileDefault | clicontract.ProfileFlywheel | + clicontract.ProfileLegacy | clicontract.ProfileCombined, + Args: clicontract.ArgsPolicy{Name: "minimum-2", Validate: cobra.MinimumNArgs(2)}, + Output: clicontract.OutputText, + Effects: clicontract.EffectFilesystem, + ExitClasses: map[int]clicontract.ExitClass{ + 0: clicontract.ExitSuccess, 1: clicontract.ExitFailure, + councilgate.ExitCouncil: clicontract.ExitConflict, councilgate.ExitDisagree: clicontract.ExitConflict, + }, + } +} + +func (module Module) Command() *cobra.Command { return module.command() } +func (module Module) LegacyCommand() *cobra.Command { return module.command() } + +func (module Module) command() *cobra.Command { + command := &cobra.Command{ + Use: "council-gate [...]", + Short: "Fail-closed two-plus judge verdict aggregation", + Args: cobra.MinimumNArgs(2), + } + command.RunE = func(command *cobra.Command, paths []string) error { + result := module.useCases.Evaluate(command.Context(), councilgate.Request{ + Paths: append([]string(nil), paths...), Stdin: command.InOrStdin(), + }) + if result.Outcome == councilgate.OutcomePass { + _, err := fmt.Fprintf(command.OutOrStdout(), "COUNCIL PASS: %d/%d judges unanimous across %d distinct contexts (%d model families)\n", result.Pass, result.Total, result.Contexts, result.Families) + return err + } + command.SilenceErrors = true + command.SilenceUsage = true + return renderFailure(command, result) + } + return command +} + +type exitError struct{ code int } + +func (failure *exitError) Error() string { return "" } +func (failure *exitError) ExitCode() int { return failure.code } + +func renderFailure(command *cobra.Command, result councilgate.Result) error { + writer := command.ErrOrStderr() + switch result.Outcome { + case councilgate.OutcomeUnverified: + fmt.Fprintf(writer, "FAIL-CLOSED: %d/%d verdict(s) unverified (no COMMANDS RUN / identity gap)\n", result.Unverified, result.Total) + return &exitError{code: councilgate.ExitCouncil} + case councilgate.OutcomeDuplicateContext: + if result.Duplicate != "" { + fmt.Fprintf(writer, "FAIL-CLOSED: duplicate judge context %q does not count as an independent judge\n", result.Duplicate) + } else { + fmt.Fprintf(writer, "FAIL-CLOSED: PASS quorum has %d distinct judge context; need at least 2 independent contexts\n", result.Contexts) + } + return &exitError{code: councilgate.ExitCouncil} + case councilgate.OutcomeDuplicateJudge: + fmt.Fprintf(writer, "FAIL-CLOSED: duplicate judge %q does not count as an independent judge\n", result.Duplicate) + return &exitError{code: councilgate.ExitCouncil} + case councilgate.OutcomeCrossFamily: + fmt.Fprintf(writer, "FAIL-CLOSED: --require-cross-family set but PASS quorum spans %d model family; need at least 2 (cross-family)\n", result.Families) + return &exitError{code: councilgate.ExitCouncil} + case councilgate.OutcomeAllFail: + fmt.Fprintf(writer, "FAIL-CLOSED: %d/%d judges FAIL\n", result.Fail, result.Total) + return &exitError{code: councilgate.ExitCouncil} + default: + fmt.Fprintf(writer, "DISAGREEMENT: %d PASS / %d FAIL - fail-closed; dispatch tie-break\n", result.Pass, result.Fail) + return &exitError{code: councilgate.ExitDisagree} + } +} diff --git a/cli/internal/commands/council_gate/module_test.go b/cli/internal/commands/council_gate/module_test.go new file mode 100644 index 000000000..1a9b2bead --- /dev/null +++ b/cli/internal/commands/council_gate/module_test.go @@ -0,0 +1,49 @@ +package council_gate + +import ( + "bytes" + "context" + "testing" + + "github.com/boshu2/agentops/cli/internal/clicontract" + "github.com/boshu2/agentops/cli/internal/councilgate" +) + +type fakeUseCases struct{ result councilgate.Result } + +func (fake fakeUseCases) Evaluate(context.Context, councilgate.Request) councilgate.Result { + return fake.result +} + +func TestModuleRendersPassAndDeclaresContract(t *testing.T) { + module := NewModule(fakeUseCases{result: councilgate.Result{Outcome: councilgate.OutcomePass, Total: 2, Pass: 2, Contexts: 2, Families: 1}}) + command := module.Command() + var stdout bytes.Buffer + command.SetOut(&stdout) + command.SetArgs([]string{"a", "b"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if got := stdout.String(); got != "COUNCIL PASS: 2/2 judges unanimous across 2 distinct contexts (1 model families)\n" { + t.Fatalf("stdout = %q", got) + } + if err := clicontract.ValidateContract(module.Contract()); err != nil { + t.Fatal(err) + } +} + +func TestModuleMapsDisagreementToEight(t *testing.T) { + module := NewModule(fakeUseCases{result: councilgate.Result{Outcome: councilgate.OutcomeDisagreement, Total: 2, Pass: 1, Fail: 1}}) + command := module.Command() + var stderr bytes.Buffer + command.SetErr(&stderr) + command.SetArgs([]string{"a", "b"}) + err := command.Execute() + exit, ok := err.(interface{ ExitCode() int }) + if !ok || exit.ExitCode() != councilgate.ExitDisagree { + t.Fatalf("error = %#v", err) + } + if got := stderr.String(); got != "DISAGREEMENT: 1 PASS / 1 FAIL - fail-closed; dispatch tie-break\n" { + t.Fatalf("stderr = %q", got) + } +} diff --git a/cli/internal/councilgate/service.go b/cli/internal/councilgate/service.go new file mode 100644 index 000000000..ac7ddf6aa --- /dev/null +++ b/cli/internal/councilgate/service.go @@ -0,0 +1,115 @@ +// Package councilgate owns fail-closed verdict aggregation policy. +package councilgate + +import ( + "context" + "io" + + "github.com/boshu2/agentops/cli/internal/liveness" + "github.com/boshu2/agentops/cli/internal/verdict" +) + +const ( + ExitCouncil = 6 + ExitDisagree = 8 +) + +type Outcome uint8 + +const ( + OutcomePass Outcome = iota + OutcomeUnverified + OutcomeDuplicateContext + OutcomeDuplicateJudge + OutcomeCrossFamily + OutcomeAllFail + OutcomeDisagreement +) + +type Request struct { + Paths []string + Stdin io.Reader +} + +type Result struct { + Outcome Outcome + Total int + Pass int + Fail int + Unverified int + Contexts int + Families int + Duplicate string +} + +type Policy struct { + RequireCrossFamily bool +} + +type Reader interface { + Read(context.Context, string, io.Reader) (string, error) +} + +type Service struct { + reader Reader + policy Policy +} + +func NewService(reader Reader, policy Policy) Service { + return Service{reader: reader, policy: policy} +} + +func (service Service) Evaluate(ctx context.Context, request Request) Result { + result := Result{Total: len(request.Paths)} + families := map[string]bool{} + judges := map[string]bool{} + contexts := map[string]bool{} + for _, path := range request.Paths { + text, err := service.reader.Read(ctx, path, request.Stdin) + identity, gaps := verdict.Identity(text) + if err != nil || !verdict.HasCommandsRun(text) || len(gaps) > 0 { + result.Unverified++ + continue + } + canonicalContext := liveness.CanonicalizeContextID(identity.ContextID) + if contexts[canonicalContext] { + result.Outcome = OutcomeDuplicateContext + result.Duplicate = identity.ContextID + return result + } + contexts[canonicalContext] = true + if judges[identity.JudgeName] { + result.Outcome = OutcomeDuplicateJudge + result.Duplicate = identity.JudgeName + return result + } + judges[identity.JudgeName] = true + pass, fail := verdict.TokenCounts(text) + switch { + case pass == 1 && fail == 0: + result.Pass++ + families[identity.JudgeModelFamily] = true + case fail == 1 && pass == 0: + result.Fail++ + default: + result.Unverified++ + } + } + result.Contexts = len(contexts) + result.Families = len(families) + switch { + case result.Unverified > 0: + result.Outcome = OutcomeUnverified + case result.Pass == result.Total && result.Contexts < 2: + result.Outcome = OutcomeDuplicateContext + case result.Pass == result.Total && service.policy.RequireCrossFamily && result.Families < 2: + result.Outcome = OutcomeCrossFamily + case result.Pass == result.Total: + result.Outcome = OutcomePass + case result.Fail == result.Total: + result.Outcome = OutcomeAllFail + default: + result.Outcome = OutcomeDisagreement + } + return result +} diff --git a/cli/internal/councilgate/service_test.go b/cli/internal/councilgate/service_test.go new file mode 100644 index 000000000..c3127541b --- /dev/null +++ b/cli/internal/councilgate/service_test.go @@ -0,0 +1,48 @@ +package councilgate + +import ( + "context" + "io" + "testing" +) + +type mapReader map[string]string + +func (reader mapReader) Read(_ context.Context, path string, _ io.Reader) (string, error) { + return reader[path], nil +} + +func testVerdict(judge, contextID, family, token string) string { + return "author: worker\njudge: " + judge + "\njudge_program: codex\njudge_model_family: " + family + "\ncontext_id: " + contextID + "\nVERDICT: " + token + "\nCOMMANDS RUN:\n go test ./...\n" +} + +func TestServiceAggregatesIndependentContexts(t *testing.T) { + reader := mapReader{"a": testVerdict("a", "ctx-a", "gpt", "PASS"), "b": testVerdict("b", "ctx-b", "gpt", "PASS")} + result := NewService(reader, Policy{}).Evaluate(context.Background(), Request{Paths: []string{"a", "b"}}) + if result.Outcome != OutcomePass || result.Pass != 2 || result.Contexts != 2 || result.Families != 1 { + t.Fatalf("result = %+v", result) + } +} + +func TestServiceFailsClosedForMixedAndDuplicateContext(t *testing.T) { + reader := mapReader{ + "pass": testVerdict("a", "ctx-a", "gpt", "PASS"), + "fail": testVerdict("b", "ctx-b", "gpt", "FAIL"), + "dupe": testVerdict("b", "CTX-A", "gpt", "PASS"), + } + service := NewService(reader, Policy{}) + if result := service.Evaluate(context.Background(), Request{Paths: []string{"pass", "fail"}}); result.Outcome != OutcomeDisagreement { + t.Fatalf("mixed result = %+v", result) + } + if result := service.Evaluate(context.Background(), Request{Paths: []string{"pass", "dupe"}}); result.Outcome != OutcomeDuplicateContext { + t.Fatalf("duplicate result = %+v", result) + } +} + +func TestServiceOptionalCrossFamilyPolicy(t *testing.T) { + reader := mapReader{"a": testVerdict("a", "ctx-a", "gpt", "PASS"), "b": testVerdict("b", "ctx-b", "gpt", "PASS")} + result := NewService(reader, Policy{RequireCrossFamily: true}).Evaluate(context.Background(), Request{Paths: []string{"a", "b"}}) + if result.Outcome != OutcomeCrossFamily { + t.Fatalf("result = %+v", result) + } +} diff --git a/cli/internal/verdict/parser.go b/cli/internal/verdict/parser.go new file mode 100644 index 000000000..4a420b0eb --- /dev/null +++ b/cli/internal/verdict/parser.go @@ -0,0 +1,170 @@ +// Package verdict parses independent-review verdict artifacts without owning +// filesystem or command concerns. +package verdict + +import ( + "bufio" + "regexp" + "strings" +) + +// LineCap bounds a single verdict line. Artifacts beyond the cap fail closed. +const LineCap = 1 << 20 // 1 MiB + +var ( + commandsHeader = regexp.MustCompile(`(?i)commands[ _-]*run`) + reasonsHeader = regexp.MustCompile(`^\s*reasons`) + passToken = regexp.MustCompile(`(?i)^\s*VERDICT:\s*PASS\b`) + failToken = regexp.MustCompile(`(?i)^\s*VERDICT:\s*FAIL\b`) +) + +// IdentityInfo is the normalized identity tuple carried by a verdict. +type IdentityInfo struct { + Author string + JudgeName string + JudgeProgram string + JudgeModelFamily string + ContextID string + AuthorContextID string +} + +// ScanLines returns every artifact line or an error when a line exceeds LineCap. +func ScanLines(text string) ([]string, error) { + scanner := bufio.NewScanner(strings.NewReader(text)) + scanner.Buffer(make([]byte, 0, 64*1024), LineCap) + var lines []string + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return lines, nil +} + +// HasCommandsRun reports whether the artifact cites at least one command. +func HasCommandsRun(text string) bool { + lines, err := ScanLines(text) + if err != nil { + return false + } + inBlock := false + for _, line := range lines { + lower := strings.ToLower(line) + if commandsHeader.MatchString(line) { + inBlock = true + if index := strings.Index(line, ":"); index >= 0 && strings.TrimSpace(line[index+1:]) != "" { + return true + } + continue + } + if inBlock && reasonsHeader.MatchString(lower) { + inBlock = false + } + if inBlock && strings.TrimSpace(line) != "" { + return true + } + } + return false +} + +// Identity returns the normalized tuple and every reason it cannot establish +// an independent judge. +func Identity(text string) (IdentityInfo, []string) { + info := IdentityInfo{ + Author: NormalizeIdentityValue(MetadataValue(text, "author", "author_id", "author-id", "author id", "author_name", "author-name", "author name")), + JudgeName: NormalizeIdentityValue(MetadataValue(text, "judge", "judge_name", "judge-name", "judge name", "judge_id", "judge-id", "judge id")), + JudgeProgram: NormalizeIdentityValue(MetadataValue(text, "judge_program", "judge-program", "judge program", "program", "validator_program", "validator-program", "validator program")), + JudgeModelFamily: NormalizeModelFamily(MetadataValue(text, "judge_model_family", "judge-model-family", "judge model family", "model_family", "model-family", "model family", "family")), + ContextID: NormalizeIdentityValue(MetadataValue(text, "context_id", "context-id", "context id", "validator_session", "validator-session", "validator session")), + AuthorContextID: NormalizeIdentityValue(MetadataValue(text, "author_context_id", "author-context-id", "author context id")), + } + var gaps []string + if info.Author == "" { + gaps = append(gaps, "missing author") + } + if info.JudgeName == "" { + gaps = append(gaps, "missing judge.name") + } + if info.JudgeProgram == "" { + gaps = append(gaps, "missing judge.program") + } + if info.JudgeModelFamily == "" { + gaps = append(gaps, "missing judge.model_family") + } else if UnknownModelFamily(info.JudgeModelFamily) { + gaps = append(gaps, "judge.model_family is unknown") + } + if info.ContextID == "" { + gaps = append(gaps, "missing judge.context_id") + } + if info.Author != "" && info.JudgeName != "" && info.Author == info.JudgeName { + gaps = append(gaps, "judge.name equals author") + } + if info.AuthorContextID != "" && info.ContextID != "" && info.AuthorContextID == info.ContextID { + gaps = append(gaps, "judge.context_id equals author context") + } + if MetadataValue(text, "allow_self", "allow-self", "allow self", "self_waiver", "self-waiver", "self waiver") != "" { + gaps = append(gaps, "self-judge waiver must be external and principal-logged, not verdict-authored") + } + return info, gaps +} + +// MetadataValue returns the first matching metadata field. +func MetadataValue(text string, keys ...string) string { + wanted := map[string]bool{} + for _, key := range keys { + wanted[NormalizeMetadataKey(key)] = true + } + lines, err := ScanLines(text) + if err != nil { + return "" + } + for _, raw := range lines { + line := strings.TrimSpace(raw) + line = strings.TrimPrefix(line, "-") + line = strings.TrimPrefix(strings.TrimSpace(line), "*") + line = strings.TrimSpace(line) + index := strings.Index(line, ":") + if index < 0 || !wanted[NormalizeMetadataKey(line[:index])] { + continue + } + return strings.Trim(strings.TrimSpace(line[index+1:]), "`\"'") + } + return "" +} + +func NormalizeMetadataKey(key string) string { + key = strings.ToLower(strings.TrimSpace(key)) + key = strings.ReplaceAll(key, "-", "_") + return strings.ReplaceAll(key, " ", "_") +} + +func NormalizeIdentityValue(value string) string { return strings.TrimSpace(value) } +func NormalizeModelFamily(value string) string { return strings.ToLower(strings.TrimSpace(value)) } + +func UnknownModelFamily(family string) bool { + switch strings.ToLower(strings.TrimSpace(family)) { + case "", "unknown", "none", "n/a", "na", "unset": + return true + default: + return false + } +} + +// TokenCounts counts exact PASS and FAIL verdict tokens. Unscannable artifacts +// return zero counts so callers fail closed. +func TokenCounts(text string) (pass, fail int) { + lines, err := ScanLines(text) + if err != nil { + return 0, 0 + } + for _, line := range lines { + if passToken.MatchString(line) { + pass++ + } + if failToken.MatchString(line) { + fail++ + } + } + return pass, fail +} diff --git a/cli/internal/verdict/parser_test.go b/cli/internal/verdict/parser_test.go new file mode 100644 index 000000000..5b99b0e2b --- /dev/null +++ b/cli/internal/verdict/parser_test.go @@ -0,0 +1,24 @@ +package verdict + +import ( + "strings" + "testing" +) + +func TestParserFailsClosedOnOversizedLine(t *testing.T) { + body := "author: worker\njudge: judge-a\njudge_program: codex\njudge_model_family: gpt\ncontext_id: ctx-a\nVERDICT: PASS\nCOMMANDS RUN:\n go test ./...\n" + strings.Repeat("x", LineCap+1) + if HasCommandsRun(body) { + t.Fatal("oversized artifact retained a trusted command block") + } + if pass, fail := TokenCounts(body); pass != 0 || fail != 0 { + t.Fatalf("oversized token counts = %d/%d, want 0/0", pass, fail) + } +} + +func TestIdentityRejectsSelfJudgeContext(t *testing.T) { + body := "author: worker\njudge: judge-a\njudge_program: codex\njudge_model_family: gpt\ncontext_id: ctx-a\nauthor_context_id: ctx-a\n" + _, gaps := Identity(body) + if len(gaps) != 1 || gaps[0] != "judge.context_id equals author context" { + t.Fatalf("gaps = %v", gaps) + } +} From 41f5f07eb23ba3fc6ebbcdef5bdfa82dad8f9d54 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:48:55 -0400 Subject: [PATCH 058/108] chore(cli): seal council-gate migration lineage (age-nw28h.2.6) --- .../compatibility-baseline/families/council-gate/lineage.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/testdata/compatibility-baseline/families/council-gate/lineage.json b/cli/testdata/compatibility-baseline/families/council-gate/lineage.json index ea5c30898..723d80a3d 100644 --- a/cli/testdata/compatibility-baseline/families/council-gate/lineage.json +++ b/cli/testdata/compatibility-baseline/families/council-gate/lineage.json @@ -6,5 +6,6 @@ "fixture_sha256": "be0e78852c998cf61f76ac635fa0fa8c86c071969b0012168ee47ecc78b40247", "ownership_sha256": "8677bfaf108ee9da2fde85527c10ca1e10167612f49ddf9254ff60155a4d3d6b", "scope_sha256": "d0bfb45b0a3592d67d411c21645bf83c65a2dc085c4d55b17c4d58a8ac8c737b", - "migration_state": "migrating" + "migration_state": "migrated", + "accepted_sha": "e38c499458012d159a6397e5b56af5be5ba5d3ba" } From 5db00edf944fde06bbf660dd7743a799840b1e1a Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:51:49 -0400 Subject: [PATCH 059/108] chore(provenance): bind pawl CONFIRMED verdict for age-nw28h.2.6 #trivial --- docs/provenance/ledger.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/provenance/ledger.jsonl b/docs/provenance/ledger.jsonl index 0a9222d2b..585812ce4 100644 --- a/docs/provenance/ledger.jsonl +++ b/docs/provenance/ledger.jsonl @@ -521,3 +521,4 @@ {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.3@2172c77","from_type":"verdict","to_id":"2172c77f05692169948f3b034970c90d64a366c1","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.3 disposition=CONFIRMED","bead_id":"age-nw28h.2.3","trust_tier":"inferred","ts":"2026-07-11T20:30:36Z","reviewer_family":"gpt","rounds":1,"evidence_path":"/Users/bo/dev/agentops-wt/go-cli-clean-architecture/.agents/evidence/go-cli-clean-architecture/family-claim-independent-verdict-2172c77f0.txt","prev_hash":"9739edafbb15e8a5d7e0ac15be295093bb14ebf60ffc13f92d46db35942e5aca","payload_hash":"7d01c886ce25d14d82b1cabbf379bc4e777f083328ee4695ca8344979eae9855","hash":"be2caecd5f64c47b68e12dd6a220eacac168450f3c6c0160f4428f4a917e92d0"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.4@0782b56","from_type":"verdict","to_id":"0782b566af8be3b013b649f95f607b5bbebc4fc5","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.4 disposition=CONFIRMED","bead_id":"age-nw28h.2.4","trust_tier":"inferred","ts":"2026-07-11T22:04:47Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-close-independent-verdict-0782b566a.txt","prev_hash":"be2caecd5f64c47b68e12dd6a220eacac168450f3c6c0160f4428f4a917e92d0","payload_hash":"a451e922b8fb7db3c5d35d3815169a837ec3e0b650486fdfd1c1a1c976e9d168","hash":"671a7c4580ed2345d62f32c92074f974f51455e3ea341f4eb371445a8b8f395b"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.5@c55ebba","from_type":"verdict","to_id":"c55ebbacb90fe44bc7097b45240965c5f21abd30","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.5 disposition=CONFIRMED","bead_id":"age-nw28h.2.5","trust_tier":"inferred","ts":"2026-07-11T22:26:32Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-config-independent-verdict-c55ebbacb.txt","prev_hash":"671a7c4580ed2345d62f32c92074f974f51455e3ea341f4eb371445a8b8f395b","payload_hash":"65b9734effd05e76746ac5c29fad79f7eba85a0613aa220fa4669f0744507f9e","hash":"0a925b381ecc0937e340c884fc83a9f992a30a6fc49c0c4f94efc4aa500d1dc8"} +{"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.6@e38c499","from_type":"verdict","to_id":"e38c499458012d159a6397e5b56af5be5ba5d3ba","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.6 disposition=CONFIRMED","bead_id":"age-nw28h.2.6","trust_tier":"inferred","ts":"2026-07-11T22:51:49Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-council-gate-independent-verdict-e38c49945.txt","prev_hash":"0a925b381ecc0937e340c884fc83a9f992a30a6fc49c0c4f94efc4aa500d1dc8","payload_hash":"2f39b3dd97dcdad748e0d7a7a01cfa2cd94464e4efe9ad65d9d267d4811c54e0","hash":"94d27584f6296479beddda872b4124eeeab38e82c9f95eda412832c746345864"} From fc3b298598a30dbca31b64cf1386bc91421fd194 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:06:28 -0400 Subject: [PATCH 060/108] refactor(doctor): extract legacy and reviewer health services (age-nw28h.2.7.1) --- cli/cmd/ao/codex.go | 13 +- cli/cmd/ao/doctor.go | 143 +------- cli/cmd/ao/doctor_helpers_legacy_test.go | 79 +++++ cli/cmd/ao/doctor_legacy_composition.go | 11 + cli/cmd/ao/doctor_wedge.go | 321 ------------------ cli/cmd/ao/doctor_wedge_legacy_test.go | 54 +++ cli/cmd/ao/gate_runner_adapter.go | 3 +- cli/cmd/ao/quickstart_verdict.go | 7 +- cli/cmd/ao/reviewer_health_composition.go | 12 + cli/cmd/ao/session_bootstrap.go | 5 +- cli/internal/adapters/doctor/legacy.go | 185 ++++++++++ cli/internal/adapters/reviewerhealth/probe.go | 56 +++ .../adapters/reviewerhealth/probe_test.go | 30 ++ cli/internal/doctor/legacy_service.go | 25 ++ cli/internal/doctor/legacy_service_test.go | 27 ++ cli/internal/reviewerhealth/service.go | 60 ++++ cli/internal/reviewerhealth/service_test.go | 40 +++ 17 files changed, 597 insertions(+), 474 deletions(-) create mode 100644 cli/cmd/ao/doctor_helpers_legacy_test.go create mode 100644 cli/cmd/ao/doctor_legacy_composition.go delete mode 100644 cli/cmd/ao/doctor_wedge.go create mode 100644 cli/cmd/ao/doctor_wedge_legacy_test.go create mode 100644 cli/cmd/ao/reviewer_health_composition.go create mode 100644 cli/internal/adapters/doctor/legacy.go create mode 100644 cli/internal/adapters/reviewerhealth/probe.go create mode 100644 cli/internal/adapters/reviewerhealth/probe_test.go create mode 100644 cli/internal/doctor/legacy_service.go create mode 100644 cli/internal/doctor/legacy_service_test.go create mode 100644 cli/internal/reviewerhealth/service.go create mode 100644 cli/internal/reviewerhealth/service_test.go diff --git a/cli/cmd/ao/codex.go b/cli/cmd/ao/codex.go index d77bb2bf1..3298800b1 100644 --- a/cli/cmd/ao/codex.go +++ b/cli/cmd/ao/codex.go @@ -24,6 +24,7 @@ import ( "github.com/boshu2/agentops/cli/internal/adapters/vendorimage/codexruntime" "github.com/boshu2/agentops/cli/internal/bridge" "github.com/boshu2/agentops/cli/internal/pool" + "github.com/boshu2/agentops/cli/internal/quality" "github.com/boshu2/agentops/cli/internal/ratchet" cliRPI "github.com/boshu2/agentops/cli/internal/rpi" "github.com/boshu2/agentops/cli/internal/storage" @@ -423,7 +424,7 @@ func codexStartAlreadyStarted(state *codexLifecycleState, sessionID string) bool if startupContextPath == "" { return false } - return fileExists(startupContextPath) + return quality.FileExists(startupContextPath) } func performCodexStart(cwd string) (codexStartResult, error) { @@ -901,13 +902,13 @@ func codexImageHealthMissingScript(cwd string, command []string) string { } scriptPath := command[1] if filepath.IsAbs(scriptPath) { - if !fileExists(scriptPath) { + if !quality.FileExists(scriptPath) { return scriptPath } return "" } abs := filepath.Join(cwd, scriptPath) - if !fileExists(abs) { + if !quality.FileExists(abs) { return scriptPath } return "" @@ -2012,7 +2013,7 @@ func writeCodexStartupOperatorModel(sb *strings.Builder, cwd, agentsRoot string) sb.WriteString("- Canonical primitives: `fitness gradient`, `stateful environment`, `replaceable actors`, `stigmergic traces`, `selection gates`, `evolutionary promotion`, `governance`\n") sb.WriteString("- Treat the control plane as the product; actors are replaceable executors and the environment carries memory, coordination, trust, and adaptation.\n") operatorModelPath := filepath.Join(agentsRoot, "knowledge", "operator-model.md") - if fileExists(operatorModelPath) { + if quality.FileExists(operatorModelPath) { fmt.Fprintf(sb, "- Doctrine: `%s`\n", displayKnowledgeContextPath(cwd, operatorModelPath)) } } @@ -2111,10 +2112,10 @@ func codexStartupSourceLinks(cwd, agentsRoot string, briefings []codexArtifactRe links := make([]string, 0, 8) operatorModelPath := filepath.Join(agentsRoot, "knowledge", "operator-model.md") beliefBookPath := filepath.Join(agentsRoot, "knowledge", "book-of-beliefs.md") - if fileExists(operatorModelPath) { + if quality.FileExists(operatorModelPath) { links = append(links, fmt.Sprintf("Doctrine: `%s`", displayKnowledgeContextPath(cwd, operatorModelPath))) } - if fileExists(beliefBookPath) { + if quality.FileExists(beliefBookPath) { links = append(links, fmt.Sprintf("Beliefs: `%s`", displayKnowledgeContextPath(cwd, beliefBookPath))) } for _, item := range briefings { diff --git a/cli/cmd/ao/doctor.go b/cli/cmd/ao/doctor.go index 3803134fc..9200782a5 100644 --- a/cli/cmd/ao/doctor.go +++ b/cli/cmd/ao/doctor.go @@ -2,17 +2,10 @@ package main import ( - "io" - "os" - "os/exec" - "path/filepath" - "time" - "github.com/spf13/cobra" "github.com/boshu2/agentops/cli/internal/doctor" "github.com/boshu2/agentops/cli/internal/quality" - "github.com/boshu2/agentops/cli/internal/storage" ) var ( @@ -44,37 +37,11 @@ func init() { rootCmd.AddCommand(doctorCmd) } -// Type aliases — canonical types live in internal/quality. -type doctorCheck = quality.Check -type doctorOutput = quality.DoctorOutput - // gatherDoctorChecks runs all doctor checks and returns the results. -func gatherDoctorChecks() []doctorCheck { - checks := []doctorCheck{ - {Name: "ao CLI", Status: "pass", Detail: formatVersion(version), Required: true}, - checkCLIDependencies(), - checkKnowledgeBase(), - checkKnowledgeFreshness(), - checkSearchIndex(), - checkFlywheelHealth(), - checkSkills(), - checkCodexSync(), - checkSkillIntegrity(), - checkStaleReferences(), - checkOptionalCLI("codex", "needed for --mixed council"), - } - // Wedge/verification preflight (reviewer reachability, cross-family - // capability, binary freshness, ledger health, LAW-0 guard) — additive. - return append(checks, wedgeDoctorChecks()...) +func gatherDoctorChecks() []quality.Check { + return newLegacyDoctorService().Checks(rootCmd.Context()) } -// Thin wrappers — delegate to quality package, kept for test compatibility. -func doctorStatusIcon(status string) string { return quality.StatusIcon(status) } -func hasRequiredFailure(checks []doctorCheck) bool { return quality.HasRequiredFailure(checks) } -func renderDoctorTable(w io.Writer, output doctorOutput) { quality.RenderTable(w, output) } -func newestFileModTime(entries []os.DirEntry) time.Time { return quality.NewestFileModTime(entries) } -func countEstablished(dir string) int { return quality.CountEstablished(dir) } - func runDoctor(cmd *cobra.Command, args []string) error { // Engine-flag invocations (--fix, --explain, --robot-triage) route entirely // through the diagnose-and-repair engine. @@ -131,109 +98,3 @@ func appendEngineFindings(cmd *cobra.Command) error { renderEngineFindings(cmd, rep) return exitErr(rep.ExitCode, "doctor findings present") } - -func checkCLIDependencies() doctorCheck { - return quality.CheckCLIDependencies(exec.LookPath) -} - -func checkKnowledgeBase() doctorCheck { - cwd, err := os.Getwd() - if err != nil { - return doctorCheck{Name: "Knowledge Base", Status: "fail", Detail: "cannot determine working directory", Required: true} - } - return quality.CheckKnowledgeBase(filepath.Join(cwd, storage.DefaultBaseDir)) -} - -func checkKnowledgeFreshness() doctorCheck { - cwd, err := os.Getwd() - if err != nil { - return doctorCheck{Name: "Knowledge Freshness", Status: "warn", Detail: "cannot determine working directory", Required: false} - } - return quality.CheckKnowledgeFreshness(filepath.Join(cwd, storage.DefaultBaseDir, "sessions")) -} - -// Thin wrappers for pure functions — delegate to quality package. -func formatVersion(v string) string { return quality.FormatVersion(v) } -func formatDuration(d time.Duration) string { return quality.FormatDuration(d) } -func formatNumber(n int) string { return quality.FormatNumber(n) } -func countFileLines(path string) int { return quality.CountFileLines(path) } - -func checkSearchIndex() doctorCheck { - cwd, err := os.Getwd() - if err != nil { - return doctorCheck{Name: "Search Index", Status: "warn", Detail: "cannot determine working directory", Required: false} - } - return quality.CheckSearchIndex(filepath.Join(cwd, IndexDir, IndexFileName)) -} - -func checkFlywheelHealth(baseDir ...string) doctorCheck { - dir := "" - if len(baseDir) > 0 && baseDir[0] != "" { - dir = baseDir[0] - } else { - var err error - dir, err = os.Getwd() - if err != nil { - return doctorCheck{Name: "Flywheel Health", Status: "warn", Detail: "cannot determine working directory", Required: false} - } - } - return quality.CheckFlywheelHealth(filepath.Join(dir, storage.DefaultBaseDir)) -} - -// Thin wrappers — delegate Codex/skill checks to internal/quality. -func checkSkills() doctorCheck { return quality.CheckSkills() } -func checkCodexSync() doctorCheck { return quality.CheckCodexSync() } -func checkSkillIntegrity() doctorCheck { return quality.CheckSkillIntegrity() } -func checkOptionalCLI(name, reason string) doctorCheck { - return quality.CheckOptionalCLI(name, reason) -} -func findHealScript() string { return quality.FindHealScript() } -func sha256File(path string) (string, error) { return quality.SHA256File(path) } - -// fileExists is used by other cmd/ao files. -func fileExists(path string) bool { return quality.FileExists(path) } - -// Test-compatibility wrappers for skill/codex helpers. -func skillOverlapWarning(base map[string]struct{}, primaryCount int, primary, msgFmt string, others ...map[string]struct{}) *doctorCheck { - return quality.SkillOverlapWarning(base, primaryCount, primary, msgFmt, others...) -} -func scanSkillDir(dir string) map[string]struct{} { return quality.ScanSkillDir(dir) } -func overlappingSkillNames(base map[string]struct{}, others ...map[string]struct{}) []string { - return quality.OverlappingSkillNames(base, others...) -} - -// Type aliases for stale reference types. -var deprecatedCommands = quality.DeprecatedCommands - -type staleReference = quality.StaleReference - -func checkStaleReferences() doctorCheck { - return quality.CheckStaleReferences([]string{ - "skills/*/SKILL.md", - "skills/*/references/*.md", - "skills-codex/*/SKILL.md", - "skills-codex-overrides/*/SKILL.md", - "docs/*.md", - "scripts/*.sh", - "docs/contracts/*.md", - "docs/plans/*.md", - }) -} - -// Thin wrappers for test compatibility. -func scanFileForDeprecatedCommands(path string) []staleReference { - return quality.ScanFileForDeprecatedCommands(path) -} -func countUniqueFiles(refs []staleReference) int { return quality.CountUniqueFiles(refs) } - -func countHealFindings(output string) int { return quality.CountHealFindings(output) } - -func countFiles(dir string) int { return quality.CountFiles(dir) } -func countLearningFiles(dir string) int { return quality.CountLearningFiles(dir) } -func countCheckStatuses(checks []doctorCheck) (int, int, int) { - return quality.CountCheckStatuses(checks) -} -func buildDoctorSummary(passes, fails, warns, total int) string { - return quality.BuildSummary(passes, fails, warns, total) -} -func computeResult(checks []doctorCheck) doctorOutput { return quality.ComputeResult(checks) } diff --git a/cli/cmd/ao/doctor_helpers_legacy_test.go b/cli/cmd/ao/doctor_helpers_legacy_test.go new file mode 100644 index 000000000..d663f0f48 --- /dev/null +++ b/cli/cmd/ao/doctor_helpers_legacy_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "io" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/boshu2/agentops/cli/internal/quality" + "github.com/boshu2/agentops/cli/internal/storage" +) + +type doctorCheck = quality.Check +type doctorOutput = quality.DoctorOutput +type staleReference = quality.StaleReference + +var deprecatedCommands = quality.DeprecatedCommands + +func doctorStatusIcon(status string) string { return quality.StatusIcon(status) } +func hasRequiredFailure(checks []doctorCheck) bool { return quality.HasRequiredFailure(checks) } +func renderDoctorTable(writer io.Writer, output doctorOutput) { quality.RenderTable(writer, output) } +func newestFileModTime(entries []os.DirEntry) time.Time { return quality.NewestFileModTime(entries) } +func countEstablished(directory string) int { return quality.CountEstablished(directory) } +func formatVersion(value string) string { return quality.FormatVersion(value) } +func formatDuration(value time.Duration) string { return quality.FormatDuration(value) } +func formatNumber(value int) string { return quality.FormatNumber(value) } +func countFileLines(path string) int { return quality.CountFileLines(path) } +func checkCLIDependencies() doctorCheck { return quality.CheckCLIDependencies(exec.LookPath) } +func checkKnowledgeBase() doctorCheck { + cwd, _ := os.Getwd() + return quality.CheckKnowledgeBase(filepath.Join(cwd, storage.DefaultBaseDir)) +} +func checkKnowledgeFreshness() doctorCheck { + cwd, _ := os.Getwd() + return quality.CheckKnowledgeFreshness(filepath.Join(cwd, storage.DefaultBaseDir, "sessions")) +} +func checkSearchIndex() doctorCheck { + cwd, _ := os.Getwd() + return quality.CheckSearchIndex(filepath.Join(cwd, IndexDir, IndexFileName)) +} +func checkFlywheelHealth(base ...string) doctorCheck { + cwd, _ := os.Getwd() + if len(base) > 0 && base[0] != "" { + cwd = base[0] + } + return quality.CheckFlywheelHealth(filepath.Join(cwd, storage.DefaultBaseDir)) +} +func checkSkills() doctorCheck { return quality.CheckSkills() } +func checkCodexSync() doctorCheck { return quality.CheckCodexSync() } +func checkSkillIntegrity() doctorCheck { return quality.CheckSkillIntegrity() } +func checkOptionalCLI(name, reason string) doctorCheck { return quality.CheckOptionalCLI(name, reason) } +func findHealScript() string { return quality.FindHealScript() } +func sha256File(path string) (string, error) { return quality.SHA256File(path) } +func fileExists(path string) bool { return quality.FileExists(path) } +func skillOverlapWarning(base map[string]struct{}, count int, primary, format string, others ...map[string]struct{}) *doctorCheck { + return quality.SkillOverlapWarning(base, count, primary, format, others...) +} +func scanSkillDir(path string) map[string]struct{} { return quality.ScanSkillDir(path) } +func overlappingSkillNames(base map[string]struct{}, others ...map[string]struct{}) []string { + return quality.OverlappingSkillNames(base, others...) +} +func checkStaleReferences() doctorCheck { + return quality.CheckStaleReferences([]string{"skills/*/SKILL.md", "skills/*/references/*.md", "skills-codex/*/SKILL.md", "skills-codex-overrides/*/SKILL.md", "docs/*.md", "scripts/*.sh", "docs/contracts/*.md", "docs/plans/*.md"}) +} +func scanFileForDeprecatedCommands(path string) []staleReference { + return quality.ScanFileForDeprecatedCommands(path) +} +func countUniqueFiles(refs []staleReference) int { return quality.CountUniqueFiles(refs) } +func countHealFindings(output string) int { return quality.CountHealFindings(output) } +func countFiles(path string) int { return quality.CountFiles(path) } +func countLearningFiles(path string) int { return quality.CountLearningFiles(path) } +func countCheckStatuses(checks []doctorCheck) (int, int, int) { + return quality.CountCheckStatuses(checks) +} +func buildDoctorSummary(passes, fails, warns, total int) string { + return quality.BuildSummary(passes, fails, warns, total) +} +func computeResult(checks []doctorCheck) doctorOutput { return quality.ComputeResult(checks) } diff --git a/cli/cmd/ao/doctor_legacy_composition.go b/cli/cmd/ao/doctor_legacy_composition.go new file mode 100644 index 000000000..034694cdd --- /dev/null +++ b/cli/cmd/ao/doctor_legacy_composition.go @@ -0,0 +1,11 @@ +package main + +import ( + doctoradapter "github.com/boshu2/agentops/cli/internal/adapters/doctor" + doctorapp "github.com/boshu2/agentops/cli/internal/doctor" +) + +func newLegacyDoctorService() doctorapp.LegacyService { + checks := doctoradapter.SystemLegacyChecks(version, IndexDir, IndexFileName, resolveLedgerPath, reviewerHealthService) + return doctorapp.NewLegacyService(version, checks) +} diff --git a/cli/cmd/ao/doctor_wedge.go b/cli/cmd/ao/doctor_wedge.go deleted file mode 100644 index ffb75f9fa..000000000 --- a/cli/cmd/ao/doctor_wedge.go +++ /dev/null @@ -1,321 +0,0 @@ -// practices: [sre, resilience-patterns] -package main - -import ( - "context" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - "time" - - "github.com/boshu2/agentops/cli/internal/provenancegraph" -) - -// wedgeReviewerTimeout bounds each live reviewer probe so a hanging reviewer -// binary can never wedge `ao doctor` itself: on expiry the reviewer is -// reported unreachable and doctor moves on. -const wedgeReviewerTimeout = 10 * time.Second - -// reviewerProbe describes one cross-family reviewer CLI probed by the wedge -// preflight, with the exact install command a failing check must teach. -type reviewerProbe struct { - name string - installCmd string -} - -// wedgeReviewers is the closed set of reviewer families the verification -// wedge can route reviews through. Claude is the producer family, so -// cross-family capability requires at least one of these to answer live. -// LAW 0: claude print-mode is never a reviewer path and never appears here. -var wedgeReviewers = []reviewerProbe{ - {name: "codex", installCmd: "npm install -g @openai/codex && codex login"}, - {name: "agy", installCmd: "install the AGY CLI, then verify with 'agy models'"}, -} - -// wedgeDoctorChecks runs the wedge/verification preflight: reviewer -// reachability, cross-family capability, binary freshness, ledger health, -// and the LAW-0 guard. Purely additive to the legacy doctor check table and -// serialized with the same quality.Check schema. -func wedgeDoctorChecks() []doctorCheck { - checks, live := reviewerReachabilityChecks(wedgeReviewerTimeout) - checks = append(checks, - crossFamilyCheck(live), - checkBinaryFreshness(), - checkLedgerHealth(resolveLedgerPath()), - checkLaw0Guard(os.Environ()), - ) - return checks -} - -// reviewerReachabilityChecks probes every reviewer family and returns the -// per-family checks plus the names of the families that answered live. An -// absent binary is a degraded warn (CI-safe: no invocation, no hang); a -// present-but-unresponsive binary fails after the hard timeout. -func reviewerReachabilityChecks(timeout time.Duration) ([]doctorCheck, []string) { - checks := make([]doctorCheck, 0, len(wedgeReviewers)) - var live []string - for _, r := range wedgeReviewers { - c, ok := checkReviewerReachable(r, timeout) - if ok { - live = append(live, r.name) - } - checks = append(checks, c) - } - return checks, live -} - -// checkReviewerReachable reports whether one reviewer CLI is on PATH and -// answers a trivial `--version` invocation within timeout. The invocation -// runs under exec.CommandContext so a hanging binary is killed at the -// deadline and reported unreachable instead of wedging doctor. -func checkReviewerReachable(r reviewerProbe, timeout time.Duration) (doctorCheck, bool) { - name := "Reviewer: " + r.name - if _, err := exec.LookPath(r.name); err != nil { - return doctorCheck{ - Name: name, - Status: "warn", - Detail: fmt.Sprintf("not found on PATH — install: %s", r.installCmd), - Required: false, - }, false - } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - start := time.Now() - err := exec.CommandContext(ctx, r.name, "--version").Run() - elapsed := time.Since(start).Round(time.Millisecond) - switch { - case errors.Is(ctx.Err(), context.DeadlineExceeded): - return doctorCheck{ - Name: name, - Status: "fail", - Detail: fmt.Sprintf("unreachable: '%s --version' timed out after %s — check for a hung or unauthenticated binary: run '%s --version' manually", r.name, timeout, r.name), - Required: false, - }, false - case err != nil: - return doctorCheck{ - Name: name, - Status: "fail", - Detail: fmt.Sprintf("'%s --version' failed (%v) — reinstall: %s", r.name, err, r.installCmd), - Required: false, - }, false - } - return doctorCheck{ - Name: name, - Status: "pass", - Detail: fmt.Sprintf("reachable ('--version' answered in %s)", elapsed), - Required: false, - }, true -} - -// crossFamilyCheck summarizes reviewer reachability into the one line agents -// and operators need: can a cross-family review be routed right now? -func crossFamilyCheck(live []string) doctorCheck { - name := "Cross-Family Review" - if len(live) > 0 { - return doctorCheck{ - Name: name, - Status: "pass", - Detail: fmt.Sprintf("cross-family capable: yes (live families: %s)", strings.Join(live, ", ")), - Required: false, - } - } - return doctorCheck{ - Name: name, - Status: "warn", - Detail: "cross-family capable: no (no reviewer CLI reachable) — install one: npm install -g @openai/codex && codex login", - Required: false, - } -} - -// agentopsModuleLine identifies THE agentops checkout, the same way session -// bootstrap does (sessionBootstrapIsAgentopsRepo). -const agentopsModuleLine = "module github.com/boshu2/agentops/cli" - -// repoVersionRe extracts the fallback version literal from cli/cmd/ao/main.go -// (the var release builds override via -X main.version ldflags). -var repoVersionRe = regexp.MustCompile(`var version = "([^"]+)"`) - -// checkBinaryFreshness compares the running binary's embedded version against -// the agentops repo's declared version when doctor runs inside the repo. This -// catches the stale-installed-binary trap (operator running a build hundreds -// of commits behind the checkout). Outside the repo it reports version only. -func checkBinaryFreshness() doctorCheck { - cwd, err := os.Getwd() - if err != nil { - return doctorCheck{Name: "Binary Freshness", Status: "warn", Detail: "cannot determine working directory", Required: false} - } - return binaryFreshnessCheck(cwd, version) -} - -// binaryFreshnessCheck is the testable core of checkBinaryFreshness: dir is -// where doctor runs, running is the embedded version of the live binary. -func binaryFreshnessCheck(dir, running string) doctorCheck { - name := "Binary Freshness" - root, ok := findAgentopsRepoRoot(dir) - if !ok { - return doctorCheck{ - Name: name, - Status: "pass", - Detail: fmt.Sprintf("ao %s (outside the agentops repo — freshness not applicable)", running), - Required: false, - } - } - repoVer, ok := repoDeclaredVersion(root) - if !ok { - return doctorCheck{ - Name: name, - Status: "warn", - Detail: "cannot read the repo's declared version (cli/cmd/ao/main.go) — rebuild to be safe: scripts/preflight-uat-binary.sh", - Required: false, - } - } - if repoVer == running { - return doctorCheck{ - Name: name, - Status: "pass", - Detail: fmt.Sprintf("ao %s matches the repo's declared version", running), - Required: false, - } - } - return doctorCheck{ - Name: name, - Status: "warn", - Detail: fmt.Sprintf("running ao %s but the repo declares %s (stale installed binary) — rebuild+install: scripts/preflight-uat-binary.sh (or: brew upgrade agentops)", running, repoVer), - Required: false, - } -} - -// findAgentopsRepoRoot walks up from dir looking for THE agentops checkout: -// a directory whose cli/go.mod declares the agentops module path. -func findAgentopsRepoRoot(dir string) (string, bool) { - for i := 0; i < 12; i++ { - data, ok := readRegularFileCapped(filepath.Join(dir, "cli", "go.mod"), 1<<16) - if ok && strings.Contains(string(data), agentopsModuleLine) { - return dir, true - } - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent - } - return "", false -} - -// repoDeclaredVersion parses the version literal out of the repo's -// cli/cmd/ao/main.go. -func repoDeclaredVersion(root string) (string, bool) { - data, ok := readRegularFileCapped(filepath.Join(root, "cli", "cmd", "ao", "main.go"), 1<<16) - if !ok { - return "", false - } - m := repoVersionRe.FindSubmatch(data) - if m == nil { - return "", false - } - return string(m[1]), true -} - -// checkLedgerHealth verifies the provenance ledger in place through the SAME -// code path as `ao provenance verify` (provenancegraph VerifyFile — never a -// reimplementation) and reports feed recency informationally. -func checkLedgerHealth(path string) doctorCheck { - name := "Ledger Health" - store := provenancegraph.NewStore(path) - res, err := store.VerifyFile() - if err != nil { - return doctorCheck{ - Name: name, - Status: "fail", - Detail: fmt.Sprintf("cannot read ledger at %s (%v) — check the path and permissions: ls -l %s", path, err, path), - Required: true, - } - } - if !res.Pass { - return doctorCheck{ - Name: name, - Status: "fail", - Detail: fmt.Sprintf("chain breaks at line %d: %s — inspect: ao provenance verify", res.FirstBrokenLine, res.Message), - Required: true, - } - } - if res.RecordCount == 0 { - return doctorCheck{ - Name: name, - Status: "pass", - Detail: "no ledger records yet (an empty or absent ledger is an intact chain)", - Required: false, - } - } - return doctorCheck{ - Name: name, - Status: "pass", - Detail: fmt.Sprintf("chain intact (%d records%s)", res.RecordCount, ledgerRecencySuffix(store)), - Required: false, - } -} - -// ledgerRecencySuffix returns ", newest record old" for the final -// ledger record, or "" when the timestamp cannot be read — recency is -// informational only and never fails the check. -func ledgerRecencySuffix(store *provenancegraph.Store) string { - edges, err := store.Read() - if err != nil || len(edges) == 0 { - return "" - } - ts, perr := time.Parse(time.RFC3339, edges[len(edges)-1].TS) - if perr != nil { - return "" - } - return fmt.Sprintf(", newest record %s old", formatDuration(time.Since(ts))) -} - -// law0Needles are the forbidden reviewer invocations (LAW 0): no reviewer -// path may ever shell out to claude print-mode (the forbidden -p/--print invocations). -// Needles are assembled so this source never contains the contiguous forbidden -// pattern the door9 gate greps for. -var law0Needles = []string{"claude" + " -p", "claude" + " --print"} - -// checkLaw0Guard scans reviewer-scoped environment variables (PAWL_*, -// AGENTOPS_*, AO_*, and any *REVIEWER* name) for a claude-p reviewer path. -// There is no reviewer config-file surface today — env vars (PAWL_NO_SERVICE -// and friends) are the only runtime reviewer knobs, so a clean env scan is a -// clean report. -func checkLaw0Guard(environ []string) doctorCheck { - name := "LAW-0 Guard" - for _, kv := range environ { - k, v, ok := strings.Cut(kv, "=") - if !ok || !law0RelevantEnv(k) { - continue - } - for _, needle := range law0Needles { - if strings.Contains(v, needle) { - return doctorCheck{ - Name: name, - Status: "fail", - Detail: fmt.Sprintf("LAW-0 violation: $%s routes a reviewer through %q — remove it: unset %s", k, needle, k), - Required: true, - } - } - } - } - return doctorCheck{ - Name: name, - Status: "pass", - Detail: "no reviewer path configured through claude print-mode (scanned PAWL_*/AGENTOPS_*/AO_*/*REVIEWER* env)", - Required: true, - } -} - -// law0RelevantEnv reports whether an environment variable name is a reviewer -// configuration surface the LAW-0 guard must scan. -func law0RelevantEnv(name string) bool { - return strings.HasPrefix(name, "PAWL_") || - strings.HasPrefix(name, "AGENTOPS_") || - strings.HasPrefix(name, "AO_") || - strings.Contains(name, "REVIEWER") -} diff --git a/cli/cmd/ao/doctor_wedge_legacy_test.go b/cli/cmd/ao/doctor_wedge_legacy_test.go new file mode 100644 index 000000000..26e8721ed --- /dev/null +++ b/cli/cmd/ao/doctor_wedge_legacy_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "time" + + doctoradapter "github.com/boshu2/agentops/cli/internal/adapters/doctor" + revieweradapter "github.com/boshu2/agentops/cli/internal/adapters/reviewerhealth" + "github.com/boshu2/agentops/cli/internal/reviewerhealth" +) + +const wedgeReviewerTimeout = reviewerProbeTimeout + +type reviewerProbe struct { + name string + installCmd string +} + +var wedgeReviewers = func() []reviewerProbe { + catalog := reviewerhealth.DefaultCatalog() + out := make([]reviewerProbe, 0, len(catalog)) + for _, reviewer := range catalog { + out = append(out, reviewerProbe{name: reviewer.Name, installCmd: reviewer.InstallCommand}) + } + return out +}() + +func reviewerReachabilityChecks(timeout time.Duration) ([]doctorCheck, []string) { + return reviewerHealthService.Check(context.Background(), timeout) +} + +func checkReviewerReachable(reviewer reviewerProbe, timeout time.Duration) (doctorCheck, bool) { + service := reviewerhealth.NewService([]reviewerhealth.Reviewer{{ + Name: reviewer.name, InstallCommand: reviewer.installCmd, + }}, revieweradapter.SystemProbe()) + checks, live := service.Check(context.Background(), timeout) + return checks[0], len(live) == 1 +} + +const agentopsModuleLine = "module github.com/boshu2/agentops/cli" + +func wedgeDoctorChecks() []doctorCheck { + return newLegacyDoctorService().Checks(context.Background())[11:] +} +func crossFamilyCheck(live []string) doctorCheck { return doctoradapter.CrossFamilyCheck(live) } +func binaryFreshnessCheck(dir, running string) doctorCheck { + return doctoradapter.BinaryFreshnessCheck(dir, running) +} +func checkLedgerHealth(path string) doctorCheck { + return doctoradapter.CheckLedgerHealth(path, time.Now) +} +func checkLaw0Guard(environment []string) doctorCheck { + return doctoradapter.CheckLaw0Guard(environment) +} diff --git a/cli/cmd/ao/gate_runner_adapter.go b/cli/cmd/ao/gate_runner_adapter.go index 8c8771bea..5c354ce1a 100644 --- a/cli/cmd/ao/gate_runner_adapter.go +++ b/cli/cmd/ao/gate_runner_adapter.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/boshu2/agentops/cli/internal/ports" + "github.com/boshu2/agentops/cli/internal/quality" ) // productionGateRunner satisfies ports.GateRunnerPort by invoking @@ -51,7 +52,7 @@ func (g *productionGateRunner) Run(ctx context.Context, req ports.GateRunRequest return ports.GateVerdict{}, fmt.Errorf("productionGateRunner: repoRoot required") } scriptPath := filepath.Join(g.repoRoot, "scripts", "check-"+string(req.Name)+".sh") - if !fileExists(scriptPath) { + if !quality.FileExists(scriptPath) { return ports.GateVerdict{ Status: ports.GateStatusUnknown, Reason: fmt.Sprintf("no script for gate %q at %s", req.Name, scriptPath), diff --git a/cli/cmd/ao/quickstart_verdict.go b/cli/cmd/ao/quickstart_verdict.go index 1618ad987..277f60ca8 100644 --- a/cli/cmd/ao/quickstart_verdict.go +++ b/cli/cmd/ao/quickstart_verdict.go @@ -2,6 +2,7 @@ package main import ( + "context" "fmt" "os" "path/filepath" @@ -42,7 +43,7 @@ func prepareFirstVerdict() *firstVerdictInfo { if err := os.MkdirAll(filepath.Dir(ledgerPath), 0o755); err == nil { info.LedgerReady = true } - _, live := reviewerReachabilityChecks(wedgeReviewerTimeout) + _, live := reviewerHealthService.Check(context.Background(), reviewerProbeTimeout) info.ReviewerLive = live if len(live) > 0 { info.NextCommand = firstVerdictCommand @@ -51,8 +52,8 @@ func prepareFirstVerdict() *firstVerdictInfo { // No reviewer reachable: teach the exact doctor corrective install // one-liners (wedgeReviewers.installCmd is the shared source of truth), // never a pointer into diffuse docs. - for _, r := range wedgeReviewers { - info.ReviewerInstall = append(info.ReviewerInstall, r.name+": "+r.installCmd) + for _, reviewer := range reviewerHealthService.Catalog() { + info.ReviewerInstall = append(info.ReviewerInstall, reviewer.Name+": "+reviewer.InstallCommand) } return info } diff --git a/cli/cmd/ao/reviewer_health_composition.go b/cli/cmd/ao/reviewer_health_composition.go new file mode 100644 index 000000000..2b1d886a7 --- /dev/null +++ b/cli/cmd/ao/reviewer_health_composition.go @@ -0,0 +1,12 @@ +package main + +import ( + "time" + + revieweradapter "github.com/boshu2/agentops/cli/internal/adapters/reviewerhealth" + "github.com/boshu2/agentops/cli/internal/reviewerhealth" +) + +const reviewerProbeTimeout = 10 * time.Second + +var reviewerHealthService = reviewerhealth.NewService(reviewerhealth.DefaultCatalog(), revieweradapter.SystemProbe()) diff --git a/cli/cmd/ao/session_bootstrap.go b/cli/cmd/ao/session_bootstrap.go index 66fd36a98..a1a074e69 100644 --- a/cli/cmd/ao/session_bootstrap.go +++ b/cli/cmd/ao/session_bootstrap.go @@ -41,6 +41,7 @@ import ( "time" "github.com/boshu2/agentops/cli/internal/promptsafe" + "github.com/boshu2/agentops/cli/internal/quality" "github.com/boshu2/agentops/cli/internal/search" "github.com/boshu2/agentops/cli/internal/types" @@ -180,9 +181,9 @@ func computeBootstrapStatus(ctx context.Context, cwd string, noMail bool) Sessio status.BeadsDir = beadsDir.Path status.BeadsDirSource = beadsDir.Source - status.AgentsMDRead = fileExists(filepath.Join(cwd, "AGENTS.md")) + status.AgentsMDRead = quality.FileExists(filepath.Join(cwd, "AGENTS.md")) for _, sib := range agentsMDSiblings { - if fileExists(filepath.Join(cwd, sib)) { + if quality.FileExists(filepath.Join(cwd, sib)) { status.AgentsSiblingsRead = append(status.AgentsSiblingsRead, sib) } } diff --git a/cli/internal/adapters/doctor/legacy.go b/cli/internal/adapters/doctor/legacy.go new file mode 100644 index 000000000..c32577817 --- /dev/null +++ b/cli/internal/adapters/doctor/legacy.go @@ -0,0 +1,185 @@ +// Package doctor contains concrete runtime adapters for doctor use cases. +package doctor + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/boshu2/agentops/cli/internal/provenancegraph" + "github.com/boshu2/agentops/cli/internal/quality" + "github.com/boshu2/agentops/cli/internal/reviewerhealth" + "github.com/boshu2/agentops/cli/internal/storage" +) + +const reviewerTimeout = 10 * time.Second + +type LegacyChecks struct { + ToolVersion string + IndexDir string + IndexFile string + LedgerPath func() string + Reviewers reviewerhealth.Service + WorkingDir func() (string, error) + Environment func() []string + LookPath func(string) (string, error) + Now func() time.Time +} + +func (adapter LegacyChecks) Checks(ctx context.Context) []quality.Check { + checks := []quality.Check{quality.CheckCLIDependencies(adapter.LookPath)} + cwd, cwdErr := adapter.WorkingDir() + if cwdErr != nil { + checks = append(checks, + quality.Check{Name: "Knowledge Base", Status: "fail", Detail: "cannot determine working directory", Required: true}, + quality.Check{Name: "Knowledge Freshness", Status: "warn", Detail: "cannot determine working directory"}, + quality.Check{Name: "Search Index", Status: "warn", Detail: "cannot determine working directory"}, + quality.Check{Name: "Flywheel Health", Status: "warn", Detail: "cannot determine working directory"}, + ) + } else { + base := filepath.Join(cwd, storage.DefaultBaseDir) + checks = append(checks, + quality.CheckKnowledgeBase(base), + quality.CheckKnowledgeFreshness(filepath.Join(base, "sessions")), + quality.CheckSearchIndex(filepath.Join(cwd, adapter.IndexDir, adapter.IndexFile)), + quality.CheckFlywheelHealth(base), + ) + } + checks = append(checks, + quality.CheckSkills(), quality.CheckCodexSync(), quality.CheckSkillIntegrity(), + quality.CheckStaleReferences([]string{ + "skills/*/SKILL.md", "skills/*/references/*.md", "skills-codex/*/SKILL.md", + "skills-codex-overrides/*/SKILL.md", "docs/*.md", "scripts/*.sh", + "docs/contracts/*.md", "docs/plans/*.md", + }), + quality.CheckOptionalCLI("codex", "needed for --mixed council"), + ) + reviewerChecks, live := adapter.Reviewers.Check(ctx, reviewerTimeout) + checks = append(checks, reviewerChecks...) + checks = append(checks, CrossFamilyCheck(live)) + if cwdErr != nil { + checks = append(checks, quality.Check{Name: "Binary Freshness", Status: "warn", Detail: "cannot determine working directory"}) + } else { + checks = append(checks, BinaryFreshnessCheck(cwd, adapter.ToolVersion)) + } + ledgerPath := "" + if adapter.LedgerPath != nil { + ledgerPath = adapter.LedgerPath() + } + checks = append(checks, CheckLedgerHealth(ledgerPath, adapter.Now), CheckLaw0Guard(adapter.Environment())) + return checks +} + +func SystemLegacyChecks(toolVersion, indexDir, indexFile string, ledgerPath func() string, reviewers reviewerhealth.Service) LegacyChecks { + return LegacyChecks{ + ToolVersion: toolVersion, IndexDir: indexDir, IndexFile: indexFile, LedgerPath: ledgerPath, Reviewers: reviewers, + WorkingDir: os.Getwd, Environment: os.Environ, LookPath: exec.LookPath, Now: time.Now, + } +} + +func CrossFamilyCheck(live []string) quality.Check { + if len(live) > 0 { + return quality.Check{Name: "Cross-Family Review", Status: "pass", Detail: fmt.Sprintf("cross-family capable: yes (live families: %s)", strings.Join(live, ", "))} + } + return quality.Check{Name: "Cross-Family Review", Status: "warn", Detail: "cross-family capable: no (no reviewer CLI reachable) — install one: npm install -g @openai/codex && codex login"} +} + +const agentopsModuleLine = "module github.com/boshu2/agentops/cli" + +var repoVersionPattern = regexp.MustCompile(`var version = "([^"]+)"`) + +func BinaryFreshnessCheck(dir, running string) quality.Check { + name := "Binary Freshness" + root, ok := FindAgentopsRepoRoot(dir) + if !ok { + return quality.Check{Name: name, Status: "pass", Detail: fmt.Sprintf("ao %s (outside the agentops repo — freshness not applicable)", running)} + } + repoVersion, ok := RepoDeclaredVersion(root) + if !ok { + return quality.Check{Name: name, Status: "warn", Detail: "cannot read the repo's declared version (cli/cmd/ao/main.go) — rebuild to be safe: scripts/preflight-uat-binary.sh"} + } + if repoVersion == running { + return quality.Check{Name: name, Status: "pass", Detail: fmt.Sprintf("ao %s matches the repo's declared version", running)} + } + return quality.Check{Name: name, Status: "warn", Detail: fmt.Sprintf("running ao %s but the repo declares %s (stale installed binary) — rebuild+install: scripts/preflight-uat-binary.sh (or: brew upgrade agentops)", running, repoVersion)} +} + +func FindAgentopsRepoRoot(dir string) (string, bool) { + for range 12 { + data, err := os.ReadFile(filepath.Join(dir, "cli", "go.mod")) + if err == nil && len(data) <= 1<<16 && strings.Contains(string(data), agentopsModuleLine) { + return dir, true + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + return "", false +} + +func RepoDeclaredVersion(root string) (string, bool) { + data, err := os.ReadFile(filepath.Join(root, "cli", "cmd", "ao", "main.go")) + if err != nil || len(data) > 1<<16 { + return "", false + } + match := repoVersionPattern.FindSubmatch(data) + return stringMatch(match) +} + +func stringMatch(match [][]byte) (string, bool) { + if len(match) != 2 { + return "", false + } + return string(match[1]), true +} + +func CheckLedgerHealth(path string, now func() time.Time) quality.Check { + name := "Ledger Health" + store := provenancegraph.NewStore(path) + result, err := store.VerifyFile() + if err != nil { + return quality.Check{Name: name, Status: "fail", Detail: fmt.Sprintf("cannot read ledger at %s (%v) — check the path and permissions: ls -l %s", path, err, path), Required: true} + } + if !result.Pass { + return quality.Check{Name: name, Status: "fail", Detail: fmt.Sprintf("chain breaks at line %d: %s — inspect: ao provenance verify", result.FirstBrokenLine, result.Message), Required: true} + } + if result.RecordCount == 0 { + return quality.Check{Name: name, Status: "pass", Detail: "no ledger records yet (an empty or absent ledger is an intact chain)"} + } + detail := fmt.Sprintf("chain intact (%d records", result.RecordCount) + edges, readErr := store.Read() + if readErr == nil && len(edges) > 0 && now != nil { + if timestamp, parseErr := time.Parse(time.RFC3339, edges[len(edges)-1].TS); parseErr == nil { + detail += fmt.Sprintf(", newest record %s old", quality.FormatDuration(now().Sub(timestamp))) + } + } + return quality.Check{Name: name, Status: "pass", Detail: detail + ")"} +} + +var law0Needles = []string{"claude" + " -p", "claude" + " --print"} + +func CheckLaw0Guard(environment []string) quality.Check { + for _, item := range environment { + key, value, ok := strings.Cut(item, "=") + if !ok || !law0RelevantEnv(key) { + continue + } + for _, needle := range law0Needles { + if strings.Contains(value, needle) { + return quality.Check{Name: "LAW-0 Guard", Status: "fail", Detail: fmt.Sprintf("LAW-0 violation: $%s routes a reviewer through %q — remove it: unset %s", key, needle, key), Required: true} + } + } + } + return quality.Check{Name: "LAW-0 Guard", Status: "pass", Detail: "no reviewer path configured through claude print-mode (scanned PAWL_*/AGENTOPS_*/AO_*/*REVIEWER* env)", Required: true} +} + +func law0RelevantEnv(name string) bool { + return strings.HasPrefix(name, "PAWL_") || strings.HasPrefix(name, "AGENTOPS_") || strings.HasPrefix(name, "AO_") || strings.Contains(name, "REVIEWER") +} diff --git a/cli/internal/adapters/reviewerhealth/probe.go b/cli/internal/adapters/reviewerhealth/probe.go new file mode 100644 index 000000000..4336105ad --- /dev/null +++ b/cli/internal/adapters/reviewerhealth/probe.go @@ -0,0 +1,56 @@ +// Package reviewerhealth contains process adapters for reviewer reachability. +package reviewerhealth + +import ( + "context" + "errors" + "fmt" + "os/exec" + "time" + + reviewerapp "github.com/boshu2/agentops/cli/internal/reviewerhealth" +) + +type Probe struct { + LookPath func(string) (string, error) + Run func(context.Context, string, ...string) error + Now func() time.Time +} + +func SystemProbe() Probe { + return Probe{ + LookPath: exec.LookPath, + Run: func(ctx context.Context, name string, args ...string) error { + return exec.CommandContext(ctx, name, args...).Run() + }, + Now: time.Now, + } +} + +func (probe Probe) Check(ctx context.Context, reviewer reviewerapp.Reviewer, timeout time.Duration) reviewerapp.ProbeResult { + if _, err := probe.LookPath(reviewer.Name); err != nil { + return reviewerapp.ProbeResult{ + Status: "warn", Detail: fmt.Sprintf("not found on PATH — install: %s", reviewer.InstallCommand), + } + } + checkCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + start := probe.Now() + err := probe.Run(checkCtx, reviewer.Name, "--version") + elapsed := probe.Now().Sub(start).Round(time.Millisecond) + switch { + case errors.Is(checkCtx.Err(), context.DeadlineExceeded): + return reviewerapp.ProbeResult{ + Status: "fail", + Detail: fmt.Sprintf("unreachable: '%s --version' timed out after %s — check for a hung or unauthenticated binary: run '%s --version' manually", reviewer.Name, timeout, reviewer.Name), + } + case err != nil: + return reviewerapp.ProbeResult{ + Status: "fail", Detail: fmt.Sprintf("'%s --version' failed (%v) — reinstall: %s", reviewer.Name, err, reviewer.InstallCommand), + } + default: + return reviewerapp.ProbeResult{ + Status: "pass", Detail: fmt.Sprintf("reachable ('--version' answered in %s)", elapsed), Live: true, + } + } +} diff --git a/cli/internal/adapters/reviewerhealth/probe_test.go b/cli/internal/adapters/reviewerhealth/probe_test.go new file mode 100644 index 000000000..444654d39 --- /dev/null +++ b/cli/internal/adapters/reviewerhealth/probe_test.go @@ -0,0 +1,30 @@ +package reviewerhealth + +import ( + "context" + "errors" + "testing" + "time" + + reviewerapp "github.com/boshu2/agentops/cli/internal/reviewerhealth" +) + +func TestProbeMissingAndReachable(t *testing.T) { + reviewer := reviewerapp.Reviewer{Name: "codex", InstallCommand: "install codex"} + missing := Probe{ + LookPath: func(string) (string, error) { return "", errors.New("missing") }, + Run: func(context.Context, string, ...string) error { return nil }, Now: time.Now, + } + if result := missing.Check(context.Background(), reviewer, time.Second); result.Status != "warn" || result.Live { + t.Fatalf("missing result = %+v", result) + } + now := time.Unix(0, 0) + reachable := Probe{ + LookPath: func(string) (string, error) { return "/bin/codex", nil }, + Run: func(context.Context, string, ...string) error { return nil }, + Now: func() time.Time { current := now; now = now.Add(25 * time.Millisecond); return current }, + } + if result := reachable.Check(context.Background(), reviewer, time.Second); result.Status != "pass" || !result.Live { + t.Fatalf("reachable result = %+v", result) + } +} diff --git a/cli/internal/doctor/legacy_service.go b/cli/internal/doctor/legacy_service.go new file mode 100644 index 000000000..b9caeb877 --- /dev/null +++ b/cli/internal/doctor/legacy_service.go @@ -0,0 +1,25 @@ +package doctor + +import ( + "context" + + "github.com/boshu2/agentops/cli/internal/quality" +) + +type LegacyChecks interface { + Checks(context.Context) []quality.Check +} + +type LegacyService struct { + version string + checks LegacyChecks +} + +func NewLegacyService(version string, checks LegacyChecks) LegacyService { + return LegacyService{version: version, checks: checks} +} + +func (service LegacyService) Checks(ctx context.Context) []quality.Check { + checks := []quality.Check{{Name: "ao CLI", Status: "pass", Detail: quality.FormatVersion(service.version), Required: true}} + return append(checks, service.checks.Checks(ctx)...) +} diff --git a/cli/internal/doctor/legacy_service_test.go b/cli/internal/doctor/legacy_service_test.go new file mode 100644 index 000000000..97064601c --- /dev/null +++ b/cli/internal/doctor/legacy_service_test.go @@ -0,0 +1,27 @@ +package doctor + +import ( + "context" + "testing" + + "github.com/boshu2/agentops/cli/internal/quality" +) + +type fakeLegacyChecks struct{ checks []quality.Check } + +func (fake fakeLegacyChecks) Checks(context.Context) []quality.Check { + return append([]quality.Check(nil), fake.checks...) +} + +func TestLegacyServicePrependsCLIVersionAndPreservesAdapterOrder(t *testing.T) { + service := NewLegacyService("3.0.0", fakeLegacyChecks{checks: []quality.Check{ + {Name: "CLI Dependencies"}, {Name: "Knowledge Base"}, {Name: "Reviewer: codex"}, + }}) + checks := service.Checks(context.Background()) + if len(checks) != 4 || checks[0].Name != "ao CLI" || checks[0].Detail != "v3.0.0" { + t.Fatalf("checks = %+v", checks) + } + if checks[1].Name != "CLI Dependencies" || checks[3].Name != "Reviewer: codex" { + t.Fatalf("order changed: %+v", checks) + } +} diff --git a/cli/internal/reviewerhealth/service.go b/cli/internal/reviewerhealth/service.go new file mode 100644 index 000000000..772223389 --- /dev/null +++ b/cli/internal/reviewerhealth/service.go @@ -0,0 +1,60 @@ +// Package reviewerhealth owns the shared reviewer catalog and reachability +// policy used by doctor and quick-start. +package reviewerhealth + +import ( + "context" + "time" + + "github.com/boshu2/agentops/cli/internal/quality" +) + +type Reviewer struct { + Name string + InstallCommand string +} + +type ProbeResult struct { + Status string + Detail string + Live bool +} + +type Probe interface { + Check(context.Context, Reviewer, time.Duration) ProbeResult +} + +type Service struct { + catalog []Reviewer + probe Probe +} + +func DefaultCatalog() []Reviewer { + return []Reviewer{ + {Name: "codex", InstallCommand: "npm install -g @openai/codex && codex login"}, + {Name: "agy", InstallCommand: "install the AGY CLI, then verify with 'agy models'"}, + } +} + +func NewService(catalog []Reviewer, probe Probe) Service { + return Service{catalog: append([]Reviewer(nil), catalog...), probe: probe} +} + +func (service Service) Check(ctx context.Context, timeout time.Duration) ([]quality.Check, []string) { + checks := make([]quality.Check, 0, len(service.catalog)) + var live []string + for _, reviewer := range service.catalog { + result := service.probe.Check(ctx, reviewer, timeout) + checks = append(checks, quality.Check{ + Name: "Reviewer: " + reviewer.Name, Status: result.Status, Detail: result.Detail, Required: false, + }) + if result.Live { + live = append(live, reviewer.Name) + } + } + return checks, live +} + +func (service Service) Catalog() []Reviewer { + return append([]Reviewer(nil), service.catalog...) +} diff --git a/cli/internal/reviewerhealth/service_test.go b/cli/internal/reviewerhealth/service_test.go new file mode 100644 index 000000000..cf1a63d55 --- /dev/null +++ b/cli/internal/reviewerhealth/service_test.go @@ -0,0 +1,40 @@ +package reviewerhealth + +import ( + "context" + "testing" + "time" +) + +type fakeProbe map[string]ProbeResult + +func (probe fakeProbe) Check(_ context.Context, reviewer Reviewer, _ time.Duration) ProbeResult { + return probe[reviewer.Name] +} + +func TestServiceChecksCatalogInOrderAndReportsLiveReviewers(t *testing.T) { + catalog := []Reviewer{{Name: "codex", InstallCommand: "install codex"}, {Name: "agy", InstallCommand: "install agy"}} + service := NewService(catalog, fakeProbe{ + "codex": {Status: "pass", Detail: "reachable", Live: true}, + "agy": {Status: "warn", Detail: "missing"}, + }) + checks, live := service.Check(context.Background(), time.Second) + if len(checks) != 2 || checks[0].Name != "Reviewer: codex" || checks[1].Name != "Reviewer: agy" { + t.Fatalf("checks = %+v", checks) + } + if len(live) != 1 || live[0] != "codex" { + t.Fatalf("live = %v", live) + } +} + +func TestDefaultCatalogPreservesInstallGuidance(t *testing.T) { + catalog := DefaultCatalog() + if len(catalog) != 2 || catalog[0].Name != "codex" || catalog[1].Name != "agy" { + t.Fatalf("catalog = %+v", catalog) + } + for _, reviewer := range catalog { + if reviewer.InstallCommand == "" { + t.Fatalf("missing install command: %+v", reviewer) + } + } +} From d845aa394a2040e8b52575bd820af888c6711e46 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:07:40 -0400 Subject: [PATCH 061/108] test(doctor): preserve bounded regular-file freshness reads --- cli/internal/adapters/doctor/legacy.go | 23 +++++++++--- cli/internal/adapters/doctor/legacy_test.go | 40 +++++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 cli/internal/adapters/doctor/legacy_test.go diff --git a/cli/internal/adapters/doctor/legacy.go b/cli/internal/adapters/doctor/legacy.go index c32577817..4ffa2e9c6 100644 --- a/cli/internal/adapters/doctor/legacy.go +++ b/cli/internal/adapters/doctor/legacy.go @@ -4,6 +4,7 @@ package doctor import ( "context" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -111,8 +112,8 @@ func BinaryFreshnessCheck(dir, running string) quality.Check { func FindAgentopsRepoRoot(dir string) (string, bool) { for range 12 { - data, err := os.ReadFile(filepath.Join(dir, "cli", "go.mod")) - if err == nil && len(data) <= 1<<16 && strings.Contains(string(data), agentopsModuleLine) { + data, ok := readRegularFileCapped(filepath.Join(dir, "cli", "go.mod"), 1<<16) + if ok && strings.Contains(string(data), agentopsModuleLine) { return dir, true } parent := filepath.Dir(dir) @@ -125,14 +126,28 @@ func FindAgentopsRepoRoot(dir string) (string, bool) { } func RepoDeclaredVersion(root string) (string, bool) { - data, err := os.ReadFile(filepath.Join(root, "cli", "cmd", "ao", "main.go")) - if err != nil || len(data) > 1<<16 { + data, ok := readRegularFileCapped(filepath.Join(root, "cli", "cmd", "ao", "main.go"), 1<<16) + if !ok { return "", false } match := repoVersionPattern.FindSubmatch(data) return stringMatch(match) } +func readRegularFileCapped(path string, maxBytes int64) ([]byte, bool) { + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() { + return nil, false + } + file, err := os.Open(path) + if err != nil { + return nil, false + } + defer func() { _ = file.Close() }() + data, err := io.ReadAll(io.LimitReader(file, maxBytes)) + return data, err == nil +} + func stringMatch(match [][]byte) (string, bool) { if len(match) != 2 { return "", false diff --git a/cli/internal/adapters/doctor/legacy_test.go b/cli/internal/adapters/doctor/legacy_test.go new file mode 100644 index 000000000..99f2dc283 --- /dev/null +++ b/cli/internal/adapters/doctor/legacy_test.go @@ -0,0 +1,40 @@ +package doctor + +import ( + "os" + "path/filepath" + "testing" +) + +func TestBinaryFreshnessRejectsSymlinkedRepoMarkers(t *testing.T) { + root := t.TempDir() + external := filepath.Join(t.TempDir(), "go.mod") + if err := os.WriteFile(external, []byte(agentopsModuleLine+"\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "cli"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(external, filepath.Join(root, "cli", "go.mod")); err != nil { + t.Fatal(err) + } + if _, ok := FindAgentopsRepoRoot(root); ok { + t.Fatal("symlinked go.mod identified an AgentOps checkout") + } +} + +func TestRepoDeclaredVersionRejectsOversizedFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "cli", "cmd", "ao") + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + body := make([]byte, (1<<16)+100) + copy(body[(1<<16)+1:], `var version = "forged"`) + if err := os.WriteFile(filepath.Join(path, "main.go"), body, 0o644); err != nil { + t.Fatal(err) + } + root := filepath.Clean(filepath.Join(path, "..", "..", "..")) + if _, ok := RepoDeclaredVersion(root); ok { + t.Fatal("oversized main.go yielded a version") + } +} From 012e446c8a6519cb90d1bf65e9c10eafe940c4a0 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:13:38 -0400 Subject: [PATCH 062/108] chore(provenance): bind pawl CONFIRMED verdict for age-nw28h.2.7.1 #trivial --- docs/provenance/ledger.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/provenance/ledger.jsonl b/docs/provenance/ledger.jsonl index 585812ce4..a67691432 100644 --- a/docs/provenance/ledger.jsonl +++ b/docs/provenance/ledger.jsonl @@ -522,3 +522,4 @@ {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.4@0782b56","from_type":"verdict","to_id":"0782b566af8be3b013b649f95f607b5bbebc4fc5","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.4 disposition=CONFIRMED","bead_id":"age-nw28h.2.4","trust_tier":"inferred","ts":"2026-07-11T22:04:47Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-close-independent-verdict-0782b566a.txt","prev_hash":"be2caecd5f64c47b68e12dd6a220eacac168450f3c6c0160f4428f4a917e92d0","payload_hash":"a451e922b8fb7db3c5d35d3815169a837ec3e0b650486fdfd1c1a1c976e9d168","hash":"671a7c4580ed2345d62f32c92074f974f51455e3ea341f4eb371445a8b8f395b"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.5@c55ebba","from_type":"verdict","to_id":"c55ebbacb90fe44bc7097b45240965c5f21abd30","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.5 disposition=CONFIRMED","bead_id":"age-nw28h.2.5","trust_tier":"inferred","ts":"2026-07-11T22:26:32Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-config-independent-verdict-c55ebbacb.txt","prev_hash":"671a7c4580ed2345d62f32c92074f974f51455e3ea341f4eb371445a8b8f395b","payload_hash":"65b9734effd05e76746ac5c29fad79f7eba85a0613aa220fa4669f0744507f9e","hash":"0a925b381ecc0937e340c884fc83a9f992a30a6fc49c0c4f94efc4aa500d1dc8"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.6@e38c499","from_type":"verdict","to_id":"e38c499458012d159a6397e5b56af5be5ba5d3ba","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.6 disposition=CONFIRMED","bead_id":"age-nw28h.2.6","trust_tier":"inferred","ts":"2026-07-11T22:51:49Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-council-gate-independent-verdict-e38c49945.txt","prev_hash":"0a925b381ecc0937e340c884fc83a9f992a30a6fc49c0c4f94efc4aa500d1dc8","payload_hash":"2f39b3dd97dcdad748e0d7a7a01cfa2cd94464e4efe9ad65d9d267d4811c54e0","hash":"94d27584f6296479beddda872b4124eeeab38e82c9f95eda412832c746345864"} +{"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.7.1@d845aa3","from_type":"verdict","to_id":"d845aa394a2040e8b52575bd820af888c6711e46","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.7.1 disposition=CONFIRMED","bead_id":"age-nw28h.2.7.1","trust_tier":"inferred","ts":"2026-07-11T23:13:38Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/doctor-slice1-independent-verdict-d845aa394.txt","prev_hash":"94d27584f6296479beddda872b4124eeeab38e82c9f95eda412832c746345864","payload_hash":"226ba023b25b089c62219aaabd1ae01549eb8c8c7d1e932484f54ba83c95ec90","hash":"3dcc42d990f0bd92fc89cdcc3be75ecc782832e93c3fff25926758dec72934f2"} From 7e4a14127817bf41c79fa2f8ee12ed1fd584fdc8 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:24:59 -0400 Subject: [PATCH 063/108] refactor(doctor): extract read service and make diff read-only (age-nw28h.2.7.2) --- cli/cmd/ao/doctor.go | 8 +- cli/cmd/ao/doctor_legacy_composition.go | 2 + cli/cmd/ao/doctor_read_contract_test.go | 18 ++++ cli/cmd/ao/doctor_surface.go | 56 ++++++------ cli/internal/adapters/doctor/read.go | 52 +++++++++++ cli/internal/adapters/doctor/read_test.go | 24 +++++ cli/internal/doctor/diff_readonly_test.go | 35 ++++++++ cli/internal/doctor/engine.go | 72 ++++++++++++++- cli/internal/doctor/read_service.go | 101 ++++++++++++++++++++++ cli/internal/doctor/read_service_test.go | 81 +++++++++++++++++ 10 files changed, 410 insertions(+), 39 deletions(-) create mode 100644 cli/cmd/ao/doctor_read_contract_test.go create mode 100644 cli/internal/adapters/doctor/read.go create mode 100644 cli/internal/adapters/doctor/read_test.go create mode 100644 cli/internal/doctor/diff_readonly_test.go create mode 100644 cli/internal/doctor/read_service.go create mode 100644 cli/internal/doctor/read_service_test.go diff --git a/cli/cmd/ao/doctor.go b/cli/cmd/ao/doctor.go index 9200782a5..8263b44da 100644 --- a/cli/cmd/ao/doctor.go +++ b/cli/cmd/ao/doctor.go @@ -45,7 +45,7 @@ func gatherDoctorChecks() []quality.Check { func runDoctor(cmd *cobra.Command, args []string) error { // Engine-flag invocations (--fix, --explain, --robot-triage) route entirely // through the diagnose-and-repair engine. - if doctorFix || doctorExplainFlag != "" || doctorRobotTriage { + if doctorFix || doctorExplainFlag != "" || doctorRobotTriage || doctorSince != "" { return runDoctorEngineDefault(cmd) } @@ -87,11 +87,7 @@ func appendEngineFindings(cmd *cobra.Command) error { if doctorWantsJSON() { return nil } - opts, err := doctorEngineOptions() - if err != nil { - return nil // never let the engine break the legacy command - } - rep, derr := doctor.Diagnose(opts) + rep, derr := doctorReadService.Diagnose(cmd.Context(), doctorReadRequest()) if derr != nil || rep == nil || len(rep.Findings) == 0 { return nil } diff --git a/cli/cmd/ao/doctor_legacy_composition.go b/cli/cmd/ao/doctor_legacy_composition.go index 034694cdd..dc59eaeb5 100644 --- a/cli/cmd/ao/doctor_legacy_composition.go +++ b/cli/cmd/ao/doctor_legacy_composition.go @@ -9,3 +9,5 @@ func newLegacyDoctorService() doctorapp.LegacyService { checks := doctoradapter.SystemLegacyChecks(version, IndexDir, IndexFileName, resolveLedgerPath, reviewerHealthService) return doctorapp.NewLegacyService(version, checks) } + +var doctorReadService = doctorapp.NewReadService(version, doctoradapter.ReadRuntime{ToolVersion: version}, doctoradapter.ReadGateway{}) diff --git a/cli/cmd/ao/doctor_read_contract_test.go b/cli/cmd/ao/doctor_read_contract_test.go new file mode 100644 index 000000000..82b1fa247 --- /dev/null +++ b/cli/cmd/ao/doctor_read_contract_test.go @@ -0,0 +1,18 @@ +package main + +import "testing" + +func TestDoctorReadCommandsRejectJunkArguments(t *testing.T) { + if err := doctorCmd.ValidateArgs([]string{"junk"}); err == nil { + t.Fatal("doctor accepted junk positional argument") + } + for _, name := range []string{"capabilities", "health", "robot-docs", "ls", "diff"} { + child, _, err := doctorCmd.Find([]string{name}) + if err != nil { + t.Fatal(err) + } + if err := child.ValidateArgs([]string{"junk"}); err == nil { + t.Fatalf("doctor %s accepted junk positional argument", name) + } + } +} diff --git a/cli/cmd/ao/doctor_surface.go b/cli/cmd/ao/doctor_surface.go index 7a74e2abe..f1d452601 100644 --- a/cli/cmd/ao/doctor_surface.go +++ b/cli/cmd/ao/doctor_surface.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" + "errors" "fmt" "os" "time" @@ -55,7 +56,7 @@ func registerDoctorSurface() { f.BoolVar(&doctorOnline, "online", false, "Enable network probes (default: offline-only)") f.BoolVar(&doctorQuick, "quick", false, "Run only fast-path detectors (< 200ms)") f.StringVar(&doctorSeverity, "severity", "P3", "Minimum severity to emit (P0|P1|P2|P3)") - f.BoolVar(&doctorRobot, "robot", false, "Alias for --json with structured wrapper") + f.BoolVar(&doctorRobot, "robot", false, "Alias for stable JSON report output") f.BoolVar(&doctorRobotTriage, "robot-triage", false, "Emit the mega-command triage JSON") f.StringVar(&doctorExplainFlag, "explain", "", "Expand a single finding by id") @@ -111,6 +112,7 @@ func doctorEngineOptions() (doctor.Options, error) { Severity: doctorSeverity, DryRun: doctorDryRun, JSON: doctorWantsJSON(), + Since: doctorSince, Now: time.Now(), }, nil } @@ -120,6 +122,14 @@ func doctorWantsJSON() bool { return doctorJSON || doctorRobot || jsonFlag } +func doctorReadRequest() doctor.ReadRequest { + return doctor.ReadRequest{ + Only: append([]string(nil), doctorOnly...), Skip: append([]string(nil), doctorSkip...), + Quick: doctorQuick, Online: doctorOnline, Severity: doctorSeverity, + DryRun: doctorDryRun, JSON: doctorWantsJSON(), Since: doctorSince, + } +} + // printDoctorJSON marshals v as indented JSON to stdout. func printDoctorJSON(cmd *cobra.Command, v any) error { data, err := json.MarshalIndent(v, "", " ") @@ -144,12 +154,8 @@ func runDoctorEngineDefault(cmd *cobra.Command) error { if doctorExplainFlag != "" { return runDoctorExplain(cmd, doctorExplainFlag) } - opts, err := doctorEngineOptions() - if err != nil { - return err - } if doctorRobotTriage { - triage, rep, terr := doctor.RobotTriage(opts) + triage, rep, terr := doctorReadService.Triage(cmd.Context(), doctorReadRequest()) if terr != nil { return &doctorExitError{code: doctor.ExitIOError, msg: terr.Error()} } @@ -159,9 +165,13 @@ func runDoctorEngineDefault(cmd *cobra.Command) error { return exitErr(rep.ExitCode, "doctor findings present") } if doctorFix { + opts, err := doctorEngineOptions() + if err != nil { + return err + } return runDoctorFix(cmd, opts) } - rep, derr := doctor.Diagnose(opts) + rep, derr := doctorReadService.Diagnose(cmd.Context(), doctorReadRequest()) if derr != nil { return &doctorExitError{code: doctor.ExitIOError, msg: derr.Error()} } @@ -211,12 +221,12 @@ func runDoctorFix(cmd *cobra.Command, opts doctor.Options) error { // runDoctorExplain expands a single finding. func runDoctorExplain(cmd *cobra.Command, findingID string) error { - cwd, err := os.Getwd() - if err != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: err.Error()} - } - finding, ferr := doctor.Explain(cwd, findingID) + finding, ferr := doctorReadService.Explain(cmd.Context(), findingID) if ferr != nil { + var runtimeFailure *doctor.RuntimeError + if errors.As(ferr, &runtimeFailure) { + return &doctorExitError{code: doctor.ExitIOError, msg: ferr.Error()} + } return &doctorExitError{code: doctor.ExitNoInput, msg: ferr.Error()} } if doctorWantsJSON() { @@ -293,7 +303,7 @@ func newDoctorCapabilitiesCmd() *cobra.Command { Use: "capabilities", Short: "Print the machine-readable doctor contract (JSON)", RunE: func(cmd *cobra.Command, _ []string) error { - caps := doctor.NewCapabilities(version) + caps := doctorReadService.Capabilities(cmd.Context()) return printDoctorJSON(cmd, caps) }, } @@ -304,11 +314,7 @@ func newDoctorHealthCmd() *cobra.Command { Use: "health", Short: "Cheap one-line liveness summary", RunE: func(cmd *cobra.Command, _ []string) error { - cwd, err := os.Getwd() - if err != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: err.Error()} - } - line, hr, herr := doctor.Health(cwd, version) + line, hr, herr := doctorReadService.Health(cmd.Context()) if herr != nil { return &doctorExitError{code: doctor.ExitIOError, msg: herr.Error()} } @@ -329,7 +335,7 @@ func newDoctorRobotDocsCmd() *cobra.Command { Use: "robot-docs", Short: "Print the paste-ready agent handbook (Markdown)", RunE: func(cmd *cobra.Command, _ []string) error { - fmt.Fprint(cmd.OutOrStdout(), doctor.RobotDocs()) + fmt.Fprint(cmd.OutOrStdout(), doctorReadService.RobotDocs(cmd.Context())) return nil }, } @@ -370,11 +376,7 @@ func newDoctorLsCmd() *cobra.Command { Use: "ls", Short: "List runs in .doctor/runs/", RunE: func(cmd *cobra.Command, _ []string) error { - cwd, err := os.Getwd() - if err != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: err.Error()} - } - runs, lerr := doctor.Ls(cwd) + runs, lerr := doctorReadService.List(cmd.Context()) if lerr != nil { return &doctorExitError{code: doctor.ExitIOError, msg: lerr.Error()} } @@ -399,11 +401,7 @@ func newDoctorDiffCmd() *cobra.Command { Use: "diff", Short: "Show what --fix would change (read-only)", RunE: func(cmd *cobra.Command, _ []string) error { - opts, err := doctorEngineOptions() - if err != nil { - return err - } - rep, derr := doctor.Diff(opts) + rep, derr := doctorReadService.Diff(cmd.Context(), doctorReadRequest()) if derr != nil { return &doctorExitError{code: doctor.ExitIOError, msg: derr.Error()} } diff --git a/cli/internal/adapters/doctor/read.go b/cli/internal/adapters/doctor/read.go new file mode 100644 index 000000000..4eaf2b218 --- /dev/null +++ b/cli/internal/adapters/doctor/read.go @@ -0,0 +1,52 @@ +package doctor + +import ( + "context" + "os" + "time" + + doctorapp "github.com/boshu2/agentops/cli/internal/doctor" +) + +type ReadRuntime struct{ ToolVersion string } + +func (runtime ReadRuntime) RepoRoot(context.Context) (string, error) { return os.Getwd() } + +func (runtime ReadRuntime) Options(ctx context.Context, request doctorapp.ReadRequest) (doctorapp.Options, error) { + root, err := runtime.RepoRoot(ctx) + if err != nil { + return doctorapp.Options{}, err + } + home, _ := os.UserHomeDir() + return doctorapp.Options{ + RepoRoot: root, CWD: root, HomeDir: home, ToolVersion: runtime.ToolVersion, + Only: append([]string(nil), request.Only...), Skip: append([]string(nil), request.Skip...), + Quick: request.Quick, Online: request.Online, Severity: request.Severity, + DryRun: request.DryRun, JSON: request.JSON, Since: request.Since, Now: time.Now(), + }, nil +} + +type ReadGateway struct{} + +func (ReadGateway) Diagnose(_ context.Context, options doctorapp.Options) (*doctorapp.Report, error) { + return doctorapp.Diagnose(options) +} +func (ReadGateway) Triage(_ context.Context, options doctorapp.Options) (*doctorapp.RobotTriageResult, *doctorapp.Report, error) { + return doctorapp.RobotTriage(options) +} +func (ReadGateway) Explain(_ context.Context, root, id string) (*doctorapp.Finding, error) { + return doctorapp.Explain(root, id) +} +func (ReadGateway) Capabilities(_ context.Context, version string) *doctorapp.Capabilities { + return doctorapp.NewCapabilities(version) +} +func (ReadGateway) Health(_ context.Context, root, version string) (string, *doctorapp.HealthResult, error) { + return doctorapp.Health(root, version) +} +func (ReadGateway) RobotDocs(context.Context) string { return doctorapp.RobotDocs() } +func (ReadGateway) List(_ context.Context, root string) ([]doctorapp.RunSummary, error) { + return doctorapp.Ls(root) +} +func (ReadGateway) Diff(_ context.Context, options doctorapp.Options) (*doctorapp.Report, error) { + return doctorapp.Diff(options) +} diff --git a/cli/internal/adapters/doctor/read_test.go b/cli/internal/adapters/doctor/read_test.go new file mode 100644 index 000000000..d9707d882 --- /dev/null +++ b/cli/internal/adapters/doctor/read_test.go @@ -0,0 +1,24 @@ +package doctor + +import ( + "context" + "testing" + + doctorapp "github.com/boshu2/agentops/cli/internal/doctor" +) + +func TestReadRuntimeMapsApplicationRequest(t *testing.T) { + options, err := (ReadRuntime{ToolVersion: "3.0.0"}).Options(context.Background(), doctorapp.ReadRequest{ + Only: []string{"one"}, Skip: []string{"two"}, Quick: true, Online: true, + Severity: "P2", JSON: true, Since: "prior", + }) + if err != nil { + t.Fatal(err) + } + if options.ToolVersion != "3.0.0" || options.Since != "prior" || !options.Quick || !options.Online || !options.JSON { + t.Fatalf("options = %+v", options) + } + if len(options.Only) != 1 || options.Only[0] != "one" || len(options.Skip) != 1 || options.Skip[0] != "two" { + t.Fatalf("options = %+v", options) + } +} diff --git a/cli/internal/doctor/diff_readonly_test.go b/cli/internal/doctor/diff_readonly_test.go new file mode 100644 index 000000000..447dbc34f --- /dev/null +++ b/cli/internal/doctor/diff_readonly_test.go @@ -0,0 +1,35 @@ +package doctor + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestDiffDoesNotCreateDoctorArtifactsOrEditGitignore(t *testing.T) { + root := t.TempDir() + gitignore := filepath.Join(root, ".gitignore") + if err := os.WriteFile(gitignore, []byte("existing\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Diff(Options{RepoRoot: root, CWD: root, HomeDir: root, ToolVersion: "test", Now: time.Unix(1_700_000_000, 0)}); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(root, ".doctor")); !os.IsNotExist(err) { + t.Fatalf("diff created .doctor: %v", err) + } + body, err := os.ReadFile(gitignore) + if err != nil || string(body) != "existing\n" { + t.Fatalf("gitignore body=%q err=%v", body, err) + } +} + +func TestFindingsSinceReturnsOnlyNewFindingIDs(t *testing.T) { + current := []Finding{{ID: "same"}, {ID: "new"}} + prior := []Finding{{ID: "same"}, {ID: "gone"}} + got := findingsSince(current, prior) + if len(got) != 1 || got[0].ID != "new" { + t.Fatalf("findingsSince = %+v", got) + } +} diff --git a/cli/internal/doctor/engine.go b/cli/internal/doctor/engine.go index 9b5be3582..bc7073a41 100644 --- a/cli/internal/doctor/engine.go +++ b/cli/internal/doctor/engine.go @@ -40,6 +40,7 @@ type Options struct { Severity string // minimum severity: P0|P1|P2|P3 DryRun bool JSON bool + Since string Now time.Time } @@ -194,12 +195,30 @@ func summarize(findings []Finding) ReportSummary { // the report with the correct exit code (0 healthy / 1 findings / 4 refused / // 6 online-required). func Diagnose(opts Options) (*Report, error) { + return diagnose(opts, true) +} + +func diagnose(opts Options, persist bool) (*Report, error) { now := opts.Now if now.IsZero() { now = time.Now() } sha := targetSHA(opts.RepoRoot) - ra, err := NewRunArtifact(opts.RepoRoot, sha, now) + var prior []Finding + if strings.TrimSpace(opts.Since) != "" { + var err error + prior, err = reportFindings(opts.RepoRoot, opts.Since) + if err != nil { + return nil, err + } + } + var ra *RunArtifact + var err error + if persist { + ra, err = NewRunArtifact(opts.RepoRoot, sha, now) + } else { + ra = transientRunArtifact(opts.RepoRoot, sha, now) + } if err != nil { return nil, fmt.Errorf("doctor: %w", err) } @@ -216,6 +235,9 @@ func Diagnose(opts Options) (*Report, error) { if err != nil { return nil, err } + if prior != nil { + findings = findingsSince(findings, prior) + } rep := buildReport(ra, opts.ToolVersion, sha, now, findings) rep.Summary.OnlineRequired = onlineSkipped @@ -226,12 +248,54 @@ func Diagnose(opts Options) (*Report, error) { } rep.OK = rep.ExitCode == ExitHealthy rep.NextSteps = append(diagnoseNextSteps(findings), onlineCoverageNote(onlineSkipped, opts.Online)...) - if err := persistRun(ra, opts, rep, 0); err != nil { - return rep, err + if persist { + if err := persistRun(ra, opts, rep, 0); err != nil { + return rep, err + } } return rep, nil } +func transientRunArtifact(repoRoot, targetSHA string, now time.Time) *RunArtifact { + started := now.UTC().Truncate(time.Second) + runID := RunID(targetSHA, started) + doctorDir := filepath.Join(repoRoot, ".doctor") + return &RunArtifact{ + RepoRoot: repoRoot, DoctorDir: doctorDir, RunID: runID, + RunDir: filepath.Join(doctorDir, "runs", runDirName(started, runID)), StartedAt: started, + } +} + +func reportFindings(repoRoot, runID string) ([]Finding, error) { + runDir, err := resolveRunDir(repoRoot, runID) + if err != nil { + return nil, fmt.Errorf("doctor: --since %q: %w", runID, err) + } + data, err := os.ReadFile(filepath.Join(runDir, "report.json")) + if err != nil { + return nil, fmt.Errorf("doctor: --since %q: read report: %w", runID, err) + } + var report Report + if err := json.Unmarshal(data, &report); err != nil { + return nil, fmt.Errorf("doctor: --since %q: parse report: %w", runID, err) + } + return report.Findings, nil +} + +func findingsSince(current, prior []Finding) []Finding { + seen := make(map[string]bool, len(prior)) + for _, finding := range prior { + seen[finding.ID] = true + } + out := make([]Finding, 0, len(current)) + for _, finding := range current { + if !seen[finding.ID] { + out = append(out, finding) + } + } + return out +} + // buildReport assembles a Report shell from a finding set. func buildReport(ra *RunArtifact, toolVersion, sha string, now time.Time, findings []Finding) *Report { finished := time.Now() @@ -783,7 +847,7 @@ func Explain(repoRoot, findingID string) (*Finding, error) { // empty registry it always reports a clean diff. func Diff(opts Options) (*Report, error) { opts.DryRun = true - return Diagnose(opts) + return diagnose(opts, false) } // RobotTriageResult is the mega-command JSON for --robot-triage. diff --git a/cli/internal/doctor/read_service.go b/cli/internal/doctor/read_service.go new file mode 100644 index 000000000..14e4f1121 --- /dev/null +++ b/cli/internal/doctor/read_service.go @@ -0,0 +1,101 @@ +package doctor + +import "context" + +type RuntimeError struct{ Err error } + +func (failure *RuntimeError) Error() string { return failure.Err.Error() } +func (failure *RuntimeError) Unwrap() error { return failure.Err } + +type ReadRequest struct { + Only []string + Skip []string + Quick bool + Online bool + Severity string + DryRun bool + JSON bool + Since string +} + +type ReadRuntime interface { + Options(context.Context, ReadRequest) (Options, error) + RepoRoot(context.Context) (string, error) +} + +type ReadGateway interface { + Diagnose(context.Context, Options) (*Report, error) + Triage(context.Context, Options) (*RobotTriageResult, *Report, error) + Explain(context.Context, string, string) (*Finding, error) + Capabilities(context.Context, string) *Capabilities + Health(context.Context, string, string) (string, *HealthResult, error) + RobotDocs(context.Context) string + List(context.Context, string) ([]RunSummary, error) + Diff(context.Context, Options) (*Report, error) +} + +type ReadService struct { + toolVersion string + runtime ReadRuntime + gateway ReadGateway +} + +func NewReadService(toolVersion string, runtime ReadRuntime, gateway ReadGateway) ReadService { + return ReadService{toolVersion: toolVersion, runtime: runtime, gateway: gateway} +} + +func (service ReadService) Diagnose(ctx context.Context, request ReadRequest) (*Report, error) { + options, err := service.runtime.Options(ctx, request) + if err != nil { + return nil, &RuntimeError{Err: err} + } + return service.gateway.Diagnose(ctx, options) +} + +func (service ReadService) Triage(ctx context.Context, request ReadRequest) (*RobotTriageResult, *Report, error) { + options, err := service.runtime.Options(ctx, request) + if err != nil { + return nil, nil, &RuntimeError{Err: err} + } + return service.gateway.Triage(ctx, options) +} + +func (service ReadService) Explain(ctx context.Context, findingID string) (*Finding, error) { + root, err := service.runtime.RepoRoot(ctx) + if err != nil { + return nil, &RuntimeError{Err: err} + } + return service.gateway.Explain(ctx, root, findingID) +} + +func (service ReadService) Capabilities(ctx context.Context) *Capabilities { + return service.gateway.Capabilities(ctx, service.toolVersion) +} + +func (service ReadService) Health(ctx context.Context) (string, *HealthResult, error) { + root, err := service.runtime.RepoRoot(ctx) + if err != nil { + return "", nil, &RuntimeError{Err: err} + } + return service.gateway.Health(ctx, root, service.toolVersion) +} + +func (service ReadService) RobotDocs(ctx context.Context) string { + return service.gateway.RobotDocs(ctx) +} + +func (service ReadService) List(ctx context.Context) ([]RunSummary, error) { + root, err := service.runtime.RepoRoot(ctx) + if err != nil { + return nil, &RuntimeError{Err: err} + } + return service.gateway.List(ctx, root) +} + +func (service ReadService) Diff(ctx context.Context, request ReadRequest) (*Report, error) { + options, err := service.runtime.Options(ctx, request) + if err != nil { + return nil, &RuntimeError{Err: err} + } + return service.gateway.Diff(ctx, options) +} diff --git a/cli/internal/doctor/read_service_test.go b/cli/internal/doctor/read_service_test.go new file mode 100644 index 000000000..cd01556e6 --- /dev/null +++ b/cli/internal/doctor/read_service_test.go @@ -0,0 +1,81 @@ +package doctor + +import ( + "context" + "errors" + "testing" +) + +type fakeReadRuntime struct { + options Options + request *ReadRequest +} + +func (runtime fakeReadRuntime) Options(_ context.Context, request ReadRequest) (Options, error) { + if runtime.request != nil { + *runtime.request = request + } + return runtime.options, nil +} +func (runtime fakeReadRuntime) RepoRoot(context.Context) (string, error) { + return runtime.options.RepoRoot, nil +} + +type fakeReadGateway struct{ diagnosed, diffed bool } + +func (gateway *fakeReadGateway) Diagnose(context.Context, Options) (*Report, error) { + gateway.diagnosed = true + return &Report{}, nil +} +func (gateway *fakeReadGateway) Triage(context.Context, Options) (*RobotTriageResult, *Report, error) { + return &RobotTriageResult{}, &Report{}, nil +} +func (gateway *fakeReadGateway) Explain(context.Context, string, string) (*Finding, error) { + return &Finding{}, nil +} +func (gateway *fakeReadGateway) Capabilities(context.Context, string) *Capabilities { + return &Capabilities{} +} +func (gateway *fakeReadGateway) Health(context.Context, string, string) (string, *HealthResult, error) { + return "ok", &HealthResult{}, nil +} +func (gateway *fakeReadGateway) RobotDocs(context.Context) string { return "docs" } +func (gateway *fakeReadGateway) List(context.Context, string) ([]RunSummary, error) { return nil, nil } +func (gateway *fakeReadGateway) Diff(context.Context, Options) (*Report, error) { + gateway.diffed = true + return &Report{}, nil +} + +func TestReadServiceDelegatesDiagnoseAndDiffThroughPorts(t *testing.T) { + gateway := &fakeReadGateway{} + var captured ReadRequest + service := NewReadService("3.0.0", fakeReadRuntime{options: Options{RepoRoot: "/repo"}, request: &captured}, gateway) + if _, err := service.Diagnose(context.Background(), ReadRequest{Since: "old"}); err != nil { + t.Fatal(err) + } + if captured.Since != "old" { + t.Fatalf("request = %+v", captured) + } + if _, err := service.Diff(context.Background(), ReadRequest{}); err != nil { + t.Fatal(err) + } + if !gateway.diagnosed || !gateway.diffed { + t.Fatalf("gateway calls diagnose=%t diff=%t", gateway.diagnosed, gateway.diffed) + } +} + +type failingReadRuntime struct{} + +func (failingReadRuntime) Options(context.Context, ReadRequest) (Options, error) { + return Options{}, errors.New("cwd") +} +func (failingReadRuntime) RepoRoot(context.Context) (string, error) { return "", errors.New("cwd") } + +func TestReadServiceMarksRuntimeFailures(t *testing.T) { + service := NewReadService("3.0.0", failingReadRuntime{}, &fakeReadGateway{}) + _, err := service.Explain(context.Background(), "finding") + var runtimeFailure *RuntimeError + if !errors.As(err, &runtimeFailure) { + t.Fatalf("error = %#v, want RuntimeError", err) + } +} From c199e972557a4f3d720520bcd999b1581de96d9d Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:29:00 -0400 Subject: [PATCH 064/108] fix(doctor): keep transient diff reports truthful and schema-stable --- cli/internal/doctor/diff_readonly_test.go | 18 +++++++++++++++++- cli/internal/doctor/engine.go | 11 +++++++++-- cli/internal/doctor/runartifact.go | 2 ++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/cli/internal/doctor/diff_readonly_test.go b/cli/internal/doctor/diff_readonly_test.go index 447dbc34f..f958c80e9 100644 --- a/cli/internal/doctor/diff_readonly_test.go +++ b/cli/internal/doctor/diff_readonly_test.go @@ -1,6 +1,7 @@ package doctor import ( + "encoding/json" "os" "path/filepath" "testing" @@ -13,9 +14,24 @@ func TestDiffDoesNotCreateDoctorArtifactsOrEditGitignore(t *testing.T) { if err := os.WriteFile(gitignore, []byte("existing\n"), 0o644); err != nil { t.Fatal(err) } - if _, err := Diff(Options{RepoRoot: root, CWD: root, HomeDir: root, ToolVersion: "test", Now: time.Unix(1_700_000_000, 0)}); err != nil { + report, err := Diff(Options{RepoRoot: root, CWD: root, HomeDir: root, ToolVersion: "test", Now: time.Unix(1_700_000_000, 0)}) + if err != nil { t.Fatal(err) } + if report.Findings == nil || report.RunID != "" || report.RunDir != "" { + t.Fatalf("transient report = %+v", report) + } + encoded, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + var shape map[string]any + if err := json.Unmarshal(encoded, &shape); err != nil { + t.Fatal(err) + } + if _, ok := shape["findings"].([]any); !ok { + t.Fatalf("diff JSON findings = %#v", shape["findings"]) + } if _, err := os.Stat(filepath.Join(root, ".doctor")); !os.IsNotExist(err) { t.Fatalf("diff created .doctor: %v", err) } diff --git a/cli/internal/doctor/engine.go b/cli/internal/doctor/engine.go index bc7073a41..1d701b482 100644 --- a/cli/internal/doctor/engine.go +++ b/cli/internal/doctor/engine.go @@ -299,13 +299,20 @@ func findingsSince(current, prior []Finding) []Finding { // buildReport assembles a Report shell from a finding set. func buildReport(ra *RunArtifact, toolVersion, sha string, now time.Time, findings []Finding) *Report { finished := time.Now() + if findings == nil { + findings = []Finding{} + } + runID, runDir := ra.RunID, filepath.Join(".doctor", "runs", filepath.Base(ra.RunDir)) + if !ra.Persisted { + runID, runDir = "", "" + } return &Report{ SchemaVersion: SchemaVersion, Tool: ToolName, ToolVersion: toolVersion, DoctorVersion: DoctorVersion, - RunID: ra.RunID, - RunDir: filepath.Join(".doctor", "runs", filepath.Base(ra.RunDir)), + RunID: runID, + RunDir: runDir, StartedAt: ra.StartedAt.Format(time.RFC3339), FinishedAt: finished.UTC().Format(time.RFC3339Nano), DurationMS: finished.Sub(now).Milliseconds(), diff --git a/cli/internal/doctor/runartifact.go b/cli/internal/doctor/runartifact.go index 7db97417d..87e8e303a 100644 --- a/cli/internal/doctor/runartifact.go +++ b/cli/internal/doctor/runartifact.go @@ -43,6 +43,7 @@ type RunArtifact struct { RunID string // 6-char short id RunDir string // absolute path to .doctor/runs/__ StartedAt time.Time + Persisted bool } // NewRunArtifact creates a fresh run directory under /.doctor/runs/, @@ -78,6 +79,7 @@ func NewRunArtifact(repoRoot, targetSHA string, now time.Time) (*RunArtifact, er RunID: runID, RunDir: runDir, StartedAt: ts, + Persisted: true, } if err := ra.updateLatestSymlink(); err != nil { return nil, err From d62cc714a976091aac3dfb4cb9f4649db45571d7 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:30:55 -0400 Subject: [PATCH 065/108] fix(doctor): keep since comparisons transient --- cli/internal/doctor/diff_readonly_test.go | 44 +++++++++++++++++++++++ cli/internal/doctor/engine.go | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/cli/internal/doctor/diff_readonly_test.go b/cli/internal/doctor/diff_readonly_test.go index f958c80e9..4f12e9835 100644 --- a/cli/internal/doctor/diff_readonly_test.go +++ b/cli/internal/doctor/diff_readonly_test.go @@ -49,3 +49,47 @@ func TestFindingsSinceReturnsOnlyNewFindingIDs(t *testing.T) { t.Fatalf("findingsSince = %+v", got) } } + +func TestDiagnoseSinceDoesNotReplaceLatestSnapshot(t *testing.T) { + root := t.TempDir() + runName := "2026-01-01T00-00-00Z__prior1" + runDir := filepath.Join(root, ".doctor", "runs", runName) + if err := os.MkdirAll(runDir, 0o755); err != nil { + t.Fatal(err) + } + prior := Report{ + SchemaVersion: SchemaVersion, RunID: "prior1", StartedAt: "2026-01-01T00:00:00Z", + Summary: ReportSummary{TotalFindings: 1, BySeverity: map[string]int{"P0": 0, "P1": 1, "P2": 0, "P3": 0}}, + Findings: []Finding{{ID: "persistent", Severity: "P1"}}, ExitCode: ExitFindings, + } + data, err := json.Marshal(prior) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(runDir, "report.json"), data, 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join("runs", runName), filepath.Join(root, ".doctor", "latest")); err != nil { + t.Fatal(err) + } + + report, err := Diagnose(Options{RepoRoot: root, CWD: root, HomeDir: root, ToolVersion: "test", Since: "prior1", Now: time.Unix(1_700_000_000, 0)}) + if err != nil { + t.Fatal(err) + } + if report.RunID != "" || report.RunDir != "" { + t.Fatalf("delta report persisted identity: %+v", report) + } + target, err := os.Readlink(filepath.Join(root, ".doctor", "latest")) + if err != nil || target != filepath.Join("runs", runName) { + t.Fatalf("latest=%q err=%v", target, err) + } + entries, err := os.ReadDir(filepath.Join(root, ".doctor", "runs")) + if err != nil || len(entries) != 1 { + t.Fatalf("runs=%v err=%v", entries, err) + } + _, health, err := Health(root, "test") + if err != nil || health.Findings != 1 || health.ExitCode != ExitFindings { + t.Fatalf("health=%+v err=%v", health, err) + } +} diff --git a/cli/internal/doctor/engine.go b/cli/internal/doctor/engine.go index 1d701b482..dcc8c089a 100644 --- a/cli/internal/doctor/engine.go +++ b/cli/internal/doctor/engine.go @@ -195,7 +195,7 @@ func summarize(findings []Finding) ReportSummary { // the report with the correct exit code (0 healthy / 1 findings / 4 refused / // 6 online-required). func Diagnose(opts Options) (*Report, error) { - return diagnose(opts, true) + return diagnose(opts, strings.TrimSpace(opts.Since) == "") } func diagnose(opts Options, persist bool) (*Report, error) { From ca0137a4d77842ade48b8b344e87625481c00776 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:33:29 -0400 Subject: [PATCH 066/108] chore(provenance): bind pawl CONFIRMED verdict for age-nw28h.2.7.2 #trivial --- docs/provenance/ledger.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/provenance/ledger.jsonl b/docs/provenance/ledger.jsonl index a67691432..e0a51a110 100644 --- a/docs/provenance/ledger.jsonl +++ b/docs/provenance/ledger.jsonl @@ -523,3 +523,4 @@ {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.5@c55ebba","from_type":"verdict","to_id":"c55ebbacb90fe44bc7097b45240965c5f21abd30","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.5 disposition=CONFIRMED","bead_id":"age-nw28h.2.5","trust_tier":"inferred","ts":"2026-07-11T22:26:32Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-config-independent-verdict-c55ebbacb.txt","prev_hash":"671a7c4580ed2345d62f32c92074f974f51455e3ea341f4eb371445a8b8f395b","payload_hash":"65b9734effd05e76746ac5c29fad79f7eba85a0613aa220fa4669f0744507f9e","hash":"0a925b381ecc0937e340c884fc83a9f992a30a6fc49c0c4f94efc4aa500d1dc8"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.6@e38c499","from_type":"verdict","to_id":"e38c499458012d159a6397e5b56af5be5ba5d3ba","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.6 disposition=CONFIRMED","bead_id":"age-nw28h.2.6","trust_tier":"inferred","ts":"2026-07-11T22:51:49Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-council-gate-independent-verdict-e38c49945.txt","prev_hash":"0a925b381ecc0937e340c884fc83a9f992a30a6fc49c0c4f94efc4aa500d1dc8","payload_hash":"2f39b3dd97dcdad748e0d7a7a01cfa2cd94464e4efe9ad65d9d267d4811c54e0","hash":"94d27584f6296479beddda872b4124eeeab38e82c9f95eda412832c746345864"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.7.1@d845aa3","from_type":"verdict","to_id":"d845aa394a2040e8b52575bd820af888c6711e46","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.7.1 disposition=CONFIRMED","bead_id":"age-nw28h.2.7.1","trust_tier":"inferred","ts":"2026-07-11T23:13:38Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/doctor-slice1-independent-verdict-d845aa394.txt","prev_hash":"94d27584f6296479beddda872b4124eeeab38e82c9f95eda412832c746345864","payload_hash":"226ba023b25b089c62219aaabd1ae01549eb8c8c7d1e932484f54ba83c95ec90","hash":"3dcc42d990f0bd92fc89cdcc3be75ecc782832e93c3fff25926758dec72934f2"} +{"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.7.2@d62cc71","from_type":"verdict","to_id":"d62cc714a976091aac3dfb4cb9f4649db45571d7","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.7.2 disposition=CONFIRMED","bead_id":"age-nw28h.2.7.2","trust_tier":"inferred","ts":"2026-07-11T23:33:28Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/doctor-slice2-independent-verdict-d62cc714a.txt","prev_hash":"3dcc42d990f0bd92fc89cdcc3be75ecc782832e93c3fff25926758dec72934f2","payload_hash":"faa83455e0c399af9a22b257fbbf3fe2ea188fba2b3dbbcb976d649de67ebd12","hash":"60ad2a9db73ba023c0f9008a460e74f09d28cbf8ee2b9f9d72fb44c2a67d9c56"} From 0795c408449f41a6df01e386d160a34695d31e65 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:33:42 -0400 Subject: [PATCH 067/108] chore(provenance): bind pawl CONFIRMED verdict for age-nw28h.2.7.2 #trivial --- docs/provenance/ledger.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/provenance/ledger.jsonl b/docs/provenance/ledger.jsonl index e0a51a110..b739df4e1 100644 --- a/docs/provenance/ledger.jsonl +++ b/docs/provenance/ledger.jsonl @@ -524,3 +524,4 @@ {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.6@e38c499","from_type":"verdict","to_id":"e38c499458012d159a6397e5b56af5be5ba5d3ba","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.6 disposition=CONFIRMED","bead_id":"age-nw28h.2.6","trust_tier":"inferred","ts":"2026-07-11T22:51:49Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/family-council-gate-independent-verdict-e38c49945.txt","prev_hash":"0a925b381ecc0937e340c884fc83a9f992a30a6fc49c0c4f94efc4aa500d1dc8","payload_hash":"2f39b3dd97dcdad748e0d7a7a01cfa2cd94464e4efe9ad65d9d267d4811c54e0","hash":"94d27584f6296479beddda872b4124eeeab38e82c9f95eda412832c746345864"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.7.1@d845aa3","from_type":"verdict","to_id":"d845aa394a2040e8b52575bd820af888c6711e46","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.7.1 disposition=CONFIRMED","bead_id":"age-nw28h.2.7.1","trust_tier":"inferred","ts":"2026-07-11T23:13:38Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/doctor-slice1-independent-verdict-d845aa394.txt","prev_hash":"94d27584f6296479beddda872b4124eeeab38e82c9f95eda412832c746345864","payload_hash":"226ba023b25b089c62219aaabd1ae01549eb8c8c7d1e932484f54ba83c95ec90","hash":"3dcc42d990f0bd92fc89cdcc3be75ecc782832e93c3fff25926758dec72934f2"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.7.2@d62cc71","from_type":"verdict","to_id":"d62cc714a976091aac3dfb4cb9f4649db45571d7","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.7.2 disposition=CONFIRMED","bead_id":"age-nw28h.2.7.2","trust_tier":"inferred","ts":"2026-07-11T23:33:28Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/doctor-slice2-independent-verdict-d62cc714a.txt","prev_hash":"3dcc42d990f0bd92fc89cdcc3be75ecc782832e93c3fff25926758dec72934f2","payload_hash":"faa83455e0c399af9a22b257fbbf3fe2ea188fba2b3dbbcb976d649de67ebd12","hash":"60ad2a9db73ba023c0f9008a460e74f09d28cbf8ee2b9f9d72fb44c2a67d9c56"} +{"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.7.2@ca0137a","from_type":"verdict","to_id":"ca0137a4d77842ade48b8b344e87625481c00776","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.7.2 disposition=CONFIRMED","bead_id":"age-nw28h.2.7.2","trust_tier":"inferred","ts":"2026-07-11T23:33:42Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/doctor-slice2-independent-verdict-d62cc714a.txt","prev_hash":"60ad2a9db73ba023c0f9008a460e74f09d28cbf8ee2b9f9d72fb44c2a67d9c56","payload_hash":"fd520c5bf41a440beb3fdd887884e66666c5635842146fc9b3ac27dc024bedf7","hash":"ff704fc5e563e0af2a024b1a803cbf619b17df25ee76cff4db5750779ce1f84d"} From b38057f73cde23dfad3953a92ea61c095bee6a50 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:36:32 -0400 Subject: [PATCH 068/108] fix(doctor): make fix dry-run truly transient (age-nw28h.2.7.3) --- cli/internal/doctor/diff_readonly_test.go | 43 +++++++++++++++++++++++ cli/internal/doctor/engine.go | 37 +++++++++++++------ cli/internal/doctor/mutate.go | 10 +++--- 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/cli/internal/doctor/diff_readonly_test.go b/cli/internal/doctor/diff_readonly_test.go index 4f12e9835..005d04696 100644 --- a/cli/internal/doctor/diff_readonly_test.go +++ b/cli/internal/doctor/diff_readonly_test.go @@ -41,6 +41,49 @@ func TestDiffDoesNotCreateDoctorArtifactsOrEditGitignore(t *testing.T) { } } +func TestFixDryRunDoesNotCreateDoctorArtifactsOrEditGitignore(t *testing.T) { + root := t.TempDir() + gitignore := filepath.Join(root, ".gitignore") + if err := os.WriteFile(gitignore, []byte("existing\n"), 0o644); err != nil { + t.Fatal(err) + } + configDir := filepath.Join(root, ".agentops") + if err := os.MkdirAll(configDir, 0o755); err != nil { + t.Fatal(err) + } + configPath := filepath.Join(configDir, "config.yaml") + brokenConfig := []byte("broken: [\n") + if err := os.WriteFile(configPath, brokenConfig, 0o644); err != nil { + t.Fatal(err) + } + + report, err := Fix(Options{ + RepoRoot: root, CWD: root, HomeDir: root, ToolVersion: "test", + Only: []string{fmInvalidConfigYAML}, DryRun: true, + Now: time.Unix(1_700_000_000, 0), + }) + if err != nil { + t.Fatal(err) + } + if report.ExitCode != ExitHealthy || report.Summary.TotalFindings != 1 { + t.Fatalf("dry-run report = %+v", report) + } + if report.RunID != "" || report.RunDir != "" || report.ActionsPath != "" || report.BackupsDir != "" || report.UndoCommand != "" { + t.Fatalf("dry-run report advertises persisted artifacts: %+v", report) + } + if _, err := os.Stat(filepath.Join(root, ".doctor")); !os.IsNotExist(err) { + t.Fatalf("fix --dry-run created .doctor: %v", err) + } + body, err := os.ReadFile(gitignore) + if err != nil || string(body) != "existing\n" { + t.Fatalf("gitignore body=%q err=%v", body, err) + } + body, err = os.ReadFile(configPath) + if err != nil || string(body) != string(brokenConfig) { + t.Fatalf("config body=%q err=%v", body, err) + } +} + func TestFindingsSinceReturnsOnlyNewFindingIDs(t *testing.T) { current := []Finding{{ID: "same"}, {ID: "new"}} prior := []Finding{{ID: "same"}, {ID: "gone"}} diff --git a/cli/internal/doctor/engine.go b/cli/internal/doctor/engine.go index dcc8c089a..f08422ef6 100644 --- a/cli/internal/doctor/engine.go +++ b/cli/internal/doctor/engine.go @@ -367,9 +367,15 @@ func Fix(opts Options) (*Report, error) { now = time.Now() } sha := targetSHA(opts.RepoRoot) - ra, err := NewRunArtifact(opts.RepoRoot, sha, now) - if err != nil { - return nil, fmt.Errorf("doctor: %w", err) + var ra *RunArtifact + var err error + if opts.DryRun { + ra = transientRunArtifact(opts.RepoRoot, sha, now) + } else { + ra, err = NewRunArtifact(opts.RepoRoot, sha, now) + if err != nil { + return nil, fmt.Errorf("doctor: %w", err) + } } env := &DetectEnv{ RepoRoot: opts.RepoRoot, CWD: opts.CWD, HomeDir: opts.HomeDir, @@ -388,14 +394,20 @@ func Fix(opts Options) (*Report, error) { rep.OK = true rep.NextSteps = append([]string{"No findings. Nothing to fix."}, onlineCoverageNote(onlineSkipped, opts.Online)...) + if opts.DryRun { + return rep, nil + } return rep, persistRun(ra, opts, rep, 0) } - actionsFile, err := ra.OpenActionsFile() - if err != nil { - return rep, err + var actionsFile *os.File + if !opts.DryRun { + actionsFile, err = ra.OpenActionsFile() + if err != nil { + return rep, err + } + defer func() { _ = actionsFile.Close() }() } - defer func() { _ = actionsFile.Close() }() caps := NewCapabilities(opts.ToolVersion) locks := NewLockManager(filepath.Join(opts.RepoRoot, ".doctor", "locks")) @@ -404,9 +416,11 @@ func Fix(opts Options) (*Report, error) { totalActions, fixedCount, failed, skipped := applyFixers(opts.RepoRoot, mctx, env, findings) rep.ActionsTaken = totalActions - rep.ActionsPath = filepath.Join(rep.RunDir, "actions.jsonl") - rep.BackupsDir = filepath.Join(rep.RunDir, "backups") - rep.UndoCommand = fmt.Sprintf("ao doctor undo %s", filepath.Base(ra.RunDir)) + if !opts.DryRun { + rep.ActionsPath = filepath.Join(rep.RunDir, "actions.jsonl") + rep.BackupsDir = filepath.Join(rep.RunDir, "backups") + rep.UndoCommand = fmt.Sprintf("ao doctor undo %s", filepath.Base(ra.RunDir)) + } rep.ExitCode = fixExitCode(len(findings), fixedCount, failed) if opts.DryRun { // A dry-run executes nothing; per the doctor CLI contract @@ -424,6 +438,9 @@ func Fix(opts Options) (*Report, error) { rep.NextSteps = append(rep.NextSteps, "refused (ambiguous, resolve by hand): "+s) } rep.OK = rep.ExitCode == ExitHealthy + if opts.DryRun { + return rep, nil + } return rep, persistRun(ra, opts, rep, totalActions) } diff --git a/cli/internal/doctor/mutate.go b/cli/internal/doctor/mutate.go index 291d8faf5..50567c942 100644 --- a/cli/internal/doctor/mutate.go +++ b/cli/internal/doctor/mutate.go @@ -156,11 +156,13 @@ func cmpStrict(a, b string) error { // allowed to write to user-state disk under fix. func Mutate(ctx *MutateContext, path string, op Op) (ActionResult, error) { // Step 1 — per-path advisory lock. - guard, err := ctx.Locks.Acquire(path) - if err != nil { - return ActionResult{Err: err}, err + if !ctx.DryRun { + guard, err := ctx.Locks.Acquire(path) + if err != nil { + return ActionResult{Err: err}, err + } + defer func() { _ = guard.Release() }() } - defer func() { _ = guard.Release() }() // Step 2 — before_hash. beforeBytes, err := readOrEmpty(path) From 8cf46a670c3f861f6e5ba51dba64b043d796f74a Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:39:55 -0400 Subject: [PATCH 069/108] refactor(doctor): extract mutation application ports (age-nw28h.2.7.3) --- cli/cmd/ao/doctor_legacy_composition.go | 1 + cli/cmd/ao/doctor_surface.go | 51 ++++----------- cli/internal/adapters/doctor/mutation.go | 27 ++++++++ cli/internal/adapters/doctor/mutation_test.go | 51 +++++++++++++++ cli/internal/doctor/mutation_service.go | 44 +++++++++++++ cli/internal/doctor/mutation_service_test.go | 64 +++++++++++++++++++ 6 files changed, 199 insertions(+), 39 deletions(-) create mode 100644 cli/internal/adapters/doctor/mutation.go create mode 100644 cli/internal/adapters/doctor/mutation_test.go create mode 100644 cli/internal/doctor/mutation_service.go create mode 100644 cli/internal/doctor/mutation_service_test.go diff --git a/cli/cmd/ao/doctor_legacy_composition.go b/cli/cmd/ao/doctor_legacy_composition.go index dc59eaeb5..4fc11bf32 100644 --- a/cli/cmd/ao/doctor_legacy_composition.go +++ b/cli/cmd/ao/doctor_legacy_composition.go @@ -11,3 +11,4 @@ func newLegacyDoctorService() doctorapp.LegacyService { } var doctorReadService = doctorapp.NewReadService(version, doctoradapter.ReadRuntime{ToolVersion: version}, doctoradapter.ReadGateway{}) +var doctorMutationService = doctorapp.NewMutationService(doctoradapter.MutationRuntime{ToolVersion: version}, doctoradapter.MutationGateway{}) diff --git a/cli/cmd/ao/doctor_surface.go b/cli/cmd/ao/doctor_surface.go index f1d452601..498fcd1e5 100644 --- a/cli/cmd/ao/doctor_surface.go +++ b/cli/cmd/ao/doctor_surface.go @@ -90,38 +90,19 @@ func registerDoctorSurface() { }) } -// doctorEngineOptions builds doctor.Options from the current process context. -func doctorEngineOptions() (doctor.Options, error) { - cwd, err := os.Getwd() - if err != nil { - return doctor.Options{}, &doctorExitError{code: doctor.ExitIOError, msg: err.Error()} - } - home, err := os.UserHomeDir() - if err != nil { - home = "" - } - return doctor.Options{ - RepoRoot: cwd, - CWD: cwd, - HomeDir: home, - ToolVersion: version, - Only: doctorOnly, - Skip: doctorSkip, - Quick: doctorQuick, - Online: doctorOnline, - Severity: doctorSeverity, - DryRun: doctorDryRun, - JSON: doctorWantsJSON(), - Since: doctorSince, - Now: time.Now(), - }, nil -} - // doctorWantsJSON reports whether the caller asked for JSON output. func doctorWantsJSON() bool { return doctorJSON || doctorRobot || jsonFlag } +func doctorMutationRequest() doctor.MutationRequest { + return doctor.MutationRequest{ + Only: append([]string(nil), doctorOnly...), Skip: append([]string(nil), doctorSkip...), + Quick: doctorQuick, Online: doctorOnline, Severity: doctorSeverity, + DryRun: doctorDryRun, JSON: doctorWantsJSON(), + } +} + func doctorReadRequest() doctor.ReadRequest { return doctor.ReadRequest{ Only: append([]string(nil), doctorOnly...), Skip: append([]string(nil), doctorSkip...), @@ -165,11 +146,7 @@ func runDoctorEngineDefault(cmd *cobra.Command) error { return exitErr(rep.ExitCode, "doctor findings present") } if doctorFix { - opts, err := doctorEngineOptions() - if err != nil { - return err - } - return runDoctorFix(cmd, opts) + return runDoctorFix(cmd, doctorMutationRequest()) } rep, derr := doctorReadService.Diagnose(cmd.Context(), doctorReadRequest()) if derr != nil { @@ -199,8 +176,8 @@ func renderEngineFindings(cmd *cobra.Command, rep *doctor.Report) { } // runDoctorFix runs the fix engine and renders/exits accordingly. -func runDoctorFix(cmd *cobra.Command, opts doctor.Options) error { - rep, err := doctor.Fix(opts) +func runDoctorFix(cmd *cobra.Command, request doctor.MutationRequest) error { + rep, err := doctorMutationService.Fix(cmd.Context(), request) if err != nil { return &doctorExitError{code: doctor.ExitIOError, msg: err.Error()} } @@ -243,11 +220,7 @@ func newDoctorFixCmd() *cobra.Command { Use: "fix", Short: "Run detectors, then apply fixers (backs up before every mutation)", RunE: func(cmd *cobra.Command, _ []string) error { - opts, err := doctorEngineOptions() - if err != nil { - return err - } - return runDoctorFix(cmd, opts) + return runDoctorFix(cmd, doctorMutationRequest()) }, } } diff --git a/cli/internal/adapters/doctor/mutation.go b/cli/internal/adapters/doctor/mutation.go new file mode 100644 index 000000000..42b95e2ec --- /dev/null +++ b/cli/internal/adapters/doctor/mutation.go @@ -0,0 +1,27 @@ +package doctor + +import ( + "context" + + doctorapp "github.com/boshu2/agentops/cli/internal/doctor" +) + +// MutationRuntime snapshots process state for doctor mutation requests. +type MutationRuntime struct{ ToolVersion string } + +// Options maps an application mutation request to engine options. +func (runtime MutationRuntime) Options(ctx context.Context, request doctorapp.MutationRequest) (doctorapp.Options, error) { + return ReadRuntime(runtime).Options(ctx, doctorapp.ReadRequest{ + Only: append([]string(nil), request.Only...), Skip: append([]string(nil), request.Skip...), + Quick: request.Quick, Online: request.Online, Severity: request.Severity, + DryRun: request.DryRun, JSON: request.JSON, + }) +} + +// MutationGateway adapts the application service to the existing fix engine. +type MutationGateway struct{} + +// Fix executes the doctor fix engine. +func (MutationGateway) Fix(_ context.Context, options doctorapp.Options) (*doctorapp.Report, error) { + return doctorapp.Fix(options) +} diff --git a/cli/internal/adapters/doctor/mutation_test.go b/cli/internal/adapters/doctor/mutation_test.go new file mode 100644 index 000000000..8bf78802a --- /dev/null +++ b/cli/internal/adapters/doctor/mutation_test.go @@ -0,0 +1,51 @@ +package doctor + +import ( + "context" + "os" + "path/filepath" + "testing" + + doctorapp "github.com/boshu2/agentops/cli/internal/doctor" +) + +func TestMutationRuntimeMapsApplicationRequest(t *testing.T) { + options, err := (MutationRuntime{ToolVersion: "3.0.0"}).Options(context.Background(), doctorapp.MutationRequest{ + Only: []string{"one"}, Skip: []string{"two"}, Quick: true, Online: true, + Severity: "P2", DryRun: true, JSON: true, + }) + if err != nil { + t.Fatal(err) + } + if options.ToolVersion != "3.0.0" || !options.Quick || !options.Online || !options.DryRun || !options.JSON { + t.Fatalf("options=%+v", options) + } + if len(options.Only) != 1 || options.Only[0] != "one" || len(options.Skip) != 1 || options.Skip[0] != "two" { + t.Fatalf("options=%+v", options) + } +} + +func TestMutationServiceDryRunThroughRealAdaptersDoesNotPersist(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + configDir := filepath.Join(root, ".agentops") + if err := os.MkdirAll(configDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(configDir, "config.yaml"), []byte("broken: [\n"), 0o644); err != nil { + t.Fatal(err) + } + service := doctorapp.NewMutationService(MutationRuntime{ToolVersion: "test"}, MutationGateway{}) + report, err := service.Fix(context.Background(), doctorapp.MutationRequest{ + Only: []string{"fm-cli-config-invalid-config-yaml-swallowed"}, DryRun: true, + }) + if err != nil { + t.Fatal(err) + } + if report.ExitCode != doctorapp.ExitHealthy || report.Summary.TotalFindings != 1 { + t.Fatalf("report=%+v", report) + } + if _, err := os.Stat(filepath.Join(root, ".doctor")); !os.IsNotExist(err) { + t.Fatalf("service fix --dry-run created .doctor: %v", err) + } +} diff --git a/cli/internal/doctor/mutation_service.go b/cli/internal/doctor/mutation_service.go new file mode 100644 index 000000000..614cd064c --- /dev/null +++ b/cli/internal/doctor/mutation_service.go @@ -0,0 +1,44 @@ +package doctor + +import "context" + +// MutationRequest carries caller-owned options for a doctor fix invocation. +type MutationRequest struct { + Only []string + Skip []string + Quick bool + Online bool + Severity string + DryRun bool + JSON bool +} + +// MutationRuntime snapshots process state needed to execute a mutation request. +type MutationRuntime interface { + Options(context.Context, MutationRequest) (Options, error) +} + +// MutationGateway executes the doctor mutation engine. +type MutationGateway interface { + Fix(context.Context, Options) (*Report, error) +} + +// MutationService owns doctor fix application orchestration. +type MutationService struct { + runtime MutationRuntime + gateway MutationGateway +} + +// NewMutationService constructs the doctor mutation application service. +func NewMutationService(runtime MutationRuntime, gateway MutationGateway) MutationService { + return MutationService{runtime: runtime, gateway: gateway} +} + +// Fix resolves runtime state and executes a doctor fix request. +func (service MutationService) Fix(ctx context.Context, request MutationRequest) (*Report, error) { + options, err := service.runtime.Options(ctx, request) + if err != nil { + return nil, &RuntimeError{Err: err} + } + return service.gateway.Fix(ctx, options) +} diff --git a/cli/internal/doctor/mutation_service_test.go b/cli/internal/doctor/mutation_service_test.go new file mode 100644 index 000000000..6f30d48c8 --- /dev/null +++ b/cli/internal/doctor/mutation_service_test.go @@ -0,0 +1,64 @@ +package doctor + +import ( + "context" + "errors" + "testing" +) + +type fakeMutationRuntime struct { + options Options + request *MutationRequest + err error +} + +func (runtime fakeMutationRuntime) Options(_ context.Context, request MutationRequest) (Options, error) { + if runtime.request != nil { + *runtime.request = request + } + return runtime.options, runtime.err +} + +type fakeMutationGateway struct { + options Options + report *Report +} + +func (gateway *fakeMutationGateway) Fix(_ context.Context, options Options) (*Report, error) { + gateway.options = options + return gateway.report, nil +} + +func TestMutationServiceDelegatesFixThroughPorts(t *testing.T) { + var captured MutationRequest + want := &Report{ExitCode: ExitFixPartial} + gateway := &fakeMutationGateway{report: want} + service := NewMutationService( + fakeMutationRuntime{options: Options{RepoRoot: "/repo", DryRun: true}, request: &captured}, + gateway, + ) + + request := MutationRequest{Only: []string{"one"}, DryRun: true, JSON: true} + got, err := service.Fix(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if got != want || gateway.options.RepoRoot != "/repo" || !gateway.options.DryRun { + t.Fatalf("report=%+v options=%+v", got, gateway.options) + } + if len(captured.Only) != 1 || captured.Only[0] != "one" || !captured.DryRun || !captured.JSON { + t.Fatalf("request=%+v", captured) + } +} + +func TestMutationServiceMarksRuntimeFailures(t *testing.T) { + service := NewMutationService( + fakeMutationRuntime{err: errors.New("cwd")}, + &fakeMutationGateway{}, + ) + _, err := service.Fix(context.Background(), MutationRequest{}) + var runtimeFailure *RuntimeError + if !errors.As(err, &runtimeFailure) { + t.Fatalf("error=%#v, want RuntimeError", err) + } +} From 8a9c3293a893e3c1186c89137123c8fb0616b698 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:42:37 -0400 Subject: [PATCH 070/108] test(doctor): cover mutation command wiring (age-nw28h.2.7.3) --- cli/cmd/ao/doctor_test.go | 33 +++++++++++++++++++++++++++++++++ cli/docs/COMMANDS.md | 2 +- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/cli/cmd/ao/doctor_test.go b/cli/cmd/ao/doctor_test.go index fe20eb54d..43c0a3e66 100644 --- a/cli/cmd/ao/doctor_test.go +++ b/cli/cmd/ao/doctor_test.go @@ -15,6 +15,39 @@ import ( "github.com/boshu2/agentops/cli/internal/doctor" ) +func TestDoctorMutationRequestCopiesCommandFlags(t *testing.T) { + oldOnly, oldSkip := doctorOnly, doctorSkip + oldQuick, oldOnline := doctorQuick, doctorOnline + oldSeverity, oldDryRun := doctorSeverity, doctorDryRun + oldRobot := doctorRobot + t.Cleanup(func() { + doctorOnly, doctorSkip = oldOnly, oldSkip + doctorQuick, doctorOnline = oldQuick, oldOnline + doctorSeverity, doctorDryRun = oldSeverity, oldDryRun + doctorRobot = oldRobot + }) + + doctorOnly = []string{"one"} + doctorSkip = []string{"two"} + doctorQuick = true + doctorOnline = true + doctorSeverity = "P2" + doctorDryRun = true + doctorRobot = true + + request := doctorMutationRequest() + if len(request.Only) != 1 || request.Only[0] != "one" || len(request.Skip) != 1 || request.Skip[0] != "two" { + t.Fatalf("request=%+v", request) + } + if !request.Quick || !request.Online || request.Severity != "P2" || !request.DryRun || !request.JSON { + t.Fatalf("request=%+v", request) + } + doctorOnly[0] = "changed" + if request.Only[0] != "one" { + t.Fatalf("request aliases command flags: %+v", request) + } +} + func TestComputeResult(t *testing.T) { tests := []struct { name string diff --git a/cli/docs/COMMANDS.md b/cli/docs/COMMANDS.md index 2c8953e84..2e70d4cc2 100644 --- a/cli/docs/COMMANDS.md +++ b/cli/docs/COMMANDS.md @@ -143,7 +143,7 @@ ao doctor [command] --online Enable network probes (default: offline-only) --only strings Scope to a subset of detectors or subsystems --quick Run only fast-path detectors (< 200ms) - --robot Alias for --json with structured wrapper + --robot Alias for stable JSON report output --robot-triage Emit the mega-command triage JSON --severity string Minimum severity to emit (P0|P1|P2|P3) (default "P3") --since string Diff findings against an earlier run From 41b50155fdbbbb789be06ab442fa36b490ec38b8 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:47:06 -0400 Subject: [PATCH 071/108] chore(provenance): bind pawl CONFIRMED verdict for age-nw28h.2.7.3 #trivial --- docs/provenance/ledger.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/provenance/ledger.jsonl b/docs/provenance/ledger.jsonl index b739df4e1..ec2de1744 100644 --- a/docs/provenance/ledger.jsonl +++ b/docs/provenance/ledger.jsonl @@ -525,3 +525,4 @@ {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.7.1@d845aa3","from_type":"verdict","to_id":"d845aa394a2040e8b52575bd820af888c6711e46","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.7.1 disposition=CONFIRMED","bead_id":"age-nw28h.2.7.1","trust_tier":"inferred","ts":"2026-07-11T23:13:38Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/doctor-slice1-independent-verdict-d845aa394.txt","prev_hash":"94d27584f6296479beddda872b4124eeeab38e82c9f95eda412832c746345864","payload_hash":"226ba023b25b089c62219aaabd1ae01549eb8c8c7d1e932484f54ba83c95ec90","hash":"3dcc42d990f0bd92fc89cdcc3be75ecc782832e93c3fff25926758dec72934f2"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.7.2@d62cc71","from_type":"verdict","to_id":"d62cc714a976091aac3dfb4cb9f4649db45571d7","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.7.2 disposition=CONFIRMED","bead_id":"age-nw28h.2.7.2","trust_tier":"inferred","ts":"2026-07-11T23:33:28Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/doctor-slice2-independent-verdict-d62cc714a.txt","prev_hash":"3dcc42d990f0bd92fc89cdcc3be75ecc782832e93c3fff25926758dec72934f2","payload_hash":"faa83455e0c399af9a22b257fbbf3fe2ea188fba2b3dbbcb976d649de67ebd12","hash":"60ad2a9db73ba023c0f9008a460e74f09d28cbf8ee2b9f9d72fb44c2a67d9c56"} {"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.7.2@ca0137a","from_type":"verdict","to_id":"ca0137a4d77842ade48b8b344e87625481c00776","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.7.2 disposition=CONFIRMED","bead_id":"age-nw28h.2.7.2","trust_tier":"inferred","ts":"2026-07-11T23:33:42Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/doctor-slice2-independent-verdict-d62cc714a.txt","prev_hash":"60ad2a9db73ba023c0f9008a460e74f09d28cbf8ee2b9f9d72fb44c2a67d9c56","payload_hash":"fd520c5bf41a440beb3fdd887884e66666c5635842146fc9b3ac27dc024bedf7","hash":"ff704fc5e563e0af2a024b1a803cbf619b17df25ee76cff4db5750779ce1f84d"} +{"schema_version":"agentops-sdlc-provenance.v1","from_id":"age-nw28h.2.7.3@8a9c329","from_type":"verdict","to_id":"8a9c3293a893e3c1186c89137123c8fb0616b698","to_type":"commit","relation":"wasDerivedFrom","evidence_ref":"pawl-verdict age-nw28h.2.7.3 disposition=CONFIRMED","bead_id":"age-nw28h.2.7.3","trust_tier":"inferred","ts":"2026-07-11T23:47:06Z","reviewer_family":"gpt","rounds":1,"evidence_path":".agents/evidence/go-cli-clean-architecture/doctor-slice3-independent-verdict-8a9c3293a.txt","prev_hash":"ff704fc5e563e0af2a024b1a803cbf619b17df25ee76cff4db5750779ce1f84d","payload_hash":"917b122a9a9a8bfd53dc264aab61b7fa056fcec0cbd544a9541f7e3a6422c835","hash":"5f32d511b63a3e5b7a73ffba6a9c1c9c202e3ec5e733fa0b257f425a775d3a05"} From 6a5ae2694d4947de5942d1968ad4b307f81eaf86 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:52:57 -0400 Subject: [PATCH 072/108] fix(doctor): make GC dry-run non-destructive (age-nw28h.2.7.4) --- cli/cmd/ao/doctor_legacy_composition.go | 1 + cli/cmd/ao/doctor_surface.go | 41 +++++----- cli/internal/adapters/doctor/maintenance.go | 29 +++++++ .../adapters/doctor/maintenance_test.go | 18 +++++ cli/internal/doctor/doctor_test.go | 4 +- cli/internal/doctor/engine.go | 32 +++++--- cli/internal/doctor/maintenance_service.go | 75 +++++++++++++++++++ .../doctor/maintenance_service_test.go | 73 ++++++++++++++++++ cli/internal/doctor/maintenance_test.go | 59 +++++++++++++++ 9 files changed, 298 insertions(+), 34 deletions(-) create mode 100644 cli/internal/adapters/doctor/maintenance.go create mode 100644 cli/internal/adapters/doctor/maintenance_test.go create mode 100644 cli/internal/doctor/maintenance_service.go create mode 100644 cli/internal/doctor/maintenance_service_test.go create mode 100644 cli/internal/doctor/maintenance_test.go diff --git a/cli/cmd/ao/doctor_legacy_composition.go b/cli/cmd/ao/doctor_legacy_composition.go index 4fc11bf32..20471b2ff 100644 --- a/cli/cmd/ao/doctor_legacy_composition.go +++ b/cli/cmd/ao/doctor_legacy_composition.go @@ -12,3 +12,4 @@ func newLegacyDoctorService() doctorapp.LegacyService { var doctorReadService = doctorapp.NewReadService(version, doctoradapter.ReadRuntime{ToolVersion: version}, doctoradapter.ReadGateway{}) var doctorMutationService = doctorapp.NewMutationService(doctoradapter.MutationRuntime{ToolVersion: version}, doctoradapter.MutationGateway{}) +var doctorMaintenanceService = doctorapp.NewMaintenanceService(doctoradapter.MaintenanceRuntime{}, doctoradapter.MaintenanceGateway{}) diff --git a/cli/cmd/ao/doctor_surface.go b/cli/cmd/ao/doctor_surface.go index 498fcd1e5..ace0fe1ec 100644 --- a/cli/cmd/ao/doctor_surface.go +++ b/cli/cmd/ao/doctor_surface.go @@ -5,8 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "os" - "time" "github.com/spf13/cobra" @@ -231,11 +229,9 @@ func newDoctorUndoCmd() *cobra.Command { Short: "Restore from .doctor/runs//backups/ (run-id may be 'latest')", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - cwd, err := os.Getwd() - if err != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: err.Error()} - } - res, uerr := doctor.Undo(cwd, args[0], doctorUndoStrict, doctorDryRun) + res, uerr := doctorMaintenanceService.Undo(cmd.Context(), doctor.UndoRequest{ + RunID: args[0], Strict: doctorUndoStrict, DryRun: doctorDryRun, + }) if uerr != nil { if doctorWantsJSON() { _ = printDoctorJSON(cmd, map[string]any{ @@ -319,23 +315,24 @@ func newDoctorGcCmd() *cobra.Command { Use: "gc", Short: "Prune old runs (requires --yes and --before )", RunE: func(cmd *cobra.Command, _ []string) error { - cwd, err := os.Getwd() - if err != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: err.Error()} - } - var cutoff time.Time - if doctorGCBefore != "" { - parsed, perr := time.Parse("2006-01-02", doctorGCBefore) - if perr != nil { - return &doctorExitError{code: doctor.ExitUsage, msg: "invalid --before date (want YYYY-MM-DD)"} - } - cutoff = parsed - } - pruned, gerr := doctor.GC(cwd, cutoff, doctorGCYes) + result, gerr := doctorMaintenanceService.GC(cmd.Context(), doctor.GCRequest{ + Before: doctorGCBefore, Yes: doctorGCYes, DryRun: doctorDryRun, + }) if gerr != nil { - return &doctorExitError{code: doctor.ExitUsage, msg: gerr.Error()} + var usageFailure *doctor.UsageError + if errors.As(gerr, &usageFailure) { + return &doctorExitError{code: doctor.ExitUsage, msg: gerr.Error()} + } + return &doctorExitError{code: doctor.ExitIOError, msg: gerr.Error()} } - fmt.Fprintf(cmd.OutOrStdout(), "pruned %d run(s)\n", pruned) + if doctorWantsJSON() { + return printDoctorJSON(cmd, result) + } + if result.DryRun { + fmt.Fprintf(cmd.OutOrStdout(), "would prune %d run(s)\n", result.Matched) + return nil + } + fmt.Fprintf(cmd.OutOrStdout(), "pruned %d run(s)\n", result.Pruned) return nil }, } diff --git a/cli/internal/adapters/doctor/maintenance.go b/cli/internal/adapters/doctor/maintenance.go new file mode 100644 index 000000000..f4e7c711a --- /dev/null +++ b/cli/internal/adapters/doctor/maintenance.go @@ -0,0 +1,29 @@ +package doctor + +import ( + "context" + "time" + + doctorapp "github.com/boshu2/agentops/cli/internal/doctor" +) + +// MaintenanceRuntime snapshots process state for doctor maintenance requests. +type MaintenanceRuntime struct{} + +// RepoRoot returns the current doctor target directory. +func (MaintenanceRuntime) RepoRoot(ctx context.Context) (string, error) { + return (ReadRuntime{}).RepoRoot(ctx) +} + +// MaintenanceGateway adapts maintenance use cases to the existing engine. +type MaintenanceGateway struct{} + +// Undo executes the doctor undo engine. +func (MaintenanceGateway) Undo(_ context.Context, root string, request doctorapp.UndoRequest) (*doctorapp.UndoResult, error) { + return doctorapp.Undo(root, request.RunID, request.Strict, request.DryRun) +} + +// GC executes the doctor garbage-collection engine. +func (MaintenanceGateway) GC(_ context.Context, root string, cutoff time.Time, yes, dryRun bool) (doctorapp.GCResult, error) { + return doctorapp.GC(root, cutoff, yes, dryRun) +} diff --git a/cli/internal/adapters/doctor/maintenance_test.go b/cli/internal/adapters/doctor/maintenance_test.go new file mode 100644 index 000000000..66681340a --- /dev/null +++ b/cli/internal/adapters/doctor/maintenance_test.go @@ -0,0 +1,18 @@ +package doctor + +import ( + "context" + "testing" +) + +func TestMaintenanceRuntimeReturnsCurrentDirectory(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + got, err := (MaintenanceRuntime{}).RepoRoot(context.Background()) + if err != nil { + t.Fatal(err) + } + if got != root { + t.Fatalf("root=%q, want %q", got, root) + } +} diff --git a/cli/internal/doctor/doctor_test.go b/cli/internal/doctor/doctor_test.go index ad998fb22..eafe67d33 100644 --- a/cli/internal/doctor/doctor_test.go +++ b/cli/internal/doctor/doctor_test.go @@ -345,10 +345,10 @@ func TestFix_NoDetectorsSelectedNoFindings(t *testing.T) { // TestGC_RefusesWithoutGates verifies gc never deletes silently. func TestGC_RefusesWithoutGates(t *testing.T) { repo := t.TempDir() - if _, err := GC(repo, time.Time{}, true); err == nil { + if _, err := GC(repo, time.Time{}, true, false); err == nil { t.Fatal("gc accepted a zero cutoff") } - if _, err := GC(repo, time.Now(), false); err == nil { + if _, err := GC(repo, time.Now(), false, false); err == nil { t.Fatal("gc accepted without --yes") } } diff --git a/cli/internal/doctor/engine.go b/cli/internal/doctor/engine.go index f08422ef6..673864e5a 100644 --- a/cli/internal/doctor/engine.go +++ b/cli/internal/doctor/engine.go @@ -913,20 +913,28 @@ func RobotTriage(opts Options) (*RobotTriageResult, *Report, error) { }, rep, nil } +// GCResult describes a doctor run garbage-collection operation. +type GCResult struct { + Matched int `json:"matched"` + Pruned int `json:"pruned"` + DryRun bool `json:"dry_run"` +} + // GC prunes run directories whose started_at is before the cutoff. It refuses -// unless yes is true and cutoff is non-zero (never deletes silently). -func GC(repoRoot string, cutoff time.Time, yes bool) (int, error) { +// unless yes is true and cutoff is non-zero (never deletes silently). Dry-run +// reports matching runs without removing them. +func GC(repoRoot string, cutoff time.Time, yes, dryRun bool) (GCResult, error) { + result := GCResult{DryRun: dryRun} if !yes || cutoff.IsZero() { - return 0, fmt.Errorf("doctor: gc requires --yes and --before ") + return result, fmt.Errorf("doctor: gc requires --yes and --before ") } entries, err := os.ReadDir(runsDir(repoRoot)) if err != nil { if os.IsNotExist(err) { - return 0, nil + return result, nil } - return 0, fmt.Errorf("doctor: read runs dir: %w", err) + return result, fmt.Errorf("doctor: read runs dir: %w", err) } - pruned := 0 for _, e := range entries { if !e.IsDir() { continue @@ -936,12 +944,16 @@ func GC(repoRoot string, cutoff time.Time, yes bool) (int, error) { if started.IsZero() || !started.Before(cutoff) { continue } - if err := os.RemoveAll(dir); err != nil { - return pruned, fmt.Errorf("doctor: prune %s: %w", e.Name(), err) + result.Matched++ + if dryRun { + continue } - pruned++ + if err := os.RemoveAll(dir); err != nil { + return result, fmt.Errorf("doctor: prune %s: %w", e.Name(), err) + } + result.Pruned++ } - return pruned, nil + return result, nil } // runStartedAt reads a run's started_at, falling back to dir mtime. diff --git a/cli/internal/doctor/maintenance_service.go b/cli/internal/doctor/maintenance_service.go new file mode 100644 index 000000000..8cd7b7273 --- /dev/null +++ b/cli/internal/doctor/maintenance_service.go @@ -0,0 +1,75 @@ +package doctor + +import ( + "context" + "fmt" + "strings" + "time" +) + +// UsageError marks invalid maintenance command input. +type UsageError struct{ Err error } + +func (failure *UsageError) Error() string { return failure.Err.Error() } +func (failure *UsageError) Unwrap() error { return failure.Err } + +// UndoRequest carries caller-owned options for a doctor undo invocation. +type UndoRequest struct { + RunID string + Strict bool + DryRun bool +} + +// GCRequest carries caller-owned options for a doctor GC invocation. +type GCRequest struct { + Before string + Yes bool + DryRun bool +} + +// MaintenanceRuntime snapshots process state needed by maintenance use cases. +type MaintenanceRuntime interface { + RepoRoot(context.Context) (string, error) +} + +// MaintenanceGateway executes doctor undo and garbage collection. +type MaintenanceGateway interface { + Undo(context.Context, string, UndoRequest) (*UndoResult, error) + GC(context.Context, string, time.Time, bool, bool) (GCResult, error) +} + +// MaintenanceService owns doctor undo and GC application orchestration. +type MaintenanceService struct { + runtime MaintenanceRuntime + gateway MaintenanceGateway +} + +// NewMaintenanceService constructs the doctor maintenance application service. +func NewMaintenanceService(runtime MaintenanceRuntime, gateway MaintenanceGateway) MaintenanceService { + return MaintenanceService{runtime: runtime, gateway: gateway} +} + +// Undo resolves the repository root and executes an undo request. +func (service MaintenanceService) Undo(ctx context.Context, request UndoRequest) (*UndoResult, error) { + root, err := service.runtime.RepoRoot(ctx) + if err != nil { + return nil, &RuntimeError{Err: err} + } + return service.gateway.Undo(ctx, root, request) +} + +// GC validates command input, resolves the repository root, and executes GC. +func (service MaintenanceService) GC(ctx context.Context, request GCRequest) (GCResult, error) { + if !request.Yes || strings.TrimSpace(request.Before) == "" { + return GCResult{DryRun: request.DryRun}, &UsageError{Err: fmt.Errorf("doctor: gc requires --yes and --before ")} + } + cutoff, err := time.Parse("2006-01-02", request.Before) + if err != nil { + return GCResult{DryRun: request.DryRun}, &UsageError{Err: fmt.Errorf("invalid --before date (want YYYY-MM-DD): %w", err)} + } + root, err := service.runtime.RepoRoot(ctx) + if err != nil { + return GCResult{DryRun: request.DryRun}, &RuntimeError{Err: err} + } + return service.gateway.GC(ctx, root, cutoff, request.Yes, request.DryRun) +} diff --git a/cli/internal/doctor/maintenance_service_test.go b/cli/internal/doctor/maintenance_service_test.go new file mode 100644 index 000000000..d27373e71 --- /dev/null +++ b/cli/internal/doctor/maintenance_service_test.go @@ -0,0 +1,73 @@ +package doctor + +import ( + "context" + "errors" + "testing" + "time" +) + +type fakeMaintenanceRuntime struct { + root string + err error +} + +func (runtime fakeMaintenanceRuntime) RepoRoot(context.Context) (string, error) { + return runtime.root, runtime.err +} + +type fakeMaintenanceGateway struct { + undoRoot string + undoRequest UndoRequest + gcRoot string + gcCutoff time.Time + gcYes bool + gcDryRun bool +} + +func (gateway *fakeMaintenanceGateway) Undo(_ context.Context, root string, request UndoRequest) (*UndoResult, error) { + gateway.undoRoot, gateway.undoRequest = root, request + return &UndoResult{RunID: request.RunID}, nil +} + +func (gateway *fakeMaintenanceGateway) GC(_ context.Context, root string, cutoff time.Time, yes, dryRun bool) (GCResult, error) { + gateway.gcRoot, gateway.gcCutoff = root, cutoff + gateway.gcYes, gateway.gcDryRun = yes, dryRun + return GCResult{Matched: 2, DryRun: dryRun}, nil +} + +func TestMaintenanceServiceDelegatesUndoAndGCThroughPorts(t *testing.T) { + gateway := &fakeMaintenanceGateway{} + service := NewMaintenanceService(fakeMaintenanceRuntime{root: "/repo"}, gateway) + undo, err := service.Undo(context.Background(), UndoRequest{RunID: "latest", Strict: true, DryRun: true}) + if err != nil { + t.Fatal(err) + } + if undo.RunID != "latest" || gateway.undoRoot != "/repo" || !gateway.undoRequest.Strict || !gateway.undoRequest.DryRun { + t.Fatalf("undo=%+v gateway=%+v", undo, gateway) + } + gc, err := service.GC(context.Background(), GCRequest{Before: "2026-01-02", Yes: true, DryRun: true}) + if err != nil { + t.Fatal(err) + } + wantCutoff := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC) + if gc.Matched != 2 || gateway.gcRoot != "/repo" || !gateway.gcCutoff.Equal(wantCutoff) || !gateway.gcYes || !gateway.gcDryRun { + t.Fatalf("gc=%+v gateway=%+v", gc, gateway) + } +} + +func TestMaintenanceServiceClassifiesRuntimeAndUsageFailures(t *testing.T) { + service := NewMaintenanceService(fakeMaintenanceRuntime{err: errors.New("cwd")}, &fakeMaintenanceGateway{}) + _, err := service.Undo(context.Background(), UndoRequest{RunID: "latest"}) + var runtimeFailure *RuntimeError + if !errors.As(err, &runtimeFailure) { + t.Fatalf("undo error=%#v, want RuntimeError", err) + } + + service = NewMaintenanceService(fakeMaintenanceRuntime{root: "/repo"}, &fakeMaintenanceGateway{}) + _, err = service.GC(context.Background(), GCRequest{Before: "not-a-date", Yes: true}) + var usageFailure *UsageError + if !errors.As(err, &usageFailure) { + t.Fatalf("gc error=%#v, want UsageError", err) + } +} diff --git a/cli/internal/doctor/maintenance_test.go b/cli/internal/doctor/maintenance_test.go new file mode 100644 index 000000000..faeafea14 --- /dev/null +++ b/cli/internal/doctor/maintenance_test.go @@ -0,0 +1,59 @@ +package doctor + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" +) + +func TestGCDryRunReportsCandidatesWithoutDeleting(t *testing.T) { + root := t.TempDir() + oldDir := writeGCTestRun(t, root, "old", time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)) + newDir := writeGCTestRun(t, root, "new", time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)) + cutoff := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + result, err := GC(root, cutoff, true, true) + if err != nil { + t.Fatal(err) + } + if result.Matched != 1 || result.Pruned != 0 || !result.DryRun { + t.Fatalf("dry-run result=%+v", result) + } + for _, dir := range []string{oldDir, newDir} { + if info, err := os.Stat(dir); err != nil || !info.IsDir() { + t.Fatalf("dry-run removed %s: %v", dir, err) + } + } + + result, err = GC(root, cutoff, true, false) + if err != nil { + t.Fatal(err) + } + if result.Matched != 1 || result.Pruned != 1 || result.DryRun { + t.Fatalf("real result=%+v", result) + } + if _, err := os.Stat(oldDir); !os.IsNotExist(err) { + t.Fatalf("old run still exists: %v", err) + } + if info, err := os.Stat(newDir); err != nil || !info.IsDir() { + t.Fatalf("new run missing: %v", err) + } +} + +func writeGCTestRun(t *testing.T, root, name string, started time.Time) string { + t.Helper() + dir := filepath.Join(root, ".doctor", "runs", name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + data, err := json.Marshal(Report{StartedAt: started.Format(time.RFC3339)}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "report.json"), data, 0o644); err != nil { + t.Fatal(err) + } + return dir +} From 69775c2777907694ca7723f5fbc58af75c910b56 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:14:53 -0400 Subject: [PATCH 073/108] test(doctor): freeze compatibility family before Cobra cutover (age-nw28h.2.7.4) --- .../families/doctor/case.json | 25 +++++++++++++++++++ .../families/doctor/ownership.json | 23 +++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/doctor/case.json create mode 100644 cli/testdata/compatibility-baseline/families/doctor/ownership.json diff --git a/cli/testdata/compatibility-baseline/families/doctor/case.json b/cli/testdata/compatibility-baseline/families/doctor/case.json new file mode 100644 index 000000000..29c4d8e60 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/doctor/case.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "family": "doctor", + "checks": [ + { + "id": "doctor-black-box", + "command": "tmp=$(mktemp -d /tmp/ao-doctor.XXXXXX); (cd cli && go build -o \"$tmp/ao\" ./cmd/ao); \"$tmp/ao\" doctor --help >\"$tmp/help\"; grep -Fq 'ao doctor [flags]' \"$tmp/help\"; for child in capabilities diff explain fix gc health ls robot-docs undo; do grep -Eq \"^[[:space:]]+$child[[:space:]]\" \"$tmp/help\"; done; mkdir -p \"$tmp/repo/.doctor/runs/old\"; printf '{\"started_at\":\"2025-01-01T00:00:00Z\"}\\n' >\"$tmp/repo/.doctor/runs/old/report.json\"; set +e; \"$tmp/ao\" doctor junk >\"$tmp/junk.out\" 2>\"$tmp/junk.err\"; junk=$?; set -e; test \"$junk\" -eq 1; (cd \"$tmp/repo\" && \"$tmp/ao\" doctor gc --before 2026-01-01 --yes >\"$tmp/gc.real\"); grep -qx 'pruned 1 run(s)' \"$tmp/gc.real\"; test ! -e \"$tmp/repo/.doctor/runs/old\"" + }, + { + "id": "doctor-go-behavior", + "command": "cd cli && go test ./internal/doctor ./internal/adapters/doctor ./cmd/ao -run 'Doctor|GC|Maintenance|Mutate_DryRun|DiffDoesNotCreate|DiagnoseSince' -count=1" + } + ], + "dimensions": { + "path": {"status":"evidence","evidence":["doctor-black-box"]}, + "aliases": {"status":"not_applicable","reason":"The doctor family declares no aliases."}, + "accepted_args": {"status":"evidence","evidence":["doctor-black-box","doctor-go-behavior"]}, + "rejected_args": {"status":"evidence","evidence":["doctor-black-box"]}, + "stdout": {"status":"evidence","evidence":["doctor-black-box","doctor-go-behavior"]}, + "stderr": {"status":"evidence","evidence":["doctor-black-box"]}, + "exit_classes": {"status":"evidence","evidence":["doctor-black-box","doctor-go-behavior"]}, + "tracker_selection": {"status":"not_applicable","reason":"Doctor inspects health and repair state but does not select the repository issue tracker."}, + "ordered_effects": {"status":"evidence","evidence":["doctor-black-box","doctor-go-behavior"]} + } +} diff --git a/cli/testdata/compatibility-baseline/families/doctor/ownership.json b/cli/testdata/compatibility-baseline/families/doctor/ownership.json new file mode 100644 index 000000000..cd2c9e713 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/doctor/ownership.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "family": "doctor", + "profiles": {"default":"present","flywheel":"present","legacy":"present","combined":"present"}, + "legacy_symbols": [ + "doctorCmd", "doctorExitError", "doctorJSON", "doctorFix", "doctorDryRun", + "doctorOnly", "doctorSkip", "doctorSince", "doctorOnline", "doctorQuick", + "doctorSeverity", "doctorRobot", "doctorRobotTriage", "doctorExplainFlag", + "doctorUndoStrict", "doctorGCBefore", "doctorGCYes", "registerDoctorSurface", + "runDoctor", "runDoctorEngineDefault", "runDoctorFix", "runDoctorExplain" + ], + "live_owner": "cli/internal/commands/doctor", + "allowed_paths": [ + "cli/cmd/ao/doctor*.go", "cli/cmd/ao/root.go", "cli/cmd/ao/validate.go", + "cli/cmd/ao/cobra_commands_test.go", "cli/cmd/ao/testutil_test.go", "cli/cmd/ao/json_validity_test.go", + "cli/cmd/ao/quality_test_support_test.go", + "cli/cmd/ao/quickstart_verdict_test.go", "cli/internal/doctor/**", + "cli/internal/adapters/doctor/**", "cli/internal/reviewerhealth/**", + "cli/internal/adapters/reviewerhealth/**", "cli/internal/commands/doctor/**", + "cli/internal/quality/**", "cli/docs/COMMANDS.md", + "cli/testdata/compatibility-baseline/families/doctor/lineage.json" + ] +} From 191bbf548d5213c9e8c2499612630debf676d03c Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:10:07 -0400 Subject: [PATCH 074/108] refactor(doctor): atomically cut over Cobra ownership (age-nw28h.2.7.4) --- cli/cmd/ao/cobra_commands_test.go | 3 - cli/cmd/ao/doctor.go | 96 -- cli/cmd/ao/doctor_helpers_legacy_test.go | 79 - cli/cmd/ao/doctor_helpers_test.go | 27 - cli/cmd/ao/doctor_legacy_composition.go | 15 - cli/cmd/ao/doctor_module.go | 40 + cli/cmd/ao/doctor_module_integration_test.go | 48 + cli/cmd/ao/doctor_read_contract_test.go | 4 +- cli/cmd/ao/doctor_surface.go | 388 ----- cli/cmd/ao/doctor_test.go | 1599 ------------------ cli/cmd/ao/doctor_wedge_legacy_test.go | 54 - cli/cmd/ao/doctor_wedge_test.go | 461 ----- cli/cmd/ao/json_validity_test.go | 6 +- cli/cmd/ao/quality_test_support_test.go | 36 + cli/cmd/ao/quickstart_verdict_test.go | 18 +- cli/cmd/ao/root.go | 3 +- cli/cmd/ao/testutil_test.go | 3 - cli/cmd/ao/validate.go | 2 +- cli/internal/commands/doctor/module.go | 402 +++++ cli/internal/commands/doctor/module_test.go | 95 ++ 20 files changed, 643 insertions(+), 2736 deletions(-) delete mode 100644 cli/cmd/ao/doctor.go delete mode 100644 cli/cmd/ao/doctor_helpers_legacy_test.go delete mode 100644 cli/cmd/ao/doctor_helpers_test.go delete mode 100644 cli/cmd/ao/doctor_legacy_composition.go create mode 100644 cli/cmd/ao/doctor_module.go create mode 100644 cli/cmd/ao/doctor_module_integration_test.go delete mode 100644 cli/cmd/ao/doctor_surface.go delete mode 100644 cli/cmd/ao/doctor_test.go delete mode 100644 cli/cmd/ao/doctor_wedge_legacy_test.go delete mode 100644 cli/cmd/ao/doctor_wedge_test.go create mode 100644 cli/cmd/ao/quality_test_support_test.go create mode 100644 cli/internal/commands/doctor/module.go create mode 100644 cli/internal/commands/doctor/module_test.go diff --git a/cli/cmd/ao/cobra_commands_test.go b/cli/cmd/ao/cobra_commands_test.go index 3275b6e91..f740bf8c2 100644 --- a/cli/cmd/ao/cobra_commands_test.go +++ b/cli/cmd/ao/cobra_commands_test.go @@ -77,7 +77,6 @@ func executeCommand(args ...string) (string, error) { origContextPacketFlags := contextPacketFlags origContextExplainFlags := contextExplainFlags origContextPacketStatusFlags := contextPacketStatusFlags - origDoctorJSON := doctorJSON origMaturityApply := maturityApply origMaturityScan := maturityScan origMaturityCurate := maturityCurate @@ -147,7 +146,6 @@ func executeCommand(args ...string) (string, error) { contextPacketFlags = origContextPacketFlags contextExplainFlags = origContextExplainFlags contextPacketStatusFlags = origContextPacketStatusFlags - doctorJSON = origDoctorJSON maturityApply = origMaturityApply maturityScan = origMaturityScan maturityCurate = origMaturityCurate @@ -229,7 +227,6 @@ func executeCommand(args ...string) (string, error) { evalCoverageDomains = []string{"cli", "hook", "skill", "rpi", "runtime", "retrieval", "scenario", "mixed", "security"} evalCoverageDims = []string{"correctness", "process_adherence", "artifact_quality", "runtime_compatibility", "efficiency", "safety", "learning_closure"} evalCoverageRuntimes = []string{"static", "shell", "mock"} - doctorJSON = false maturityApply = false maturityScan = false maturityCurate = false diff --git a/cli/cmd/ao/doctor.go b/cli/cmd/ao/doctor.go deleted file mode 100644 index 8263b44da..000000000 --- a/cli/cmd/ao/doctor.go +++ /dev/null @@ -1,96 +0,0 @@ -// practices: [sre, resilience-patterns] -package main - -import ( - "github.com/spf13/cobra" - - "github.com/boshu2/agentops/cli/internal/doctor" - "github.com/boshu2/agentops/cli/internal/quality" -) - -var ( - doctorJSON bool -) - -var doctorCmd = &cobra.Command{ - Use: "doctor", - Short: "Check AgentOps health", - Long: `Run health checks on your AgentOps installation. - -Validates that all required components are present and configured. -Optional components are reported as warnings but do not cause failure. - -Examples: - ao doctor - ao doctor --json`, - RunE: runDoctor, -} - -func init() { - doctorCmd.GroupID = "core" - doctorCmd.Flags().BoolVar(&doctorJSON, "json", false, "Output results as JSON") - // Attach the diagnose-and-repair engine surface: additive flags - // (--fix, --dry-run, --only, ...) plus subcommands (fix, undo, explain, - // capabilities, health, robot-docs, gc, ls, diff). The legacy 16-check - // behavior is preserved when no engine flag/subcommand is used. - registerDoctorSurface() - rootCmd.AddCommand(doctorCmd) -} - -// gatherDoctorChecks runs all doctor checks and returns the results. -func gatherDoctorChecks() []quality.Check { - return newLegacyDoctorService().Checks(rootCmd.Context()) -} - -func runDoctor(cmd *cobra.Command, args []string) error { - // Engine-flag invocations (--fix, --explain, --robot-triage) route entirely - // through the diagnose-and-repair engine. - if doctorFix || doctorExplainFlag != "" || doctorRobotTriage || doctorSince != "" { - return runDoctorEngineDefault(cmd) - } - - // `ao doctor --json` is the engine's machine surface: a single diagnose - // Report (schema_version, exit_code, findings). - if doctorWantsJSON() { - return runDoctorEngineDefault(cmd) - } - - // Human-readable form: the legacy check table, with engine findings - // appended additively so existing output never regresses. - checks := gatherDoctorChecks() - if err := quality.RunDoctor(quality.DoctorOptions{ - JSON: doctorJSON, - Checks: checks, - Stdout: cmd.OutOrStdout(), - }); err != nil { - return err - } - // Additive: also run the registered failure-mode detectors and surface - // their findings without disturbing the legacy `checks` output above. With - // the FOUNDATION wave's empty registry this is a no-op; later waves light up. - return appendEngineFindings(cmd) -} - -// appendEngineFindings runs the doctor engine's detectors and appends their -// findings to the human-readable `ao doctor` output. It never alters the legacy -// `checks` output or the bare command's exit semantics — a doctorExitError is -// only returned when engine findings exist (mapped to exit 1). -func appendEngineFindings(cmd *cobra.Command) error { - // With no detectors registered there is nothing to add and no reason to - // create a run directory; skip entirely so the legacy command stays - // side-effect-free. - if len(doctor.Detectors()) == 0 { - return nil - } - // JSON callers receive the engine Report from the dedicated --json path in - // runDoctor; never emit a second JSON document here. - if doctorWantsJSON() { - return nil - } - rep, derr := doctorReadService.Diagnose(cmd.Context(), doctorReadRequest()) - if derr != nil || rep == nil || len(rep.Findings) == 0 { - return nil - } - renderEngineFindings(cmd, rep) - return exitErr(rep.ExitCode, "doctor findings present") -} diff --git a/cli/cmd/ao/doctor_helpers_legacy_test.go b/cli/cmd/ao/doctor_helpers_legacy_test.go deleted file mode 100644 index d663f0f48..000000000 --- a/cli/cmd/ao/doctor_helpers_legacy_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -import ( - "io" - "os" - "os/exec" - "path/filepath" - "time" - - "github.com/boshu2/agentops/cli/internal/quality" - "github.com/boshu2/agentops/cli/internal/storage" -) - -type doctorCheck = quality.Check -type doctorOutput = quality.DoctorOutput -type staleReference = quality.StaleReference - -var deprecatedCommands = quality.DeprecatedCommands - -func doctorStatusIcon(status string) string { return quality.StatusIcon(status) } -func hasRequiredFailure(checks []doctorCheck) bool { return quality.HasRequiredFailure(checks) } -func renderDoctorTable(writer io.Writer, output doctorOutput) { quality.RenderTable(writer, output) } -func newestFileModTime(entries []os.DirEntry) time.Time { return quality.NewestFileModTime(entries) } -func countEstablished(directory string) int { return quality.CountEstablished(directory) } -func formatVersion(value string) string { return quality.FormatVersion(value) } -func formatDuration(value time.Duration) string { return quality.FormatDuration(value) } -func formatNumber(value int) string { return quality.FormatNumber(value) } -func countFileLines(path string) int { return quality.CountFileLines(path) } -func checkCLIDependencies() doctorCheck { return quality.CheckCLIDependencies(exec.LookPath) } -func checkKnowledgeBase() doctorCheck { - cwd, _ := os.Getwd() - return quality.CheckKnowledgeBase(filepath.Join(cwd, storage.DefaultBaseDir)) -} -func checkKnowledgeFreshness() doctorCheck { - cwd, _ := os.Getwd() - return quality.CheckKnowledgeFreshness(filepath.Join(cwd, storage.DefaultBaseDir, "sessions")) -} -func checkSearchIndex() doctorCheck { - cwd, _ := os.Getwd() - return quality.CheckSearchIndex(filepath.Join(cwd, IndexDir, IndexFileName)) -} -func checkFlywheelHealth(base ...string) doctorCheck { - cwd, _ := os.Getwd() - if len(base) > 0 && base[0] != "" { - cwd = base[0] - } - return quality.CheckFlywheelHealth(filepath.Join(cwd, storage.DefaultBaseDir)) -} -func checkSkills() doctorCheck { return quality.CheckSkills() } -func checkCodexSync() doctorCheck { return quality.CheckCodexSync() } -func checkSkillIntegrity() doctorCheck { return quality.CheckSkillIntegrity() } -func checkOptionalCLI(name, reason string) doctorCheck { return quality.CheckOptionalCLI(name, reason) } -func findHealScript() string { return quality.FindHealScript() } -func sha256File(path string) (string, error) { return quality.SHA256File(path) } -func fileExists(path string) bool { return quality.FileExists(path) } -func skillOverlapWarning(base map[string]struct{}, count int, primary, format string, others ...map[string]struct{}) *doctorCheck { - return quality.SkillOverlapWarning(base, count, primary, format, others...) -} -func scanSkillDir(path string) map[string]struct{} { return quality.ScanSkillDir(path) } -func overlappingSkillNames(base map[string]struct{}, others ...map[string]struct{}) []string { - return quality.OverlappingSkillNames(base, others...) -} -func checkStaleReferences() doctorCheck { - return quality.CheckStaleReferences([]string{"skills/*/SKILL.md", "skills/*/references/*.md", "skills-codex/*/SKILL.md", "skills-codex-overrides/*/SKILL.md", "docs/*.md", "scripts/*.sh", "docs/contracts/*.md", "docs/plans/*.md"}) -} -func scanFileForDeprecatedCommands(path string) []staleReference { - return quality.ScanFileForDeprecatedCommands(path) -} -func countUniqueFiles(refs []staleReference) int { return quality.CountUniqueFiles(refs) } -func countHealFindings(output string) int { return quality.CountHealFindings(output) } -func countFiles(path string) int { return quality.CountFiles(path) } -func countLearningFiles(path string) int { return quality.CountLearningFiles(path) } -func countCheckStatuses(checks []doctorCheck) (int, int, int) { - return quality.CountCheckStatuses(checks) -} -func buildDoctorSummary(passes, fails, warns, total int) string { - return quality.BuildSummary(passes, fails, warns, total) -} -func computeResult(checks []doctorCheck) doctorOutput { return quality.ComputeResult(checks) } diff --git a/cli/cmd/ao/doctor_helpers_test.go b/cli/cmd/ao/doctor_helpers_test.go deleted file mode 100644 index 9105dd379..000000000 --- a/cli/cmd/ao/doctor_helpers_test.go +++ /dev/null @@ -1,27 +0,0 @@ -// practices: [sre, resilience-patterns] -package main - -import ( - "testing" -) - -// Tests for doctor.go helper functions - -func TestFormatVersion(t *testing.T) { - tests := []struct { - input string - want string - }{ - {"dev", "vdev"}, - {"v2.18.2", "v2.18.2"}, - {"v2.18.2-20-g3fef2f4-dirty", "v2.18.2-20-g3fef2f4-dirty"}, - {"2.18.2", "v2.18.2"}, - {"", "v"}, - } - for _, tt := range tests { - got := formatVersion(tt.input) - if got != tt.want { - t.Errorf("formatVersion(%q) = %q, want %q", tt.input, got, tt.want) - } - } -} diff --git a/cli/cmd/ao/doctor_legacy_composition.go b/cli/cmd/ao/doctor_legacy_composition.go deleted file mode 100644 index 20471b2ff..000000000 --- a/cli/cmd/ao/doctor_legacy_composition.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -import ( - doctoradapter "github.com/boshu2/agentops/cli/internal/adapters/doctor" - doctorapp "github.com/boshu2/agentops/cli/internal/doctor" -) - -func newLegacyDoctorService() doctorapp.LegacyService { - checks := doctoradapter.SystemLegacyChecks(version, IndexDir, IndexFileName, resolveLedgerPath, reviewerHealthService) - return doctorapp.NewLegacyService(version, checks) -} - -var doctorReadService = doctorapp.NewReadService(version, doctoradapter.ReadRuntime{ToolVersion: version}, doctoradapter.ReadGateway{}) -var doctorMutationService = doctorapp.NewMutationService(doctoradapter.MutationRuntime{ToolVersion: version}, doctoradapter.MutationGateway{}) -var doctorMaintenanceService = doctorapp.NewMaintenanceService(doctoradapter.MaintenanceRuntime{}, doctoradapter.MaintenanceGateway{}) diff --git a/cli/cmd/ao/doctor_module.go b/cli/cmd/ao/doctor_module.go new file mode 100644 index 000000000..3c6828c05 --- /dev/null +++ b/cli/cmd/ao/doctor_module.go @@ -0,0 +1,40 @@ +// practices: [hexagonal-architecture, sre, resilience-patterns] +package main + +import ( + "github.com/spf13/cobra" + + doctoradapter "github.com/boshu2/agentops/cli/internal/adapters/doctor" + doctorcommands "github.com/boshu2/agentops/cli/internal/commands/doctor" + doctorapp "github.com/boshu2/agentops/cli/internal/doctor" +) + +func newLegacyDoctorService() doctorapp.LegacyService { + checks := doctoradapter.SystemLegacyChecks(version, IndexDir, IndexFileName, resolveLedgerPath, reviewerHealthService) + return doctorapp.NewLegacyService(version, checks) +} + +var doctorReadService = doctorapp.NewReadService(version, doctoradapter.ReadRuntime{ToolVersion: version}, doctoradapter.ReadGateway{}) +var doctorMutationService = doctorapp.NewMutationService(doctoradapter.MutationRuntime{ToolVersion: version}, doctoradapter.MutationGateway{}) +var doctorMaintenanceService = doctorapp.NewMaintenanceService(doctoradapter.MaintenanceRuntime{}, doctoradapter.MaintenanceGateway{}) + +var doctorModule = doctorcommands.NewModule(doctorcommands.UseCases{ + LegacyChecks: newLegacyDoctorService().Checks, + Read: doctorReadService, + Mutation: doctorMutationService, + Maintenance: doctorMaintenanceService, + DetectorCount: func() int { return len(doctorapp.Detectors()) }, +}, doctorcommands.HostOptions{ + Globals: func(command *cobra.Command) doctorcommands.GlobalOptions { + app := AppFromContext(command.Context()) + return doctorcommands.GlobalOptions{DryRun: app.DryRun, JSON: app.JSON, Output: app.Output} + }, + EnrichFlagErr: flagErrorWithSuggestion, +}) + +var doctorCommand = doctorModule.Command() + +func init() { + doctorCommand.GroupID = "core" + rootCmd.AddCommand(doctorCommand) +} diff --git a/cli/cmd/ao/doctor_module_integration_test.go b/cli/cmd/ao/doctor_module_integration_test.go new file mode 100644 index 000000000..95dcc9663 --- /dev/null +++ b/cli/cmd/ao/doctor_module_integration_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/boshu2/agentops/cli/internal/doctor" +) + +func TestDoctorModuleTableOutput(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + t.Setenv("HOME", t.TempDir()) + if err := os.MkdirAll(filepath.Join(root, ".agents", "ao"), 0o755); err != nil { + t.Fatal(err) + } + command := doctorModule.Command() + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + _ = command.Execute() + if !strings.Contains(output.String(), "ao doctor") { + t.Fatalf("output=%q", output.String()) + } +} + +func TestDoctorModuleJSONOutputIsSingleReport(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + t.Setenv("HOME", t.TempDir()) + command := doctorModule.Command() + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + command.SetArgs([]string{"--json"}) + _ = command.Execute() + var report doctor.Report + if err := json.Unmarshal(output.Bytes(), &report); err != nil { + t.Fatalf("json=%q err=%v", output.String(), err) + } + if report.SchemaVersion != doctor.SchemaVersion { + t.Fatalf("report=%+v", report) + } +} diff --git a/cli/cmd/ao/doctor_read_contract_test.go b/cli/cmd/ao/doctor_read_contract_test.go index 82b1fa247..10f08e997 100644 --- a/cli/cmd/ao/doctor_read_contract_test.go +++ b/cli/cmd/ao/doctor_read_contract_test.go @@ -3,11 +3,11 @@ package main import "testing" func TestDoctorReadCommandsRejectJunkArguments(t *testing.T) { - if err := doctorCmd.ValidateArgs([]string{"junk"}); err == nil { + if err := doctorCommand.ValidateArgs([]string{"junk"}); err == nil { t.Fatal("doctor accepted junk positional argument") } for _, name := range []string{"capabilities", "health", "robot-docs", "ls", "diff"} { - child, _, err := doctorCmd.Find([]string{name}) + child, _, err := doctorCommand.Find([]string{name}) if err != nil { t.Fatal(err) } diff --git a/cli/cmd/ao/doctor_surface.go b/cli/cmd/ao/doctor_surface.go deleted file mode 100644 index ace0fe1ec..000000000 --- a/cli/cmd/ao/doctor_surface.go +++ /dev/null @@ -1,388 +0,0 @@ -// practices: [sre, resilience-patterns] -package main - -import ( - "encoding/json" - "errors" - "fmt" - - "github.com/spf13/cobra" - - "github.com/boshu2/agentops/cli/internal/doctor" -) - -// doctorExitError carries a doctor exit code out through cobra's RunE so that -// Execute() can map it to os.Exit. It reuses the established typed-error + -// errors.As pattern (see AgentsLintError). -type doctorExitError struct { - code int - msg string -} - -func (e *doctorExitError) Error() string { return e.msg } - -// ExitCode returns the process exit code this error maps to. -func (e *doctorExitError) ExitCode() int { return e.code } - -// doctorEngineFlags holds the additive flags for the new doctor engine surface. -var ( - doctorFix bool - doctorDryRun bool - doctorOnly []string - doctorSkip []string - doctorSince string - doctorOnline bool - doctorQuick bool - doctorSeverity string - doctorRobot bool - doctorRobotTriage bool - doctorExplainFlag string - doctorUndoStrict bool - doctorGCBefore string - doctorGCYes bool -) - -// registerDoctorSurface attaches the engine flags and subcommands to doctorCmd. -// It is invoked from doctor.go's init after doctorCmd is constructed. -func registerDoctorSurface() { - f := doctorCmd.Flags() - f.BoolVar(&doctorFix, "fix", false, "Apply fixers for findings (routes through mutate())") - f.BoolVar(&doctorDryRun, "dry-run", false, "With --fix: print the plan, change nothing") - f.StringSliceVar(&doctorOnly, "only", nil, "Scope to a subset of detectors or subsystems") - f.StringSliceVar(&doctorSkip, "skip", nil, "Inverse of --only") - f.StringVar(&doctorSince, "since", "", "Diff findings against an earlier run") - f.BoolVar(&doctorOnline, "online", false, "Enable network probes (default: offline-only)") - f.BoolVar(&doctorQuick, "quick", false, "Run only fast-path detectors (< 200ms)") - f.StringVar(&doctorSeverity, "severity", "P3", "Minimum severity to emit (P0|P1|P2|P3)") - f.BoolVar(&doctorRobot, "robot", false, "Alias for stable JSON report output") - f.BoolVar(&doctorRobotTriage, "robot-triage", false, "Emit the mega-command triage JSON") - f.StringVar(&doctorExplainFlag, "explain", "", "Expand a single finding by id") - - doctorCmd.AddCommand( - newDoctorFixCmd(), - newDoctorUndoCmd(), - newDoctorExplainCmd(), - newDoctorCapabilitiesCmd(), - newDoctorHealthCmd(), - newDoctorRobotDocsCmd(), - newDoctorGcCmd(), - newDoctorLsCmd(), - newDoctorDiffCmd(), - ) - - // Doctor commands carry their result in the exit code. Findings (exit 1) - // are a normal diagnostic outcome, not a failure, so cobra must not print - // "Error: ..." to stderr. Genuine failures are surfaced by Execute() - // instead (see root.go). SilenceErrors does not inherit — set it per cmd. - doctorCmd.SilenceErrors = true - for _, sub := range doctorCmd.Commands() { - sub.SilenceErrors = true - } - // SilenceErrors also mutes flag-parse errors (e.g. an unknown flag), which - // ARE genuine usage failures the caller must see. Restore them via an - // explicit flag-error func (inherited by subcommands). - doctorCmd.SetFlagErrorFunc(func(c *cobra.Command, err error) error { - enriched := flagErrorWithSuggestion(c, err) - fmt.Fprintln(c.ErrOrStderr(), "Error:", enriched.Error()) - return enriched - }) -} - -// doctorWantsJSON reports whether the caller asked for JSON output. -func doctorWantsJSON() bool { - return doctorJSON || doctorRobot || jsonFlag -} - -func doctorMutationRequest() doctor.MutationRequest { - return doctor.MutationRequest{ - Only: append([]string(nil), doctorOnly...), Skip: append([]string(nil), doctorSkip...), - Quick: doctorQuick, Online: doctorOnline, Severity: doctorSeverity, - DryRun: doctorDryRun, JSON: doctorWantsJSON(), - } -} - -func doctorReadRequest() doctor.ReadRequest { - return doctor.ReadRequest{ - Only: append([]string(nil), doctorOnly...), Skip: append([]string(nil), doctorSkip...), - Quick: doctorQuick, Online: doctorOnline, Severity: doctorSeverity, - DryRun: doctorDryRun, JSON: doctorWantsJSON(), Since: doctorSince, - } -} - -// printDoctorJSON marshals v as indented JSON to stdout. -func printDoctorJSON(cmd *cobra.Command, v any) error { - data, err := json.MarshalIndent(v, "", " ") - if err != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: err.Error()} - } - fmt.Fprintln(cmd.OutOrStdout(), string(data)) - return nil -} - -// exitErr wraps an exit code into a doctorExitError unless the code is success. -func exitErr(code int, msg string) error { - if code == doctor.ExitHealthy { - return nil - } - return &doctorExitError{code: code, msg: msg} -} - -// runDoctorEngineDefault runs diagnose (or fix / explain / triage) for the -// default `ao doctor` invocation when an engine flag is present. -func runDoctorEngineDefault(cmd *cobra.Command) error { - if doctorExplainFlag != "" { - return runDoctorExplain(cmd, doctorExplainFlag) - } - if doctorRobotTriage { - triage, rep, terr := doctorReadService.Triage(cmd.Context(), doctorReadRequest()) - if terr != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: terr.Error()} - } - if err := printDoctorJSON(cmd, triage); err != nil { - return err - } - return exitErr(rep.ExitCode, "doctor findings present") - } - if doctorFix { - return runDoctorFix(cmd, doctorMutationRequest()) - } - rep, derr := doctorReadService.Diagnose(cmd.Context(), doctorReadRequest()) - if derr != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: derr.Error()} - } - if doctorWantsJSON() { - if err := printDoctorJSON(cmd, rep); err != nil { - return err - } - } else { - renderEngineFindings(cmd, rep) - } - return exitErr(rep.ExitCode, "doctor findings present") -} - -// renderEngineFindings appends a findings section to human-readable output. -func renderEngineFindings(cmd *cobra.Command, rep *doctor.Report) { - w := cmd.OutOrStdout() - if len(rep.Findings) == 0 { - return - } - fmt.Fprintln(w) - fmt.Fprintf(w, "Failure-mode findings (%d):\n", len(rep.Findings)) - for _, f := range rep.Findings { - fmt.Fprintf(w, " [%s] %s — %s\n", f.Severity, f.ID, f.Title) - } -} - -// runDoctorFix runs the fix engine and renders/exits accordingly. -func runDoctorFix(cmd *cobra.Command, request doctor.MutationRequest) error { - rep, err := doctorMutationService.Fix(cmd.Context(), request) - if err != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: err.Error()} - } - if doctorWantsJSON() { - if jerr := printDoctorJSON(cmd, rep); jerr != nil { - return jerr - } - } else { - w := cmd.OutOrStdout() - fmt.Fprintf(w, "ao doctor fix — run %s\n", rep.RunID) - fmt.Fprintf(w, " findings: %d actions: %d\n", rep.Summary.TotalFindings, rep.ActionsTaken) - if rep.UndoCommand != "" { - fmt.Fprintf(w, " undo: %s\n", rep.UndoCommand) - } - } - return exitErr(rep.ExitCode, "doctor fix incomplete") -} - -// runDoctorExplain expands a single finding. -func runDoctorExplain(cmd *cobra.Command, findingID string) error { - finding, ferr := doctorReadService.Explain(cmd.Context(), findingID) - if ferr != nil { - var runtimeFailure *doctor.RuntimeError - if errors.As(ferr, &runtimeFailure) { - return &doctorExitError{code: doctor.ExitIOError, msg: ferr.Error()} - } - return &doctorExitError{code: doctor.ExitNoInput, msg: ferr.Error()} - } - if doctorWantsJSON() { - return printDoctorJSON(cmd, finding) - } - w := cmd.OutOrStdout() - fmt.Fprintf(w, "%s [%s] (%s)\n%s\n", finding.ID, finding.Severity, finding.Subsystem, finding.Title) - fmt.Fprintf(w, "Remediation: %s\n", finding.Remediation.Command) - return nil -} - -func newDoctorFixCmd() *cobra.Command { - return &cobra.Command{ - Use: "fix", - Short: "Run detectors, then apply fixers (backs up before every mutation)", - RunE: func(cmd *cobra.Command, _ []string) error { - return runDoctorFix(cmd, doctorMutationRequest()) - }, - } -} - -func newDoctorUndoCmd() *cobra.Command { - c := &cobra.Command{ - Use: "undo ", - Short: "Restore from .doctor/runs//backups/ (run-id may be 'latest')", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - res, uerr := doctorMaintenanceService.Undo(cmd.Context(), doctor.UndoRequest{ - RunID: args[0], Strict: doctorUndoStrict, DryRun: doctorDryRun, - }) - if uerr != nil { - if doctorWantsJSON() { - _ = printDoctorJSON(cmd, map[string]any{ - "run_id": args[0], - "exit_code": doctor.ExitIOError, - "error": uerr.Error(), - }) - } else { - fmt.Fprintf(cmd.ErrOrStderr(), "undo failed: %v\n", uerr) - } - return &doctorExitError{code: doctor.ExitIOError, msg: uerr.Error()} - } - if doctorWantsJSON() { - return printDoctorJSON(cmd, res) - } - fmt.Fprintf(cmd.OutOrStdout(), "undo %s: restored=%d skipped=%d\n", res.RunID, res.Restored, res.Skipped) - return nil - }, - } - c.Flags().BoolVar(&doctorUndoStrict, "strict", true, "Refuse if any backup is missing or hash-mismatched") - c.Flags().BoolVar(&doctorDryRun, "dry-run", false, "Print the restore plan; do not execute") - return c -} - -func newDoctorExplainCmd() *cobra.Command { - return &cobra.Command{ - Use: "explain ", - Short: "Expand a single finding with full evidence", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - return runDoctorExplain(cmd, args[0]) - }, - } -} - -func newDoctorCapabilitiesCmd() *cobra.Command { - return &cobra.Command{ - Use: "capabilities", - Short: "Print the machine-readable doctor contract (JSON)", - RunE: func(cmd *cobra.Command, _ []string) error { - caps := doctorReadService.Capabilities(cmd.Context()) - return printDoctorJSON(cmd, caps) - }, - } -} - -func newDoctorHealthCmd() *cobra.Command { - return &cobra.Command{ - Use: "health", - Short: "Cheap one-line liveness summary", - RunE: func(cmd *cobra.Command, _ []string) error { - line, hr, herr := doctorReadService.Health(cmd.Context()) - if herr != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: herr.Error()} - } - if doctorWantsJSON() { - if jerr := printDoctorJSON(cmd, hr); jerr != nil { - return jerr - } - } else { - fmt.Fprintln(cmd.OutOrStdout(), line) - } - return exitErr(hr.ExitCode, "doctor health: not ok") - }, - } -} - -func newDoctorRobotDocsCmd() *cobra.Command { - return &cobra.Command{ - Use: "robot-docs", - Short: "Print the paste-ready agent handbook (Markdown)", - RunE: func(cmd *cobra.Command, _ []string) error { - fmt.Fprint(cmd.OutOrStdout(), doctorReadService.RobotDocs(cmd.Context())) - return nil - }, - } -} - -func newDoctorGcCmd() *cobra.Command { - c := &cobra.Command{ - Use: "gc", - Short: "Prune old runs (requires --yes and --before )", - RunE: func(cmd *cobra.Command, _ []string) error { - result, gerr := doctorMaintenanceService.GC(cmd.Context(), doctor.GCRequest{ - Before: doctorGCBefore, Yes: doctorGCYes, DryRun: doctorDryRun, - }) - if gerr != nil { - var usageFailure *doctor.UsageError - if errors.As(gerr, &usageFailure) { - return &doctorExitError{code: doctor.ExitUsage, msg: gerr.Error()} - } - return &doctorExitError{code: doctor.ExitIOError, msg: gerr.Error()} - } - if doctorWantsJSON() { - return printDoctorJSON(cmd, result) - } - if result.DryRun { - fmt.Fprintf(cmd.OutOrStdout(), "would prune %d run(s)\n", result.Matched) - return nil - } - fmt.Fprintf(cmd.OutOrStdout(), "pruned %d run(s)\n", result.Pruned) - return nil - }, - } - c.Flags().StringVar(&doctorGCBefore, "before", "", "Prune runs started before this date (YYYY-MM-DD)") - c.Flags().BoolVar(&doctorGCYes, "yes", false, "Confirm pruning (required)") - return c -} - -func newDoctorLsCmd() *cobra.Command { - return &cobra.Command{ - Use: "ls", - Short: "List runs in .doctor/runs/", - RunE: func(cmd *cobra.Command, _ []string) error { - runs, lerr := doctorReadService.List(cmd.Context()) - if lerr != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: lerr.Error()} - } - if doctorWantsJSON() { - return printDoctorJSON(cmd, map[string]any{"runs": runs}) - } - w := cmd.OutOrStdout() - if len(runs) == 0 { - fmt.Fprintln(w, "no doctor runs") - return nil - } - for _, r := range runs { - fmt.Fprintf(w, "%s exit=%d actions=%d\n", r.RunID, r.ExitCode, r.ActionCount) - } - return nil - }, - } -} - -func newDoctorDiffCmd() *cobra.Command { - return &cobra.Command{ - Use: "diff", - Short: "Show what --fix would change (read-only)", - RunE: func(cmd *cobra.Command, _ []string) error { - rep, derr := doctorReadService.Diff(cmd.Context(), doctorReadRequest()) - if derr != nil { - return &doctorExitError{code: doctor.ExitIOError, msg: derr.Error()} - } - if doctorWantsJSON() { - return printDoctorJSON(cmd, rep) - } - renderEngineFindings(cmd, rep) - if len(rep.Findings) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "clean diff: --fix would change nothing") - } - return nil - }, - } -} diff --git a/cli/cmd/ao/doctor_test.go b/cli/cmd/ao/doctor_test.go deleted file mode 100644 index 43c0a3e66..000000000 --- a/cli/cmd/ao/doctor_test.go +++ /dev/null @@ -1,1599 +0,0 @@ -// practices: [sre, resilience-patterns] -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/boshu2/agentops/cli/internal/doctor" -) - -func TestDoctorMutationRequestCopiesCommandFlags(t *testing.T) { - oldOnly, oldSkip := doctorOnly, doctorSkip - oldQuick, oldOnline := doctorQuick, doctorOnline - oldSeverity, oldDryRun := doctorSeverity, doctorDryRun - oldRobot := doctorRobot - t.Cleanup(func() { - doctorOnly, doctorSkip = oldOnly, oldSkip - doctorQuick, doctorOnline = oldQuick, oldOnline - doctorSeverity, doctorDryRun = oldSeverity, oldDryRun - doctorRobot = oldRobot - }) - - doctorOnly = []string{"one"} - doctorSkip = []string{"two"} - doctorQuick = true - doctorOnline = true - doctorSeverity = "P2" - doctorDryRun = true - doctorRobot = true - - request := doctorMutationRequest() - if len(request.Only) != 1 || request.Only[0] != "one" || len(request.Skip) != 1 || request.Skip[0] != "two" { - t.Fatalf("request=%+v", request) - } - if !request.Quick || !request.Online || request.Severity != "P2" || !request.DryRun || !request.JSON { - t.Fatalf("request=%+v", request) - } - doctorOnly[0] = "changed" - if request.Only[0] != "one" { - t.Fatalf("request aliases command flags: %+v", request) - } -} - -func TestComputeResult(t *testing.T) { - tests := []struct { - name string - checks []doctorCheck - wantResult string - wantFails bool - }{ - { - name: "all pass", - checks: []doctorCheck{ - {Name: "a", Status: "pass", Required: true}, - {Name: "b", Status: "pass", Required: true}, - }, - wantResult: "HEALTHY", - wantFails: false, - }, - { - name: "one failure", - checks: []doctorCheck{ - {Name: "a", Status: "pass", Required: true}, - {Name: "b", Status: "fail", Required: true}, - }, - wantResult: "UNHEALTHY", - wantFails: true, - }, - { - name: "warnings only", - checks: []doctorCheck{ - {Name: "a", Status: "pass", Required: true}, - {Name: "b", Status: "warn", Required: false}, - }, - wantResult: "DEGRADED", - wantFails: false, - }, - { - name: "mixed failures and warnings", - checks: []doctorCheck{ - {Name: "a", Status: "fail", Required: true}, - {Name: "b", Status: "warn", Required: false}, - {Name: "c", Status: "pass", Required: true}, - }, - wantResult: "UNHEALTHY", - wantFails: true, - }, - { - name: "empty checks", - checks: []doctorCheck{}, - wantResult: "HEALTHY", - wantFails: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - output := computeResult(tt.checks) - if output.Result != tt.wantResult { - t.Errorf("computeResult() result = %q, want %q", output.Result, tt.wantResult) - } - if tt.wantFails && output.Summary == "all checks passed" { - t.Error("expected failure in summary") - } - if !tt.wantFails && len(tt.checks) > 0 && !hasWarns(tt.checks) { - expected := fmt.Sprintf("%d/%d checks passed", len(tt.checks), len(tt.checks)) - if output.Summary != expected { - t.Errorf("expected %q, got %q", expected, output.Summary) - } - } - }) - } -} - -func hasWarns(checks []doctorCheck) bool { - for _, c := range checks { - if c.Status == "warn" { - return true - } - } - return false -} - -func TestCountFiles(t *testing.T) { - tmpDir := t.TempDir() - - t.Run("empty directory", func(t *testing.T) { - got := countFiles(tmpDir) - if got != 0 { - t.Errorf("countFiles(empty) = %d, want 0", got) - } - }) - - t.Run("with files", func(t *testing.T) { - if err := os.WriteFile(filepath.Join(tmpDir, "a.md"), []byte("test"), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmpDir, "b.md"), []byte("test"), 0644); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(tmpDir, "subdir"), 0755); err != nil { - t.Fatal(err) - } - - got := countFiles(tmpDir) - if got != 2 { - t.Errorf("countFiles() = %d, want 2 (should not count directories)", got) - } - }) - - t.Run("nonexistent directory", func(t *testing.T) { - got := countFiles(filepath.Join(tmpDir, "nonexistent")) - if got != 0 { - t.Errorf("countFiles(nonexistent) = %d, want 0", got) - } - }) -} - -// --- Integration tests for doctor check functions --- - -// chdirTemp moved to testutil_test.go. - -func TestCheckKnowledgeBase(t *testing.T) { - t.Run("initialized", func(t *testing.T) { - tmp := chdirTemp(t) - if err := os.MkdirAll(filepath.Join(tmp, ".agents", "ao"), 0755); err != nil { - t.Fatal(err) - } - result := checkKnowledgeBase() - if result.Status != "pass" { - t.Errorf("status=%q, want pass (detail: %s)", result.Status, result.Detail) - } - }) - - t.Run("not initialized", func(t *testing.T) { - chdirTemp(t) - result := checkKnowledgeBase() - if result.Status != "fail" { - t.Errorf("status=%q, want fail (detail: %s)", result.Status, result.Detail) - } - }) -} - -func TestCheckKnowledgeFreshness(t *testing.T) { - t.Run("recent session", func(t *testing.T) { - tmp := chdirTemp(t) - sessDir := filepath.Join(tmp, ".agents", "ao", "sessions") - if err := os.MkdirAll(sessDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(sessDir, "session-1.md"), []byte("recent"), 0644); err != nil { - t.Fatal(err) - } - - result := checkKnowledgeFreshness() - if result.Status != "pass" { - t.Errorf("status=%q, want pass (detail: %s)", result.Status, result.Detail) - } - }) - - t.Run("no sessions", func(t *testing.T) { - tmp := chdirTemp(t) - if err := os.MkdirAll(filepath.Join(tmp, ".agents", "ao", "sessions"), 0755); err != nil { - t.Fatal(err) - } - - result := checkKnowledgeFreshness() - if result.Status != "warn" { - t.Errorf("status=%q, want warn (detail: %s)", result.Status, result.Detail) - } - }) - - t.Run("no sessions dir", func(t *testing.T) { - chdirTemp(t) - result := checkKnowledgeFreshness() - if result.Status != "warn" { - t.Errorf("status=%q, want warn (detail: %s)", result.Status, result.Detail) - } - }) -} - -func TestCheckSearchIndex(t *testing.T) { - t.Run("index exists with content", func(t *testing.T) { - tmp := chdirTemp(t) - indexDir := filepath.Join(tmp, IndexDir) - if err := os.MkdirAll(indexDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(indexDir, IndexFileName), []byte("{\"term\":\"hello\"}\n{\"term\":\"world\"}\n"), 0644); err != nil { - t.Fatal(err) - } - - result := checkSearchIndex() - if result.Status != "pass" { - t.Errorf("status=%q, want pass (detail: %s)", result.Status, result.Detail) - } - }) - - t.Run("empty index", func(t *testing.T) { - tmp := chdirTemp(t) - indexDir := filepath.Join(tmp, IndexDir) - if err := os.MkdirAll(indexDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(indexDir, IndexFileName), []byte(""), 0644); err != nil { - t.Fatal(err) - } - - result := checkSearchIndex() - if result.Status != "warn" { - t.Errorf("status=%q, want warn (detail: %s)", result.Status, result.Detail) - } - }) - - t.Run("no index", func(t *testing.T) { - chdirTemp(t) - result := checkSearchIndex() - if result.Status != "warn" { - t.Errorf("status=%q, want warn (detail: %s)", result.Status, result.Detail) - } - }) -} - -func TestCheckFlywheelHealth(t *testing.T) { - t.Run("with learnings", func(t *testing.T) { - tmp := t.TempDir() - learningsDir := filepath.Join(tmp, ".agents", "ao", "learnings") - if err := os.MkdirAll(learningsDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(learningsDir, "L1.md"), []byte("learning 1"), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(learningsDir, "L2.md"), []byte("learning 2"), 0644); err != nil { - t.Fatal(err) - } - - result := checkFlywheelHealth(tmp) - if result.Status != "pass" { - t.Errorf("status=%q, want pass (detail: %s)", result.Status, result.Detail) - } - }) - - t.Run("no learnings", func(t *testing.T) { - tmp := t.TempDir() - result := checkFlywheelHealth(tmp) - if result.Status != "warn" { - t.Errorf("status=%q, want warn (detail: %s)", result.Status, result.Detail) - } - }) - - t.Run("alt path learnings", func(t *testing.T) { - tmp := t.TempDir() - altDir := filepath.Join(tmp, ".agents", "learnings") - if err := os.MkdirAll(altDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(altDir, "L1.md"), []byte("learning"), 0644); err != nil { - t.Fatal(err) - } - - result := checkFlywheelHealth(tmp) - if result.Status != "pass" { - t.Errorf("status=%q, want pass (detail: %s)", result.Status, result.Detail) - } - }) -} - -func TestCountFileLines(t *testing.T) { - tmp := t.TempDir() - - t.Run("file with lines", func(t *testing.T) { - path := filepath.Join(tmp, "test.jsonl") - if err := os.WriteFile(path, []byte("{\"a\":1}\n{\"b\":2}\n{\"c\":3}\n"), 0644); err != nil { - t.Fatal(err) - } - got := countFileLines(path) - if got != 3 { - t.Errorf("countFileLines() = %d, want 3", got) - } - }) - - t.Run("empty file", func(t *testing.T) { - path := filepath.Join(tmp, "empty.jsonl") - if err := os.WriteFile(path, []byte(""), 0644); err != nil { - t.Fatal(err) - } - got := countFileLines(path) - if got != 0 { - t.Errorf("countFileLines() = %d, want 0", got) - } - }) - - t.Run("nonexistent file", func(t *testing.T) { - got := countFileLines(filepath.Join(tmp, "nope")) - if got != 0 { - t.Errorf("countFileLines(nonexistent) = %d, want 0", got) - } - }) - - t.Run("blank lines ignored", func(t *testing.T) { - path := filepath.Join(tmp, "blanks.jsonl") - if err := os.WriteFile(path, []byte("line1\n\n \nline2\n"), 0644); err != nil { - t.Fatal(err) - } - got := countFileLines(path) - if got != 2 { - t.Errorf("countFileLines() = %d, want 2", got) - } - }) -} - -func TestFormatNumber(t *testing.T) { - tests := []struct { - input int - want string - }{ - {0, "0"}, - {42, "42"}, - {999, "999"}, - {1000, "1,000"}, - {1247, "1,247"}, - {1000000, "1,000,000"}, - } - for _, tt := range tests { - t.Run(fmt.Sprintf("%d", tt.input), func(t *testing.T) { - got := formatNumber(tt.input) - if got != tt.want { - t.Errorf("formatNumber(%d) = %q, want %q", tt.input, got, tt.want) - } - }) - } -} - -func TestFormatDuration(t *testing.T) { - tests := []struct { - name string - input time.Duration - want string - }{ - {"seconds", 30 * time.Second, "30s"}, - {"minutes", 5 * time.Minute, "5m"}, - {"hours", 3 * time.Hour, "3h"}, - {"days", 48 * time.Hour, "2d"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := formatDuration(tt.input) - if got != tt.want { - t.Errorf("formatDuration() = %q, want %q", got, tt.want) - } - }) - } -} - -func TestCountHealFindings(t *testing.T) { - tests := []struct { - name string - output string - want int - }{ - { - name: "no findings", - output: "All clean. No findings.", - want: 0, - }, - { - name: "report format findings", - output: "[MISSING_NAME] skills/foo: No name field in frontmatter\n[DEAD_REF] skills/bar: SKILL.md references non-existent references/old.md\n\n2 finding(s) detected.\n", - want: 2, - }, - { - name: "summary only fallback", - output: "some noise\n5 finding(s) detected.\n", - want: 5, - }, - { - name: "empty output", - output: "", - want: 0, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := countHealFindings(tt.output) - if got != tt.want { - t.Errorf("countHealFindings() = %d, want %d", got, tt.want) - } - }) - } -} - -func TestCheckSkillIntegrity_NoHealScript(t *testing.T) { - // When run from a temp dir with no heal.sh, should warn gracefully. - // Override HOME so findHealScript() can't find ~/.claude/skills/heal-skill/. - // soc-y1bk: t.Setenv auto-restores prior HOME on cleanup; no manual reset needed. - chdirTemp(t) - t.Setenv("HOME", t.TempDir()) - result := checkSkillIntegrity() - if result.Status != "warn" { - t.Errorf("status=%q, want warn (detail: %s)", result.Status, result.Detail) - } - if result.Required { - t.Error("Skill Integrity should not be required") - } -} - -func TestCheckSkillIntegrity_WithHealScript(t *testing.T) { - // Create a minimal heal.sh that exits 0 (all clean) - tmp := chdirTemp(t) - skillDir := filepath.Join(tmp, "skills", "heal-skill", "scripts") - if err := os.MkdirAll(skillDir, 0755); err != nil { - t.Fatal(err) - } - healScript := filepath.Join(skillDir, "heal.sh") - if err := os.WriteFile(healScript, []byte("#!/usr/bin/env bash\necho 'All clean. No findings.'\nexit 0\n"), 0755); err != nil { - t.Fatal(err) - } - - result := checkSkillIntegrity() - if result.Status != "pass" { - t.Errorf("status=%q, want pass (detail: %s)", result.Status, result.Detail) - } -} - -func TestSkillOverlapWarning_NoOverlap(t *testing.T) { - base := map[string]struct{}{"a": {}, "b": {}} - other := map[string]struct{}{"c": {}, "d": {}} - result := skillOverlapWarning(base, 10, "primary", "%d found in %s; %d overlap: %s", other) - if result != nil { - t.Errorf("expected nil for non-overlapping sets, got %+v", result) - } -} - -func TestSkillOverlapWarning_WithOverlap(t *testing.T) { - base := map[string]struct{}{"a": {}, "b": {}, "c": {}} - other := map[string]struct{}{"b": {}, "c": {}, "d": {}} - result := skillOverlapWarning(base, 10, "primary", "%d found in %s; %d overlap: %s", other) - if result == nil { - t.Fatal("expected warning for overlapping sets, got nil") - } - if result.Status != "warn" { - t.Errorf("status=%q, want warn", result.Status) - } - if !strings.Contains(result.Detail, "2 overlap") { - t.Errorf("detail=%q, want '2 overlap' substring", result.Detail) - } -} - -func TestScanSkillDir_NonexistentDir(t *testing.T) { - result := scanSkillDir("/nonexistent/path/that/does/not/exist") - if result != nil { - t.Errorf("expected nil for nonexistent dir, got %v", result) - } -} - -func TestScanSkillDir_WithSkills(t *testing.T) { - tmp := t.TempDir() - // Create a skill directory with SKILL.md - skillDir := filepath.Join(tmp, "my-skill") - if err := os.MkdirAll(skillDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# skill"), 0644); err != nil { - t.Fatal(err) - } - // Create a non-skill directory (no SKILL.md) - if err := os.MkdirAll(filepath.Join(tmp, "not-a-skill"), 0755); err != nil { - t.Fatal(err) - } - - result := scanSkillDir(tmp) - if result == nil { - t.Fatal("expected non-nil result") - } - if _, ok := result["my-skill"]; !ok { - t.Error("expected 'my-skill' in result") - } - if _, ok := result["not-a-skill"]; ok { - t.Error("did not expect 'not-a-skill' in result (no SKILL.md)") - } -} - -func TestCheckSkillIntegrity_WithFindings(t *testing.T) { - // Create a heal.sh that reports findings and exits 1 - tmp := chdirTemp(t) - skillDir := filepath.Join(tmp, "skills", "heal-skill", "scripts") - if err := os.MkdirAll(skillDir, 0755); err != nil { - t.Fatal(err) - } - healScript := filepath.Join(skillDir, "heal.sh") - script := `#!/usr/bin/env bash -echo "[MISSING_NAME] skills/foo: No name field" -echo "[DEAD_REF] skills/bar: references non-existent file" -echo "" -echo "2 finding(s) detected." -exit 1 -` - if err := os.WriteFile(healScript, []byte(script), 0755); err != nil { - t.Fatal(err) - } - - result := checkSkillIntegrity() - if result.Status != "warn" { - t.Errorf("status=%q, want warn (detail: %s)", result.Status, result.Detail) - } -} - -func TestFileExists(t *testing.T) { - tmp := t.TempDir() - f := filepath.Join(tmp, "exists.txt") - if err := os.WriteFile(f, []byte("hi"), 0644); err != nil { - t.Fatal(err) - } - if !fileExists(f) { - t.Error("expected fileExists to return true for existing file") - } - if fileExists(filepath.Join(tmp, "nope.txt")) { - t.Error("expected fileExists to return false for non-existent file") - } -} - -func TestDoctorStaleReplacementsExist(t *testing.T) { - // Every replacement (value) in deprecatedCommands should resolve to a - // real Cobra command registered under rootCmd. - for old, newCmd := range deprecatedCommands { - parts := strings.Fields(newCmd) - if len(parts) < 2 { - t.Errorf("deprecatedCommands[%q] = %q — too few parts", old, newCmd) - continue - } - // Walk rootCmd to find the command (skip "ao" prefix). rootCmd.Find - // returns rootCmd itself when nothing matches — a dead replacement. - cmd, _, err := rootCmd.Find(parts[1:]) - if err != nil || cmd == rootCmd { - // The replacement is not registered in this build. In the spine - // (default) build some replacements point to commands archived - // behind a build tag (e.g. "ao curate" → //go:build flywheel, - // age-nzwo). Tolerate those here; the flywheel/legacy run of this - // test — where every archived command is compiled back in — is the - // strict check that still catches a genuinely dead replacement. - if len(archiveBuildTags) == 0 { - t.Logf("deprecatedCommands[%q] = %q — replacement absent from spine build (archived behind a build tag); checked by the archive-tag run", old, newCmd) - continue - } - t.Errorf("deprecatedCommands[%q] = %q — command not found (err=%v)", old, newCmd, err) - } - } -} - -func TestRenderDoctorTable_Healthy(t *testing.T) { - output := doctorOutput{ - Checks: []doctorCheck{ - {Name: "ao CLI", Status: "pass", Detail: "v2.0.0", Required: true}, - {Name: "Knowledge", Status: "pass", Detail: "ok", Required: true}, - }, - Result: "HEALTHY", - Summary: "2/2 checks passed", - } - var buf bytes.Buffer - renderDoctorTable(&buf, output) - rendered := buf.String() - - if !strings.Contains(rendered, "ao doctor") { - t.Error("expected header 'ao doctor'") - } - if !strings.Contains(rendered, "ao CLI") { - t.Error("expected check name 'ao CLI'") - } - if !strings.Contains(rendered, "2/2 checks passed") { - t.Error("expected summary in output") - } -} - -func TestRenderDoctorTable_WithWarnings(t *testing.T) { - output := doctorOutput{ - Checks: []doctorCheck{ - {Name: "Pass Check", Status: "pass", Detail: "ok", Required: true}, - {Name: "Warn Check", Status: "warn", Detail: "degraded", Required: false}, - {Name: "Fail Check", Status: "fail", Detail: "broken", Required: true}, - {Name: "Unknown", Status: "unknown", Detail: "???", Required: false}, - }, - Result: "UNHEALTHY", - Summary: "1/4 checks passed, 1 warning, 1 failed", - } - var buf bytes.Buffer - renderDoctorTable(&buf, output) - rendered := buf.String() - - // All status icons should appear - if !strings.Contains(rendered, "\u2713") { - t.Error("expected pass icon") - } - if !strings.Contains(rendered, "!") { - t.Error("expected warn icon") - } - if !strings.Contains(rendered, "\u2717") { - t.Error("expected fail icon") - } - if !strings.Contains(rendered, "?") { - t.Error("expected unknown icon") - } -} - -func TestRenderDoctorTable_Empty(t *testing.T) { - output := doctorOutput{ - Checks: []doctorCheck{}, - Result: "HEALTHY", - Summary: "0/0 checks passed", - } - var buf bytes.Buffer - renderDoctorTable(&buf, output) - if buf.Len() == 0 { - t.Error("expected non-empty output even with no checks") - } -} - -// --------------------------------------------------------------------------- -// doctorStatusIcon (already 100%, but test unknown branch explicitly) -// --------------------------------------------------------------------------- - -func TestDoctorStatusIcon_Unknown(t *testing.T) { - got := doctorStatusIcon("bogus") - if got != "?" { - t.Errorf("doctorStatusIcon(bogus) = %q, want ?", got) - } -} - -// --------------------------------------------------------------------------- -// checkOptionalCLI (0%) -// --------------------------------------------------------------------------- - -func TestCheckOptionalCLI_Available(t *testing.T) { - // "go" should be in PATH during tests - result := checkOptionalCLI("go", "test builds") - if result.Status != "pass" { - t.Errorf("status=%q, want pass for 'go' CLI (detail: %s)", result.Status, result.Detail) - } - if result.Required { - t.Error("optional CLI should not be required") - } - if !strings.Contains(result.Detail, "available") { - t.Errorf("expected 'available' in detail, got %q", result.Detail) - } -} - -func TestCheckOptionalCLI_NotAvailable(t *testing.T) { - result := checkOptionalCLI("nonexistent_cli_xyz_999", "test feature") - if result.Status != "warn" { - t.Errorf("status=%q, want warn for missing CLI (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "not found") { - t.Errorf("expected 'not found' in detail, got %q", result.Detail) - } - if !strings.Contains(result.Detail, "test feature") { - t.Errorf("expected reason in detail, got %q", result.Detail) - } -} - -// --------------------------------------------------------------------------- -// checkCLIDependencies (0%) -// --------------------------------------------------------------------------- - -func TestCheckCLIDependencies(t *testing.T) { - // We can't guarantee gt/bd availability, but we can exercise the function. - result := checkCLIDependencies() - // It should return either pass or warn, never fail - if result.Status != "pass" && result.Status != "warn" { - t.Errorf("status=%q, expected pass or warn", result.Status) - } - if result.Name != "CLI Dependencies" { - t.Errorf("name=%q, want 'CLI Dependencies'", result.Name) - } - if result.Required { - t.Error("CLI Dependencies should not be required") - } -} - -// --------------------------------------------------------------------------- -// checkSkills (0%) — needs a fake HOME -// --------------------------------------------------------------------------- - -func TestCheckSkills_NoSkills(t *testing.T) { - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - result := checkSkills() - if result.Status != "warn" { - t.Errorf("status=%q, want warn when no skills installed (detail: %s)", result.Status, result.Detail) - } -} - -func TestCheckSkills_WithSkills(t *testing.T) { - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - // Create a fake skill directory with SKILL.md - skillDir := filepath.Join(fakeHome, ".claude", "skills", "fake-skill") - if err := os.MkdirAll(skillDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# Fake Skill"), 0644); err != nil { - t.Fatal(err) - } - - result := checkSkills() - if result.Status != "pass" { - t.Errorf("status=%q, want pass when skills are found (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "1 skills found in ~/.claude/skills") { - t.Errorf("expected install path in detail, got %q", result.Detail) - } -} - -func TestCheckSkills_WithNativeCodexPlugin(t *testing.T) { - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - skillsRoot := filepath.Join(fakeHome, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local", "skills-codex") - skillDir := filepath.Join(skillsRoot, "fake-skill") - if err := os.MkdirAll(skillDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# Fake Skill"), 0644); err != nil { - t.Fatal(err) - } - manifestPath := filepath.Join(skillsRoot, ".agentops-manifest.json") - if err := os.WriteFile(manifestPath, []byte(`{"skills":[{"name":"fake-skill"}]}`), 0644); err != nil { - t.Fatal(err) - } - manifestHash, err := sha256File(manifestPath) - if err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(fakeHome, ".codex"), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(fakeHome, ".codex", ".agentops-codex-install.json"), []byte(fmt.Sprintf(`{"install_mode":"native-plugin","plugin_root":"%s","manifest_hash":"%s","skill_count":1}`, filepath.Join(fakeHome, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local"), manifestHash)), 0644); err != nil { - t.Fatal(err) - } - - result := checkSkills() - if result.Status != "pass" { - t.Errorf("status=%q, want pass when native plugin skills are found (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "~/.codex/plugins/cache/agentops-marketplace/agentops/local/skills-codex") || !strings.Contains(result.Detail, "native manifest OK") { - t.Errorf("expected native plugin path in detail, got %q", result.Detail) - } -} - -func TestCheckSkills_NativeCodexPluginMissingManifestWarns(t *testing.T) { - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - skillDir := filepath.Join(fakeHome, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local", "skills-codex", "fake-skill") - if err := os.MkdirAll(skillDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# Fake Skill"), 0644); err != nil { - t.Fatal(err) - } - - result := checkSkills() - if result.Status != "warn" { - t.Fatalf("status=%q, want warn when native plugin manifest is missing (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, ".agentops-manifest.json") { - t.Fatalf("expected manifest warning in detail, got %q", result.Detail) - } -} - -func TestCheckSkills_NativeCodexPluginManifestMismatchWarns(t *testing.T) { - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - skillsRoot := filepath.Join(fakeHome, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local", "skills-codex") - skillDir := filepath.Join(skillsRoot, "fake-skill") - if err := os.MkdirAll(skillDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# Fake Skill"), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(skillsRoot, ".agentops-manifest.json"), []byte(`{"skills":[{"name":"fake-skill"}]}`), 0644); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(fakeHome, ".codex"), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(fakeHome, ".codex", ".agentops-codex-install.json"), []byte(fmt.Sprintf(`{"install_mode":"native-plugin","plugin_root":"%s","manifest_hash":"deadbeef","skill_count":1}`, filepath.Join(fakeHome, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local"))), 0644); err != nil { - t.Fatal(err) - } - - result := checkSkills() - if result.Status != "warn" { - t.Fatalf("status=%q, want warn when native plugin manifest mismatches (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "manifest hash does not match") { - t.Fatalf("expected manifest mismatch warning, got %q", result.Detail) - } -} - -func TestCheckSkills_AltPath(t *testing.T) { - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - // Create skills under .agents/skills/ (alt path) - skillDir := filepath.Join(fakeHome, ".agents", "skills", "alt-skill") - if err := os.MkdirAll(skillDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# Alt Skill"), 0644); err != nil { - t.Fatal(err) - } - - result := checkSkills() - if result.Status != "pass" { - t.Errorf("status=%q, want pass for alt-path skills (detail: %s)", result.Status, result.Detail) - } -} - -func TestCheckSkills_UserSkillsOverlapWarnsWithoutPluginCache(t *testing.T) { - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - for _, dir := range []string{ - filepath.Join(fakeHome, ".codex", "skills", "research"), - filepath.Join(fakeHome, ".codex", "skills", "vibe"), - filepath.Join(fakeHome, ".agents", "skills", "research"), - } { - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte("# Skill"), 0644); err != nil { - t.Fatal(err) - } - } - - result := checkSkills() - if result.Status != "warn" { - t.Fatalf("status=%q, want warn for duplicate raw installs (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "duplicate raw skill install") { - t.Fatalf("expected duplicate raw install warning, got %q", result.Detail) - } - if !strings.Contains(result.Detail, "~/.agents/skills") { - t.Fatalf("expected legacy path in detail, got %q", result.Detail) - } - if !strings.Contains(result.Detail, "research") { - t.Fatalf("expected overlapping skill sample in detail, got %q", result.Detail) - } -} - -func TestCheckSkills_PluginCacheAndUserSkillsWarn(t *testing.T) { - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - for _, dir := range []string{ - filepath.Join(fakeHome, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local", "skills-codex", "research"), - filepath.Join(fakeHome, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local", "skills-codex", "vibe"), - filepath.Join(fakeHome, ".agents", "skills", "research"), - filepath.Join(fakeHome, ".agents", "skills", "vibe"), - } { - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte("# Skill"), 0644); err != nil { - t.Fatal(err) - } - } - - pluginRoot := filepath.Join(fakeHome, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local") - manifestPath := filepath.Join(pluginRoot, "skills-codex", ".agentops-manifest.json") - if err := os.WriteFile(manifestPath, []byte(`{"skills":[{"name":"research"},{"name":"vibe"}]}`), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(pluginRoot, ".agentops-codex-state.json"), []byte(`{"manifest_hash":"abc123","skill_count":2}`), 0644); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(fakeHome, ".codex"), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(fakeHome, ".codex", ".agentops-codex-install.json"), []byte(`{"install_mode":"native-plugin","plugin_root":"`+pluginRoot+`","skill_count":2}`), 0644); err != nil { - t.Fatal(err) - } - - result := checkSkills() - if result.Status != "warn" { - t.Fatalf("status=%q, want warn when plugin cache and ~/.agents/skills overlap (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "duplicate raw skill install") || !strings.Contains(result.Detail, "~/.agents/skills") { - t.Fatalf("expected duplicate ~/.agents/skills warning, got %q", result.Detail) - } -} - -func TestCheckSkills_RawCodexOverlapWarns(t *testing.T) { - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - for _, dir := range []string{ - filepath.Join(fakeHome, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local", "skills-codex", "research"), - filepath.Join(fakeHome, ".codex", "skills", "research"), - } { - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte("# Skill"), 0644); err != nil { - t.Fatal(err) - } - } - - result := checkSkills() - if result.Status != "warn" { - t.Fatalf("status=%q, want warn for duplicate raw Codex installs (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "duplicate raw Codex install") { - t.Fatalf("expected duplicate raw Codex warning, got %q", result.Detail) - } -} - -func TestCheckCodexSync_PassWhenRepoMatchesInstall(t *testing.T) { - fixture := setupCodexSyncFixture(t) - pluginRoot := filepath.Join(fixture.home, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local") - writeCodexInstallMeta(t, fixture.home, fixture.version, fixture.manifestHash, pluginRoot) - - result := checkCodexSync() - if result.Status != "pass" { - t.Fatalf("status=%q, want pass (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "matches repo") { - t.Fatalf("expected match detail, got %q", result.Detail) - } -} - -func TestCheckCodexSync_WarnsOnManifestDriftAtSameVersion(t *testing.T) { - repo := chdirTemp(t) - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - if err := exec.Command("git", "-C", repo, "init").Run(); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "config", "user.email", "test@example.com").Run(); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "config", "user.name", "Test").Run(); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(repo, "skills-codex"), 0755); err != nil { - t.Fatal(err) - } - manifestPath := filepath.Join(repo, "skills-codex", ".agentops-manifest.json") - if err := os.WriteFile(manifestPath, []byte(`{"skills":[{"name":"research"}]}`), 0644); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "add", ".").Run(); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "commit", "-m", "fixture").Run(); err != nil { - t.Fatal(err) - } - - versionOut, err := exec.Command("git", "-C", repo, "rev-parse", "--short", "HEAD").Output() - if err != nil { - t.Fatal(err) - } - version := strings.TrimSpace(string(versionOut)) - - metaDir := filepath.Join(fakeHome, ".codex") - if err := os.MkdirAll(metaDir, 0755); err != nil { - t.Fatal(err) - } - meta := fmt.Sprintf(`{"install_mode":"native-plugin","version":"%s","manifest_hash":"stale-hash"}`, version) - if err := os.WriteFile(filepath.Join(metaDir, ".agentops-codex-install.json"), []byte(meta), 0644); err != nil { - t.Fatal(err) - } - - result := checkCodexSync() - if result.Status != "warn" { - t.Fatalf("status=%q, want warn (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "manifest differs from repo") { - t.Fatalf("expected manifest drift detail, got %q", result.Detail) - } - if strings.Contains(result.Detail, " -> ") { - t.Fatalf("expected non-version drift detail, got %q", result.Detail) - } -} - -func TestCheckCodexSync_WarnsWhenRepoIsNewer(t *testing.T) { - repo := chdirTemp(t) - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - if err := exec.Command("git", "-C", repo, "init").Run(); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "config", "user.email", "test@example.com").Run(); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "config", "user.name", "Test").Run(); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(repo, "skills-codex"), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(repo, "skills-codex", ".agentops-manifest.json"), []byte(`{"skills":[{"name":"research"}]}`), 0644); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "add", ".").Run(); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "commit", "-m", "fixture").Run(); err != nil { - t.Fatal(err) - } - - metaDir := filepath.Join(fakeHome, ".codex") - if err := os.MkdirAll(metaDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(metaDir, ".agentops-codex-install.json"), []byte(`{"install_mode":"native-plugin","version":"oldsha","manifest_hash":"deadbeef"}`), 0644); err != nil { - t.Fatal(err) - } - - result := checkCodexSync() - if result.Status != "warn" { - t.Fatalf("status=%q, want warn (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "refresh-codex-local.sh") { - t.Fatalf("expected repair command in detail, got %q", result.Detail) - } -} - -type codexSyncFixture struct { - repo string - home string - version string - manifestHash string -} - -func setupCodexSyncFixture(t *testing.T) codexSyncFixture { - t.Helper() - - repo := chdirTemp(t) - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - if err := exec.Command("git", "-C", repo, "init").Run(); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "config", "user.email", "test@example.com").Run(); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "config", "user.name", "Test").Run(); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(repo, "skills-codex"), 0755); err != nil { - t.Fatal(err) - } - manifestPath := filepath.Join(repo, "skills-codex", ".agentops-manifest.json") - if err := os.WriteFile(manifestPath, []byte(`{"skills":[{"name":"research"}]}`), 0644); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "add", ".").Run(); err != nil { - t.Fatal(err) - } - if err := exec.Command("git", "-C", repo, "commit", "-m", "fixture").Run(); err != nil { - t.Fatal(err) - } - manifestHash, err := sha256File(manifestPath) - if err != nil { - t.Fatal(err) - } - versionOut, err := exec.Command("git", "-C", repo, "rev-parse", "--short", "HEAD").Output() - if err != nil { - t.Fatal(err) - } - - return codexSyncFixture{ - repo: repo, - home: fakeHome, - version: strings.TrimSpace(string(versionOut)), - manifestHash: manifestHash, - } -} - -func writeCodexInstallMeta(t *testing.T, home, version, manifestHash, pluginRoot string) { - t.Helper() - - metaDir := filepath.Join(home, ".codex") - if err := os.MkdirAll(metaDir, 0755); err != nil { - t.Fatal(err) - } - meta := fmt.Sprintf(`{"install_mode":"native-plugin","hook_runtime":"codex-native-hooks","version":"%s","manifest_hash":"%s","plugin_root":"%s"}`, - version, manifestHash, pluginRoot) - if err := os.WriteFile(filepath.Join(metaDir, ".agentops-codex-install.json"), []byte(meta), 0644); err != nil { - t.Fatal(err) - } -} - -// --------------------------------------------------------------------------- -// runDoctor (0%) — exercise via cobra command -// --------------------------------------------------------------------------- - -func TestRunDoctor_TableOutput(t *testing.T) { - tmp := chdirTemp(t) - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - // Set up knowledge base so at least one check passes - if err := os.MkdirAll(filepath.Join(tmp, ".agents", "ao"), 0755); err != nil { - t.Fatal(err) - } - - // Save and restore the global doctorJSON flag - origJSON := doctorJSON - doctorJSON = false - t.Cleanup(func() { doctorJSON = origJSON }) - - var buf bytes.Buffer - doctorCmd.SetOut(&buf) - doctorCmd.SetErr(&buf) - t.Cleanup(func() { doctorCmd.SetOut(nil); doctorCmd.SetErr(nil) }) - - // runDoctor may return error (required check failure) — we just ensure it runs - _ = runDoctor(doctorCmd, nil) - - rendered := buf.String() - if !strings.Contains(rendered, "ao doctor") { - t.Error("expected 'ao doctor' header in table output") - } -} - -func TestRunDoctor_JSONOutput(t *testing.T) { - tmp := chdirTemp(t) - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - if err := os.MkdirAll(filepath.Join(tmp, ".agents", "ao"), 0755); err != nil { - t.Fatal(err) - } - - origJSON := doctorJSON - doctorJSON = true - t.Cleanup(func() { doctorJSON = origJSON }) - - var buf bytes.Buffer - doctorCmd.SetOut(&buf) - doctorCmd.SetErr(&buf) - t.Cleanup(func() { doctorCmd.SetOut(nil); doctorCmd.SetErr(nil) }) - - _ = runDoctor(doctorCmd, nil) - - // `ao doctor --json` emits a single engine Report, not the legacy - // check-table shape. Strict unmarshal fails if a second JSON document - // (the old concatenated legacy output) leaked onto stdout. - var rep doctor.Report - if err := json.Unmarshal(buf.Bytes(), &rep); err != nil { - t.Fatalf("expected a single valid engine Report, got error: %v\nOutput: %s", err, buf.String()) - } - if rep.SchemaVersion != "1.0" { - t.Errorf("expected schema_version 1.0, got %q", rep.SchemaVersion) - } - if rep.Tool != "ao" { - t.Errorf("expected tool 'ao', got %q", rep.Tool) - } - if rep.ExitCode != 0 && rep.ExitCode != 1 { - t.Errorf("expected diagnose exit_code 0 or 1, got %d", rep.ExitCode) - } -} - -// --------------------------------------------------------------------------- -// newestFileModTime — branch for no regular files -// --------------------------------------------------------------------------- - -func TestNewestFileModTime_OnlyDirs(t *testing.T) { - tmp := t.TempDir() - if err := os.MkdirAll(filepath.Join(tmp, "subdir"), 0755); err != nil { - t.Fatal(err) - } - entries, err := os.ReadDir(tmp) - if err != nil { - t.Fatal(err) - } - newest := newestFileModTime(entries) - if !newest.IsZero() { - t.Error("expected zero time for directory-only entries") - } -} - -// --------------------------------------------------------------------------- -// countEstablished -// --------------------------------------------------------------------------- - -func TestCountEstablished(t *testing.T) { - tmp := t.TempDir() - // Create some files with various names - for _, name := range []string{ - "learning-established-001.md", - "learning-promoted-002.md", - "learning-provisional-003.md", - "unrelated.jsonl", - } { - if err := os.WriteFile(filepath.Join(tmp, name), []byte("test"), 0644); err != nil { - t.Fatal(err) - } - } - - got := countEstablished(tmp) - if got != 2 { - t.Errorf("countEstablished() = %d, want 2", got) - } -} - -func TestCountEstablished_NonexistentDir(t *testing.T) { - got := countEstablished("/nonexistent/path/xyz") - if got != 0 { - t.Errorf("countEstablished(nonexistent) = %d, want 0", got) - } -} - -// --------------------------------------------------------------------------- -// findHealScript — exercise all branches -// --------------------------------------------------------------------------- - -func TestFindHealScript_NotFound(t *testing.T) { - // Use a temp dir so in-repo heal.sh isn't found - chdirTemp(t) - t.Setenv("HOME", t.TempDir()) - - path := findHealScript() - if path != "" { - t.Errorf("expected empty path, got %q", path) - } -} - -func TestFindHealScript_InClaude(t *testing.T) { - chdirTemp(t) - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - // Create heal.sh in ~/.claude/skills/heal-skill/scripts/ - healDir := filepath.Join(fakeHome, ".claude", "skills", "heal-skill", "scripts") - if err := os.MkdirAll(healDir, 0755); err != nil { - t.Fatal(err) - } - healPath := filepath.Join(healDir, "heal.sh") - if err := os.WriteFile(healPath, []byte("#!/bin/bash\nexit 0\n"), 0755); err != nil { - t.Fatal(err) - } - - got := findHealScript() - if got != healPath { - t.Errorf("findHealScript() = %q, want %q", got, healPath) - } -} - -func TestFindHealScript_InCodexPluginCache(t *testing.T) { - chdirTemp(t) - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - healDir := filepath.Join(fakeHome, ".codex", "plugins", "cache", "agentops-marketplace", "agentops", "local", "skills-codex", "heal-skill", "scripts") - if err := os.MkdirAll(healDir, 0755); err != nil { - t.Fatal(err) - } - healPath := filepath.Join(healDir, "heal.sh") - if err := os.WriteFile(healPath, []byte("#!/bin/bash\nexit 0\n"), 0755); err != nil { - t.Fatal(err) - } - - got := findHealScript() - if got != healPath { - t.Errorf("findHealScript() = %q, want %q", got, healPath) - } -} - -func TestFindHealScript_InAgents(t *testing.T) { - chdirTemp(t) - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - // Create heal.sh in ~/.agents/skills/heal-skill/scripts/ - healDir := filepath.Join(fakeHome, ".agents", "skills", "heal-skill", "scripts") - if err := os.MkdirAll(healDir, 0755); err != nil { - t.Fatal(err) - } - healPath := filepath.Join(healDir, "heal.sh") - if err := os.WriteFile(healPath, []byte("#!/bin/bash\nexit 0\n"), 0755); err != nil { - t.Fatal(err) - } - - got := findHealScript() - if got != healPath { - t.Errorf("findHealScript() = %q, want %q", got, healPath) - } -} - -// --------------------------------------------------------------------------- -// gatherDoctorChecks (0%) — smoke test that it returns checks -// --------------------------------------------------------------------------- - -func TestGatherDoctorChecks(t *testing.T) { - tmp := chdirTemp(t) - fakeHome := t.TempDir() - t.Setenv("HOME", fakeHome) - - // Set up minimal environment - if err := os.MkdirAll(filepath.Join(tmp, ".agents", "ao"), 0755); err != nil { - t.Fatal(err) - } - - checks := gatherDoctorChecks() - if len(checks) == 0 { - t.Error("expected at least one check from gatherDoctorChecks()") - } - - // First check should be ao CLI - if checks[0].Name != "ao CLI" { - t.Errorf("first check name = %q, want 'ao CLI'", checks[0].Name) - } - if checks[0].Status != "pass" { - t.Errorf("ao CLI check status = %q, want 'pass'", checks[0].Status) - } -} - -// --------------------------------------------------------------------------- -// checkStaleReferences — stale command reference detector -// --------------------------------------------------------------------------- - -func TestCheckStaleReferences_NoFiles(t *testing.T) { - chdirTemp(t) - result := checkStaleReferences() - if result.Status != "pass" { - t.Errorf("status=%q, want pass when no hooks/skills exist (detail: %s)", result.Status, result.Detail) - } - if result.Required { - t.Error("Stale References should not be required") - } -} - -func TestCheckStaleReferences_CleanFiles(t *testing.T) { - tmp := chdirTemp(t) - - // Create a hooks file with only current flat-style commands - hooksDir := filepath.Join(tmp, "hooks") - if err := os.MkdirAll(hooksDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(hooksDir, "dispatch.sh"), []byte("#!/bin/bash\nao forge transcript\nao rpi start\n"), 0644); err != nil { - t.Fatal(err) - } - - // Create a skill file with only current flat-style commands - skillDir := filepath.Join(tmp, "skills", "test-skill") - if err := os.MkdirAll(skillDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# Test Skill\nRun `ao inject` to load context.\n"), 0644); err != nil { - t.Fatal(err) - } - - result := checkStaleReferences() - if result.Status != "pass" { - t.Errorf("status=%q, want pass for clean files (detail: %s)", result.Status, result.Detail) - } -} - -func TestCheckStaleReferences_StaleInSkills(t *testing.T) { - tmp := chdirTemp(t) - - skillDir := filepath.Join(tmp, "skills", "test-skill") - if err := os.MkdirAll(skillDir, 0755); err != nil { - t.Fatal(err) - } - // Use old namespace-style "ao know inject" instead of flat "ao inject" - if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("# Test\nRun `ao know inject` to load.\n"), 0644); err != nil { - t.Fatal(err) - } - - result := checkStaleReferences() - if result.Status != "warn" { - t.Errorf("status=%q, want warn for stale skill reference (detail: %s)", result.Status, result.Detail) - } -} - -func TestCheckStaleReferences_StaleInDocs(t *testing.T) { - tmp := chdirTemp(t) - - docsDir := filepath.Join(tmp, "docs") - if err := os.MkdirAll(docsDir, 0755); err != nil { - t.Fatal(err) - } - // Use old namespace-style "ao work rpi" instead of flat "ao rpi" - if err := os.WriteFile(filepath.Join(docsDir, "guide.md"), []byte("# Guide\nRun `ao work rpi status` to check.\n"), 0644); err != nil { - t.Fatal(err) - } - - result := checkStaleReferences() - if result.Status != "warn" { - t.Errorf("status=%q, want warn for stale docs reference (detail: %s)", result.Status, result.Detail) - } -} - -func TestCheckStaleReferences_StaleInScripts(t *testing.T) { - tmp := chdirTemp(t) - - scriptsDir := filepath.Join(tmp, "scripts") - if err := os.MkdirAll(scriptsDir, 0755); err != nil { - t.Fatal(err) - } - // Use old namespace-style "ao quality pool list" instead of flat "ao pool list" - if err := os.WriteFile(filepath.Join(scriptsDir, "smoke.sh"), []byte("#!/bin/bash\nao quality pool list\n"), 0644); err != nil { - t.Fatal(err) - } - - result := checkStaleReferences() - if result.Status != "warn" { - t.Errorf("status=%q, want warn for stale scripts reference (detail: %s)", result.Status, result.Detail) - } -} - -func TestCheckStaleReferences_SubdirsScanned(t *testing.T) { - tmp := chdirTemp(t) - - // Create subdirectories that the expanded scan should cover - contractsDir := filepath.Join(tmp, "docs", "contracts") - if err := os.MkdirAll(contractsDir, 0755); err != nil { - t.Fatal(err) - } - examplesDir := filepath.Join(tmp, "hooks", "examples") - if err := os.MkdirAll(examplesDir, 0755); err != nil { - t.Fatal(err) - } - - // Write a stale reference in docs/contracts/ - if err := os.WriteFile( - filepath.Join(contractsDir, "test-contract.md"), - []byte("# Contract\nRun `ao know search` to find references.\n"), - 0644, - ); err != nil { - t.Fatal(err) - } - - // Write a stale reference in hooks/examples/ - if err := os.WriteFile( - filepath.Join(examplesDir, "example.sh"), - []byte("#!/bin/bash\nao work rpi status\n"), - 0644, - ); err != nil { - t.Fatal(err) - } - - result := checkStaleReferences() - if result.Status != "warn" { - t.Errorf("status=%q, want warn for stale refs in subdirs (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "stale reference") { - t.Errorf("expected 'stale reference' in detail, got %q", result.Detail) - } -} - -func TestCheckStaleReferences_CodexAndReferencesScanned(t *testing.T) { - tmp := chdirTemp(t) - - codexDir := filepath.Join(tmp, "skills-codex", "fixture") - if err := os.MkdirAll(codexDir, 0755); err != nil { - t.Fatal(err) - } - refDir := filepath.Join(tmp, "skills", "fixture", "references") - if err := os.MkdirAll(refDir, 0755); err != nil { - t.Fatal(err) - } - - // skills-codex/ copies must stay on canonical surfaces - if err := os.WriteFile( - filepath.Join(codexDir, "SKILL.md"), - []byte("---\nname: fixture\n---\nRun `ao settings config` to review.\n"), - 0644, - ); err != nil { - t.Fatal(err) - } - // skills/*/references/*.md is a first-class skill surface - if err := os.WriteFile( - filepath.Join(refDir, "overview.md"), - []byte("See `ao start seed` for bootstrap.\n"), - 0644, - ); err != nil { - t.Fatal(err) - } - - result := checkStaleReferences() - if result.Status != "warn" { - t.Errorf("status=%q, want warn for stale refs in skills-codex/ and references/ (detail: %s)", result.Status, result.Detail) - } - if !strings.Contains(result.Detail, "2 stale") { - t.Errorf("expected both subdir hits counted, got %q", result.Detail) - } -} - -func TestCheckStaleReferences_NoFalsePositiveOnFlat(t *testing.T) { - tmp := chdirTemp(t) - - // "ao forge" (flat, canonical) should NOT trigger a stale reference - hooksDir := filepath.Join(tmp, "hooks") - if err := os.MkdirAll(hooksDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(hooksDir, "test.sh"), []byte("ao forge transcript\n"), 0644); err != nil { - t.Fatal(err) - } - - result := checkStaleReferences() - if result.Status != "pass" { - t.Errorf("status=%q, want pass (should not trigger on flat 'ao forge') (detail: %s)", result.Status, result.Detail) - } -} - -func TestScanFileForDeprecatedCommands(t *testing.T) { - tmp := t.TempDir() - - t.Run("nonexistent file returns nil", func(t *testing.T) { - refs := scanFileForDeprecatedCommands(filepath.Join(tmp, "nope.sh")) - if len(refs) != 0 { - t.Errorf("expected 0 refs for nonexistent file, got %d", len(refs)) - } - }) - - t.Run("file with multiple deprecated commands", func(t *testing.T) { - f := filepath.Join(tmp, "multi.sh") - content := "ao know forge transcript\nao know inject --apply-decay\nao search query\n" - if err := os.WriteFile(f, []byte(content), 0644); err != nil { - t.Fatal(err) - } - refs := scanFileForDeprecatedCommands(f) - if len(refs) < 2 { - t.Errorf("expected at least 2 stale refs, got %d", len(refs)) - } - }) -} - -func TestCountUniqueFiles(t *testing.T) { - refs := []staleReference{ - {File: "a.sh", OldCommand: "ao know forge", NewCommand: "ao forge"}, - {File: "a.sh", OldCommand: "ao know inject", NewCommand: "ao inject"}, - {File: "b.md", OldCommand: "ao know forge", NewCommand: "ao forge"}, - } - got := countUniqueFiles(refs) - if got != 2 { - t.Errorf("countUniqueFiles() = %d, want 2", got) - } -} diff --git a/cli/cmd/ao/doctor_wedge_legacy_test.go b/cli/cmd/ao/doctor_wedge_legacy_test.go deleted file mode 100644 index 26e8721ed..000000000 --- a/cli/cmd/ao/doctor_wedge_legacy_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "context" - "time" - - doctoradapter "github.com/boshu2/agentops/cli/internal/adapters/doctor" - revieweradapter "github.com/boshu2/agentops/cli/internal/adapters/reviewerhealth" - "github.com/boshu2/agentops/cli/internal/reviewerhealth" -) - -const wedgeReviewerTimeout = reviewerProbeTimeout - -type reviewerProbe struct { - name string - installCmd string -} - -var wedgeReviewers = func() []reviewerProbe { - catalog := reviewerhealth.DefaultCatalog() - out := make([]reviewerProbe, 0, len(catalog)) - for _, reviewer := range catalog { - out = append(out, reviewerProbe{name: reviewer.Name, installCmd: reviewer.InstallCommand}) - } - return out -}() - -func reviewerReachabilityChecks(timeout time.Duration) ([]doctorCheck, []string) { - return reviewerHealthService.Check(context.Background(), timeout) -} - -func checkReviewerReachable(reviewer reviewerProbe, timeout time.Duration) (doctorCheck, bool) { - service := reviewerhealth.NewService([]reviewerhealth.Reviewer{{ - Name: reviewer.name, InstallCommand: reviewer.installCmd, - }}, revieweradapter.SystemProbe()) - checks, live := service.Check(context.Background(), timeout) - return checks[0], len(live) == 1 -} - -const agentopsModuleLine = "module github.com/boshu2/agentops/cli" - -func wedgeDoctorChecks() []doctorCheck { - return newLegacyDoctorService().Checks(context.Background())[11:] -} -func crossFamilyCheck(live []string) doctorCheck { return doctoradapter.CrossFamilyCheck(live) } -func binaryFreshnessCheck(dir, running string) doctorCheck { - return doctoradapter.BinaryFreshnessCheck(dir, running) -} -func checkLedgerHealth(path string) doctorCheck { - return doctoradapter.CheckLedgerHealth(path, time.Now) -} -func checkLaw0Guard(environment []string) doctorCheck { - return doctoradapter.CheckLaw0Guard(environment) -} diff --git a/cli/cmd/ao/doctor_wedge_test.go b/cli/cmd/ao/doctor_wedge_test.go deleted file mode 100644 index f1a784249..000000000 --- a/cli/cmd/ao/doctor_wedge_test.go +++ /dev/null @@ -1,461 +0,0 @@ -// practices: [sre, resilience-patterns] -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/boshu2/agentops/cli/internal/provenancegraph" - "github.com/boshu2/agentops/cli/internal/quality" -) - -// writeFakeBin writes an executable stub named name into dir. The stub's -// process replaces itself via exec so a CommandContext kill lands on the -// actual long-running process, not a wrapping shell. -func writeFakeBin(t *testing.T, dir, name, script string) { - t.Helper() - if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil { - t.Fatalf("write fake %s: %v", name, err) - } -} - -// --------------------------------------------------------------------------- -// Reviewer reachability (L2: PATH-stubbed fake binaries, no live CLIs) -// --------------------------------------------------------------------------- - -func TestCheckReviewerReachable_LiveFakeBinary(t *testing.T) { - bin := t.TempDir() - writeFakeBin(t, bin, "codex", "#!/bin/sh\necho fake-codex 1.0\n") - t.Setenv("PATH", bin) - - c, live := checkReviewerReachable(reviewerProbe{name: "codex", installCmd: "install-codex"}, 5*time.Second) - if !live { - t.Error("live = false, want true for an instantly-answering binary") - } - if c.Status != "pass" { - t.Errorf("status = %q, want pass (detail: %s)", c.Status, c.Detail) - } - if c.Name != "Reviewer: codex" { - t.Errorf("name = %q, want 'Reviewer: codex'", c.Name) - } - if !strings.Contains(c.Detail, "reachable") { - t.Errorf("detail = %q, want it to say reachable", c.Detail) - } - if c.Required { - t.Error("reviewer check must not be Required") - } -} - -func TestCheckReviewerReachable_HangingBinaryTimesOut(t *testing.T) { - bin := t.TempDir() - // Absolute /bin/sleep: the stubbed PATH must not break the hang itself. - writeFakeBin(t, bin, "codex", "#!/bin/sh\nexec /bin/sleep 60\n") - t.Setenv("PATH", bin) - - start := time.Now() - c, live := checkReviewerReachable(reviewerProbe{name: "codex", installCmd: "install-codex"}, 300*time.Millisecond) - elapsed := time.Since(start) - - if elapsed > 10*time.Second { - t.Fatalf("doctor wedged on a hanging binary: probe took %s (timeout was 300ms)", elapsed) - } - if live { - t.Error("live = true, want false for a hanging binary") - } - if c.Status != "fail" { - t.Errorf("status = %q, want fail (detail: %s)", c.Status, c.Detail) - } - if !strings.Contains(c.Detail, "timed out") { - t.Errorf("detail = %q, want it to report the timeout", c.Detail) - } - // Directive 13: the failure must teach the exact corrective command. - if !strings.Contains(c.Detail, "codex --version") { - t.Errorf("detail = %q, want the corrective 'codex --version' command named", c.Detail) - } -} - -func TestCheckReviewerReachable_AbsentIsDegradedNotHung(t *testing.T) { - t.Setenv("PATH", t.TempDir()) // empty dir: binary absent - - start := time.Now() - c, live := checkReviewerReachable(reviewerProbe{name: "codex", installCmd: "npm install -g @openai/codex && codex login"}, 5*time.Second) - elapsed := time.Since(start) - - if elapsed > 2*time.Second { - t.Fatalf("absent binary must short-circuit, took %s", elapsed) - } - if live { - t.Error("live = true, want false when binary is absent") - } - if c.Status != "warn" { - t.Errorf("status = %q, want warn (absent = degraded, not broken)", c.Status) - } - if !strings.Contains(c.Detail, "npm install -g @openai/codex") { - t.Errorf("detail = %q, want the exact install one-liner named", c.Detail) - } -} - -func TestCheckReviewerReachable_BrokenBinaryFails(t *testing.T) { - bin := t.TempDir() - writeFakeBin(t, bin, "agy", "#!/bin/sh\nexit 7\n") - t.Setenv("PATH", bin) - - c, live := checkReviewerReachable(reviewerProbe{name: "agy", installCmd: "reinstall-agy"}, 5*time.Second) - if live { - t.Error("live = true, want false for a binary that errors") - } - if c.Status != "fail" { - t.Errorf("status = %q, want fail (detail: %s)", c.Status, c.Detail) - } - if !strings.Contains(c.Detail, "reinstall-agy") { - t.Errorf("detail = %q, want the corrective install command named", c.Detail) - } -} - -func TestReviewerReachabilityChecks_MixedFamilies(t *testing.T) { - bin := t.TempDir() - writeFakeBin(t, bin, "codex", "#!/bin/sh\nexit 0\n") - // agy deliberately absent. - t.Setenv("PATH", bin) - - checks, live := reviewerReachabilityChecks(5 * time.Second) - if len(checks) != len(wedgeReviewers) { - t.Fatalf("got %d checks, want %d (one per reviewer family)", len(checks), len(wedgeReviewers)) - } - if len(live) != 1 || live[0] != "codex" { - t.Errorf("live = %v, want [codex]", live) - } - if checks[0].Status != "pass" { - t.Errorf("codex check status = %q, want pass (detail: %s)", checks[0].Status, checks[0].Detail) - } - if checks[1].Status != "warn" { - t.Errorf("agy check status = %q, want warn for absent (detail: %s)", checks[1].Status, checks[1].Detail) - } -} - -// --------------------------------------------------------------------------- -// Cross-family summary -// --------------------------------------------------------------------------- - -func TestCrossFamilyCheck(t *testing.T) { - tests := []struct { - name string - live []string - wantStatus string - wantDetail string - }{ - {"both live", []string{"codex", "agy"}, "pass", "cross-family capable: yes (live families: codex, agy)"}, - {"one live", []string{"agy"}, "pass", "cross-family capable: yes (live families: agy)"}, - {"none live", nil, "warn", "cross-family capable: no"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := crossFamilyCheck(tt.live) - if c.Status != tt.wantStatus { - t.Errorf("status = %q, want %q", c.Status, tt.wantStatus) - } - if !strings.Contains(c.Detail, tt.wantDetail) { - t.Errorf("detail = %q, want it to contain %q", c.Detail, tt.wantDetail) - } - }) - } - // The degraded report must teach the corrective install command. - c := crossFamilyCheck(nil) - if !strings.Contains(c.Detail, "npm install -g @openai/codex") { - t.Errorf("degraded detail = %q, want the install one-liner named", c.Detail) - } -} - -// --------------------------------------------------------------------------- -// Binary freshness -// --------------------------------------------------------------------------- - -// makeFakeAgentopsRepo scaffolds the two files binary-freshness detection -// reads: cli/go.mod with the agentops module line and cli/cmd/ao/main.go -// declaring `declared` as the version. -func makeFakeAgentopsRepo(t *testing.T, declared string) string { - t.Helper() - root := t.TempDir() - aoDir := filepath.Join(root, "cli", "cmd", "ao") - if err := os.MkdirAll(aoDir, 0o755); err != nil { - t.Fatal(err) - } - gomod := agentopsModuleLine + "\n\ngo 1.26\n" - if err := os.WriteFile(filepath.Join(root, "cli", "go.mod"), []byte(gomod), 0o644); err != nil { - t.Fatal(err) - } - maingo := "package main\n\nvar version = \"" + declared + "\"\n" - if err := os.WriteFile(filepath.Join(aoDir, "main.go"), []byte(maingo), 0o644); err != nil { - t.Fatal(err) - } - return root -} - -func TestBinaryFreshnessCheck(t *testing.T) { - tests := []struct { - name string - declared string // "" = no repo (bare temp dir) - running string - wantStatus string - wantDetail string - }{ - {"match inside repo", "9.9.9", "9.9.9", "pass", "matches the repo's declared version"}, - {"stale binary inside repo", "9.9.9", "1.0.0", "warn", "scripts/preflight-uat-binary.sh"}, - {"outside repo reports version only", "", "1.0.0", "pass", "outside the agentops repo"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dir := t.TempDir() - if tt.declared != "" { - dir = makeFakeAgentopsRepo(t, tt.declared) - } - c := binaryFreshnessCheck(dir, tt.running) - if c.Status != tt.wantStatus { - t.Errorf("status = %q, want %q (detail: %s)", c.Status, tt.wantStatus, c.Detail) - } - if !strings.Contains(c.Detail, tt.wantDetail) { - t.Errorf("detail = %q, want it to contain %q", c.Detail, tt.wantDetail) - } - }) - } -} - -func TestBinaryFreshnessCheck_StaleNamesBothFixes(t *testing.T) { - root := makeFakeAgentopsRepo(t, "9.9.9") - c := binaryFreshnessCheck(root, "1.0.0") - if !strings.Contains(c.Detail, "scripts/preflight-uat-binary.sh") || !strings.Contains(c.Detail, "brew upgrade agentops") { - t.Errorf("detail = %q, want both corrective commands named", c.Detail) - } - if !strings.Contains(c.Detail, "1.0.0") || !strings.Contains(c.Detail, "9.9.9") { - t.Errorf("detail = %q, want both versions named", c.Detail) - } -} - -func TestBinaryFreshnessCheck_ResolvesRootFromSubdir(t *testing.T) { - root := makeFakeAgentopsRepo(t, "9.9.9") - sub := filepath.Join(root, "cli", "cmd", "ao") - c := binaryFreshnessCheck(sub, "9.9.9") - if c.Status != "pass" { - t.Errorf("status = %q, want pass from a repo subdir (detail: %s)", c.Status, c.Detail) - } - if !strings.Contains(c.Detail, "matches the repo's declared version") { - t.Errorf("detail = %q, want repo-match wording", c.Detail) - } -} - -// --------------------------------------------------------------------------- -// Ledger health (fixture fidelity: round-trip the PRODUCTION writer) -// --------------------------------------------------------------------------- - -// appendFixtureEdges writes n chain-valid edges through the production -// provenancegraph writer (Store.Append seals + chains), never a hand-built -// on-disk shape. -func appendFixtureEdges(t *testing.T, path string, n int) { - t.Helper() - store := provenancegraph.NewStore(path) - for i := 0; i < n; i++ { - _, err := store.Append(provenancegraph.Edge{ - FromID: fmt.Sprintf("age-test.%d", i), - FromType: "bead", - ToID: "abc1234", - ToType: "commit", - Relation: "wasGeneratedBy", - TrustTier: "authored", - TS: time.Now().UTC().Format(time.RFC3339), - }) - if err != nil { - t.Fatalf("production append %d: %v", i, err) - } - } -} - -func TestCheckLedgerHealth_IntactChainPassesWithRecency(t *testing.T) { - path := filepath.Join(t.TempDir(), "docs", "provenance", "ledger.jsonl") - appendFixtureEdges(t, path, 2) - - c := checkLedgerHealth(path) - if c.Status != "pass" { - t.Fatalf("status = %q, want pass (detail: %s)", c.Status, c.Detail) - } - if !strings.Contains(c.Detail, "2 records") { - t.Errorf("detail = %q, want the record count", c.Detail) - } - if !strings.Contains(c.Detail, "newest record") { - t.Errorf("detail = %q, want feed recency reported", c.Detail) - } -} - -func TestCheckLedgerHealth_MissingLedgerPasses(t *testing.T) { - c := checkLedgerHealth(filepath.Join(t.TempDir(), "docs", "provenance", "ledger.jsonl")) - if c.Status != "pass" { - t.Errorf("status = %q, want pass for an absent ledger (detail: %s)", c.Status, c.Detail) - } - if !strings.Contains(c.Detail, "no ledger records yet") { - t.Errorf("detail = %q, want the empty-chain wording", c.Detail) - } -} - -func TestCheckLedgerHealth_TamperedChainFailsAndTeaches(t *testing.T) { - path := filepath.Join(t.TempDir(), "docs", "provenance", "ledger.jsonl") - appendFixtureEdges(t, path, 2) - - // Tamper with committed content: alter a payload field in place. The - // verifier (the same code path as `ao provenance verify`) must catch it. - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - tampered := bytes.Replace(data, []byte(`"to_id":"abc1234"`), []byte(`"to_id":"abc1235"`), 1) - if bytes.Equal(tampered, data) { - t.Fatal("fixture did not contain the expected to_id field to tamper") - } - if err := os.WriteFile(path, tampered, 0o644); err != nil { - t.Fatal(err) - } - - c := checkLedgerHealth(path) - if c.Status != "fail" { - t.Fatalf("status = %q, want fail for a tampered chain (detail: %s)", c.Status, c.Detail) - } - if !c.Required { - t.Error("a broken ledger chain must be a Required failure") - } - if !strings.Contains(c.Detail, "chain breaks at line 1") { - t.Errorf("detail = %q, want the broken line named", c.Detail) - } - if !strings.Contains(c.Detail, "ao provenance verify") { - t.Errorf("detail = %q, want the corrective 'ao provenance verify' named", c.Detail) - } -} - -// --------------------------------------------------------------------------- -// LAW-0 guard -// --------------------------------------------------------------------------- - -func TestCheckLaw0Guard(t *testing.T) { - tests := []struct { - name string - environ []string - wantStatus string - wantDetail string - }{ - { - name: "clean env passes", - environ: []string{"PAWL_NO_SERVICE=1", "HOME=/home/x", "AGENTOPS_REPO_ROOT=/repo"}, - wantStatus: "pass", - wantDetail: "no reviewer path configured through claude print-mode", - }, - { - name: "empty env passes", - environ: nil, - wantStatus: "pass", - wantDetail: "no reviewer path configured", - }, - { - name: "pawl var with claude print-flag fails", - environ: []string{"PAWL_REVIEWER_CMD=claude" + " -p review"}, - wantStatus: "fail", - wantDetail: "unset PAWL_REVIEWER_CMD", - }, - { - name: "reviewer var with claude print-word fails", - environ: []string{"MY_REVIEWER_BIN=claude" + " --print"}, - wantStatus: "fail", - wantDetail: "unset MY_REVIEWER_BIN", - }, - { - name: "unscoped var mentioning claude -p is ignored", - environ: []string{"SHELL_HISTORY=claude -p something"}, - wantStatus: "pass", - wantDetail: "no reviewer path configured", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := checkLaw0Guard(tt.environ) - if c.Status != tt.wantStatus { - t.Errorf("status = %q, want %q (detail: %s)", c.Status, tt.wantStatus, c.Detail) - } - if !strings.Contains(c.Detail, tt.wantDetail) { - t.Errorf("detail = %q, want it to contain %q", c.Detail, tt.wantDetail) - } - if !c.Required { - t.Error("the LAW-0 guard is a Required check") - } - }) - } -} - -// --------------------------------------------------------------------------- -// JSON schema + gather integration -// --------------------------------------------------------------------------- - -// TestWedgeChecks_JSONSchemaMatchesExistingChecks proves the new checks -// serialize through the SAME quality.Check schema as every existing doctor -// check (name/status/detail/required keys). -func TestWedgeChecks_JSONSchemaMatchesExistingChecks(t *testing.T) { - checks := []doctorCheck{ - crossFamilyCheck([]string{"codex"}), - checkLaw0Guard(nil), - binaryFreshnessCheck(t.TempDir(), "1.2.3"), - } - var buf bytes.Buffer - if err := quality.RunDoctor(quality.DoctorOptions{JSON: true, Checks: checks, Stdout: &buf}); err != nil { - t.Fatalf("RunDoctor JSON: %v", err) - } - var out quality.DoctorOutput - if err := json.Unmarshal(buf.Bytes(), &out); err != nil { - t.Fatalf("unmarshal doctor JSON: %v\n%s", err, buf.String()) - } - if len(out.Checks) != 3 { - t.Fatalf("got %d checks in JSON, want 3", len(out.Checks)) - } - if out.Checks[0].Name != "Cross-Family Review" || out.Checks[0].Status != "pass" { - t.Errorf("check[0] = %+v, want Cross-Family Review pass", out.Checks[0]) - } - for _, key := range []string{`"name"`, `"status"`, `"detail"`, `"required"`} { - if !strings.Contains(buf.String(), key) { - t.Errorf("JSON output missing schema key %s:\n%s", key, buf.String()) - } - } -} - -// TestGatherDoctorChecks_IncludesWedgeSection is the L2 wire-up proof: the -// wedge checks ride the same gather path the doctor command renders. PATH is -// stubbed so no real reviewer CLI is ever invoked. -func TestGatherDoctorChecks_IncludesWedgeSection(t *testing.T) { - t.Chdir(t.TempDir()) - t.Setenv("HOME", t.TempDir()) - bin := t.TempDir() - writeFakeBin(t, bin, "codex", "#!/bin/sh\nexit 0\n") - t.Setenv("PATH", bin) - - checks := gatherDoctorChecks() - got := make(map[string]doctorCheck, len(checks)) - for _, c := range checks { - got[c.Name] = c - } - for _, want := range []string{ - "Reviewer: codex", "Reviewer: agy", "Cross-Family Review", - "Binary Freshness", "Ledger Health", "LAW-0 Guard", - } { - if _, ok := got[want]; !ok { - t.Errorf("gatherDoctorChecks missing wedge check %q", want) - } - } - if c := got["Reviewer: codex"]; c.Status != "pass" { - t.Errorf("stubbed codex status = %q, want pass (detail: %s)", c.Status, c.Detail) - } - if c := got["Reviewer: agy"]; c.Status != "warn" { - t.Errorf("absent agy status = %q, want warn (detail: %s)", c.Status, c.Detail) - } - if c := got["Cross-Family Review"]; !strings.Contains(c.Detail, "cross-family capable: yes (live families: codex)") { - t.Errorf("cross-family detail = %q, want codex live", c.Detail) - } -} diff --git a/cli/cmd/ao/json_validity_test.go b/cli/cmd/ao/json_validity_test.go index d5a36c067..31257eb93 100644 --- a/cli/cmd/ao/json_validity_test.go +++ b/cli/cmd/ao/json_validity_test.go @@ -40,9 +40,9 @@ func withGoalsJSON(t *testing.T) { // withDoctorJSON temporarily sets the doctor-specific JSON flag. func withDoctorJSON(t *testing.T) { t.Helper() - prev := doctorJSON - doctorJSON = true - t.Cleanup(func() { doctorJSON = prev }) + prevJSON, prevOutput := jsonFlag, output + jsonFlag, output = true, "json" + t.Cleanup(func() { jsonFlag, output = prevJSON, prevOutput }) } // captureJSONStdout moved to testutil_test.go. diff --git a/cli/cmd/ao/quality_test_support_test.go b/cli/cmd/ao/quality_test_support_test.go new file mode 100644 index 000000000..aa50ac22d --- /dev/null +++ b/cli/cmd/ao/quality_test_support_test.go @@ -0,0 +1,36 @@ +package main + +import ( + "io" + "os" + "path/filepath" + "time" + + "github.com/boshu2/agentops/cli/internal/quality" +) + +type doctorCheck = quality.Check +type doctorOutput = quality.DoctorOutput + +func doctorStatusIcon(status string) string { return quality.StatusIcon(status) } +func hasRequiredFailure(checks []doctorCheck) bool { return quality.HasRequiredFailure(checks) } +func countCheckStatuses(checks []doctorCheck) (int, int, int) { + return quality.CountCheckStatuses(checks) +} +func buildDoctorSummary(passes, fails, warns, total int) string { + return quality.BuildSummary(passes, fails, warns, total) +} +func computeResult(checks []doctorCheck) doctorOutput { return quality.ComputeResult(checks) } +func renderDoctorTable(writer io.Writer, output doctorOutput) { quality.RenderTable(writer, output) } +func formatNumber(value int) string { return quality.FormatNumber(value) } +func formatDuration(value time.Duration) string { return quality.FormatDuration(value) } +func countFileLines(path string) int { return quality.CountFileLines(path) } +func countHealFindings(output string) int { return quality.CountHealFindings(output) } +func newestFileModTime(entries []os.DirEntry) time.Time { return quality.NewestFileModTime(entries) } +func countFiles(path string) int { return quality.CountFiles(path) } +func countLearningFiles(path string) int { return quality.CountLearningFiles(path) } +func countEstablished(path string) int { return quality.CountEstablished(path) } +func fileExists(path string) bool { return quality.FileExists(path) } +func checkFlywheelHealth(root string) doctorCheck { + return quality.CheckFlywheelHealth(filepath.Join(root, ".agents")) +} diff --git a/cli/cmd/ao/quickstart_verdict_test.go b/cli/cmd/ao/quickstart_verdict_test.go index 2ae24cbd2..98bde367f 100644 --- a/cli/cmd/ao/quickstart_verdict_test.go +++ b/cli/cmd/ao/quickstart_verdict_test.go @@ -9,8 +9,17 @@ import ( "testing" "github.com/spf13/cobra" + + "github.com/boshu2/agentops/cli/internal/reviewerhealth" ) +func writeFakeBin(t *testing.T, dir, name, script string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil { + t.Fatalf("write fake %s: %v", name, err) + } +} + // stubReviewerPATH points PATH at a temp bin dir containing instantly-answering // fake binaries for the given reviewer families, plus the system dirs (git/sh // stay resolvable). Restoring is handled by t.Setenv. @@ -178,11 +187,12 @@ func TestPrepareFirstVerdict_InstallLinesCoverEveryFamily(t *testing.T) { chdirTo(t, t.TempDir()) stubReviewerPATH(t) // nothing reachable info := prepareFirstVerdict() - if len(info.ReviewerInstall) != len(wedgeReviewers) { - t.Fatalf("got %d install lines, want %d (one per reviewer family)", len(info.ReviewerInstall), len(wedgeReviewers)) + reviewers := reviewerhealth.DefaultCatalog() + if len(info.ReviewerInstall) != len(reviewers) { + t.Fatalf("got %d install lines, want %d (one per reviewer family)", len(info.ReviewerInstall), len(reviewers)) } - for i, r := range wedgeReviewers { - want := r.name + ": " + r.installCmd + for i, reviewer := range reviewers { + want := reviewer.Name + ": " + reviewer.InstallCommand if info.ReviewerInstall[i] != want { t.Errorf("install[%d] = %q, want %q", i, info.ReviewerInstall[i], want) } diff --git a/cli/cmd/ao/root.go b/cli/cmd/ao/root.go index f29f8c2ed..1db4643d0 100644 --- a/cli/cmd/ao/root.go +++ b/cli/cmd/ao/root.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/boshu2/agentops/cli/internal/adapters/worktreeconfig" + doctorcommands "github.com/boshu2/agentops/cli/internal/commands/doctor" "github.com/boshu2/agentops/cli/internal/doctor" ) @@ -105,7 +106,7 @@ func Execute() { if errors.As(err, &lintErr) { os.Exit(lintErr.ExitCode) } - var docErr *doctorExitError + var docErr *doctorcommands.ExitError if errors.As(err, &docErr) { // Exit 1 means findings are present — a normal diagnostic result, // not a failure, so it carries no stderr noise. Higher codes are diff --git a/cli/cmd/ao/testutil_test.go b/cli/cmd/ao/testutil_test.go index 22e4063ec..32ece1c6b 100644 --- a/cli/cmd/ao/testutil_test.go +++ b/cli/cmd/ao/testutil_test.go @@ -295,7 +295,6 @@ func resetCommandState(t *testing.T) { origContextPacketFlags := contextPacketFlags origContextExplainFlags := contextExplainFlags origContextPacketStatusFlags := contextPacketStatusFlags - origDoctorJSON := doctorJSON origMaturityApply := maturityApply origMaturityScan := maturityScan origMaturityCurate := maturityCurate @@ -366,7 +365,6 @@ func resetCommandState(t *testing.T) { contextPacketFlags = origContextPacketFlags contextExplainFlags = origContextExplainFlags contextPacketStatusFlags = origContextPacketStatusFlags - doctorJSON = origDoctorJSON maturityApply = origMaturityApply maturityScan = origMaturityScan maturityCurate = origMaturityCurate @@ -438,7 +436,6 @@ func resetCommandState(t *testing.T) { findingsPullForce = false findingsRetireBy = "" scenarioListStatus = "" - doctorJSON = false maturityApply = false maturityScan = false maturityCurate = false diff --git a/cli/cmd/ao/validate.go b/cli/cmd/ao/validate.go index 93daa4d56..2926e6679 100644 --- a/cli/cmd/ao/validate.go +++ b/cli/cmd/ao/validate.go @@ -36,7 +36,7 @@ const ( // gateExitError carries a gate exit code out through cobra's RunE so Execute() // can map it to os.Exit. Reuses the typed-error + errors.As pattern -// (see doctorExitError, AgentsLintError). +// (see the doctor command module's typed exit and AgentsLintError). type gateExitError struct { code int msg string diff --git a/cli/internal/commands/doctor/module.go b/cli/internal/commands/doctor/module.go new file mode 100644 index 000000000..ff213cf8e --- /dev/null +++ b/cli/internal/commands/doctor/module.go @@ -0,0 +1,402 @@ +// Package doctor owns Cobra presentation for the doctor command family. +package doctor + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/spf13/cobra" + + doctorapp "github.com/boshu2/agentops/cli/internal/doctor" + "github.com/boshu2/agentops/cli/internal/quality" +) + +type ReadUseCases interface { + Diagnose(context.Context, doctorapp.ReadRequest) (*doctorapp.Report, error) + Triage(context.Context, doctorapp.ReadRequest) (*doctorapp.RobotTriageResult, *doctorapp.Report, error) + Explain(context.Context, string) (*doctorapp.Finding, error) + Capabilities(context.Context) *doctorapp.Capabilities + Health(context.Context) (string, *doctorapp.HealthResult, error) + RobotDocs(context.Context) string + List(context.Context) ([]doctorapp.RunSummary, error) + Diff(context.Context, doctorapp.ReadRequest) (*doctorapp.Report, error) +} + +type MutationUseCases interface { + Fix(context.Context, doctorapp.MutationRequest) (*doctorapp.Report, error) +} + +type MaintenanceUseCases interface { + Undo(context.Context, doctorapp.UndoRequest) (*doctorapp.UndoResult, error) + GC(context.Context, doctorapp.GCRequest) (doctorapp.GCResult, error) +} + +type GlobalOptions struct { + DryRun bool + JSON bool + Output string +} + +type UseCases struct { + LegacyChecks func(context.Context) []quality.Check + Read ReadUseCases + Mutation MutationUseCases + Maintenance MaintenanceUseCases + DetectorCount func() int +} + +type HostOptions struct { + Globals func(*cobra.Command) GlobalOptions + EnrichFlagErr func(*cobra.Command, error) error +} + +type Module struct { + useCases UseCases + host HostOptions +} + +func NewModule(useCases UseCases, host HostOptions) Module { + return Module{useCases: useCases, host: host} +} + +type rootOptions struct { + json, fix, dryRun, online, quick, robot, robotTriage bool + only, skip []string + since, severity, explain string +} + +type undoOptions struct{ strict, dryRun bool } +type gcOptions struct { + before string + yes bool +} + +// ExitError carries a doctor process status through Cobra. +type ExitError struct { + Code int + Message string +} + +func (failure *ExitError) Error() string { return failure.Message } +func (failure *ExitError) ExitCode() int { return failure.Code } + +func (module Module) globals(command *cobra.Command) GlobalOptions { + if module.host.Globals == nil { + return GlobalOptions{} + } + return module.host.Globals(command) +} + +func (module Module) Command() *cobra.Command { + var options rootOptions + command := &cobra.Command{ + Use: "doctor", Short: "Check AgentOps health", Args: nil, + Long: `Run health checks on your AgentOps installation. + +Validates that all required components are present and configured. +Optional components are reported as warnings but do not cause failure. + +Examples: + ao doctor + ao doctor --json`, + } + flags := command.Flags() + flags.BoolVar(&options.json, "json", false, "Output results as JSON") + flags.BoolVar(&options.fix, "fix", false, "Apply fixers for findings (routes through mutate())") + flags.BoolVar(&options.dryRun, "dry-run", false, "With --fix: print the plan, change nothing") + flags.StringSliceVar(&options.only, "only", nil, "Scope to a subset of detectors or subsystems") + flags.StringSliceVar(&options.skip, "skip", nil, "Inverse of --only") + flags.StringVar(&options.since, "since", "", "Diff findings against an earlier run") + flags.BoolVar(&options.online, "online", false, "Enable network probes (default: offline-only)") + flags.BoolVar(&options.quick, "quick", false, "Run only fast-path detectors (< 200ms)") + flags.StringVar(&options.severity, "severity", "P3", "Minimum severity to emit (P0|P1|P2|P3)") + flags.BoolVar(&options.robot, "robot", false, "Alias for stable JSON report output") + flags.BoolVar(&options.robotTriage, "robot-triage", false, "Emit the mega-command triage JSON") + flags.StringVar(&options.explain, "explain", "", "Expand a single finding by id") + command.RunE = func(command *cobra.Command, _ []string) error { return module.runRoot(command, options) } + command.AddCommand( + module.fixCommand(&options), module.undoCommand(&options), module.explainCommand(&options), + module.capabilitiesCommand(), module.healthCommand(&options), module.robotDocsCommand(), + module.gcCommand(&options), module.listCommand(&options), module.diffCommand(&options), + ) + command.SilenceErrors = true + command.SilenceUsage = true + for _, child := range command.Commands() { + child.SilenceErrors = true + } + if module.host.EnrichFlagErr != nil { + command.SetFlagErrorFunc(func(command *cobra.Command, err error) error { + err = module.host.EnrichFlagErr(command, err) + fmt.Fprintln(command.ErrOrStderr(), "Error:", err) + return err + }) + } + return command +} + +func (module Module) wantsJSON(command *cobra.Command, options *rootOptions) bool { + global := module.globals(command) + return options.json || options.robot || global.JSON || global.Output == "json" +} + +func (module Module) effectiveDryRun(command *cobra.Command, options *rootOptions, local bool) bool { + return local || options.dryRun || module.globals(command).DryRun +} + +func readRequest(options rootOptions, dryRun, jsonOutput bool) doctorapp.ReadRequest { + return doctorapp.ReadRequest{Only: append([]string(nil), options.only...), Skip: append([]string(nil), options.skip...), + Quick: options.quick, Online: options.online, Severity: options.severity, + DryRun: dryRun, JSON: jsonOutput, Since: options.since} +} + +func mutationRequest(options rootOptions, dryRun, jsonOutput bool) doctorapp.MutationRequest { + return doctorapp.MutationRequest{Only: append([]string(nil), options.only...), Skip: append([]string(nil), options.skip...), + Quick: options.quick, Online: options.online, Severity: options.severity, DryRun: dryRun, JSON: jsonOutput} +} + +func (module Module) runRoot(command *cobra.Command, options rootOptions) error { + jsonOutput := module.wantsJSON(command, &options) + dryRun := module.effectiveDryRun(command, &options, false) + request := readRequest(options, dryRun, jsonOutput) + if options.explain != "" { + return module.runExplain(command, options.explain, jsonOutput) + } + if options.robotTriage { + triage, report, err := module.useCases.Read.Triage(command.Context(), request) + if err != nil { + return exit(doctorapp.ExitIOError, err.Error()) + } + if err := writeJSON(command, triage); err != nil { + return err + } + return resultExit(report.ExitCode, "doctor findings present") + } + if options.fix { + return module.runFix(command, mutationRequest(options, dryRun, jsonOutput), jsonOutput) + } + if jsonOutput { + report, err := module.useCases.Read.Diagnose(command.Context(), request) + if err != nil { + return exit(doctorapp.ExitIOError, err.Error()) + } + if err := writeJSON(command, report); err != nil { + return err + } + return resultExit(report.ExitCode, "doctor findings present") + } + checks := module.useCases.LegacyChecks(command.Context()) + if err := quality.RunDoctor(quality.DoctorOptions{Checks: checks, Stdout: command.OutOrStdout()}); err != nil { + return err + } + if module.useCases.DetectorCount != nil && module.useCases.DetectorCount() == 0 { + return nil + } + report, err := module.useCases.Read.Diagnose(command.Context(), request) + if err != nil || report == nil || len(report.Findings) == 0 { + return nil + } + renderFindings(command, report) + return resultExit(report.ExitCode, "doctor findings present") +} + +func exit(code int, message string) error { return &ExitError{Code: code, Message: message} } +func resultExit(code int, message string) error { + if code == doctorapp.ExitHealthy { + return nil + } + return exit(code, message) +} + +func writeJSON(command *cobra.Command, value any) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return exit(doctorapp.ExitIOError, err.Error()) + } + _, err = fmt.Fprintln(command.OutOrStdout(), string(data)) + return err +} + +func renderFindings(command *cobra.Command, report *doctorapp.Report) { + if len(report.Findings) == 0 { + return + } + w := command.OutOrStdout() + fmt.Fprintln(w) + fmt.Fprintf(w, "Failure-mode findings (%d):\n", len(report.Findings)) + for _, finding := range report.Findings { + fmt.Fprintf(w, " [%s] %s — %s\n", finding.Severity, finding.ID, finding.Title) + } +} + +func (module Module) runFix(command *cobra.Command, request doctorapp.MutationRequest, jsonOutput bool) error { + report, err := module.useCases.Mutation.Fix(command.Context(), request) + if err != nil { + return exit(doctorapp.ExitIOError, err.Error()) + } + if jsonOutput { + if err := writeJSON(command, report); err != nil { + return err + } + } else { + fmt.Fprintf(command.OutOrStdout(), "ao doctor fix — run %s\n", report.RunID) + fmt.Fprintf(command.OutOrStdout(), " findings: %d actions: %d\n", report.Summary.TotalFindings, report.ActionsTaken) + if report.UndoCommand != "" { + fmt.Fprintf(command.OutOrStdout(), " undo: %s\n", report.UndoCommand) + } + } + return resultExit(report.ExitCode, "doctor fix incomplete") +} + +func (module Module) runExplain(command *cobra.Command, id string, jsonOutput bool) error { + finding, err := module.useCases.Read.Explain(command.Context(), id) + if err != nil { + var runtimeFailure *doctorapp.RuntimeError + if errors.As(err, &runtimeFailure) { + return exit(doctorapp.ExitIOError, err.Error()) + } + return exit(doctorapp.ExitNoInput, err.Error()) + } + if jsonOutput { + return writeJSON(command, finding) + } + fmt.Fprintf(command.OutOrStdout(), "%s [%s] (%s)\n%s\n", finding.ID, finding.Severity, finding.Subsystem, finding.Title) + fmt.Fprintf(command.OutOrStdout(), "Remediation: %s\n", finding.Remediation.Command) + return nil +} + +func (module Module) fixCommand(options *rootOptions) *cobra.Command { + return &cobra.Command{Use: "fix", Short: "Run detectors, then apply fixers (backs up before every mutation)", RunE: func(command *cobra.Command, _ []string) error { + jsonOutput := module.wantsJSON(command, options) + dryRun := module.effectiveDryRun(command, options, false) + return module.runFix(command, mutationRequest(*options, dryRun, jsonOutput), jsonOutput) + }} +} + +func (module Module) undoCommand(options *rootOptions) *cobra.Command { + local := undoOptions{strict: true} + command := &cobra.Command{Use: "undo ", Short: "Restore from .doctor/runs//backups/ (run-id may be 'latest')", Args: cobra.ExactArgs(1)} + command.Flags().BoolVar(&local.strict, "strict", true, "Refuse if any backup is missing or hash-mismatched") + command.Flags().BoolVar(&local.dryRun, "dry-run", false, "Print the restore plan; do not execute") + command.RunE = func(command *cobra.Command, args []string) error { + jsonOutput := module.wantsJSON(command, options) + request := doctorapp.UndoRequest{RunID: args[0], Strict: local.strict, DryRun: module.effectiveDryRun(command, options, local.dryRun)} + result, err := module.useCases.Maintenance.Undo(command.Context(), request) + if err != nil { + if jsonOutput { + _ = writeJSON(command, map[string]any{"run_id": args[0], "exit_code": doctorapp.ExitIOError, "error": err.Error()}) + } else { + fmt.Fprintf(command.ErrOrStderr(), "undo failed: %v\n", err) + } + return exit(doctorapp.ExitIOError, err.Error()) + } + if jsonOutput { + return writeJSON(command, result) + } + fmt.Fprintf(command.OutOrStdout(), "undo %s: restored=%d skipped=%d\n", result.RunID, result.Restored, result.Skipped) + return nil + } + return command +} + +func (module Module) explainCommand(options *rootOptions) *cobra.Command { + return &cobra.Command{Use: "explain ", Short: "Expand a single finding with full evidence", Args: cobra.ExactArgs(1), RunE: func(command *cobra.Command, args []string) error { + return module.runExplain(command, args[0], module.wantsJSON(command, options)) + }} +} + +func (module Module) capabilitiesCommand() *cobra.Command { + return &cobra.Command{Use: "capabilities", Short: "Print the machine-readable doctor contract (JSON)", RunE: func(command *cobra.Command, _ []string) error { + return writeJSON(command, module.useCases.Read.Capabilities(command.Context())) + }} +} + +func (module Module) healthCommand(options *rootOptions) *cobra.Command { + return &cobra.Command{Use: "health", Short: "Cheap one-line liveness summary", RunE: func(command *cobra.Command, _ []string) error { + line, result, err := module.useCases.Read.Health(command.Context()) + if err != nil { + return exit(doctorapp.ExitIOError, err.Error()) + } + if module.wantsJSON(command, options) { + if err := writeJSON(command, result); err != nil { + return err + } + } else { + fmt.Fprintln(command.OutOrStdout(), line) + } + return resultExit(result.ExitCode, "doctor health: not ok") + }} +} + +func (module Module) robotDocsCommand() *cobra.Command { + return &cobra.Command{Use: "robot-docs", Short: "Print the paste-ready agent handbook (Markdown)", RunE: func(command *cobra.Command, _ []string) error { + _, err := fmt.Fprint(command.OutOrStdout(), module.useCases.Read.RobotDocs(command.Context())) + return err + }} +} + +func (module Module) gcCommand(options *rootOptions) *cobra.Command { + var local gcOptions + command := &cobra.Command{Use: "gc", Short: "Prune old runs (requires --yes and --before )"} + command.Flags().StringVar(&local.before, "before", "", "Prune runs started before this date (YYYY-MM-DD)") + command.Flags().BoolVar(&local.yes, "yes", false, "Confirm pruning (required)") + command.RunE = func(command *cobra.Command, _ []string) error { + result, err := module.useCases.Maintenance.GC(command.Context(), doctorapp.GCRequest{Before: local.before, Yes: local.yes, DryRun: module.effectiveDryRun(command, options, false)}) + if err != nil { + var usageFailure *doctorapp.UsageError + if errors.As(err, &usageFailure) { + return exit(doctorapp.ExitUsage, err.Error()) + } + return exit(doctorapp.ExitIOError, err.Error()) + } + if module.wantsJSON(command, options) { + return writeJSON(command, result) + } + if result.DryRun { + fmt.Fprintf(command.OutOrStdout(), "would prune %d run(s)\n", result.Matched) + } else { + fmt.Fprintf(command.OutOrStdout(), "pruned %d run(s)\n", result.Pruned) + } + return nil + } + return command +} + +func (module Module) listCommand(options *rootOptions) *cobra.Command { + return &cobra.Command{Use: "ls", Short: "List runs in .doctor/runs/", RunE: func(command *cobra.Command, _ []string) error { + runs, err := module.useCases.Read.List(command.Context()) + if err != nil { + return exit(doctorapp.ExitIOError, err.Error()) + } + if module.wantsJSON(command, options) { + return writeJSON(command, map[string]any{"runs": runs}) + } + if len(runs) == 0 { + fmt.Fprintln(command.OutOrStdout(), "no doctor runs") + return nil + } + for _, run := range runs { + fmt.Fprintf(command.OutOrStdout(), "%s exit=%d actions=%d\n", run.RunID, run.ExitCode, run.ActionCount) + } + return nil + }} +} + +func (module Module) diffCommand(options *rootOptions) *cobra.Command { + return &cobra.Command{Use: "diff", Short: "Show what --fix would change (read-only)", RunE: func(command *cobra.Command, _ []string) error { + jsonOutput := module.wantsJSON(command, options) + report, err := module.useCases.Read.Diff(command.Context(), readRequest(*options, true, jsonOutput)) + if err != nil { + return exit(doctorapp.ExitIOError, err.Error()) + } + if jsonOutput { + return writeJSON(command, report) + } + renderFindings(command, report) + if len(report.Findings) == 0 { + fmt.Fprintln(command.OutOrStdout(), "clean diff: --fix would change nothing") + } + return nil + }} +} diff --git a/cli/internal/commands/doctor/module_test.go b/cli/internal/commands/doctor/module_test.go new file mode 100644 index 000000000..9e4f77fca --- /dev/null +++ b/cli/internal/commands/doctor/module_test.go @@ -0,0 +1,95 @@ +package doctor + +import ( + "context" + "testing" + + "github.com/spf13/cobra" + + doctorapp "github.com/boshu2/agentops/cli/internal/doctor" + "github.com/boshu2/agentops/cli/internal/quality" +) + +type fakeRead struct{} + +func (fakeRead) Diagnose(context.Context, doctorapp.ReadRequest) (*doctorapp.Report, error) { + return &doctorapp.Report{}, nil +} +func (fakeRead) Triage(context.Context, doctorapp.ReadRequest) (*doctorapp.RobotTriageResult, *doctorapp.Report, error) { + return &doctorapp.RobotTriageResult{}, &doctorapp.Report{}, nil +} +func (fakeRead) Explain(context.Context, string) (*doctorapp.Finding, error) { + return &doctorapp.Finding{}, nil +} +func (fakeRead) Capabilities(context.Context) *doctorapp.Capabilities { + return &doctorapp.Capabilities{} +} +func (fakeRead) Health(context.Context) (string, *doctorapp.HealthResult, error) { + return "ok", &doctorapp.HealthResult{}, nil +} +func (fakeRead) RobotDocs(context.Context) string { return "docs" } +func (fakeRead) List(context.Context) ([]doctorapp.RunSummary, error) { return nil, nil } +func (fakeRead) Diff(context.Context, doctorapp.ReadRequest) (*doctorapp.Report, error) { + return &doctorapp.Report{}, nil +} + +type fakeMutation struct{ request doctorapp.MutationRequest } + +func (fake *fakeMutation) Fix(_ context.Context, request doctorapp.MutationRequest) (*doctorapp.Report, error) { + fake.request = request + return &doctorapp.Report{ExitCode: doctorapp.ExitHealthy}, nil +} + +type fakeMaintenance struct { + undo doctorapp.UndoRequest + gc doctorapp.GCRequest +} + +func (fake *fakeMaintenance) Undo(_ context.Context, request doctorapp.UndoRequest) (*doctorapp.UndoResult, error) { + fake.undo = request + return &doctorapp.UndoResult{RunID: request.RunID}, nil +} +func (fake *fakeMaintenance) GC(_ context.Context, request doctorapp.GCRequest) (doctorapp.GCResult, error) { + fake.gc = request + return doctorapp.GCResult{Matched: 2, DryRun: request.DryRun}, nil +} + +func testModule(mutation *fakeMutation, maintenance *fakeMaintenance, globals GlobalOptions) Module { + return NewModule(UseCases{ + LegacyChecks: func(context.Context) []quality.Check { return nil }, + Read: fakeRead{}, Mutation: mutation, Maintenance: maintenance, + DetectorCount: func() int { return 0 }, + }, HostOptions{Globals: func(*cobra.Command) GlobalOptions { return globals }}) +} + +func TestInheritedDryRunProtectsFixGCAndUndo(t *testing.T) { + mutation, maintenance := &fakeMutation{}, &fakeMaintenance{} + for _, args := range [][]string{{"fix"}, {"gc", "--before", "2026-01-01", "--yes"}, {"undo", "latest"}} { + command := testModule(mutation, maintenance, GlobalOptions{DryRun: true}).Command() + command.SetArgs(args) + if err := command.Execute(); err != nil { + t.Fatalf("%v: %v", args, err) + } + } + if !mutation.request.DryRun || !maintenance.gc.DryRun || !maintenance.undo.DryRun { + t.Fatalf("fix=%+v gc=%+v undo=%+v", mutation.request, maintenance.gc, maintenance.undo) + } +} + +func TestDoctorFlagOwnershipRemainsLocal(t *testing.T) { + command := testModule(&fakeMutation{}, &fakeMaintenance{}, GlobalOptions{}).Command() + if command.Flags().Lookup("dry-run") == nil || command.Flags().Lookup("json") == nil { + t.Fatal("doctor local flags missing") + } + gc, _, _ := command.Find([]string{"gc"}) + if gc.Flags().Lookup("before") == nil || gc.Flags().Lookup("yes") == nil { + t.Fatal("gc local flags missing") + } + if gc.Flags().Lookup("dry-run") != nil { + t.Fatal("gc must inherit, not own, dry-run") + } + undo, _, _ := command.Find([]string{"undo"}) + if undo.Flags().Lookup("dry-run") == nil || undo.Flags().Lookup("strict") == nil { + t.Fatal("undo local flags missing") + } +} From 45ee0f46ea8495608e1f58770ac8ba9799dacd6c Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:12:02 -0400 Subject: [PATCH 075/108] fix(doctor): preserve immutable help contract (age-nw28h.2.7.4) --- cli/internal/commands/doctor/module.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/internal/commands/doctor/module.go b/cli/internal/commands/doctor/module.go index ff213cf8e..340b6fe2e 100644 --- a/cli/internal/commands/doctor/module.go +++ b/cli/internal/commands/doctor/module.go @@ -112,7 +112,7 @@ Examples: flags.BoolVar(&options.online, "online", false, "Enable network probes (default: offline-only)") flags.BoolVar(&options.quick, "quick", false, "Run only fast-path detectors (< 200ms)") flags.StringVar(&options.severity, "severity", "P3", "Minimum severity to emit (P0|P1|P2|P3)") - flags.BoolVar(&options.robot, "robot", false, "Alias for stable JSON report output") + flags.BoolVar(&options.robot, "robot", false, "Alias for --json with structured wrapper") flags.BoolVar(&options.robotTriage, "robot-triage", false, "Emit the mega-command triage JSON") flags.StringVar(&options.explain, "explain", "", "Expand a single finding by id") command.RunE = func(command *cobra.Command, _ []string) error { return module.runRoot(command, options) } From 856accd106bba7e0f86b2609d995f249b7ea0465 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:18:13 -0400 Subject: [PATCH 076/108] fix(doctor): declare command contract and scoped test owner (age-nw28h.2.7.4) --- cli/cmd/ao/doctor_module.go | 4 ++++ ...st.go => doctor_quality_test_support_test.go} | 0 cli/internal/commands/doctor/module.go | 16 ++++++++++++++++ 3 files changed, 20 insertions(+) rename cli/cmd/ao/{quality_test_support_test.go => doctor_quality_test_support_test.go} (100%) diff --git a/cli/cmd/ao/doctor_module.go b/cli/cmd/ao/doctor_module.go index 3c6828c05..a21585c2c 100644 --- a/cli/cmd/ao/doctor_module.go +++ b/cli/cmd/ao/doctor_module.go @@ -5,6 +5,7 @@ import ( "github.com/spf13/cobra" doctoradapter "github.com/boshu2/agentops/cli/internal/adapters/doctor" + "github.com/boshu2/agentops/cli/internal/clicontract" doctorcommands "github.com/boshu2/agentops/cli/internal/commands/doctor" doctorapp "github.com/boshu2/agentops/cli/internal/doctor" ) @@ -36,5 +37,8 @@ var doctorCommand = doctorModule.Command() func init() { doctorCommand.GroupID = "core" + if err := clicontract.Attach(doctorCommand, doctorModule.Contract()); err != nil { + panic(err) + } rootCmd.AddCommand(doctorCommand) } diff --git a/cli/cmd/ao/quality_test_support_test.go b/cli/cmd/ao/doctor_quality_test_support_test.go similarity index 100% rename from cli/cmd/ao/quality_test_support_test.go rename to cli/cmd/ao/doctor_quality_test_support_test.go diff --git a/cli/internal/commands/doctor/module.go b/cli/internal/commands/doctor/module.go index 340b6fe2e..e33ff89d6 100644 --- a/cli/internal/commands/doctor/module.go +++ b/cli/internal/commands/doctor/module.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" + "github.com/boshu2/agentops/cli/internal/clicontract" doctorapp "github.com/boshu2/agentops/cli/internal/doctor" "github.com/boshu2/agentops/cli/internal/quality" ) @@ -61,6 +62,21 @@ func NewModule(useCases UseCases, host HostOptions) Module { return Module{useCases: useCases, host: host} } +func (Module) Contract() clicontract.CommandContract { + return clicontract.CommandContract{ + ID: "ao.doctor", + Profiles: clicontract.ProfileDefault | clicontract.ProfileFlywheel | + clicontract.ProfileLegacy | clicontract.ProfileCombined, + Args: clicontract.ArgsPolicy{Name: "range", Validate: cobra.ArbitraryArgs}, + Output: clicontract.OutputNone, + Effects: clicontract.EffectFilesystem | clicontract.EffectProcess | clicontract.EffectNetwork | clicontract.EffectEnvironment | clicontract.EffectClock, + ExitClasses: map[int]clicontract.ExitClass{ + 0: clicontract.ExitSuccess, + 1: clicontract.ExitFailure, + }, + } +} + type rootOptions struct { json, fix, dryRun, online, quick, robot, robotTriage bool only, skip []string From 0ad37dffc39034883e5b8288dba109f385b6fccc Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:22:55 -0400 Subject: [PATCH 077/108] fix(doctor): reject junk positional arguments (age-nw28h.2.7.4) --- cli/internal/commands/doctor/module.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cli/internal/commands/doctor/module.go b/cli/internal/commands/doctor/module.go index e33ff89d6..4a138171b 100644 --- a/cli/internal/commands/doctor/module.go +++ b/cli/internal/commands/doctor/module.go @@ -67,7 +67,7 @@ func (Module) Contract() clicontract.CommandContract { ID: "ao.doctor", Profiles: clicontract.ProfileDefault | clicontract.ProfileFlywheel | clicontract.ProfileLegacy | clicontract.ProfileCombined, - Args: clicontract.ArgsPolicy{Name: "range", Validate: cobra.ArbitraryArgs}, + Args: clicontract.ArgsPolicy{Name: "range", Validate: cobra.NoArgs}, Output: clicontract.OutputNone, Effects: clicontract.EffectFilesystem | clicontract.EffectProcess | clicontract.EffectNetwork | clicontract.EffectEnvironment | clicontract.EffectClock, ExitClasses: map[int]clicontract.ExitClass{ @@ -108,7 +108,7 @@ func (module Module) globals(command *cobra.Command) GlobalOptions { func (module Module) Command() *cobra.Command { var options rootOptions command := &cobra.Command{ - Use: "doctor", Short: "Check AgentOps health", Args: nil, + Use: "doctor", Short: "Check AgentOps health", Args: cobra.NoArgs, Long: `Run health checks on your AgentOps installation. Validates that all required components are present and configured. @@ -287,7 +287,7 @@ func (module Module) fixCommand(options *rootOptions) *cobra.Command { jsonOutput := module.wantsJSON(command, options) dryRun := module.effectiveDryRun(command, options, false) return module.runFix(command, mutationRequest(*options, dryRun, jsonOutput), jsonOutput) - }} + }, Args: cobra.NoArgs} } func (module Module) undoCommand(options *rootOptions) *cobra.Command { @@ -323,13 +323,13 @@ func (module Module) explainCommand(options *rootOptions) *cobra.Command { } func (module Module) capabilitiesCommand() *cobra.Command { - return &cobra.Command{Use: "capabilities", Short: "Print the machine-readable doctor contract (JSON)", RunE: func(command *cobra.Command, _ []string) error { + return &cobra.Command{Use: "capabilities", Short: "Print the machine-readable doctor contract (JSON)", Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { return writeJSON(command, module.useCases.Read.Capabilities(command.Context())) }} } func (module Module) healthCommand(options *rootOptions) *cobra.Command { - return &cobra.Command{Use: "health", Short: "Cheap one-line liveness summary", RunE: func(command *cobra.Command, _ []string) error { + return &cobra.Command{Use: "health", Short: "Cheap one-line liveness summary", Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { line, result, err := module.useCases.Read.Health(command.Context()) if err != nil { return exit(doctorapp.ExitIOError, err.Error()) @@ -346,7 +346,7 @@ func (module Module) healthCommand(options *rootOptions) *cobra.Command { } func (module Module) robotDocsCommand() *cobra.Command { - return &cobra.Command{Use: "robot-docs", Short: "Print the paste-ready agent handbook (Markdown)", RunE: func(command *cobra.Command, _ []string) error { + return &cobra.Command{Use: "robot-docs", Short: "Print the paste-ready agent handbook (Markdown)", Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { _, err := fmt.Fprint(command.OutOrStdout(), module.useCases.Read.RobotDocs(command.Context())) return err }} @@ -354,7 +354,7 @@ func (module Module) robotDocsCommand() *cobra.Command { func (module Module) gcCommand(options *rootOptions) *cobra.Command { var local gcOptions - command := &cobra.Command{Use: "gc", Short: "Prune old runs (requires --yes and --before )"} + command := &cobra.Command{Use: "gc", Short: "Prune old runs (requires --yes and --before )", Args: cobra.NoArgs} command.Flags().StringVar(&local.before, "before", "", "Prune runs started before this date (YYYY-MM-DD)") command.Flags().BoolVar(&local.yes, "yes", false, "Confirm pruning (required)") command.RunE = func(command *cobra.Command, _ []string) error { @@ -380,7 +380,7 @@ func (module Module) gcCommand(options *rootOptions) *cobra.Command { } func (module Module) listCommand(options *rootOptions) *cobra.Command { - return &cobra.Command{Use: "ls", Short: "List runs in .doctor/runs/", RunE: func(command *cobra.Command, _ []string) error { + return &cobra.Command{Use: "ls", Short: "List runs in .doctor/runs/", Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { runs, err := module.useCases.Read.List(command.Context()) if err != nil { return exit(doctorapp.ExitIOError, err.Error()) @@ -400,7 +400,7 @@ func (module Module) listCommand(options *rootOptions) *cobra.Command { } func (module Module) diffCommand(options *rootOptions) *cobra.Command { - return &cobra.Command{Use: "diff", Short: "Show what --fix would change (read-only)", RunE: func(command *cobra.Command, _ []string) error { + return &cobra.Command{Use: "diff", Short: "Show what --fix would change (read-only)", Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { jsonOutput := module.wantsJSON(command, options) report, err := module.useCases.Read.Diff(command.Context(), readRequest(*options, true, jsonOutput)) if err != nil { From fe67359bc4676a547ac8db72ef8c3ed052177f24 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:23:31 -0400 Subject: [PATCH 078/108] test(doctor): seal accepted compatibility lineage (age-nw28h.2.7.4) --- .../families/doctor/lineage.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/doctor/lineage.json diff --git a/cli/testdata/compatibility-baseline/families/doctor/lineage.json b/cli/testdata/compatibility-baseline/families/doctor/lineage.json new file mode 100644 index 000000000..b5201520c --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/doctor/lineage.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "family": "doctor", + "old_implementation_sha": "6a5ae2694d4947de5942d1968ad4b307f81eaf86", + "freeze_sha": "69775c2777907694ca7723f5fbc58af75c910b56", + "fixture_sha256": "b49b9432a87f6d73cf216abbff3c728816d18bcc4982a5b1d132827c5c0ff8de", + "ownership_sha256": "c8220109d1ca1c8cd069aad0cf0f212da7057a5a4cff50fd2aee4b3a3569e6aa", + "scope_sha256": "4845ae371d9a39227eeacca258bf3cf016c00eec17ca8c405f7cf2893d76dd1f", + "migration_state": "migrated", + "accepted_sha": "0ad37dffc39034883e5b8288dba109f385b6fccc" +} From 64e192453f070ed21672ffa31f40dfaa2019a4df Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:42:28 -0400 Subject: [PATCH 079/108] test(doctor): restore owner-side coverage (age-nw28h.2.7.4) --- cli/cmd/ao/cobra_commands_test.go | 329 ------------------ cli/cmd/ao/codex_test.go | 8 +- .../ao/doctor_quality_test_support_test.go | 36 -- cli/cmd/ao/error_snapshot_test.go | 10 +- cli/cmd/ao/golden_output_test.go | 17 +- cli/cmd/ao/handoff_test.go | 14 +- cli/cmd/ao/helpers_test.go | 185 ---------- cli/cmd/ao/json_validity_test.go | 5 +- cli/cmd/ao/regression_test.go | 13 +- cli/cmd/ao/session_bootstrap_test.go | 4 +- cli/internal/adapters/doctor/legacy_test.go | 160 +++++++++ .../adapters/reviewerhealth/probe_test.go | 54 +++ cli/internal/commands/doctor/module_test.go | 78 +++++ cli/internal/quality/skills_codex_test.go | 191 ++++++++++ 14 files changed, 524 insertions(+), 580 deletions(-) delete mode 100644 cli/cmd/ao/doctor_quality_test_support_test.go create mode 100644 cli/internal/quality/skills_codex_test.go diff --git a/cli/cmd/ao/cobra_commands_test.go b/cli/cmd/ao/cobra_commands_test.go index f740bf8c2..ec56a7fdd 100644 --- a/cli/cmd/ao/cobra_commands_test.go +++ b/cli/cmd/ao/cobra_commands_test.go @@ -1427,232 +1427,6 @@ func TestCobraOutputValidateResult(t *testing.T) { }) } -// TestCobraDoctorHelpers exercises doctor helper functions. -func TestCobraDoctorHelpers(t *testing.T) { - assertDoctorStatusIcons(t) - assertDoctorRequiredFailure(t) - assertDoctorStatusCounts(t) - assertDoctorSummaries(t) - assertDoctorResults(t) - assertDoctorTableRendering(t) - assertDoctorFormatNumbers(t) - assertDoctorFormatDurations(t) - assertDoctorFileLineCounts(t) - assertDoctorHealFindingCounts(t) -} - -func assertDoctorStatusIcons(t *testing.T) { - t.Helper() - - t.Run("doctorStatusIcon", func(t *testing.T) { - cases := []struct { - status string - want string - }{ - {"pass", "\u2713"}, - {"warn", "!"}, - {"fail", "\u2717"}, - {"unknown", "?"}, - } - for _, tc := range cases { - got := doctorStatusIcon(tc.status) - if got != tc.want { - t.Errorf("doctorStatusIcon(%q) = %q, want %q", tc.status, got, tc.want) - } - } - }) -} - -func assertDoctorRequiredFailure(t *testing.T) { - t.Helper() - - t.Run("hasRequiredFailure", func(t *testing.T) { - checks := []doctorCheck{ - {Name: "a", Status: "pass", Required: true}, - {Name: "b", Status: "warn", Required: false}, - } - if hasRequiredFailure(checks) { - t.Error("expected no required failure") - } - - checks = append(checks, doctorCheck{Name: "c", Status: "fail", Required: true}) - if !hasRequiredFailure(checks) { - t.Error("expected required failure") - } - }) -} - -func assertDoctorStatusCounts(t *testing.T) { - t.Helper() - - t.Run("countCheckStatuses", func(t *testing.T) { - checks := []doctorCheck{ - {Status: "pass"}, {Status: "pass"}, {Status: "warn"}, {Status: "fail"}, - } - p, f, w := countCheckStatuses(checks) - if p != 2 || f != 1 || w != 1 { - t.Errorf("countCheckStatuses = (%d, %d, %d), want (2, 1, 1)", p, f, w) - } - }) -} - -func assertDoctorSummaries(t *testing.T) { - t.Helper() - - t.Run("buildDoctorSummary", func(t *testing.T) { - assertDoctorSummaryEquals(t, buildDoctorSummary(5, 0, 0, 5), "5/5 checks passed") - assertDoctorSummaryContains(t, buildDoctorSummary(4, 0, 1, 5), "1 warning") - assertDoctorSummaryContains(t, buildDoctorSummary(3, 1, 1, 5), "1 failed") - }) -} - -func assertDoctorSummaryEquals(t *testing.T, got string, want string) { - t.Helper() - - if got != want { - t.Errorf("got %q", got) - } -} - -func assertDoctorSummaryContains(t *testing.T, got string, want string) { - t.Helper() - - if !strings.Contains(got, want) { - t.Errorf("expected %q, got %q", want, got) - } -} - -func assertDoctorResults(t *testing.T) { - t.Helper() - - t.Run("computeResult", func(t *testing.T) { - cases := []struct { - checks []doctorCheck - want string - }{ - {checks: []doctorCheck{{Status: "pass"}}, want: "HEALTHY"}, - {checks: []doctorCheck{{Status: "warn"}}, want: "DEGRADED"}, - {checks: []doctorCheck{{Status: "fail"}}, want: "UNHEALTHY"}, - } - for _, tc := range cases { - result := computeResult(tc.checks) - if result.Result != tc.want { - t.Errorf("expected %s, got %s", tc.want, result.Result) - } - } - }) -} - -func assertDoctorTableRendering(t *testing.T) { - t.Helper() - - t.Run("renderDoctorTable", func(t *testing.T) { - buf := new(bytes.Buffer) - out := doctorOutput{ - Checks: []doctorCheck{ - {Name: "Test Check", Status: "pass", Detail: "OK"}, - }, - Result: "HEALTHY", - Summary: "1/1 checks passed", - } - renderDoctorTable(buf, out) - s := buf.String() - if !strings.Contains(s, "Test Check") || !strings.Contains(s, "1/1") { - t.Errorf("renderDoctorTable output missing expected content: %s", s) - } - }) -} - -func assertDoctorFormatNumbers(t *testing.T) { - t.Helper() - - t.Run("formatNumber", func(t *testing.T) { - cases := []struct { - n int - want string - }{ - {0, "0"}, - {42, "42"}, - {999, "999"}, - {1000, "1,000"}, - {1247, "1,247"}, - {1000000, "1,000,000"}, - } - for _, tc := range cases { - got := formatNumber(tc.n) - if got != tc.want { - t.Errorf("formatNumber(%d) = %q, want %q", tc.n, got, tc.want) - } - } - }) -} - -func assertDoctorFormatDurations(t *testing.T) { - t.Helper() - - t.Run("formatDuration", func(t *testing.T) { - cases := []struct { - d time.Duration - want string - }{ - {30 * time.Second, "30s"}, - {5 * time.Minute, "5m"}, - {3 * time.Hour, "3h"}, - {48 * time.Hour, "2d"}, - } - for _, tc := range cases { - got := formatDuration(tc.d) - if got != tc.want { - t.Errorf("formatDuration(%v) = %q, want %q", tc.d, got, tc.want) - } - } - }) -} - -func assertDoctorFileLineCounts(t *testing.T) { - t.Helper() - - t.Run("countFileLines", func(t *testing.T) { - tmp := t.TempDir() - path := filepath.Join(tmp, "test.txt") - content := "line1\nline2\n\nline3\n" - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatal(err) - } - got := countFileLines(path) - if got != 3 { - t.Errorf("countFileLines = %d, want 3", got) - } - }) - - t.Run("countFileLines_nonexistent", func(t *testing.T) { - got := countFileLines("/nonexistent/path") - if got != 0 { - t.Errorf("countFileLines(nonexistent) = %d, want 0", got) - } - }) -} - -func assertDoctorHealFindingCounts(t *testing.T) { - t.Helper() - - t.Run("countHealFindings", func(t *testing.T) { - output := "[CODE] path/to/file: finding 1\n[CODE] other/file: finding 2\nSummary line\n" - got := countHealFindings(output) - if got != 2 { - t.Errorf("countHealFindings = %d, want 2", got) - } - }) - - t.Run("countHealFindings_from_summary", func(t *testing.T) { - output := "Some output\n5 finding(s) detected.\n" - got := countHealFindings(output) - if got != 5 { - t.Errorf("countHealFindings = %d, want 5", got) - } - }) -} - // TestCobraStatusHelpers exercises status helper functions. func TestCobraStatusHelpers(t *testing.T) { t.Run("truncateStatus", func(t *testing.T) { @@ -2412,93 +2186,6 @@ func TestCobraQuickstartHelpers(t *testing.T) { }) } -// TestCobraNewestFileModTime exercises newestFileModTime. -func TestCobraNewestFileModTime(t *testing.T) { - tmp := t.TempDir() - // Create files with different times - f1 := filepath.Join(tmp, "a.txt") - f2 := filepath.Join(tmp, "b.txt") - if err := os.WriteFile(f1, []byte("a"), 0644); err != nil { - t.Fatal(err) - } - time.Sleep(10 * time.Millisecond) // ensure different modtimes - if err := os.WriteFile(f2, []byte("b"), 0644); err != nil { - t.Fatal(err) - } - - entries, err := os.ReadDir(tmp) - if err != nil { - t.Fatal(err) - } - - newest := newestFileModTime(entries) - if newest.IsZero() { - t.Error("expected non-zero newest time") - } -} - -// TestCobraCountFiles exercises countFiles. -func TestCobraCountFiles(t *testing.T) { - tmp := t.TempDir() - if err := os.WriteFile(filepath.Join(tmp, "a.txt"), []byte("a"), 0644); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(tmp, "subdir"), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmp, "b.txt"), []byte("b"), 0644); err != nil { - t.Fatal(err) - } - - got := countFiles(tmp) - if got != 2 { - t.Errorf("countFiles = %d, want 2", got) - } - - got = countFiles("/nonexistent/path") - if got != 0 { - t.Errorf("countFiles(nonexistent) = %d, want 0", got) - } -} - -// TestCobraCountLearningFiles exercises countLearningFiles. -func TestCobraCountLearningFiles(t *testing.T) { - tmp := t.TempDir() - if err := os.WriteFile(filepath.Join(tmp, "a.md"), []byte("md"), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmp, "b.jsonl"), []byte("jsonl"), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmp, "c.txt"), []byte("txt"), 0644); err != nil { - t.Fatal(err) - } - - got := countLearningFiles(tmp) - if got != 2 { - t.Errorf("countLearningFiles = %d, want 2", got) - } -} - -// TestCobraCountEstablished exercises countEstablished. -func TestCobraCountEstablished(t *testing.T) { - tmp := t.TempDir() - if err := os.WriteFile(filepath.Join(tmp, "learning-established.md"), []byte(""), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmp, "promoted-pattern.md"), []byte(""), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmp, "regular.md"), []byte(""), 0644); err != nil { - t.Fatal(err) - } - - got := countEstablished(tmp) - if got != 2 { - t.Errorf("countEstablished = %d, want 2", got) - } -} - // TestCobraResolveGoalsFile exercises resolveGoalsFile. func TestCobraResolveGoalsFile(t *testing.T) { tmp := chdirTemp(t) @@ -2546,22 +2233,6 @@ func TestCobraResolveGoalsFile(t *testing.T) { }) } -// TestCobraFileExists exercises fileExists. -func TestCobraFileExists(t *testing.T) { - tmp := t.TempDir() - path := filepath.Join(tmp, "exists.txt") - if err := os.WriteFile(path, []byte(""), 0644); err != nil { - t.Fatal(err) - } - - if !fileExists(path) { - t.Error("fileExists should return true for existing file") - } - if fileExists(filepath.Join(tmp, "nope.txt")) { - t.Error("fileExists should return false for nonexistent file") - } -} - // TestCobraRatchetParentHelp ensures ratchet parent help works. func TestCobraRatchetParentHelp(t *testing.T) { out, err := executeCommand("ratchet", "--help") diff --git a/cli/cmd/ao/codex_test.go b/cli/cmd/ao/codex_test.go index 2c5dd17a7..5f3714f4f 100644 --- a/cli/cmd/ao/codex_test.go +++ b/cli/cmd/ao/codex_test.go @@ -10,6 +10,8 @@ import ( "strings" "testing" "time" + + "github.com/boshu2/agentops/cli/internal/quality" ) func TestDetectCodexLifecycleProfile_Hookless(t *testing.T) { @@ -202,10 +204,10 @@ Use ao codex start and ao codex stop when runtime hooks are unavailable. if result.Runtime.Mode != lifecycleModeCodexHookless { t.Fatalf("runtime mode = %q, want %q", result.Runtime.Mode, lifecycleModeCodexHookless) } - if result.StartupContextPath == "" || !fileExists(result.StartupContextPath) { + if result.StartupContextPath == "" || !quality.FileExists(result.StartupContextPath) { t.Fatalf("startup context path missing or unreadable: %q", result.StartupContextPath) } - if result.StatePath == "" || !fileExists(result.StatePath) { + if result.StatePath == "" || !quality.FileExists(result.StatePath) { t.Fatalf("state path missing or unreadable: %q", result.StatePath) } citationsPath := filepath.Join(repo, ".agents", "ao", "citations.jsonl") @@ -615,7 +617,7 @@ func TestCodexStopJSONUsesHistoryFallback(t *testing.T) { if result.Session.SessionID == "" { t.Fatal("expected session close result to include a session ID") } - if !fileExists(result.TranscriptPath) { + if !quality.FileExists(result.TranscriptPath) { t.Fatalf("synthetic transcript path missing: %s", result.TranscriptPath) } } diff --git a/cli/cmd/ao/doctor_quality_test_support_test.go b/cli/cmd/ao/doctor_quality_test_support_test.go deleted file mode 100644 index aa50ac22d..000000000 --- a/cli/cmd/ao/doctor_quality_test_support_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import ( - "io" - "os" - "path/filepath" - "time" - - "github.com/boshu2/agentops/cli/internal/quality" -) - -type doctorCheck = quality.Check -type doctorOutput = quality.DoctorOutput - -func doctorStatusIcon(status string) string { return quality.StatusIcon(status) } -func hasRequiredFailure(checks []doctorCheck) bool { return quality.HasRequiredFailure(checks) } -func countCheckStatuses(checks []doctorCheck) (int, int, int) { - return quality.CountCheckStatuses(checks) -} -func buildDoctorSummary(passes, fails, warns, total int) string { - return quality.BuildSummary(passes, fails, warns, total) -} -func computeResult(checks []doctorCheck) doctorOutput { return quality.ComputeResult(checks) } -func renderDoctorTable(writer io.Writer, output doctorOutput) { quality.RenderTable(writer, output) } -func formatNumber(value int) string { return quality.FormatNumber(value) } -func formatDuration(value time.Duration) string { return quality.FormatDuration(value) } -func countFileLines(path string) int { return quality.CountFileLines(path) } -func countHealFindings(output string) int { return quality.CountHealFindings(output) } -func newestFileModTime(entries []os.DirEntry) time.Time { return quality.NewestFileModTime(entries) } -func countFiles(path string) int { return quality.CountFiles(path) } -func countLearningFiles(path string) int { return quality.CountLearningFiles(path) } -func countEstablished(path string) int { return quality.CountEstablished(path) } -func fileExists(path string) bool { return quality.FileExists(path) } -func checkFlywheelHealth(root string) doctorCheck { - return quality.CheckFlywheelHealth(filepath.Join(root, ".agents")) -} diff --git a/cli/cmd/ao/error_snapshot_test.go b/cli/cmd/ao/error_snapshot_test.go index 8ed31668c..a6b7b408b 100644 --- a/cli/cmd/ao/error_snapshot_test.go +++ b/cli/cmd/ao/error_snapshot_test.go @@ -11,6 +11,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" + + "github.com/boshu2/agentops/cli/internal/quality" ) // errorTestdataDir returns the absolute path to testdata/errors/ relative to @@ -201,18 +203,18 @@ func TestErrorRatchetRecordMissingOutput(t *testing.T) { func TestErrorDoctorEmptyDir(t *testing.T) { _ = chdirTemp(t) - // Use the computeResult + renderDoctorTable path to get deterministic output. + // Use the quality.ComputeResult + quality.RenderTable path to get deterministic output. // gatherDoctorChecks depends on filesystem state. For a controlled test, // synthesize checks matching what an empty dir produces. - checks := []doctorCheck{ + checks := []quality.Check{ {Name: "ao CLI", Status: "pass", Detail: "vdev", Required: true}, {Name: "Knowledge Base", Status: "fail", Detail: ".agents/ao not initialized", Required: true}, {Name: "Flywheel Health", Status: "warn", Detail: "No learnings found \u2014 the flywheel hasn't started", Required: false}, } - result := computeResult(checks) + result := quality.ComputeResult(checks) var buf bytes.Buffer - renderDoctorTable(&buf, result) + quality.RenderTable(&buf, result) errorGoldenTest(t, "doctor-empty-dir.txt", buf.Bytes()) } diff --git a/cli/cmd/ao/golden_output_test.go b/cli/cmd/ao/golden_output_test.go index 702ea3716..033e48fab 100644 --- a/cli/cmd/ao/golden_output_test.go +++ b/cli/cmd/ao/golden_output_test.go @@ -13,6 +13,7 @@ import ( "github.com/google/go-cmp/cmp" + "github.com/boshu2/agentops/cli/internal/quality" "github.com/boshu2/agentops/cli/internal/ratchet" ) @@ -241,7 +242,7 @@ func TestGoldenRatchetNextJSON_AllComplete(t *testing.T) { // --------------------------------------------------------------------------- func TestGoldenDoctorTableAllPass(t *testing.T) { - checks := []doctorCheck{ + checks := []quality.Check{ {Name: "ao CLI", Status: "pass", Detail: "v2.10.0", Required: true}, {Name: "CLI Dependencies", Status: "pass", Detail: "gt and bd available", Required: false}, {Name: "Hook Coverage", Status: "pass", Detail: "Full coverage: 5/5 events", Required: false}, @@ -251,10 +252,10 @@ func TestGoldenDoctorTableAllPass(t *testing.T) { {Name: "Flywheel Health", Status: "pass", Detail: "12 learnings (3 established)", Required: false}, {Name: "Plugin", Status: "pass", Detail: "18 skills found", Required: false}, } - result := computeResult(checks) + result := quality.ComputeResult(checks) var buf bytes.Buffer - renderDoctorTable(&buf, result) + quality.RenderTable(&buf, result) goldenTest(t, "doctor-all-pass.txt", buf.Bytes()) } @@ -264,17 +265,17 @@ func TestGoldenDoctorTableAllPass(t *testing.T) { // --------------------------------------------------------------------------- func TestGoldenDoctorTableMixed(t *testing.T) { - checks := []doctorCheck{ + checks := []quality.Check{ {Name: "ao CLI", Status: "pass", Detail: "v2.10.0", Required: true}, {Name: "CLI Dependencies", Status: "warn", Detail: "bd not found — install with 'brew install beads'", Required: false}, {Name: "Hook Coverage", Status: "warn", Detail: "Partial coverage: 3/5 events — run 'ao hooks install --force'", Required: false}, {Name: "Knowledge Base", Status: "fail", Detail: ".agents/ao not initialized", Required: true}, {Name: "Flywheel Health", Status: "warn", Detail: "No learnings found — the flywheel hasn't started", Required: false}, } - result := computeResult(checks) + result := quality.ComputeResult(checks) var buf bytes.Buffer - renderDoctorTable(&buf, result) + quality.RenderTable(&buf, result) goldenTest(t, "doctor-mixed.txt", buf.Bytes()) } @@ -284,11 +285,11 @@ func TestGoldenDoctorTableMixed(t *testing.T) { // --------------------------------------------------------------------------- func TestGoldenDoctorJSON(t *testing.T) { - checks := []doctorCheck{ + checks := []quality.Check{ {Name: "ao CLI", Status: "pass", Detail: "v2.10.0", Required: true}, {Name: "Knowledge Base", Status: "fail", Detail: ".agents/ao not initialized", Required: true}, } - result := computeResult(checks) + result := quality.ComputeResult(checks) data, err := json.MarshalIndent(result, "", " ") if err != nil { diff --git a/cli/cmd/ao/handoff_test.go b/cli/cmd/ao/handoff_test.go index f528c58f4..51267cd7d 100644 --- a/cli/cmd/ao/handoff_test.go +++ b/cli/cmd/ao/handoff_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/boshu2/agentops/cli/internal/quality" ) func TestRunHandoff_WritesArtifact(t *testing.T) { @@ -28,7 +30,7 @@ func TestRunHandoff_WritesArtifact(t *testing.T) { } // Verify file exists - if !fileExists(path) { + if !quality.FileExists(path) { t.Fatalf("artifact file not found at %s", path) } @@ -83,7 +85,7 @@ func TestRunHandoff_DryRun(t *testing.T) { // Verify NO file is written handoffDir := filepath.Join(dir, ".agents", "handoff") - if fileExists(handoffDir) { + if quality.FileExists(handoffDir) { t.Error("handoff dir should not exist in dry-run mode") } } @@ -106,7 +108,7 @@ func TestRunHandoff_NoKill(t *testing.T) { } // Verify artifact was written (no-kill still writes) - if !fileExists(path) { + if !quality.FileExists(path) { t.Fatal("artifact should exist with --no-kill") } } @@ -409,7 +411,7 @@ func TestWriteHandoffArtifact_AtomicWrite(t *testing.T) { // Verify no .tmp file lingering tmpPath := path + ".tmp" - if fileExists(tmpPath) { + if quality.FileExists(tmpPath) { t.Error(".tmp file should not exist after successful write") } @@ -585,11 +587,11 @@ func TestRunHandoff_DryRunViaRunHandoff(t *testing.T) { // Verify NO files were written handoffDir := filepath.Join(dir, ".agents", "handoff") - if fileExists(handoffDir) { + if quality.FileExists(handoffDir) { t.Error("handoff dir should not exist when --dry-run is set") } agentsDir := filepath.Join(dir, ".agents") - if fileExists(agentsDir) { + if quality.FileExists(agentsDir) { t.Error(".agents dir should not exist when --dry-run is set") } } diff --git a/cli/cmd/ao/helpers_test.go b/cli/cmd/ao/helpers_test.go index 9427c4844..dafce7580 100644 --- a/cli/cmd/ao/helpers_test.go +++ b/cli/cmd/ao/helpers_test.go @@ -13,184 +13,6 @@ import ( "github.com/boshu2/agentops/cli/internal/types" ) -// =========================================================================== -// doctor.go helpers -// =========================================================================== - -func TestHelper_doctorStatusIcon(t *testing.T) { - tests := []struct { - status string - want string - }{ - {"pass", "\u2713"}, - {"warn", "!"}, - {"fail", "\u2717"}, - {"unknown", "?"}, - {"", "?"}, - } - for _, tc := range tests { - t.Run(tc.status, func(t *testing.T) { - got := doctorStatusIcon(tc.status) - if got != tc.want { - t.Errorf("doctorStatusIcon(%q) = %q, want %q", tc.status, got, tc.want) - } - }) - } -} - -func TestHelper_countCheckStatuses(t *testing.T) { - tests := []struct { - name string - checks []doctorCheck - wantPasses, wantFails, wantWarns int - }{ - { - name: "empty", - checks: nil, - wantPasses: 0, wantFails: 0, wantWarns: 0, - }, - { - name: "all pass", - checks: []doctorCheck{ - {Status: "pass"}, - {Status: "pass"}, - }, - wantPasses: 2, wantFails: 0, wantWarns: 0, - }, - { - name: "mixed", - checks: []doctorCheck{ - {Status: "pass"}, - {Status: "fail"}, - {Status: "warn"}, - {Status: "warn"}, - {Status: "pass"}, - }, - wantPasses: 2, wantFails: 1, wantWarns: 2, - }, - { - name: "all fail", - checks: []doctorCheck{ - {Status: "fail"}, - {Status: "fail"}, - }, - wantPasses: 0, wantFails: 2, wantWarns: 0, - }, - { - name: "unknown status ignored", - checks: []doctorCheck{ - {Status: "pass"}, - {Status: "something_else"}, - }, - wantPasses: 1, wantFails: 0, wantWarns: 0, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - p, f, w := countCheckStatuses(tc.checks) - if p != tc.wantPasses || f != tc.wantFails || w != tc.wantWarns { - t.Errorf("countCheckStatuses() = (%d, %d, %d), want (%d, %d, %d)", - p, f, w, tc.wantPasses, tc.wantFails, tc.wantWarns) - } - }) - } -} - -func TestHelper_buildDoctorSummary(t *testing.T) { - tests := []struct { - name string - passes, fails, warns, total int - want string - }{ - { - name: "all pass no warnings", - passes: 5, fails: 0, warns: 0, total: 5, - want: "5/5 checks passed", - }, - { - name: "all pass one warning", - passes: 4, fails: 0, warns: 1, total: 5, - want: "4/5 checks passed, 1 warning", - }, - { - name: "all pass multiple warnings", - passes: 3, fails: 0, warns: 2, total: 5, - want: "3/5 checks passed, 2 warnings", - }, - { - name: "failures only", - passes: 3, fails: 2, warns: 0, total: 5, - want: "3/5 checks passed, 2 failed", - }, - { - name: "failures and warnings", - passes: 2, fails: 1, warns: 2, total: 5, - want: "2/5 checks passed, 2 warnings, 1 failed", - }, - { - name: "single failure single warning", - passes: 3, fails: 1, warns: 1, total: 5, - want: "3/5 checks passed, 1 warning, 1 failed", - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := buildDoctorSummary(tc.passes, tc.fails, tc.warns, tc.total) - if got != tc.want { - t.Errorf("buildDoctorSummary(%d,%d,%d,%d) = %q, want %q", - tc.passes, tc.fails, tc.warns, tc.total, got, tc.want) - } - }) - } -} - -func TestHelper_hasRequiredFailure(t *testing.T) { - tests := []struct { - name string - checks []doctorCheck - want bool - }{ - { - name: "empty", - checks: nil, - want: false, - }, - { - name: "no required failures", - checks: []doctorCheck{ - {Status: "pass", Required: true}, - {Status: "fail", Required: false}, - {Status: "warn", Required: true}, - }, - want: false, - }, - { - name: "has required failure", - checks: []doctorCheck{ - {Status: "pass", Required: true}, - {Status: "fail", Required: true}, - }, - want: true, - }, - { - name: "all passing required", - checks: []doctorCheck{ - {Status: "pass", Required: true}, - {Status: "pass", Required: true}, - }, - want: false, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := hasRequiredFailure(tc.checks) - if got != tc.want { - t.Errorf("hasRequiredFailure() = %v, want %v", got, tc.want) - } - }) - } -} - // =========================================================================== // feedback.go helpers // =========================================================================== @@ -859,10 +681,3 @@ func TestHelper_applyJSONLRewardFields_confidenceFormula(t *testing.T) { t.Errorf("confidence = %f, want ~%f", conf, expected) } } - -func TestHelper_buildDoctorSummary_zeroTotal(t *testing.T) { - got := buildDoctorSummary(0, 0, 0, 0) - if got != "0/0 checks passed" { - t.Errorf("buildDoctorSummary(0,0,0,0) = %q, want %q", got, "0/0 checks passed") - } -} diff --git a/cli/cmd/ao/json_validity_test.go b/cli/cmd/ao/json_validity_test.go index 31257eb93..d60edb7a9 100644 --- a/cli/cmd/ao/json_validity_test.go +++ b/cli/cmd/ao/json_validity_test.go @@ -11,6 +11,7 @@ import ( "github.com/boshu2/agentops/cli/internal/goals" "github.com/boshu2/agentops/cli/internal/pool" + "github.com/boshu2/agentops/cli/internal/quality" "github.com/boshu2/agentops/cli/internal/ratchet" "github.com/boshu2/agentops/cli/internal/types" ) @@ -710,8 +711,8 @@ func TestJSONValidity_Config(t *testing.T) { func TestJSONValidity_Doctor(t *testing.T) { withDoctorJSON(t) - result := doctorOutput{ - Checks: []doctorCheck{ + result := quality.DoctorOutput{ + Checks: []quality.Check{ {Name: "ao CLI", Status: "pass", Detail: "v0.0.0-test", Required: true}, {Name: "test check", Status: "warn", Detail: "not installed", Required: false}, }, diff --git a/cli/cmd/ao/regression_test.go b/cli/cmd/ao/regression_test.go index 4f3012c6b..f7314e58e 100644 --- a/cli/cmd/ao/regression_test.go +++ b/cli/cmd/ao/regression_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "testing" + "github.com/boshu2/agentops/cli/internal/quality" "github.com/boshu2/agentops/cli/internal/vibecheck" ) @@ -42,7 +43,7 @@ func seedAgentsFixture(t *testing.T) string { func TestDoctorFlywheelFindsFixtureArtifacts(t *testing.T) { tmp := seedAgentsFixture(t) - result := checkFlywheelHealth(tmp) + result := quality.CheckFlywheelHealth(filepath.Join(tmp, ".agents")) // Fixture has 3 .md + 1 .jsonl in learnings = 4 learning files // checkFlywheelHealth should find learnings > 0 and report pass @@ -63,13 +64,13 @@ func TestCountLearningFilesMatchesFixture(t *testing.T) { tmp := seedAgentsFixture(t) learningsDir := filepath.Join(tmp, ".agents", "learnings") - got := countLearningFiles(learningsDir) + got := quality.CountLearningFiles(learningsDir) // Fixture learnings: learn-2026-test-001.md, learn-2026-test-002.md, // learn-2026-test-003.md (3 .md) + feedback.jsonl (1 .jsonl) = 4 want := 4 if got != want { - t.Errorf("countLearningFiles() = %d, want %d", got, want) + t.Errorf("quality.CountLearningFiles() = %d, want %d", got, want) } } @@ -108,9 +109,9 @@ func TestMetricsCountArtifactsIncludesLearnings(t *testing.T) { func TestLearningCountConsistency(t *testing.T) { tmp := seedAgentsFixture(t) - // Get count from countLearningFiles (doctor.go method) + // Get count from quality.CountLearningFiles (doctor.go method) learningsDir := filepath.Join(tmp, ".agents", "learnings") - doctorCount := countLearningFiles(learningsDir) + doctorCount := quality.CountLearningFiles(learningsDir) // Get count from countArtifacts (metrics.go method) _, tierCounts, err := countArtifacts(tmp) @@ -121,7 +122,7 @@ func TestLearningCountConsistency(t *testing.T) { // Both should agree: they both count *.md + *.jsonl in the learnings directory if doctorCount != metricsCount { - t.Errorf("learning count mismatch: countLearningFiles()=%d, countArtifacts[learning]=%d", + t.Errorf("learning count mismatch: quality.CountLearningFiles()=%d, countArtifacts[learning]=%d", doctorCount, metricsCount) } } diff --git a/cli/cmd/ao/session_bootstrap_test.go b/cli/cmd/ao/session_bootstrap_test.go index fcccb5881..06a77a3ba 100644 --- a/cli/cmd/ao/session_bootstrap_test.go +++ b/cli/cmd/ao/session_bootstrap_test.go @@ -18,6 +18,8 @@ import ( "syscall" "testing" "time" + + "github.com/boshu2/agentops/cli/internal/quality" ) func TestSessionBootstrap_FullStatusJSON(t *testing.T) { @@ -397,7 +399,7 @@ func TestSessionBootstrapGateHookStatus_NeverExecutesInstaller(t *testing.T) { if got := sessionBootstrapGateHookStatus(dir); got != "inactive" { t.Fatalf("spoofed agentops repo, unwired hook: want inactive, got %q", got) } - if fileExists(sentinel) { + if quality.FileExists(sentinel) { t.Fatal("SECURITY: gate-hook status executed a working-tree installer script") } } diff --git a/cli/internal/adapters/doctor/legacy_test.go b/cli/internal/adapters/doctor/legacy_test.go index 99f2dc283..794d25b30 100644 --- a/cli/internal/adapters/doctor/legacy_test.go +++ b/cli/internal/adapters/doctor/legacy_test.go @@ -1,11 +1,171 @@ package doctor import ( + "bytes" + "context" + "errors" + "fmt" "os" "path/filepath" + "strings" "testing" + "time" + + "github.com/boshu2/agentops/cli/internal/provenancegraph" + "github.com/boshu2/agentops/cli/internal/reviewerhealth" ) +type fakeReviewerProbe map[string]reviewerhealth.ProbeResult + +func (probe fakeReviewerProbe) Check(_ context.Context, reviewer reviewerhealth.Reviewer, _ time.Duration) reviewerhealth.ProbeResult { + return probe[reviewer.Name] +} + +func TestLegacyChecksIncludesEveryDoctorSafetySection(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + t.Setenv("HOME", t.TempDir()) + reviewers := reviewerhealth.NewService(reviewerhealth.DefaultCatalog(), fakeReviewerProbe{ + "codex": {Status: "pass", Detail: "reachable", Live: true}, + "agy": {Status: "warn", Detail: "missing"}, + }) + adapter := LegacyChecks{ + ToolVersion: "1.0.0", IndexDir: ".agents/index", IndexFile: "index.json", Reviewers: reviewers, + WorkingDir: func() (string, error) { return root, nil }, + LedgerPath: func() string { return filepath.Join(root, "ledger.jsonl") }, + Environment: func() []string { return nil }, + LookPath: func(string) (string, error) { return "", errors.New("missing") }, + Now: time.Now, + } + checks := adapter.Checks(context.Background()) + byName := make(map[string]string, len(checks)) + for _, check := range checks { + byName[check.Name] = check.Status + } + for _, name := range []string{ + "Plugin", "Codex Sync", "Skill Integrity", "Reviewer: codex", "Reviewer: agy", + "Cross-Family Review", "Binary Freshness", "Ledger Health", "LAW-0 Guard", + } { + if _, ok := byName[name]; !ok { + t.Errorf("LegacyChecks missing %q: %v", name, byName) + } + } + if byName["Reviewer: codex"] != "pass" || byName["Reviewer: agy"] != "warn" || byName["Cross-Family Review"] != "pass" { + t.Fatalf("reviewer integration = %v", byName) + } +} + +func TestCrossFamilyCheck(t *testing.T) { + for _, test := range []struct { + name, status, detail string + live []string + }{ + {name: "both", live: []string{"codex", "agy"}, status: "pass", detail: "live families: codex, agy"}, + {name: "one", live: []string{"agy"}, status: "pass", detail: "live families: agy"}, + {name: "none", status: "warn", detail: "npm install -g @openai/codex"}, + } { + t.Run(test.name, func(t *testing.T) { + check := CrossFamilyCheck(test.live) + if check.Status != test.status || !strings.Contains(check.Detail, test.detail) { + t.Fatalf("check = %+v", check) + } + }) + } +} + +func makeFakeAgentopsRepo(t *testing.T, declared string) string { + t.Helper() + root := t.TempDir() + dir := filepath.Join(root, "cli", "cmd", "ao") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "cli", "go.mod"), []byte(agentopsModuleLine+"\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("package main\nvar version = \""+declared+"\"\n"), 0o644); err != nil { + t.Fatal(err) + } + return root +} + +func TestBinaryFreshnessCheck(t *testing.T) { + root := makeFakeAgentopsRepo(t, "9.9.9") + if check := BinaryFreshnessCheck(filepath.Join(root, "cli", "cmd"), "9.9.9"); check.Status != "pass" { + t.Fatalf("matching check = %+v", check) + } + check := BinaryFreshnessCheck(root, "1.0.0") + for _, want := range []string{"1.0.0", "9.9.9", "scripts/preflight-uat-binary.sh", "brew upgrade agentops"} { + if check.Status != "warn" || !strings.Contains(check.Detail, want) { + t.Fatalf("stale check lacks %q: %+v", want, check) + } + } + if check := BinaryFreshnessCheck(t.TempDir(), "1.0.0"); check.Status != "pass" || !strings.Contains(check.Detail, "outside the agentops repo") { + t.Fatalf("outside-repo check = %+v", check) + } +} + +func appendFixtureEdges(t *testing.T, path string, count int) { + t.Helper() + store := provenancegraph.NewStore(path) + for index := range count { + if _, err := store.Append(provenancegraph.Edge{ + FromID: fmt.Sprintf("age-test.%d", index), FromType: "bead", ToID: "abc1234", ToType: "commit", + Relation: "wasGeneratedBy", TrustTier: "authored", TS: time.Now().UTC().Format(time.RFC3339), + }); err != nil { + t.Fatalf("append edge %d: %v", index, err) + } + } +} + +func TestCheckLedgerHealthIntactMissingAndTampered(t *testing.T) { + missing := filepath.Join(t.TempDir(), "ledger.jsonl") + if check := CheckLedgerHealth(missing, time.Now); check.Status != "pass" || !strings.Contains(check.Detail, "no ledger records yet") { + t.Fatalf("missing ledger = %+v", check) + } + + path := filepath.Join(t.TempDir(), "ledger.jsonl") + appendFixtureEdges(t, path, 2) + if check := CheckLedgerHealth(path, time.Now); check.Status != "pass" || !strings.Contains(check.Detail, "2 records") || !strings.Contains(check.Detail, "newest record") { + t.Fatalf("intact ledger = %+v", check) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + tampered := bytes.Replace(data, []byte(`"to_id":"abc1234"`), []byte(`"to_id":"abc1235"`), 1) + if bytes.Equal(tampered, data) { + t.Fatal("fixture did not contain to_id") + } + if err := os.WriteFile(path, tampered, 0o644); err != nil { + t.Fatal(err) + } + check := CheckLedgerHealth(path, time.Now) + if check.Status != "fail" || !check.Required || !strings.Contains(check.Detail, "chain breaks at line 1") || !strings.Contains(check.Detail, "ao provenance verify") { + t.Fatalf("tampered ledger = %+v", check) + } +} + +func TestCheckLaw0Guard(t *testing.T) { + for _, test := range []struct { + name, status, detail string + environment []string + }{ + {name: "clean", environment: []string{"PAWL_NO_SERVICE=1"}, status: "pass", detail: "no reviewer path configured"}, + {name: "empty", status: "pass", detail: "no reviewer path configured"}, + {name: "pawl print flag", environment: []string{"PAWL_REVIEWER_CMD=claude" + " -p review"}, status: "fail", detail: "unset PAWL_REVIEWER_CMD"}, + {name: "reviewer print word", environment: []string{"MY_REVIEWER_BIN=claude" + " --print"}, status: "fail", detail: "unset MY_REVIEWER_BIN"}, + {name: "unscoped ignored", environment: []string{"SHELL_HISTORY=claude -p review"}, status: "pass", detail: "no reviewer path configured"}, + } { + t.Run(test.name, func(t *testing.T) { + check := CheckLaw0Guard(test.environment) + if check.Status != test.status || !check.Required || !strings.Contains(check.Detail, test.detail) { + t.Fatalf("check = %+v", check) + } + }) + } +} + func TestBinaryFreshnessRejectsSymlinkedRepoMarkers(t *testing.T) { root := t.TempDir() external := filepath.Join(t.TempDir(), "go.mod") diff --git a/cli/internal/adapters/reviewerhealth/probe_test.go b/cli/internal/adapters/reviewerhealth/probe_test.go index 444654d39..e49ad5ed8 100644 --- a/cli/internal/adapters/reviewerhealth/probe_test.go +++ b/cli/internal/adapters/reviewerhealth/probe_test.go @@ -3,6 +3,7 @@ package reviewerhealth import ( "context" "errors" + "strings" "testing" "time" @@ -28,3 +29,56 @@ func TestProbeMissingAndReachable(t *testing.T) { t.Fatalf("reachable result = %+v", result) } } + +func TestProbeHangingReviewerTimesOutWithoutWedgingDoctor(t *testing.T) { + reviewer := reviewerapp.Reviewer{Name: "codex", InstallCommand: "install codex"} + probe := Probe{ + LookPath: func(string) (string, error) { return "/bin/codex", nil }, + Run: func(ctx context.Context, _ string, _ ...string) error { + <-ctx.Done() + return ctx.Err() + }, + Now: time.Now, + } + started := time.Now() + result := probe.Check(context.Background(), reviewer, 25*time.Millisecond) + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("hanging reviewer wedged doctor for %s", elapsed) + } + if result.Status != "fail" || result.Live { + t.Fatalf("hanging result = %+v", result) + } + if !strings.Contains(result.Detail, "timed out") || !strings.Contains(result.Detail, "codex --version") { + t.Fatalf("timeout detail lacks diagnosis and repair command: %q", result.Detail) + } +} + +func TestProbeBrokenReviewerNamesInstallRepair(t *testing.T) { + reviewer := reviewerapp.Reviewer{Name: "agy", InstallCommand: "reinstall agy"} + probe := Probe{ + LookPath: func(string) (string, error) { return "/bin/agy", nil }, + Run: func(context.Context, string, ...string) error { return errors.New("exit 7") }, + Now: time.Now, + } + result := probe.Check(context.Background(), reviewer, time.Second) + if result.Status != "fail" || result.Live || !strings.Contains(result.Detail, "reinstall agy") { + t.Fatalf("broken result = %+v", result) + } +} + +func TestProbeMissingReviewerNamesExactInstallCommand(t *testing.T) { + reviewer := reviewerapp.Reviewer{Name: "codex", InstallCommand: "npm install -g @openai/codex && codex login"} + probe := Probe{ + LookPath: func(string) (string, error) { return "", errors.New("missing") }, + Run: func(context.Context, string, ...string) error { return nil }, + Now: time.Now, + } + started := time.Now() + result := probe.Check(context.Background(), reviewer, time.Second) + if elapsed := time.Since(started); elapsed > 100*time.Millisecond { + t.Fatalf("missing reviewer did not short-circuit: %s", elapsed) + } + if result.Status != "warn" || result.Live || !strings.Contains(result.Detail, reviewer.InstallCommand) { + t.Fatalf("missing result = %+v", result) + } +} diff --git a/cli/internal/commands/doctor/module_test.go b/cli/internal/commands/doctor/module_test.go index 9e4f77fca..0c1b5c7c6 100644 --- a/cli/internal/commands/doctor/module_test.go +++ b/cli/internal/commands/doctor/module_test.go @@ -1,7 +1,9 @@ package doctor import ( + "bytes" "context" + "encoding/json" "testing" "github.com/spf13/cobra" @@ -76,6 +78,82 @@ func TestInheritedDryRunProtectsFixGCAndUndo(t *testing.T) { } } +func TestDryRunUnionProtectsEveryMutationEntryPoint(t *testing.T) { + for _, test := range []struct { + name string + args []string + globals GlobalOptions + assert func(*testing.T, *fakeMutation, *fakeMaintenance) + }{ + {name: "root doctor fix and doctor-local dry-run", args: []string{"--fix", "--dry-run"}, assert: func(t *testing.T, mutation *fakeMutation, _ *fakeMaintenance) { + if !mutation.request.DryRun { + t.Fatalf("request = %+v", mutation.request) + } + }}, + {name: "fix child and root dry-run", args: []string{"fix"}, globals: GlobalOptions{DryRun: true}, assert: func(t *testing.T, mutation *fakeMutation, _ *fakeMaintenance) { + if !mutation.request.DryRun { + t.Fatalf("request = %+v", mutation.request) + } + }}, + {name: "gc child and root dry-run", args: []string{"gc", "--before", "2026-01-01", "--yes"}, globals: GlobalOptions{DryRun: true}, assert: func(t *testing.T, _ *fakeMutation, maintenance *fakeMaintenance) { + if !maintenance.gc.DryRun { + t.Fatalf("request = %+v", maintenance.gc) + } + }}, + {name: "undo child and undo-local dry-run", args: []string{"undo", "latest", "--dry-run"}, assert: func(t *testing.T, _ *fakeMutation, maintenance *fakeMaintenance) { + if !maintenance.undo.DryRun { + t.Fatalf("request = %+v", maintenance.undo) + } + }}, + {name: "undo child and root dry-run", args: []string{"undo", "latest"}, globals: GlobalOptions{DryRun: true}, assert: func(t *testing.T, _ *fakeMutation, maintenance *fakeMaintenance) { + if !maintenance.undo.DryRun { + t.Fatalf("request = %+v", maintenance.undo) + } + }}, + } { + t.Run(test.name, func(t *testing.T) { + mutation, maintenance := &fakeMutation{}, &fakeMaintenance{} + command := testModule(mutation, maintenance, test.globals).Command() + command.SetArgs(test.args) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + test.assert(t, mutation, maintenance) + }) + } +} + +func TestJSONUnionReachesRootAndChildFix(t *testing.T) { + for _, test := range []struct { + name string + args []string + globals GlobalOptions + }{ + {name: "doctor-local json", args: []string{"--fix", "--json"}}, + {name: "doctor robot alias", args: []string{"--fix", "--robot"}}, + {name: "root global json", args: []string{"fix"}, globals: GlobalOptions{JSON: true}}, + {name: "root output json", args: []string{"fix"}, globals: GlobalOptions{Output: "json"}}, + } { + t.Run(test.name, func(t *testing.T) { + mutation, maintenance := &fakeMutation{}, &fakeMaintenance{} + command := testModule(mutation, maintenance, test.globals).Command() + var stdout bytes.Buffer + command.SetOut(&stdout) + command.SetArgs(test.args) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if !mutation.request.JSON { + t.Fatalf("request = %+v", mutation.request) + } + var report doctorapp.Report + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("structured output invalid: %v\n%s", err, stdout.String()) + } + }) + } +} + func TestDoctorFlagOwnershipRemainsLocal(t *testing.T) { command := testModule(&fakeMutation{}, &fakeMaintenance{}, GlobalOptions{}).Command() if command.Flags().Lookup("dry-run") == nil || command.Flags().Lookup("json") == nil { diff --git a/cli/internal/quality/skills_codex_test.go b/cli/internal/quality/skills_codex_test.go new file mode 100644 index 000000000..4c70ec5b4 --- /dev/null +++ b/cli/internal/quality/skills_codex_test.go @@ -0,0 +1,191 @@ +package quality + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func writeSkill(t *testing.T, root, name string) { + t.Helper() + dir := filepath.Join(root, name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte("# "+name), 0o644); err != nil { + t.Fatal(err) + } +} + +func writeNativeManifest(t *testing.T, home string, names ...string) (string, string) { + t.Helper() + root := CodexNativePluginRootPath(home) + skills := filepath.Join(root, "skills-codex") + if err := os.MkdirAll(skills, 0o755); err != nil { + t.Fatal(err) + } + items := make([]string, 0, len(names)) + for _, name := range names { + writeSkill(t, skills, name) + items = append(items, fmt.Sprintf(`{"name":%q}`, name)) + } + manifest := filepath.Join(skills, ".agentops-manifest.json") + if err := os.WriteFile(manifest, []byte(`{"skills":[`+strings.Join(items, ",")+`]}`), 0o644); err != nil { + t.Fatal(err) + } + hash, err := SHA256File(manifest) + if err != nil { + t.Fatal(err) + } + return root, hash +} + +func writeInstallMeta(t *testing.T, home, root, version, hash string, count int) { + t.Helper() + if err := os.MkdirAll(filepath.Join(home, ".codex"), 0o755); err != nil { + t.Fatal(err) + } + body := fmt.Sprintf(`{"install_mode":"native-plugin","plugin_root":%q,"version":%q,"manifest_hash":%q,"skill_count":%d}`, + root, version, hash, count) + if err := os.WriteFile(CodexInstallMetaPath(home), []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestCheckSkillsReportsInstallGuidanceWhenEmpty(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + check := CheckSkills() + if check.Status != "warn" || !strings.Contains(check.Detail, "install.sh") { + t.Fatalf("check = %+v", check) + } +} + +func TestCheckSkillsValidatesNativeManifest(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + root, hash := writeNativeManifest(t, home, "research") + writeInstallMeta(t, home, root, "v1", hash, 1) + check := CheckSkills() + if check.Status != "pass" || !strings.Contains(check.Detail, "native manifest OK") { + t.Fatalf("valid native install = %+v", check) + } + + writeInstallMeta(t, home, root, "v1", "deadbeef", 1) + check = CheckSkills() + if check.Status != "warn" || !strings.Contains(check.Detail, "manifest hash does not match") { + t.Fatalf("drifted native install = %+v", check) + } +} + +func TestCheckSkillsWarnsForOverlappingInstallLayouts(t *testing.T) { + for _, test := range []struct { + name, duplicateRoot, detail string + }{ + {name: "raw Codex", duplicateRoot: filepath.Join(".codex", "skills"), detail: "duplicate raw Codex install"}, + {name: "user skills", duplicateRoot: filepath.Join(".agents", "skills"), detail: "duplicate raw skill install"}, + } { + t.Run(test.name, func(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + root, hash := writeNativeManifest(t, home, "research") + writeInstallMeta(t, home, root, "v1", hash, 1) + writeSkill(t, filepath.Join(home, test.duplicateRoot), "research") + check := CheckSkills() + if check.Status != "warn" || !strings.Contains(check.Detail, test.detail) || !strings.Contains(check.Detail, "research") { + t.Fatalf("check = %+v", check) + } + }) + } +} + +func setupCodexSyncRepo(t *testing.T) (string, string, string) { + t.Helper() + repo := t.TempDir() + for _, args := range [][]string{{"init"}, {"config", "user.email", "test@example.com"}, {"config", "user.name", "Test"}} { + if output, err := exec.Command("git", append([]string{"-C", repo}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, output) + } + } + manifest := filepath.Join(repo, "skills-codex", ".agentops-manifest.json") + if err := os.MkdirAll(filepath.Dir(manifest), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(manifest, []byte(`{"skills":[{"name":"research"}]}`), 0o644); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{{"add", "."}, {"commit", "-m", "fixture"}} { + if output, err := exec.Command("git", append([]string{"-C", repo}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, output) + } + } + versionBytes, err := exec.Command("git", "-C", repo, "rev-parse", "--short", "HEAD").Output() + if err != nil { + t.Fatal(err) + } + hash, err := SHA256File(manifest) + if err != nil { + t.Fatal(err) + } + return repo, strings.TrimSpace(string(versionBytes)), hash +} + +func TestCheckCodexSyncDetectsMatchManifestDriftAndStaleVersion(t *testing.T) { + for _, test := range []struct { + name, version, hash, status, detail string + }{ + {name: "match", status: "pass", detail: "matches repo"}, + {name: "manifest drift", hash: "deadbeef", status: "warn", detail: "manifest differs from repo"}, + {name: "stale version", version: "oldsha", hash: "deadbeef", status: "warn", detail: "refresh-codex-local.sh"}, + } { + t.Run(test.name, func(t *testing.T) { + repo, current, currentHash := setupCodexSyncRepo(t) + t.Chdir(repo) + home := t.TempDir() + t.Setenv("HOME", home) + version, hash := test.version, test.hash + if version == "" { + version = current + } + if hash == "" { + hash = currentHash + } + writeInstallMeta(t, home, CodexNativePluginRootPath(home), version, hash, 1) + check := CheckCodexSync() + if check.Status != test.status || !strings.Contains(check.Detail, test.detail) { + t.Fatalf("check = %+v", check) + } + }) + } +} + +func TestCheckSkillIntegrityAbsentCleanAndFindings(t *testing.T) { + for _, test := range []struct { + name, script, status, detail string + }{ + {name: "absent", status: "warn", detail: "not installed"}, + {name: "clean", script: "#!/bin/sh\nexit 0\n", status: "pass", detail: "passed"}, + {name: "findings", script: "#!/bin/sh\necho '[DEAD_REF] skill: broken'\nexit 1\n", status: "warn", detail: "1 skill hygiene finding"}, + } { + t.Run(test.name, func(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + t.Setenv("HOME", t.TempDir()) + if test.script != "" { + path := filepath.Join(root, "skills", "heal-skill", "scripts", "heal.sh") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(test.script), 0o755); err != nil { + t.Fatal(err) + } + } + check := CheckSkillIntegrity() + if check.Status != test.status || check.Required || !strings.Contains(check.Detail, test.detail) { + t.Fatalf("check = %+v", check) + } + }) + } +} From e100b8b13292568b92abc6094b16f57d27c6a686 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:44:26 -0400 Subject: [PATCH 080/108] docs(cli): refresh doctor command help (age-nw28h.2.7.4) --- cli/docs/COMMANDS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/docs/COMMANDS.md b/cli/docs/COMMANDS.md index 2e70d4cc2..2c8953e84 100644 --- a/cli/docs/COMMANDS.md +++ b/cli/docs/COMMANDS.md @@ -143,7 +143,7 @@ ao doctor [command] --online Enable network probes (default: offline-only) --only strings Scope to a subset of detectors or subsystems --quick Run only fast-path detectors (< 200ms) - --robot Alias for stable JSON report output + --robot Alias for --json with structured wrapper --robot-triage Emit the mega-command triage JSON --severity string Minimum severity to emit (P0|P1|P2|P3) (default "P3") --since string Diff findings against an earlier run From bff8e60ad78d3625d6ce9244164c3b0b05856698 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:58:40 -0400 Subject: [PATCH 081/108] test(done): freeze compatibility family before extraction (age-nw28h.2.8) --- .../families/done/case.json | 25 +++++++++++++++++++ .../families/done/ownership.json | 18 +++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/done/case.json create mode 100644 cli/testdata/compatibility-baseline/families/done/ownership.json diff --git a/cli/testdata/compatibility-baseline/families/done/case.json b/cli/testdata/compatibility-baseline/families/done/case.json new file mode 100644 index 000000000..45e3a7102 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/done/case.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "family": "done", + "checks": [ + { + "id": "done-black-box", + "command": "tmp=$(mktemp -d /tmp/ao-done.XXXXXX); (cd cli && go build -o \"$tmp/ao\" ./cmd/ao); \"$tmp/ao\" done --help >\"$tmp/help\"; grep -Fq 'ao done [flags]' \"$tmp/help\"; for flag in force-no-verdict json reason sha; do grep -Fq -- \"--$flag\" \"$tmp/help\"; done; set +e; \"$tmp/ao\" done age-test --sha bad >\"$tmp/out\" 2>\"$tmp/err\"; code=$?; set -e; test \"$code\" -eq 1; grep -Fq 'is not a commit sha' \"$tmp/err\"" + }, + { + "id": "done-go-behavior", + "command": "cd cli && go test ./cmd/ao -run 'Test(Done|LookupDone|ProvenanceOnly)' -count=1" + } + ], + "dimensions": { + "path": {"status":"evidence","evidence":["done-black-box"]}, + "aliases": {"status":"not_applicable","reason":"The done family declares no aliases."}, + "accepted_args": {"status":"evidence","evidence":["done-black-box","done-go-behavior"]}, + "rejected_args": {"status":"evidence","evidence":["done-black-box","done-go-behavior"]}, + "stdout": {"status":"evidence","evidence":["done-go-behavior"]}, + "stderr": {"status":"evidence","evidence":["done-black-box","done-go-behavior"]}, + "exit_classes": {"status":"evidence","evidence":["done-black-box","done-go-behavior"]}, + "tracker_selection": {"status":"evidence","evidence":["done-go-behavior"]}, + "ordered_effects": {"status":"evidence","evidence":["done-go-behavior"]} + } +} diff --git a/cli/testdata/compatibility-baseline/families/done/ownership.json b/cli/testdata/compatibility-baseline/families/done/ownership.json new file mode 100644 index 000000000..fcf9b5f61 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/done/ownership.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "family": "done", + "profiles": {"default":"present","flywheel":"present","legacy":"present","combined":"present"}, + "legacy_symbols": [ + "doneCmd", "doneSHA", "doneReason", "doneForceNoVerdct", "doneJSON", "runDone", + "doneReport", "doneVerdictLookup", "lookupDoneVerdicts", "lookupDoneVerdictsFromOrigin", + "doneStamp", "doneResolveHead", "doneCommitProvenanceOnly", "doneRefusalError" + ], + "live_owner": "cli/internal/commands/done", + "allowed_paths": [ + "cli/cmd/ao/done*.go", "cli/cmd/ao/root.go", "cli/cmd/ao/verify_prepush*.go", + "cli/cmd/ao/cobra_commands_test.go", "cli/cmd/ao/json_validity_test.go", + "cli/internal/done/**", "cli/internal/adapters/done/**", "cli/internal/commands/done/**", + "cli/docs/COMMANDS.md", + "cli/testdata/compatibility-baseline/families/done/lineage.json" + ] +} From d994c915def9a7ed5e17a914a31bdadd21733403 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:59:30 -0400 Subject: [PATCH 082/108] test(done): bind reviewed compatibility lineage (age-nw28h.2.8) --- .../compatibility-baseline/families/done/lineage.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/done/lineage.json diff --git a/cli/testdata/compatibility-baseline/families/done/lineage.json b/cli/testdata/compatibility-baseline/families/done/lineage.json new file mode 100644 index 000000000..b40a2f386 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/done/lineage.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "family": "done", + "old_implementation_sha": "e100b8b13292568b92abc6094b16f57d27c6a686", + "freeze_sha": "bff8e60ad78d3625d6ce9244164c3b0b05856698", + "fixture_sha256": "87d7b5a9e3aa9a253644ae334920532b5ca0aae4058f53984612e9bd8440194f", + "ownership_sha256": "fecf63c7ba54c784cfa33c6b82f4821bf9cd98b8d64f1f66da7ff370f9a0ce7f", + "scope_sha256": "d7c6731e492e229369af90cf48eaf0c429e5074faa9323139de01ab49a1f9eab", + "migration_state": "migrating" +} From 9bb02d59bf48845901b5d907f60476d0ff86e363 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:10:33 -0400 Subject: [PATCH 083/108] refactor(done): extract verdict-close service and adapters (age-nw28h.2.8.1) --- cli/cmd/ao/done.go | 312 ++------------------- cli/cmd/ao/done_composition.go | 15 + cli/cmd/ao/done_test.go | 38 ++- cli/cmd/ao/verify_prepush.go | 41 +-- cli/cmd/ao/verify_prepush_test.go | 9 +- cli/internal/adapters/done/runtime.go | 88 ++++++ cli/internal/adapters/done/runtime_test.go | 81 ++++++ cli/internal/done/service.go | 219 +++++++++++++++ cli/internal/done/service_test.go | 133 +++++++++ 9 files changed, 604 insertions(+), 332 deletions(-) create mode 100644 cli/cmd/ao/done_composition.go create mode 100644 cli/internal/adapters/done/runtime.go create mode 100644 cli/internal/adapters/done/runtime_test.go create mode 100644 cli/internal/done/service.go create mode 100644 cli/internal/done/service_test.go diff --git a/cli/cmd/ao/done.go b/cli/cmd/ao/done.go index a9a3d1ceb..6c88b7ff0 100644 --- a/cli/cmd/ao/done.go +++ b/cli/cmd/ao/done.go @@ -2,27 +2,14 @@ package main import ( - "bytes" "context" "encoding/json" "fmt" - "os" - "os/exec" "strings" "github.com/spf13/cobra" - "github.com/boshu2/agentops/cli/internal/provenancegraph" -) - -// Close-reason stamp dispositions. The stamp format is -// "[verdict::]" — appended to the br close reason so the -// verdict-close-rate gate (scripts/check-verdict-close-rate.sh) can measure -// how many closes reference ledger proof. -const ( - doneStampConfirmed = "CONFIRMED" - doneStampWaived = "waived-trivial" - doneStampUnverified = "UNVERIFIED" + doneapp "github.com/boshu2/agentops/cli/internal/done" ) var ( @@ -76,295 +63,30 @@ func init() { doneCmd.Flags().BoolVar(&doneJSON, "json", false, "Emit machine-readable JSON (stdout-as-data)") } -// doneReport is the structured output of ao done --json. -type doneReport struct { - BeadID string `json:"bead_id"` - CommitSHA string `json:"commit_sha"` - Disposition string `json:"disposition"` - Stamp string `json:"stamp"` - CloseReason string `json:"close_reason"` - Closed bool `json:"closed"` -} - -// doneVerdictLookup summarizes the verdict edges bound to one commit. -type doneVerdictLookup struct { - // VerdictID is the node id of the deciding verdict: the latest CONFIRMED - // one when Confirmed, else the latest verdict seen. - VerdictID string - // Confirmed reports whether any bound verdict carries disposition=CONFIRMED. - Confirmed bool - // Dispositions lists every bound verdict's disposition in ledger order - // (empty string for a verdict record without one). - Dispositions []string - // ForeignBeads lists verdict node ids bound to this commit but belonging to - // OTHER beads — never certifying, surfaced in refusal messages. - ForeignBeads []string -} - -// shaBindsCommit reports whether query (>=7 hex chars) and commitID name the -// same commit: either is a case-insensitive prefix of the other, matching the -// provenance_show prefix-resolution convention. -func shaBindsCommit(query, commitID string) bool { - q, c := strings.ToLower(query), strings.ToLower(commitID) - if len(q) < minShaPrefixLen || len(c) < minShaPrefixLen { - return false - } - if !isHexToken(q) || !isHexToken(c) { - return false - } - return strings.HasPrefix(q, c) || strings.HasPrefix(c, q) -} - -// lookupDoneVerdicts scans the ledger for verdict --wasDerivedFrom--> commit -// edges bound to sha (the shape ao provenance emit-verdict writes) AND to the -// bead being closed: verdict node ids are `@`, and a CONFIRMED -// verdict for a DIFFERENT bead on the same commit must never certify this one -// (wrong-object certification — pawl catch on this bead's own landing). A -// foreign-bead verdict is counted in ForeignBeads so the refusal can name it. -// Pure. -func lookupDoneVerdicts(edges []provenancegraph.Edge, beadID, sha string) doneVerdictLookup { - var l doneVerdictLookup - for _, e := range edges { - if e.Relation != "wasDerivedFrom" || e.FromType != "verdict" || e.ToType != "commit" { - continue - } - if !shaBindsCommit(sha, e.ToID) { - continue - } - if vb, _, ok := strings.Cut(e.FromID, "@"); !ok || vb != beadID { - l.ForeignBeads = append(l.ForeignBeads, e.FromID) - continue - } - disp := parseDisposition(e.EvidenceRef) - l.Dispositions = append(l.Dispositions, disp) - if disp == doneStampConfirmed { - l.Confirmed = true - l.VerdictID = e.FromID - } else if !l.Confirmed { - l.VerdictID = e.FromID - } - } - return l -} - -// doneOriginRef is the remote-tracking ref the origin-ledger fallback consults. -// The repo lands directly on main, so origin/main is the shared audit line. -const doneOriginRef = "origin/main" - -// lookupDoneVerdictsFromOrigin consults the already-fetched origin/main ledger -// blob for a verdict binding the commit — the stale-checkout fallback. `ao done` -// runs wherever the operator happens to be, and a verdict pushed elsewhere lands -// on origin/main before the local working tree catches up, so a local checkout -// that lags origin would otherwise refuse a genuinely-verified close. -// -// It reads `git show origin/main:docs/provenance/ledger.jsonl` — a blob read -// with NO working-tree change and NO `git fetch` (a stale origin ref simply -// misses, and the caller refuses as before, advising a fetch). Any failure (no -// origin/main ref, absent blob, unreadable/corrupt ledger) fails SOFT to -// (zero, false): the fallback can only ever HELP find a verdict, never block a -// close the local ledger would otherwise satisfy. The bool reports whether the -// origin ledger was read and decoded at all. -func lookupDoneVerdictsFromOrigin(cwd, beadID, sha string) (doneVerdictLookup, bool) { - out, err := exec.Command("git", "-C", cwd, "show", - doneOriginRef+":"+provenancegraph.LedgerRelativePath).Output() - if err != nil { - return doneVerdictLookup{}, false - } - edges, err := provenancegraph.DecodeEdges(bytes.NewReader(out)) - if err != nil { - return doneVerdictLookup{}, false - } - return lookupDoneVerdicts(edges, beadID, sha), true -} - -// doneStamp formats the close-reason stamp: "[verdict::]". -func doneStamp(sha, disposition string) string { - return "[verdict:" + shortHash7(sha) + ":" + disposition + "]" -} - -// doneResolveHead returns the full HEAD sha of the git repository at cwd. -func doneResolveHead(cwd string) (string, error) { - out, err := exec.Command("git", "-C", cwd, "rev-parse", "HEAD").Output() - if err != nil { - return "", fmt.Errorf("resolve HEAD at %s (pass --sha to name the landed commit explicitly): %w", cwd, err) - } - return strings.TrimSpace(string(out)), nil -} - -// provenanceOnlyChangedFiles is the SINGLE allowlist-discipline both the `ao -// done` waiver (doneCommitProvenanceOnly) and the pre-push #trivial waiver -// (trivialWaiver) route through, so the two can never disagree about what is -// waivable. `out` is the NUL-separated changed-file list from `git diff-tree -// --no-commit-id --no-renames --name-only -z -r `. It reports whether that -// list is non-empty AND every path is under docs/provenance/. -// -// -z is load-bearing (parsing sweep): NUL-separated output emits each path's RAW -// bytes with NO C-quoting, so the comparison is against the exact path — a path -// literally named " docs/provenance/x" (LEADING SPACE) or one with any other -// surprising byte is matched exactly, never trimmed or dequoted into the -// allowlist (the newline+CR-strip form depended on git's quoting to fail-close; -// -z removes that dependency and the ambiguity of splitting on a byte a path -// could contain). The docs/provenance/ prefix keeps its TRAILING SLASH so it is -// an exact directory boundary — "docs/provenance-evil/x" is NOT under it. -// Fail-closed on an empty list: it cannot prove triviality. -func provenanceOnlyChangedFiles(out string) bool { - var files []string - for _, f := range strings.Split(out, "\x00") { - if f != "" { - files = append(files, f) - } - } - if len(files) == 0 { - return false - } - for _, f := range files { - if !strings.HasPrefix(f, "docs/provenance/") { - return false - } - } - return true -} - -// doneCommitProvenanceOnly reports whether every changed file of the commit is -// under docs/provenance/ — the sole waiver class, mirroring the #trivial -// semantics of scripts/check-pawl-pre-push.sh. Fail-closed: a failed diff-tree -// or an empty changed-file list cannot prove triviality, so it does NOT waive. -// --no-renames forces a rename INTO docs/provenance/ to expose its -// non-allowlisted source path; -z emits raw (unquoted) NUL-separated paths for -// exact comparison. Path discipline is the shared provenanceOnlyChangedFiles -// (parity with the pre-push waiver). -func doneCommitProvenanceOnly(cwd, sha string) bool { - out, err := exec.Command("git", "-C", cwd, - "diff-tree", "--no-commit-id", "--no-renames", "--name-only", "-z", "-r", sha).Output() // #nosec G204 -- cwd is the repo; sha is a resolved commit id. - if err != nil { - return false - } - return provenanceOnlyChangedFiles(string(out)) -} - -// doneRefusalError builds the corrective refusal: no proof, no close. -func doneRefusalError(beadID, sha string, l doneVerdictLookup) error { - var found string - if len(l.Dispositions) > 0 { - found = fmt.Sprintf("verdict(s) recorded for commit %s but none CONFIRMED (found: %s)", - shortHash7(sha), strings.Join(l.Dispositions, ", ")) - } else if len(l.ForeignBeads) > 0 { - found = fmt.Sprintf("no verdict for %s on commit %s — the verdict(s) there belong to OTHER bead(s): %s (a verdict certifies its own bead only)", - beadID, shortHash7(sha), strings.Join(l.ForeignBeads, ", ")) - } else { - found = fmt.Sprintf("no verdict recorded for commit %s", shortHash7(sha)) - } - return fmt.Errorf(`%s — no verdict = not done; refusing to close %s - produce one: ao verify %s (front door — writes the commit-bound verdict on CONFIRMED) - advanced: ao pawl review %s --scope head - waiver: only a commit whose changed files are all under docs/provenance/ closes as waived-trivial - stale local? git fetch origin && git merge --ff-only origin/main (a verdict pushed elsewhere lands on origin/main first; ao done checks it, but your local ledger may still lag it) - escape hatch: ao done %s --force-no-verdict (closes with an explicit UNVERIFIED stamp)`, - found, beadID, beadID, beadID, beadID) -} - -func runDone(cmd *cobra.Command, args []string) error { - cmd.SilenceUsage = true - beadID := args[0] - out := cmd.OutOrStdout() - - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("resolve cwd: %w", err) - } - - sha := strings.TrimSpace(doneSHA) - if sha == "" { - sha, err = doneResolveHead(cwd) - if err != nil { - return err - } - } - if len(sha) < minShaPrefixLen || !isHexToken(sha) { - return fmt.Errorf("--sha %q is not a commit sha (need at least %d hex chars)", sha, minShaPrefixLen) - } - - store := provenancegraph.NewStore(resolveLedgerPath()) - edges, err := store.Read() - if err != nil { - return fmt.Errorf("read provenance ledger: %w", err) - } - - lookup := lookupDoneVerdicts(edges, beadID, sha) - var disposition string - switch { - case lookup.Confirmed: - disposition = doneStampConfirmed - case len(lookup.Dispositions) == 0 && doneCommitProvenanceOnly(cwd, sha): - disposition = doneStampWaived - default: - // The local ledger neither certifies this commit nor makes it - // waiver-eligible. Before refusing, consult the already-fetched - // origin/main ledger: a stale local checkout may simply lag a verdict - // pushed elsewhere. Adopt a CONFIRMED verdict found there and close - // exactly as if it were local; otherwise surface origin's findings in - // the refusal. Fail-closed posture unchanged — nothing here can turn a - // would-refuse into anything weaker than the local ledger allows. - if originLookup, ok := lookupDoneVerdictsFromOrigin(cwd, beadID, sha); ok { - if originLookup.Confirmed { - lookup.Confirmed = true - lookup.VerdictID = originLookup.VerdictID - } else if len(lookup.Dispositions) == 0 && len(lookup.ForeignBeads) == 0 { - // Local ledger is silent on this commit; carry origin/main's - // findings into the refusal so the message is accurate. - lookup.Dispositions = originLookup.Dispositions - lookup.ForeignBeads = originLookup.ForeignBeads - lookup.VerdictID = originLookup.VerdictID - } - } - switch { - case lookup.Confirmed: - disposition = doneStampConfirmed - case doneForceNoVerdct: - disposition = doneStampUnverified - default: - return doneRefusalError(beadID, sha, lookup) - } - } - - stamp := doneStamp(sha, disposition) - reason := strings.TrimSpace(doneReason) - if reason == "" { - reason = "Done" - } - closeReason := reason + " " + stamp - - // Shell out to the br CLI on PATH with the environment as-is (BEADS_DIR is - // respected when exported; otherwise resolved the same way `ao beads dir` - // resolves it — via git's common directory, never a hardcoded path). - ctx := cmd.Context() +func runDone(command *cobra.Command, args []string) error { + command.SilenceUsage = true + ctx := command.Context() if ctx == nil { ctx = context.Background() } - brCmd := beadsTrackerCommandContext(ctx, "close", beadID, "-r", closeReason) - brOut, err := brCmd.CombinedOutput() + result, err := newDoneService().Execute(ctx, doneapp.Request{ + BeadID: args[0], SHA: doneSHA, Reason: doneReason, ForceNoVerdict: doneForceNoVerdct, + }) if err != nil { - return fmt.Errorf("br close %s: %w\n%s", beadID, err, strings.TrimSpace(string(brOut))) + return err } - + out := command.OutOrStdout() if doneJSON { - enc := json.NewEncoder(out) - enc.SetIndent("", " ") - return enc.Encode(doneReport{ - BeadID: beadID, - CommitSHA: sha, - Disposition: disposition, - Stamp: stamp, - CloseReason: closeReason, - Closed: true, - }) + encoder := json.NewEncoder(out) + encoder.SetIndent("", " ") + return encoder.Encode(result) } - if msg := strings.TrimSpace(string(brOut)); msg != "" { - fmt.Fprintln(out, msg) + if output := strings.TrimSpace(result.TrackerOutput); output != "" { + fmt.Fprintln(out, output) } - fmt.Fprintf(out, "closed %s at %s %s\n", beadID, shortHash7(sha), stamp) - if disposition == doneStampUnverified { - fmt.Fprintf(out, "note: UNVERIFIED close — run 'ao verify %s' next time so done carries proof\n", beadID) + fmt.Fprintf(out, "closed %s at %s %s\n", result.BeadID, result.CommitSHA[:doneapp.MinimumSHAPrefix], result.Stamp) + if result.Disposition == doneapp.DispositionUnverified { + fmt.Fprintf(out, "note: UNVERIFIED close — run 'ao verify %s' next time so done carries proof\n", result.BeadID) } return nil } diff --git a/cli/cmd/ao/done_composition.go b/cli/cmd/ao/done_composition.go new file mode 100644 index 000000000..dba77d4b3 --- /dev/null +++ b/cli/cmd/ao/done_composition.go @@ -0,0 +1,15 @@ +package main + +import ( + "context" + + doneadapter "github.com/boshu2/agentops/cli/internal/adapters/done" + doneapp "github.com/boshu2/agentops/cli/internal/done" +) + +func newDoneService() doneapp.Service { + tracker := doneadapter.Tracker{Run: func(ctx context.Context, args ...string) ([]byte, error) { + return beadsTrackerCommandContext(ctx, args...).CombinedOutput() + }} + return doneapp.NewService(doneadapter.SystemRepository(), doneadapter.SystemLedger(resolveLedgerPath()), tracker) +} diff --git a/cli/cmd/ao/done_test.go b/cli/cmd/ao/done_test.go index cff59d0fe..762f63f63 100644 --- a/cli/cmd/ao/done_test.go +++ b/cli/cmd/ao/done_test.go @@ -2,6 +2,7 @@ package main import ( + "context" "encoding/json" "os" "os/exec" @@ -9,6 +10,8 @@ import ( "strings" "testing" + doneadapter "github.com/boshu2/agentops/cli/internal/adapters/done" + doneapp "github.com/boshu2/agentops/cli/internal/done" "github.com/boshu2/agentops/cli/internal/provenancegraph" ) @@ -19,6 +22,15 @@ const ( doneSHANoVerdict = "3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c" ) +func toDoneEdges(edges []provenancegraph.Edge) []doneapp.Edge { + result := make([]doneapp.Edge, 0, len(edges)) + for _, edge := range edges { + result = append(result, doneapp.Edge{FromID: edge.FromID, FromType: edge.FromType, ToID: edge.ToID, + ToType: edge.ToType, Relation: edge.Relation, EvidenceRef: edge.EvidenceRef}) + } + return result +} + // resetDoneFlags snapshots and restores the package-global done cobra flags // (test-isolation rule: package-global flag vars must not leak across tests). func resetDoneFlags(t *testing.T) { @@ -308,7 +320,7 @@ func TestDone_NonProvenanceCommitNotWaived(t *testing.T) { } } -// TestDone_JSONShape: --json emits the exact doneReport contract. +// TestDone_JSONShape: --json emits the exact done result contract. func TestDone_JSONShape(t *testing.T) { chdirRepoFixture(t) seedDoneLedger(t) @@ -321,11 +333,11 @@ func TestDone_JSONShape(t *testing.T) { if err := runDone(c, []string{"ag-done.1"}); err != nil { t.Fatalf("ao done --json: %v", err) } - var r doneReport + var r doneapp.Result if err := json.Unmarshal(out.Bytes(), &r); err != nil { - t.Fatalf("--json output not a doneReport: %v\n%s", err, out.String()) + t.Fatalf("--json output not a done result: %v\n%s", err, out.String()) } - want := doneReport{ + want := doneapp.Result{ BeadID: "ag-done.1", CommitSHA: doneSHAConfirmed, Disposition: "CONFIRMED", @@ -334,7 +346,7 @@ func TestDone_JSONShape(t *testing.T) { Closed: true, } if r != want { - t.Fatalf("doneReport = %+v, want %+v", r, want) + t.Fatalf("done result = %+v, want %+v", r, want) } } @@ -398,7 +410,7 @@ func TestDoneProvenanceOnly_LeadingSpacePathNotWaived(t *testing.T) { t.Fatalf("rev-parse: %v", err) } sha := strings.TrimSpace(string(revOut)) - if doneCommitProvenanceOnly(repo, sha) { + if doneadapter.SystemRepository().CommitProvenanceOnly(context.Background(), repo, sha) { t.Fatal("a leading-space path (' docs/provenance/...') must NOT be waived as provenance-only") } } @@ -407,7 +419,7 @@ func TestLookupDoneVerdicts_ForeignBeadNeverCertifies(t *testing.T) { edges := []provenancegraph.Edge{ buildVerdictCommitEdge(pawlVerdict{BeadID: "ag-done-1", HeadSHA: doneSHAConfirmed, Disposition: "CONFIRMED"}), } - got := lookupDoneVerdicts(edges, "ag-other", doneSHAConfirmed) + got := doneapp.LookupVerdicts(toDoneEdges(edges), "ag-other", doneSHAConfirmed) if got.Confirmed { t.Fatal("a CONFIRMED verdict for ag-done-1 must not certify ag-other") } @@ -418,7 +430,7 @@ func TestLookupDoneVerdicts_ForeignBeadNeverCertifies(t *testing.T) { t.Fatalf("ForeignBeads = %v, want the ag-done-1 verdict surfaced", got.ForeignBeads) } // Same commit, RIGHT bead still certifies. - right := lookupDoneVerdicts(edges, "ag-done-1", doneSHAConfirmed) + right := doneapp.LookupVerdicts(toDoneEdges(edges), "ag-done-1", doneSHAConfirmed) if !right.Confirmed { t.Fatal("the owning bead must still certify") } @@ -432,7 +444,7 @@ func TestLookupDoneVerdicts(t *testing.T) { buildVerdictCommitEdge(pawlVerdict{BeadID: "ag-y", HeadSHA: doneSHARefuted, Disposition: "REFUTED"}), } - got := lookupDoneVerdicts(edges, "ag-x", doneSHAConfirmed) + got := doneapp.LookupVerdicts(toDoneEdges(edges), "ag-x", doneSHAConfirmed) if !got.Confirmed { t.Fatal("sha with a CONFIRMED verdict must report Confirmed") } @@ -443,7 +455,7 @@ func TestLookupDoneVerdicts(t *testing.T) { t.Fatalf("Dispositions = %v, want [REFUTED CONFIRMED]", got.Dispositions) } - refuted := lookupDoneVerdicts(edges, "ag-y", doneSHARefuted) + refuted := doneapp.LookupVerdicts(toDoneEdges(edges), "ag-y", doneSHARefuted) if refuted.Confirmed { t.Fatal("REFUTED-only sha must not report Confirmed") } @@ -451,7 +463,7 @@ func TestLookupDoneVerdicts(t *testing.T) { t.Fatalf("Dispositions = %v, want [REFUTED]", refuted.Dispositions) } - none := lookupDoneVerdicts(edges, "ag-x", doneSHANoVerdict) + none := doneapp.LookupVerdicts(toDoneEdges(edges), "ag-x", doneSHANoVerdict) if none.Confirmed || len(none.Dispositions) != 0 { t.Fatalf("unbound sha lookup = %+v, want empty", none) } @@ -472,8 +484,8 @@ func TestShaBindsCommit(t *testing.T) { {"ag-done.1", doneSHAConfirmed, false}, } for _, tc := range cases { - if got := shaBindsCommit(tc.query, tc.commit); got != tc.want { - t.Errorf("shaBindsCommit(%q, %q) = %v, want %v", tc.query, tc.commit, got, tc.want) + if got := doneapp.SHABindsCommit(tc.query, tc.commit); got != tc.want { + t.Errorf("doneapp.SHABindsCommit(%q, %q) = %v, want %v", tc.query, tc.commit, got, tc.want) } } } diff --git a/cli/cmd/ao/verify_prepush.go b/cli/cmd/ao/verify_prepush.go index 8a62da40a..81ce69019 100644 --- a/cli/cmd/ao/verify_prepush.go +++ b/cli/cmd/ao/verify_prepush.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/cobra" + doneapp "github.com/boshu2/agentops/cli/internal/done" "github.com/boshu2/agentops/cli/internal/provenancegraph" ) @@ -440,13 +441,13 @@ func materializeLedger(content []byte) (string, func(), error) { // PARSING-DISCIPLINE SWEEP (age-rk3r.6): every verdict-edge / ledger / git-output // recognition in the gate is exact-match and fail-closed; a near-miss is // rejected, not accepted. Concretely: -// - confirmedVerdictEdgeIn: exact relation (wasDerivedFrom) + shaBindsCommit -// (hex-validated) + parseDisposition == "CONFIRMED" (exact token, never a +// - confirmedVerdictEdgeIn: exact relation (wasDerivedFrom) + doneapp.SHABindsCommit +// (hex-validated) + doneapp.ParseDisposition == "CONFIRMED" (exact token, never a // substring) — shared with the `ao done` recognizer. // - the #trivial waiver: git diff-tree --name-only -z (raw NUL-separated paths) -// - the shared provenanceOnlyChangedFiles, so a leading-space / quoted / +// - the shared doneapp.ProvenanceOnlyChangedFiles, so a leading-space / quoted / // surprising-byte path is matched exactly, never trimmed into the allowlist. -// - splitSHAList / commitRange feed shas that any downstream match (shaBindsCommit, +// - splitSHAList / commitRange feed shas that any downstream match (doneapp.SHABindsCommit, // git plumbing) re-validates, so junk is fail-closed (refused), not waived. // - isMainRef / isZeroSHA / refBranch / trunkRemoteRefs compare exact ref/sha // forms; the pre-push stdin is git-generated (not repo-controlled), and a line @@ -652,10 +653,10 @@ func hasConfirmedVerdictEdge(ledgerPath, sha string) bool { // // - Relation is required EXACTLY (wasDerivedFrom). A hash-valid row with a // different relation (e.g. wasAttributedTo) is NOT a verdict authorization. -// - shaBindsCommit is the SHARED sha match (done path): both sides hex-valid, +// - doneapp.SHABindsCommit is the SHARED sha match (done path): both sides hex-valid, // >=7 chars, one a prefix of the other — a non-hex or too-short to_id is // rejected, not accepted. -// - parseDisposition extracts the EXACT `disposition=` whitespace-delimited +// - doneapp.ParseDisposition extracts the EXACT `disposition=` whitespace-delimited // token; the check is v == "CONFIRMED", NEVER strings.Contains — so // "disposition=CONFIRMEDLY", "xdisposition=CONFIRMED", or an evidence_ref // that merely mentions the word does NOT authorize. @@ -677,10 +678,10 @@ func confirmedVerdictEdgeIn(edges []provenancegraph.Edge, sha string) bool { if e.Relation != "wasDerivedFrom" || e.FromType != "verdict" || e.ToType != "commit" { continue } - if !shaBindsCommit(sha, e.ToID) { + if !doneapp.SHABindsCommit(sha, e.ToID) { continue } - if parseDisposition(e.EvidenceRef) == doneStampConfirmed { + if doneapp.ParseDisposition(e.EvidenceRef) == doneapp.DispositionConfirmed { return true } } @@ -702,10 +703,10 @@ func reboundEdgeBoundTo(edges []provenancegraph.Edge, sha string) bool { if e.Relation != "wasDerivedFrom" || e.FromType != "verdict" || e.ToType != "commit" { continue } - if !shaBindsCommit(sha, e.ToID) { + if !doneapp.SHABindsCommit(sha, e.ToID) { continue } - if parseDisposition(e.EvidenceRef) == doneStampRebound { + if doneapp.ParseDisposition(e.EvidenceRef) == doneStampRebound { return true } } @@ -723,7 +724,7 @@ func confirmedVerdictCommitSHAs(edges []provenancegraph.Edge) []string { if e.Relation != "wasDerivedFrom" || e.FromType != "verdict" || e.ToType != "commit" { continue } - if parseDisposition(e.EvidenceRef) != doneStampConfirmed { + if doneapp.ParseDisposition(e.EvidenceRef) != doneapp.DispositionConfirmed { continue } if e.ToID == "" || seen[e.ToID] { @@ -801,7 +802,7 @@ func reboundVerdictAuthorizes(repo string, edges []provenancegraph.Edge, sha str // SECURITY (age-rk3r.18 refuter fix): the lineage to_id is fed to git as a // commit for the diff re-derivation, so it MUST be a HEX commit id resolved // as an OBJECT — never a revision expression. The direct-CONFIRMED path - // (confirmedVerdictEdgeIn → shaBindsCommit) already requires this; the + // (confirmedVerdictEdgeIn → doneapp.SHABindsCommit) already requires this; the // REBOUND lineage MUST apply the identical discipline, or a crafted ledger // with a fake CONFIRMED edge to_id="HEAD~1" (a ref alias, not a hex id) // would let `git show HEAD~1` supply a matching diff and certify an @@ -812,7 +813,7 @@ func reboundVerdictAuthorizes(repo string, edges []provenancegraph.Edge, sha str if rOID == "" { continue // non-hex / ref-alias / non-committish lineage to_id → not a valid lineage } - if shaBindsCommit(sha, rOID) { + if doneapp.SHABindsCommit(sha, rOID) { continue // a CONFIRMED on the tip itself is the confirmedVerdictEdgeIn path, not lineage } rPID := commitPatchIDGit(gitBin, repo, rOID) @@ -839,7 +840,7 @@ func reboundVerdictAuthorizes(repo string, edges []provenancegraph.Edge, sha str // author ASSERTION, so triviality is proven from the diff, not the message; an // unreadable diff cannot prove triviality and is fail-closed (not waived). // -// The path allowlist is the SHARED provenanceOnlyChangedFiles — the SAME +// The path allowlist is the SHARED doneapp.ProvenanceOnlyChangedFiles — the SAME // discipline `ao done` uses (doneCommitProvenanceOnly). Routing both through one // helper closes the parity gap: an earlier per-line strings.TrimSpace here // trimmed a leading-space path " docs/provenance/x" INTO the allowlist and @@ -854,12 +855,12 @@ func trivialWaiver(repo, sha string) (bool, error) { // --no-renames forces a rename to show as delete(old)+add(new) so a rename // FROM a non-provenance path INTO docs/provenance/ still exposes the old path. // -z emits raw, unquoted, NUL-separated paths so the allowlist check compares - // exact bytes (parsing sweep) — see provenanceOnlyChangedFiles. + // exact bytes (parsing sweep) — see doneapp.ProvenanceOnlyChangedFiles. changed, err := gitStdout(repo, "diff-tree", "--no-commit-id", "--no-renames", "--name-only", "-z", "-r", sha) if err != nil { return false, err // cannot prove triviality → fail-closed } - return provenanceOnlyChangedFiles(changed), nil + return doneapp.ProvenanceOnlyChangedFiles(changed), nil } // verifyLedgerChain runs the in-place hash-chain verification (the same @@ -923,7 +924,7 @@ func gitCommitExists(repo, ref string) bool { } // hexCommitObjectID validates that candidate is a HEX commit id (the exact -// discipline confirmedVerdictEdgeIn/shaBindsCommit applies to a bound to_id) and +// discipline confirmedVerdictEdgeIn/doneapp.SHABindsCommit applies to a bound to_id) and // resolves it to its FULL commit object id — returning "" (reject) for anything // that is not a hex, committish OBJECT. It is the fix for the age-rk3r.18 refuter // fail-open: the REBOUND lineage to_id is fed to `git show` for the diff re- @@ -934,13 +935,13 @@ func gitCommitExists(repo, ref string) bool { // // Discipline, fail-closed at every step: // - candidate must be a HEX token of >= minShaPrefixLen chars (isHexToken + -// length — the SAME predicate shaBindsCommit uses). This alone rejects every +// length — the SAME predicate doneapp.SHABindsCommit uses). This alone rejects every // revision expression, since "~", "^", ":", "/", "HEAD", branch/tag names all // carry non-hex bytes. // - it is resolved with `git rev-parse --verify --quiet ^{commit}` via the // trusted git, so a non-existent or non-committish object rejects. // - DEFENSE-IN-DEPTH against a ref NAMED like a hex prefix: the resolved full -// oid must BIND the input hex (shaBindsCommit — one a prefix of the other). A +// oid must BIND the input hex (doneapp.SHABindsCommit — one a prefix of the other). A // branch "deadbeef" whose tip oid does not start with "deadbeef" is thereby // rejected even if git resolved the name; only a genuine object-id resolution // (oid has the hex as a prefix, or vice-versa) passes. @@ -955,7 +956,7 @@ func hexCommitObjectID(gitBin, repo, candidate string) string { return "" // non-existent / non-committish object → reject } oid := strings.TrimSpace(string(out)) - if oid == "" || !shaBindsCommit(oid, candidate) { + if oid == "" || !doneapp.SHABindsCommit(oid, candidate) { return "" // resolved something that does not bind the hex (a ref alias) → reject } return oid diff --git a/cli/cmd/ao/verify_prepush_test.go b/cli/cmd/ao/verify_prepush_test.go index 4b82f1533..d5a07870c 100644 --- a/cli/cmd/ao/verify_prepush_test.go +++ b/cli/cmd/ao/verify_prepush_test.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" + doneapp "github.com/boshu2/agentops/cli/internal/done" "github.com/boshu2/agentops/cli/internal/provenancegraph" ) @@ -787,8 +788,8 @@ func TestProvenanceOnlyChangedFiles_ExactAndFailClosed(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - if got := provenanceOnlyChangedFiles(tc.out); got != tc.want { - t.Fatalf("provenanceOnlyChangedFiles(%q)=%v, want %v", tc.out, got, tc.want) + if got := doneapp.ProvenanceOnlyChangedFiles(tc.out); got != tc.want { + t.Fatalf("doneapp.ProvenanceOnlyChangedFiles(%q)=%v, want %v", tc.out, got, tc.want) } }) } @@ -1237,7 +1238,7 @@ func appendConfirmedEdgeRawToID(t *testing.T, repo, bead, rawToID string) { // REBOUND on an unreviewed commit C. The pre-fix REBOUND path fed the lineage // to_id straight to `git show`, which happily resolves "HEAD~1" and — because its // diff patch-id/content-sig matches the tip's — certified C fail-open. The direct -// CONFIRMED path always rejected the non-hex to_id (shaBindsCommit); the REBOUND +// CONFIRMED path always rejected the non-hex to_id (doneapp.SHABindsCommit); the REBOUND // path must apply the IDENTICAL hex-commit-id + object-resolution discipline. // RED-first: proven to authorize (exit 0) against the pre-fix code; REFUSES after. func TestPrePush_REBOUND_NonHexLineageRefAlias_Refused(t *testing.T) { @@ -1361,7 +1362,7 @@ func TestReboundEdgeBoundTo_ExactDispositionToken(t *testing.T) { t.Fatal("disposition=REBOUNDLY must NOT match the exact REBOUND token") } // FIRST-token contract (the CI-parity anchor, age-rk3r.18): a double-disposition - // edge whose FIRST token is REFUTED is NOT a REBOUND (parseDisposition returns + // edge whose FIRST token is REFUTED is NOT a REBOUND (doneapp.ParseDisposition returns // the first token). The CI backstop's jq dispvalue must mirror this exactly. doubleReb := rebound doubleReb.EvidenceRef = "pawl-verdict age-r disposition=REFUTED disposition=REBOUND" diff --git a/cli/internal/adapters/done/runtime.go b/cli/internal/adapters/done/runtime.go new file mode 100644 index 000000000..330d571c0 --- /dev/null +++ b/cli/internal/adapters/done/runtime.go @@ -0,0 +1,88 @@ +// Package done contains runtime adapters for the verdict-backed close use case. +package done + +import ( + "bytes" + "context" + "os" + "os/exec" + "strings" + + doneapp "github.com/boshu2/agentops/cli/internal/done" + "github.com/boshu2/agentops/cli/internal/provenancegraph" +) + +const originLedger = "origin/main:" + provenancegraph.LedgerRelativePath + +type Repository struct { + WorkingDirFunc func() (string, error) + GitOutput func(context.Context, string, ...string) ([]byte, error) +} + +func SystemRepository() Repository { + return Repository{ + WorkingDirFunc: os.Getwd, + GitOutput: func(ctx context.Context, cwd string, args ...string) ([]byte, error) { + command := exec.CommandContext(ctx, "git", append([]string{"-C", cwd}, args...)...) + return command.Output() + }, + } +} + +func (repository Repository) WorkingDir() (string, error) { return repository.WorkingDirFunc() } + +func (repository Repository) ResolveHead(ctx context.Context, cwd string) (string, error) { + output, err := repository.GitOutput(ctx, cwd, "rev-parse", "HEAD") + return strings.TrimSpace(string(output)), err +} + +func (repository Repository) CommitProvenanceOnly(ctx context.Context, cwd, sha string) bool { + output, err := repository.GitOutput(ctx, cwd, "diff-tree", "--no-commit-id", "--no-renames", "--name-only", "-z", "-r", sha) + return err == nil && doneapp.ProvenanceOnlyChangedFiles(string(output)) +} + +func (repository Repository) OriginEdges(ctx context.Context, cwd string) ([]doneapp.Edge, bool) { + output, err := repository.GitOutput(ctx, cwd, "show", originLedger) + if err != nil { + return nil, false + } + edges, err := provenancegraph.DecodeEdges(bytes.NewReader(output)) + if err != nil { + return nil, false + } + return mapEdges(edges), true +} + +type Ledger struct { + ReadEdges func() ([]provenancegraph.Edge, error) +} + +func SystemLedger(path string) Ledger { + return Ledger{ReadEdges: provenancegraph.NewStore(path).Read} +} + +func (ledger Ledger) Read(context.Context) ([]doneapp.Edge, error) { + edges, err := ledger.ReadEdges() + if err != nil { + return nil, err + } + return mapEdges(edges), nil +} + +type Tracker struct { + Run func(context.Context, ...string) ([]byte, error) +} + +func (tracker Tracker) Close(ctx context.Context, id, reason string) (string, error) { + output, err := tracker.Run(ctx, "close", id, "-r", reason) + return string(output), err +} + +func mapEdges(edges []provenancegraph.Edge) []doneapp.Edge { + result := make([]doneapp.Edge, 0, len(edges)) + for _, edge := range edges { + result = append(result, doneapp.Edge{FromID: edge.FromID, FromType: edge.FromType, ToID: edge.ToID, + ToType: edge.ToType, Relation: edge.Relation, EvidenceRef: edge.EvidenceRef}) + } + return result +} diff --git a/cli/internal/adapters/done/runtime_test.go b/cli/internal/adapters/done/runtime_test.go new file mode 100644 index 000000000..aae88529e --- /dev/null +++ b/cli/internal/adapters/done/runtime_test.go @@ -0,0 +1,81 @@ +package done + +import ( + "context" + "errors" + "strings" + "testing" + + doneapp "github.com/boshu2/agentops/cli/internal/done" + "github.com/boshu2/agentops/cli/internal/provenancegraph" +) + +func TestRepositoryMapsGitAndFailsClosed(t *testing.T) { + repository := Repository{ + WorkingDirFunc: func() (string, error) { return "/repo", nil }, + GitOutput: func(_ context.Context, cwd string, args ...string) ([]byte, error) { + if cwd != "/repo" { + t.Fatalf("cwd = %q", cwd) + } + switch args[0] { + case "rev-parse": + return []byte("abcdef0123456789\n"), nil + case "diff-tree": + return []byte("docs/provenance/ledger.jsonl\x00"), nil + case "show": + edge := provenancegraph.Edge{FromID: "age-1@abcdef0", FromType: "verdict", ToID: "abcdef0123456789", ToType: "commit", Relation: "wasDerivedFrom", EvidenceRef: "disposition=CONFIRMED"} + return []byte(mustEncodeEdge(t, edge)), nil + default: + return nil, errors.New("unexpected") + } + }, + } + if cwd, _ := repository.WorkingDir(); cwd != "/repo" { + t.Fatalf("cwd = %q", cwd) + } + if sha, err := repository.ResolveHead(context.Background(), "/repo"); err != nil || sha != "abcdef0123456789" { + t.Fatalf("sha=%q err=%v", sha, err) + } + if !repository.CommitProvenanceOnly(context.Background(), "/repo", "abcdef0") { + t.Fatal("provenance-only commit not recognized") + } + edges, ok := repository.OriginEdges(context.Background(), "/repo") + if !ok || len(edges) != 1 || edges[0].FromID != "age-1@abcdef0" { + t.Fatalf("origin edges = %+v ok=%v", edges, ok) + } + + repository.GitOutput = func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("git failed") } + if repository.CommitProvenanceOnly(context.Background(), "/repo", "abcdef0") { + t.Fatal("git failure must fail closed") + } + if _, ok := repository.OriginEdges(context.Background(), "/repo"); ok { + t.Fatal("origin read failure reported success") + } +} + +func TestLedgerAndTrackerMapPorts(t *testing.T) { + ledger := Ledger{ReadEdges: func() ([]provenancegraph.Edge, error) { + return []provenancegraph.Edge{{FromID: "v", Relation: "wasDerivedFrom"}}, nil + }} + edges, err := ledger.Read(context.Background()) + if err != nil || len(edges) != 1 || edges[0].FromID != "v" { + t.Fatalf("edges=%+v err=%v", edges, err) + } + var got []string + tracker := Tracker{Run: func(_ context.Context, args ...string) ([]byte, error) { + got = append([]string(nil), args...) + return []byte("closed\n"), nil + }} + output, err := tracker.Close(context.Background(), "age-1", "Done [verdict:abcdef0:CONFIRMED]") + if err != nil || output != "closed\n" || strings.Join(got, " ") != "close age-1 -r Done [verdict:abcdef0:CONFIRMED]" { + t.Fatalf("output=%q args=%v err=%v", output, got, err) + } + var _ doneapp.RepositoryPort = Repository{} + var _ doneapp.LedgerPort = Ledger{} + var _ doneapp.TrackerPort = Tracker{} +} + +func mustEncodeEdge(t *testing.T, edge provenancegraph.Edge) string { + t.Helper() + return `{"from_id":"` + edge.FromID + `","from_type":"` + edge.FromType + `","to_id":"` + edge.ToID + `","to_type":"` + edge.ToType + `","relation":"` + edge.Relation + `","evidence_ref":"` + edge.EvidenceRef + `"}` + "\n" +} diff --git a/cli/internal/done/service.go b/cli/internal/done/service.go new file mode 100644 index 000000000..0a3e3194e --- /dev/null +++ b/cli/internal/done/service.go @@ -0,0 +1,219 @@ +// Package done owns the verdict-backed bead-close policy. +package done + +import ( + "context" + "fmt" + "strings" +) + +const ( + DispositionConfirmed = "CONFIRMED" + DispositionWaived = "waived-trivial" + DispositionUnverified = "UNVERIFIED" + MinimumSHAPrefix = 7 +) + +type Edge struct { + FromID, FromType, ToID, ToType, Relation, EvidenceRef string +} + +type VerdictLookup struct { + VerdictID string + Confirmed bool + Dispositions []string + ForeignBeads []string +} + +type Request struct { + BeadID string + SHA string + Reason string + ForceNoVerdict bool +} + +type Result struct { + BeadID string `json:"bead_id"` + CommitSHA string `json:"commit_sha"` + Disposition string `json:"disposition"` + Stamp string `json:"stamp"` + CloseReason string `json:"close_reason"` + Closed bool `json:"closed"` + TrackerOutput string `json:"-"` +} + +type RepositoryPort interface { + WorkingDir() (string, error) + ResolveHead(context.Context, string) (string, error) + CommitProvenanceOnly(context.Context, string, string) bool + OriginEdges(context.Context, string) ([]Edge, bool) +} + +type LedgerPort interface { + Read(context.Context) ([]Edge, error) +} + +type TrackerPort interface { + Close(context.Context, string, string) (string, error) +} + +type Service struct { + repository RepositoryPort + ledger LedgerPort + tracker TrackerPort +} + +func NewService(repository RepositoryPort, ledger LedgerPort, tracker TrackerPort) Service { + return Service{repository: repository, ledger: ledger, tracker: tracker} +} + +func (service Service) Execute(ctx context.Context, request Request) (Result, error) { + cwd, err := service.repository.WorkingDir() + if err != nil { + return Result{}, fmt.Errorf("resolve cwd: %w", err) + } + sha := strings.TrimSpace(request.SHA) + if sha == "" { + sha, err = service.repository.ResolveHead(ctx, cwd) + if err != nil { + return Result{}, fmt.Errorf("resolve HEAD at %s (pass --sha to name the landed commit explicitly): %w", cwd, err) + } + } + if len(sha) < MinimumSHAPrefix || !IsHexToken(sha) { + return Result{}, fmt.Errorf("--sha %q is not a commit sha (need at least %d hex chars)", sha, MinimumSHAPrefix) + } + + edges, err := service.ledger.Read(ctx) + if err != nil { + return Result{}, fmt.Errorf("read provenance ledger: %w", err) + } + lookup := LookupVerdicts(edges, request.BeadID, sha) + disposition := "" + switch { + case lookup.Confirmed: + disposition = DispositionConfirmed + case len(lookup.Dispositions) == 0 && service.repository.CommitProvenanceOnly(ctx, cwd, sha): + disposition = DispositionWaived + default: + if originEdges, ok := service.repository.OriginEdges(ctx, cwd); ok { + origin := LookupVerdicts(originEdges, request.BeadID, sha) + if origin.Confirmed { + lookup.Confirmed = true + lookup.VerdictID = origin.VerdictID + } else if len(lookup.Dispositions) == 0 && len(lookup.ForeignBeads) == 0 { + lookup = origin + } + } + switch { + case lookup.Confirmed: + disposition = DispositionConfirmed + case request.ForceNoVerdict: + disposition = DispositionUnverified + default: + return Result{}, RefusalError(request.BeadID, sha, lookup) + } + } + + stamp := Stamp(sha, disposition) + reason := strings.TrimSpace(request.Reason) + if reason == "" { + reason = "Done" + } + closeReason := reason + " " + stamp + output, err := service.tracker.Close(ctx, request.BeadID, closeReason) + if err != nil { + return Result{}, fmt.Errorf("br close %s: %w\n%s", request.BeadID, err, strings.TrimSpace(output)) + } + return Result{BeadID: request.BeadID, CommitSHA: sha, Disposition: disposition, Stamp: stamp, + CloseReason: closeReason, Closed: true, TrackerOutput: strings.TrimSpace(output)}, nil +} + +func LookupVerdicts(edges []Edge, beadID, sha string) VerdictLookup { + var lookup VerdictLookup + for _, edge := range edges { + if edge.Relation != "wasDerivedFrom" || edge.FromType != "verdict" || edge.ToType != "commit" || !SHABindsCommit(sha, edge.ToID) { + continue + } + if verdictBead, _, ok := strings.Cut(edge.FromID, "@"); !ok || verdictBead != beadID { + lookup.ForeignBeads = append(lookup.ForeignBeads, edge.FromID) + continue + } + disposition := ParseDisposition(edge.EvidenceRef) + lookup.Dispositions = append(lookup.Dispositions, disposition) + if disposition == DispositionConfirmed { + lookup.Confirmed, lookup.VerdictID = true, edge.FromID + } else if !lookup.Confirmed { + lookup.VerdictID = edge.FromID + } + } + return lookup +} + +func SHABindsCommit(query, commitID string) bool { + query, commitID = strings.ToLower(query), strings.ToLower(commitID) + return len(query) >= MinimumSHAPrefix && len(commitID) >= MinimumSHAPrefix && IsHexToken(query) && IsHexToken(commitID) && + (strings.HasPrefix(query, commitID) || strings.HasPrefix(commitID, query)) +} + +func IsHexToken(value string) bool { + if value == "" { + return false + } + for _, character := range value { + switch { + case character >= '0' && character <= '9', character >= 'a' && character <= 'f', character >= 'A' && character <= 'F': + default: + return false + } + } + return true +} + +func ParseDisposition(reference string) string { + for _, field := range strings.Fields(reference) { + if value, ok := strings.CutPrefix(field, "disposition="); ok { + return value + } + } + return "" +} + +func Stamp(sha, disposition string) string { + return "[verdict:" + sha[:MinimumSHAPrefix] + ":" + disposition + "]" +} + +func RefusalError(beadID, sha string, lookup VerdictLookup) error { + var found string + switch { + case len(lookup.Dispositions) > 0: + found = fmt.Sprintf("verdict(s) recorded for commit %s but none CONFIRMED (found: %s)", sha[:MinimumSHAPrefix], strings.Join(lookup.Dispositions, ", ")) + case len(lookup.ForeignBeads) > 0: + found = fmt.Sprintf("no verdict for %s on commit %s — the verdict(s) there belong to OTHER bead(s): %s (a verdict certifies its own bead only)", beadID, sha[:MinimumSHAPrefix], strings.Join(lookup.ForeignBeads, ", ")) + default: + found = fmt.Sprintf("no verdict recorded for commit %s", sha[:MinimumSHAPrefix]) + } + return fmt.Errorf(`%s — no verdict = not done; refusing to close %s + produce one: ao verify %s (front door — writes the commit-bound verdict on CONFIRMED) + advanced: ao pawl review %s --scope head + waiver: only a commit whose changed files are all under docs/provenance/ closes as waived-trivial + stale local? git fetch origin && git merge --ff-only origin/main (a verdict pushed elsewhere lands on origin/main first; ao done checks it, but your local ledger may still lag it) + escape hatch: ao done %s --force-no-verdict (closes with an explicit UNVERIFIED stamp)`, found, beadID, beadID, beadID, beadID) +} + +func ProvenanceOnlyChangedFiles(output string) bool { + var files []string + for _, file := range strings.Split(output, "\x00") { + if file != "" { + files = append(files, file) + } + } + if len(files) == 0 { + return false + } + for _, file := range files { + if !strings.HasPrefix(file, "docs/provenance/") { + return false + } + } + return true +} diff --git a/cli/internal/done/service_test.go b/cli/internal/done/service_test.go new file mode 100644 index 000000000..68d78e7c5 --- /dev/null +++ b/cli/internal/done/service_test.go @@ -0,0 +1,133 @@ +package done + +import ( + "context" + "errors" + "strings" + "testing" +) + +const testSHA = "1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a" + +type fakeRepository struct { + cwd, head string + trivial bool + origin []Edge + originOK bool +} + +func (repo fakeRepository) WorkingDir() (string, error) { return repo.cwd, nil } +func (repo fakeRepository) ResolveHead(context.Context, string) (string, error) { + return repo.head, nil +} +func (repo fakeRepository) CommitProvenanceOnly(context.Context, string, string) bool { + return repo.trivial +} +func (repo fakeRepository) OriginEdges(context.Context, string) ([]Edge, bool) { + return repo.origin, repo.originOK +} + +type fakeLedger struct { + edges []Edge + err error +} + +func (ledger fakeLedger) Read(context.Context) ([]Edge, error) { return ledger.edges, ledger.err } + +type fakeTracker struct { + id, reason string + output string + err error +} + +func (tracker *fakeTracker) Close(_ context.Context, id, reason string) (string, error) { + tracker.id, tracker.reason = id, reason + return tracker.output, tracker.err +} + +func verdictEdge(bead, sha, disposition string) Edge { + return Edge{FromID: bead + "@" + sha[:7], FromType: "verdict", ToID: sha, ToType: "commit", + Relation: "wasDerivedFrom", EvidenceRef: "pawl-verdict " + bead + " disposition=" + disposition} +} + +func TestServiceConfirmedVerdictClosesWithProofStamp(t *testing.T) { + tracker := &fakeTracker{output: "closed by tracker"} + service := NewService(fakeRepository{cwd: "/repo"}, fakeLedger{edges: []Edge{verdictEdge("age-1", testSHA, DispositionConfirmed)}}, tracker) + result, err := service.Execute(context.Background(), Request{BeadID: "age-1", SHA: testSHA, Reason: "Shipped"}) + if err != nil { + t.Fatal(err) + } + if tracker.id != "age-1" || tracker.reason != "Shipped [verdict:1a1a1a1:CONFIRMED]" { + t.Fatalf("tracker close = %q %q", tracker.id, tracker.reason) + } + if !result.Closed || result.Disposition != DispositionConfirmed || result.TrackerOutput != "closed by tracker" { + t.Fatalf("result = %+v", result) + } +} + +func TestServiceRefusesMissingOrRefutedVerdict(t *testing.T) { + for _, edges := range [][]Edge{nil, {verdictEdge("age-1", testSHA, "REFUTED")}} { + tracker := &fakeTracker{} + service := NewService(fakeRepository{cwd: "/repo"}, fakeLedger{edges: edges}, tracker) + _, err := service.Execute(context.Background(), Request{BeadID: "age-1", SHA: testSHA}) + if err == nil || !strings.Contains(err.Error(), "no verdict = not done") || !strings.Contains(err.Error(), "ao verify age-1") { + t.Fatalf("refusal = %v", err) + } + if tracker.id != "" { + t.Fatal("refusal mutated tracker") + } + } +} + +func TestServiceOriginFallbackTrivialWaiverAndForcedEscape(t *testing.T) { + for _, test := range []struct { + name, disposition string + repo fakeRepository + force bool + }{ + {name: "origin", disposition: DispositionConfirmed, repo: fakeRepository{cwd: "/repo", originOK: true, origin: []Edge{verdictEdge("age-1", testSHA, DispositionConfirmed)}}}, + {name: "trivial", disposition: DispositionWaived, repo: fakeRepository{cwd: "/repo", trivial: true}}, + {name: "forced", disposition: DispositionUnverified, repo: fakeRepository{cwd: "/repo"}, force: true}, + } { + t.Run(test.name, func(t *testing.T) { + tracker := &fakeTracker{} + service := NewService(test.repo, fakeLedger{}, tracker) + result, err := service.Execute(context.Background(), Request{BeadID: "age-1", SHA: testSHA, ForceNoVerdict: test.force}) + if err != nil { + t.Fatal(err) + } + if result.Disposition != test.disposition { + t.Fatalf("result = %+v", result) + } + }) + } +} + +func TestServicePropagatesLedgerAndTrackerFailures(t *testing.T) { + tracker := &fakeTracker{err: errors.New("close failed")} + service := NewService(fakeRepository{cwd: "/repo"}, fakeLedger{err: errors.New("ledger failed")}, tracker) + if _, err := service.Execute(context.Background(), Request{BeadID: "age-1", SHA: testSHA}); err == nil || !strings.Contains(err.Error(), "read provenance ledger") { + t.Fatalf("ledger error = %v", err) + } + service = NewService(fakeRepository{cwd: "/repo", trivial: true}, fakeLedger{}, tracker) + if _, err := service.Execute(context.Background(), Request{BeadID: "age-1", SHA: testSHA}); err == nil || !strings.Contains(err.Error(), "close age-1") { + t.Fatalf("tracker error = %v", err) + } +} + +func TestProvenanceOnlyChangedFilesIsExactAndFailClosed(t *testing.T) { + for _, test := range []struct { + input string + want bool + }{ + {"docs/provenance/ledger.jsonl\x00", true}, + {"docs/provenance/a\x00docs/provenance/b\x00", true}, + {" docs/provenance/a\x00", false}, + {"docs/provenance-evil/a\x00", false}, + {"", false}, + } { + if got := ProvenanceOnlyChangedFiles(test.input); got != test.want { + t.Errorf("ProvenanceOnlyChangedFiles(%q) = %v, want %v", test.input, got, test.want) + } + } +} From e3e75d0b95906ac411c214ae7436cb75b10562f4 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:14:31 -0400 Subject: [PATCH 084/108] refactor(done): atomically cut over Cobra ownership (age-nw28h.2.8.2) --- cli/cmd/ao/done.go | 92 --- cli/cmd/ao/done_composition.go | 12 + cli/cmd/ao/done_test.go | 663 +--------------------- cli/internal/commands/done/module.go | 114 ++++ cli/internal/commands/done/module_test.go | 79 +++ cli/internal/done/service_test.go | 34 ++ 6 files changed, 254 insertions(+), 740 deletions(-) delete mode 100644 cli/cmd/ao/done.go create mode 100644 cli/internal/commands/done/module.go create mode 100644 cli/internal/commands/done/module_test.go diff --git a/cli/cmd/ao/done.go b/cli/cmd/ao/done.go deleted file mode 100644 index 6c88b7ff0..000000000 --- a/cli/cmd/ao/done.go +++ /dev/null @@ -1,92 +0,0 @@ -// practices: [design-by-contract, in-toto-provenance] -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - "github.com/spf13/cobra" - - doneapp "github.com/boshu2/agentops/cli/internal/done" -) - -var ( - doneSHA string - doneReason string - doneForceNoVerdct bool - doneJSON bool -) - -var doneCmd = &cobra.Command{ - Use: "done ", - Short: "Close a bead with a verdict-referenced stamp — no verdict = not done", - Long: `Close a bead through the membrane's bookkeeping half: the close reason is -stamped with the ledger proof that the landed commit was actually reviewed. - -Resolution order for the target commit: --sha wins; otherwise the HEAD of the -git repository at the current working directory. The commit is looked up in -the committed provenance ledger (docs/provenance/ledger.jsonl): - - CONFIRMED verdict bound to the commit - -> close via br with "[verdict::CONFIRMED]" appended to the reason. - no verdict, and every changed file of the commit is under docs/provenance/ - -> the #trivial waiver class; close with "[verdict::waived-trivial]". - no verdict otherwise - -> REFUSE (non-zero exit) and name the command that produces one - (ao verify / ao pawl review). --force-no-verdict is the escape hatch: - it closes with an explicit, greppable "[verdict::UNVERIFIED]" - stamp instead of blocking forever. - -The close itself shells out to the br CLI on PATH with the environment as-is: -run 'ao done' where 'ao beads dir' resolves the live ledger (BEADS_DIR is -respected when already exported; in linked worktrees the resolution walks -git's common directory back to the canonical _beads — no path is hardcoded). - -Posture: warn-first. Nothing intercepts a raw 'br close'; the lever is this -recommended close path plus the warn-only verdict-close-rate gate. - -Examples: - ao done ag-x31t.4 # close against HEAD's verdict - ao done ag-x31t.4 --sha 4f2a91c # close against an explicit commit - ao done ag-x31t.4 --force-no-verdict # honest UNVERIFIED escape hatch`, - Args: cobra.ExactArgs(1), - RunE: runDone, -} - -func init() { - rootCmd.AddCommand(doneCmd) - doneCmd.Flags().StringVar(&doneSHA, "sha", "", "Commit sha (or >=7-char prefix) the bead landed as (default: HEAD at cwd)") - doneCmd.Flags().StringVarP(&doneReason, "reason", "r", "Done", "Close reason prose (the verdict stamp is appended)") - doneCmd.Flags().BoolVar(&doneForceNoVerdct, "force-no-verdict", false, "Close without a verdict, stamping an explicit UNVERIFIED marker") - doneCmd.Flags().BoolVar(&doneJSON, "json", false, "Emit machine-readable JSON (stdout-as-data)") -} - -func runDone(command *cobra.Command, args []string) error { - command.SilenceUsage = true - ctx := command.Context() - if ctx == nil { - ctx = context.Background() - } - result, err := newDoneService().Execute(ctx, doneapp.Request{ - BeadID: args[0], SHA: doneSHA, Reason: doneReason, ForceNoVerdict: doneForceNoVerdct, - }) - if err != nil { - return err - } - out := command.OutOrStdout() - if doneJSON { - encoder := json.NewEncoder(out) - encoder.SetIndent("", " ") - return encoder.Encode(result) - } - if output := strings.TrimSpace(result.TrackerOutput); output != "" { - fmt.Fprintln(out, output) - } - fmt.Fprintf(out, "closed %s at %s %s\n", result.BeadID, result.CommitSHA[:doneapp.MinimumSHAPrefix], result.Stamp) - if result.Disposition == doneapp.DispositionUnverified { - fmt.Fprintf(out, "note: UNVERIFIED close — run 'ao verify %s' next time so done carries proof\n", result.BeadID) - } - return nil -} diff --git a/cli/cmd/ao/done_composition.go b/cli/cmd/ao/done_composition.go index dba77d4b3..f1a5a59b6 100644 --- a/cli/cmd/ao/done_composition.go +++ b/cli/cmd/ao/done_composition.go @@ -4,6 +4,8 @@ import ( "context" doneadapter "github.com/boshu2/agentops/cli/internal/adapters/done" + "github.com/boshu2/agentops/cli/internal/clicontract" + donecommands "github.com/boshu2/agentops/cli/internal/commands/done" doneapp "github.com/boshu2/agentops/cli/internal/done" ) @@ -13,3 +15,13 @@ func newDoneService() doneapp.Service { }} return doneapp.NewService(doneadapter.SystemRepository(), doneadapter.SystemLedger(resolveLedgerPath()), tracker) } + +var doneModule = donecommands.NewModule(newDoneService()) +var doneCommand = doneModule.Command() + +func init() { + if err := clicontract.Attach(doneCommand, doneModule.Contract()); err != nil { + panic(err) + } + rootCmd.AddCommand(doneCommand) +} diff --git a/cli/cmd/ao/done_test.go b/cli/cmd/ao/done_test.go index 762f63f63..6c8d7bb6f 100644 --- a/cli/cmd/ao/done_test.go +++ b/cli/cmd/ao/done_test.go @@ -1,663 +1,30 @@ -// practices: [design-by-contract, in-toto-provenance] package main import ( - "context" - "encoding/json" - "os" - "os/exec" - "path/filepath" "strings" "testing" - - doneadapter "github.com/boshu2/agentops/cli/internal/adapters/done" - doneapp "github.com/boshu2/agentops/cli/internal/done" - "github.com/boshu2/agentops/cli/internal/provenancegraph" ) -// Fixture shas: full 40-char commit OIDs with distinct 7-char prefixes. -const ( - doneSHAConfirmed = "1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a" - doneSHARefuted = "2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b" - doneSHANoVerdict = "3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c" -) - -func toDoneEdges(edges []provenancegraph.Edge) []doneapp.Edge { - result := make([]doneapp.Edge, 0, len(edges)) - for _, edge := range edges { - result = append(result, doneapp.Edge{FromID: edge.FromID, FromType: edge.FromType, ToID: edge.ToID, - ToType: edge.ToType, Relation: edge.Relation, EvidenceRef: edge.EvidenceRef}) +func TestDoneCommandCompositionPreservesSurface(t *testing.T) { + if doneCommand.Use != "done " || doneCommand.GroupID != "" { + t.Fatalf("done command = use %q group %q", doneCommand.Use, doneCommand.GroupID) } - return result -} - -// resetDoneFlags snapshots and restores the package-global done cobra flags -// (test-isolation rule: package-global flag vars must not leak across tests). -func resetDoneFlags(t *testing.T) { - t.Helper() - prevSHA, prevReason := doneSHA, doneReason - prevForce, prevJSON := doneForceNoVerdct, doneJSON - t.Cleanup(func() { - doneSHA, doneReason = prevSHA, prevReason - doneForceNoVerdct, doneJSON = prevForce, prevJSON - }) - doneSHA, doneReason = "", "Done" - doneForceNoVerdct, doneJSON = false, false -} - -// stubBr installs a fake `br` binary at the FRONT of PATH that records its -// argv (one arg per line) and exits with the code in exit-code (default 0). -// BEADS_DIR is pinned to a throwaway tempdir so no test can ever touch a real -// ledger even if a real br leaked onto PATH. Returns the argv capture path. -func stubBr(t *testing.T) string { - t.Helper() - dir := t.TempDir() - argvFile := filepath.Join(dir, "br-argv") - script := "#!/bin/sh\n" + - "printf '%s\\n' \"$@\" > '" + argvFile + "'\n" + - "if [ -f '" + filepath.Join(dir, "exit-code") + "' ]; then exit \"$(cat '" + filepath.Join(dir, "exit-code") + "')\"; fi\n" + - "exit 0\n" - if err := os.WriteFile(filepath.Join(dir, "br"), []byte(script), 0o755); err != nil { - t.Fatalf("write br stub: %v", err) - } - t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) - t.Setenv("BEADS_DIR", t.TempDir()) - return argvFile -} - -// readStubArgv reads the argv the br stub recorded, one arg per slice entry. -func readStubArgv(t *testing.T, argvFile string) []string { - t.Helper() - data, err := os.ReadFile(argvFile) - if err != nil { - t.Fatalf("br stub never invoked (no argv file): %v", err) - } - return strings.Split(strings.TrimRight(string(data), "\n"), "\n") -} - -// seedDoneLedger writes verdict fixtures THROUGH the production writer -// (buildVerdictCommitEdge + Store.Append — fixture-fidelity rule): one -// CONFIRMED verdict on sha-1a, one REFUTED-only verdict on sha-2b, and a -// landed-but-unreviewed edge on sha-3c. -func seedDoneLedger(t *testing.T) { - t.Helper() - store := provenancegraph.NewStore(resolveLedgerPath()) - edges := []provenancegraph.Edge{ - buildBeadCommitEdge("ag-done.1", doneSHAConfirmed), - buildVerdictCommitEdge(pawlVerdict{ - BeadID: "ag-done.1", HeadSHA: doneSHAConfirmed, Disposition: "CONFIRMED"}), - buildBeadCommitEdge("ag-done.2", doneSHARefuted), - buildVerdictCommitEdge(pawlVerdict{ - BeadID: "ag-done.2", HeadSHA: doneSHARefuted, Disposition: "REFUTED"}), - buildBeadCommitEdge("ag-done.3", doneSHANoVerdict), - } - for i, e := range edges { - e.TS = "2026-07-01T00:00:0" + string(rune('0'+i)) + "Z" - if _, err := store.Append(e); err != nil { - t.Fatalf("seed edge %d: %v", i, err) + for _, flag := range []string{"sha", "reason", "force-no-verdict", "json"} { + if doneCommand.Flags().Lookup(flag) == nil { + t.Errorf("missing flag %q", flag) } } -} - -// makeDoneGitRepo builds a throwaway git repo in a tempdir whose HEAD commit -// touches exactly the given files (on top of a base commit — a root commit -// shows no files to plain diff-tree and would fail closed, matching the pawl -// gate's semantics), chdirs into it, and returns the HEAD sha. -func makeDoneGitRepo(t *testing.T, files map[string]string) string { - t.Helper() - dir := t.TempDir() - run := func(args ...string) string { - c := exec.Command("git", args...) - c.Dir = dir - c.Env = append(os.Environ(), - "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null", - "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com", - "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com") - out, err := c.CombinedOutput() - if err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - return strings.TrimSpace(string(out)) - } - run("init", "-q") - if err := os.WriteFile(filepath.Join(dir, ".init"), []byte("base\n"), 0o644); err != nil { - t.Fatalf("write base file: %v", err) - } - run("add", "-A") - run("commit", "-q", "-m", "base") - for path, content := range files { - full := filepath.Join(dir, path) - if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { - t.Fatalf("mkdir %s: %v", path, err) - } - if err := os.WriteFile(full, []byte(content), 0o644); err != nil { - t.Fatalf("write %s: %v", path, err) - } - } - run("add", "-A") - run("commit", "-q", "-m", "fixture") - t.Chdir(dir) - return run("rev-parse", "HEAD") -} - -// TestDone_ConfirmedVerdictCloses: a CONFIRMED verdict bound to --sha closes -// the bead via br with the exact stamped close reason. -func TestDone_ConfirmedVerdictCloses(t *testing.T) { - chdirRepoFixture(t) - seedDoneLedger(t) - resetDoneFlags(t) - argvFile := stubBr(t) - doneSHA = doneSHAConfirmed - - c, out := provTestCmd() - if err := runDone(c, []string{"ag-done.1"}); err != nil { - t.Fatalf("ao done with CONFIRMED verdict: %v", err) - } - - wantArgv := []string{"close", "ag-done.1", "-r", "Done [verdict:1a1a1a1:CONFIRMED]"} - got := readStubArgv(t, argvFile) - if len(got) != len(wantArgv) { - t.Fatalf("br argv = %q, want %q", got, wantArgv) - } - for i := range wantArgv { - if got[i] != wantArgv[i] { - t.Fatalf("br argv[%d] = %q, want %q", i, got[i], wantArgv[i]) - } - } - if !strings.Contains(out.String(), "closed ag-done.1 at 1a1a1a1 [verdict:1a1a1a1:CONFIRMED]") { - t.Errorf("human output missing close line:\n%s", out.String()) + command, _, err := rootCmd.Find([]string{"done"}) + if err != nil || command != doneCommand { + t.Fatalf("root registration = %p want %p err=%v", command, doneCommand, err) } } -// TestDone_ShaPrefixResolves: a 7-char --sha prefix binds to the same verdict -// (the provenance_show prefix convention). -func TestDone_ShaPrefixResolves(t *testing.T) { - chdirRepoFixture(t) - seedDoneLedger(t) - resetDoneFlags(t) - argvFile := stubBr(t) - doneSHA = doneSHAConfirmed[:7] - - c, _ := provTestCmd() - if err := runDone(c, []string{"ag-done.1"}); err != nil { - t.Fatalf("ao done with 7-char prefix: %v", err) - } - argv := readStubArgv(t, argvFile) - if argv[3] != "Done [verdict:1a1a1a1:CONFIRMED]" { - t.Fatalf("close reason = %q, want CONFIRMED stamp", argv[3]) - } -} - -// TestDone_RefusalPaths: no verdict (and not waiver-eligible) or a -// non-CONFIRMED-only verdict must refuse with a corrective error naming -// ao verify, and br must never be invoked. -func TestDone_RefusalPaths(t *testing.T) { - cases := []struct { - name string - sha string - wantInError []string - }{ - { - name: "no verdict refuses naming ao verify", - sha: doneSHANoVerdict, - wantInError: []string{ - "no verdict recorded for commit 3c3c3c3", - "no verdict = not done", - "ao verify ag-done.3", - "ao pawl review ag-done.3", - "--force-no-verdict", - }, - }, - { - name: "REFUTED-only verdict refuses naming the disposition", - sha: doneSHARefuted, - wantInError: []string{ - "none CONFIRMED", - "REFUTED", - "ao verify", - }, - }, - { - name: "short sha prefix is a usage error", - sha: doneSHANoVerdict[:6], - wantInError: []string{"at least 7"}, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - chdirRepoFixture(t) - seedDoneLedger(t) - resetDoneFlags(t) - argvFile := stubBr(t) - doneSHA = tc.sha - - c, _ := provTestCmd() - beadID := "ag-done.3" - if tc.sha == doneSHARefuted { - beadID = "ag-done.2" - } - err := runDone(c, []string{beadID}) - if err == nil { - t.Fatal("want refusal error, got nil") - } - for _, want := range tc.wantInError { - if !strings.Contains(err.Error(), want) { - t.Errorf("refusal error missing %q:\n%s", want, err.Error()) - } - } - if _, statErr := os.Stat(argvFile); statErr == nil { - t.Error("br was invoked on a refused close — refusal must not close") - } - }) - } -} - -// TestDone_ForceNoVerdictStampsUnverified: the escape hatch closes with an -// explicit, greppable UNVERIFIED stamp instead of blocking. -func TestDone_ForceNoVerdictStampsUnverified(t *testing.T) { - chdirRepoFixture(t) - seedDoneLedger(t) - resetDoneFlags(t) - argvFile := stubBr(t) - doneSHA = doneSHANoVerdict - doneForceNoVerdct = true - doneReason = "Shipped anyway" - - c, out := provTestCmd() - if err := runDone(c, []string{"ag-done.3"}); err != nil { - t.Fatalf("ao done --force-no-verdict: %v", err) - } - argv := readStubArgv(t, argvFile) - if argv[3] != "Shipped anyway [verdict:3c3c3c3:UNVERIFIED]" { - t.Fatalf("close reason = %q, want UNVERIFIED stamp", argv[3]) - } - if !strings.Contains(out.String(), "UNVERIFIED close") { - t.Errorf("human output must flag the UNVERIFIED close:\n%s", out.String()) - } -} - -// TestDone_WaivedTrivialProvenanceOnlyCommit: with no verdict, a commit whose -// changed files are ALL under docs/provenance/ closes with the waived-trivial -// stamp — resolved from HEAD (no --sha), covering the default-sha path. -func TestDone_WaivedTrivialProvenanceOnlyCommit(t *testing.T) { - head := makeDoneGitRepo(t, map[string]string{ - "docs/provenance/ledger.jsonl": "", - }) - resetDoneFlags(t) - argvFile := stubBr(t) - - c, _ := provTestCmd() - if err := runDone(c, []string{"ag-done.4"}); err != nil { - t.Fatalf("ao done on provenance-only commit: %v", err) - } - argv := readStubArgv(t, argvFile) - want := "Done [verdict:" + head[:7] + ":waived-trivial]" - if argv[3] != want { - t.Fatalf("close reason = %q, want %q", argv[3], want) - } -} - -// TestDone_NonProvenanceCommitNotWaived: a verdict-less commit touching any -// path outside docs/provenance/ must NOT waive — it refuses. -func TestDone_NonProvenanceCommitNotWaived(t *testing.T) { - makeDoneGitRepo(t, map[string]string{ - "docs/provenance/ledger.jsonl": "", - "README.md": "code change\n", - }) - resetDoneFlags(t) - argvFile := stubBr(t) - - c, _ := provTestCmd() - err := runDone(c, []string{"ag-done.5"}) - if err == nil { - t.Fatal("want refusal on mixed commit, got nil") - } - if !strings.Contains(err.Error(), "ao verify ag-done.5") { - t.Errorf("refusal must name ao verify:\n%s", err.Error()) - } - if _, statErr := os.Stat(argvFile); statErr == nil { - t.Error("br was invoked on a refused close") - } -} - -// TestDone_JSONShape: --json emits the exact done result contract. -func TestDone_JSONShape(t *testing.T) { - chdirRepoFixture(t) - seedDoneLedger(t) - resetDoneFlags(t) - stubBr(t) - doneSHA = doneSHAConfirmed - doneJSON = true - - c, out := provTestCmd() - if err := runDone(c, []string{"ag-done.1"}); err != nil { - t.Fatalf("ao done --json: %v", err) - } - var r doneapp.Result - if err := json.Unmarshal(out.Bytes(), &r); err != nil { - t.Fatalf("--json output not a done result: %v\n%s", err, out.String()) - } - want := doneapp.Result{ - BeadID: "ag-done.1", - CommitSHA: doneSHAConfirmed, - Disposition: "CONFIRMED", - Stamp: "[verdict:1a1a1a1:CONFIRMED]", - CloseReason: "Done [verdict:1a1a1a1:CONFIRMED]", - Closed: true, - } - if r != want { - t.Fatalf("done result = %+v, want %+v", r, want) - } -} - -// TestDone_BrFailurePropagates: a non-zero br exit surfaces as an error -// carrying br's output, never a silent success. -func TestDone_BrFailurePropagates(t *testing.T) { - chdirRepoFixture(t) - seedDoneLedger(t) - resetDoneFlags(t) - argvFile := stubBr(t) - if err := os.WriteFile(filepath.Join(filepath.Dir(argvFile), "exit-code"), []byte("3"), 0o644); err != nil { - t.Fatalf("write exit-code: %v", err) - } - doneSHA = doneSHAConfirmed - - c, _ := provTestCmd() - err := runDone(c, []string{"ag-done.1"}) - if err == nil { - t.Fatal("want br failure error, got nil") - } - if !strings.Contains(err.Error(), "br close ag-done.1") { - t.Errorf("error must name the br close invocation: %s", err.Error()) - } -} - -// TestLookupDoneVerdicts: the pure ledger scan is exact about binding, -// confirmation, and disposition collection. -// TestLookupDoneVerdicts_ForeignBeadNeverCertifies pins the pawl catch on this -// bead's own landing: a CONFIRMED verdict recorded for a DIFFERENT bead on the -// same commit must never certify the bead being closed (wrong-object -// certification) — it lands in ForeignBeads and Confirmed stays false. -// TestDoneProvenanceOnly_LeadingSpacePathNotWaived pins the pawl catch: a file -// whose name begins with a space is OUTSIDE docs/provenance/ and must not be -// trimmed into the waiver allowlist. -func TestDoneProvenanceOnly_LeadingSpacePathNotWaived(t *testing.T) { - repo := t.TempDir() - gitRun := func(args ...string) { - t.Helper() - cmd := exec.Command("git", args...) - cmd.Dir = repo - cmd.Env = append(os.Environ(), - "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t") - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - } - gitRun("init", "-q") - gitRun("commit", "-q", "--allow-empty", "-m", "base") - if err := os.MkdirAll(filepath.Join(repo, " docs", "provenance"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(repo, " docs", "provenance", "ledger.jsonl"), []byte("{}\n"), 0o644); err != nil { - t.Fatal(err) - } - gitRun("add", "--", " docs") - gitRun("commit", "-q", "-m", "sneaky") - revCmd := exec.Command("git", "rev-parse", "HEAD") - revCmd.Dir = repo - revOut, err := revCmd.Output() - if err != nil { - t.Fatalf("rev-parse: %v", err) - } - sha := strings.TrimSpace(string(revOut)) - if doneadapter.SystemRepository().CommitProvenanceOnly(context.Background(), repo, sha) { - t.Fatal("a leading-space path (' docs/provenance/...') must NOT be waived as provenance-only") - } -} - -func TestLookupDoneVerdicts_ForeignBeadNeverCertifies(t *testing.T) { - edges := []provenancegraph.Edge{ - buildVerdictCommitEdge(pawlVerdict{BeadID: "ag-done-1", HeadSHA: doneSHAConfirmed, Disposition: "CONFIRMED"}), - } - got := doneapp.LookupVerdicts(toDoneEdges(edges), "ag-other", doneSHAConfirmed) - if got.Confirmed { - t.Fatal("a CONFIRMED verdict for ag-done-1 must not certify ag-other") - } - if len(got.Dispositions) != 0 { - t.Fatalf("foreign verdicts must not enter Dispositions, got %v", got.Dispositions) - } - if len(got.ForeignBeads) != 1 || !strings.HasPrefix(got.ForeignBeads[0], "ag-done-1@") { - t.Fatalf("ForeignBeads = %v, want the ag-done-1 verdict surfaced", got.ForeignBeads) - } - // Same commit, RIGHT bead still certifies. - right := doneapp.LookupVerdicts(toDoneEdges(edges), "ag-done-1", doneSHAConfirmed) - if !right.Confirmed { - t.Fatal("the owning bead must still certify") - } -} - -func TestLookupDoneVerdicts(t *testing.T) { - edges := []provenancegraph.Edge{ - buildBeadCommitEdge("ag-x", doneSHAConfirmed), - buildVerdictCommitEdge(pawlVerdict{BeadID: "ag-x", HeadSHA: doneSHAConfirmed, Disposition: "REFUTED"}), - buildVerdictCommitEdge(pawlVerdict{BeadID: "ag-x", HeadSHA: doneSHAConfirmed, Disposition: "CONFIRMED"}), - buildVerdictCommitEdge(pawlVerdict{BeadID: "ag-y", HeadSHA: doneSHARefuted, Disposition: "REFUTED"}), - } - - got := doneapp.LookupVerdicts(toDoneEdges(edges), "ag-x", doneSHAConfirmed) - if !got.Confirmed { - t.Fatal("sha with a CONFIRMED verdict must report Confirmed") - } - if got.VerdictID != "ag-x@1a1a1a1" { - t.Fatalf("VerdictID = %q, want ag-x@1a1a1a1", got.VerdictID) - } - if len(got.Dispositions) != 2 || got.Dispositions[0] != "REFUTED" || got.Dispositions[1] != "CONFIRMED" { - t.Fatalf("Dispositions = %v, want [REFUTED CONFIRMED]", got.Dispositions) - } - - refuted := doneapp.LookupVerdicts(toDoneEdges(edges), "ag-y", doneSHARefuted) - if refuted.Confirmed { - t.Fatal("REFUTED-only sha must not report Confirmed") - } - if len(refuted.Dispositions) != 1 || refuted.Dispositions[0] != "REFUTED" { - t.Fatalf("Dispositions = %v, want [REFUTED]", refuted.Dispositions) - } - - none := doneapp.LookupVerdicts(toDoneEdges(edges), "ag-x", doneSHANoVerdict) - if none.Confirmed || len(none.Dispositions) != 0 { - t.Fatalf("unbound sha lookup = %+v, want empty", none) - } -} - -// TestShaBindsCommit: prefix binding in both directions, 7-char floor, hex-only. -func TestShaBindsCommit(t *testing.T) { - cases := []struct { - query, commit string - want bool - }{ - {doneSHAConfirmed, doneSHAConfirmed, true}, - {doneSHAConfirmed[:7], doneSHAConfirmed, true}, - {doneSHAConfirmed, doneSHAConfirmed[:7], true}, - {strings.ToUpper(doneSHAConfirmed[:8]), doneSHAConfirmed, true}, - {doneSHAConfirmed[:6], doneSHAConfirmed, false}, - {doneSHARefuted, doneSHAConfirmed, false}, - {"ag-done.1", doneSHAConfirmed, false}, - } - for _, tc := range cases { - if got := doneapp.SHABindsCommit(tc.query, tc.commit); got != tc.want { - t.Errorf("doneapp.SHABindsCommit(%q, %q) = %v, want %v", tc.query, tc.commit, got, tc.want) - } - } -} - -// makeDoneOriginRepo builds a bare "origin" whose main branch carries a -// docs/provenance/ledger.jsonl seeded with originEdges, plus a local checkout -// whose WORKING-TREE ledger carries localEdges — a stale checkout that lags -// origin: its ledger is written but never committed, and it only *fetches* -// origin so origin/main is a resolvable ref without fast-forwarding the working -// tree. Edges are written THROUGH the production writer (Store.Append) for -// fixture fidelity — the same on-disk hash-chained shape `git show` returns. -// chdirs into the local checkout and returns nothing (the caller drives ao done -// from cwd). -func makeDoneOriginRepo(t *testing.T, localEdges, originEdges []provenancegraph.Edge) { - t.Helper() - root := t.TempDir() - bare := filepath.Join(root, "origin.git") - builder := filepath.Join(root, "builder") - local := filepath.Join(root, "local") - - gitEnv := append(os.Environ(), - "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null", - "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com", - "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com") - git := func(dir string, args ...string) { - t.Helper() - c := exec.Command("git", args...) - c.Dir = dir - c.Env = gitEnv - if out, err := c.CombinedOutput(); err != nil { - t.Fatalf("git -C %s %v: %v\n%s", dir, args, err, out) - } - } - writeLedger := func(dir string, edges []provenancegraph.Edge) { - t.Helper() - if len(edges) == 0 { - return - } - store := provenancegraph.NewStore(filepath.Join(dir, provenancegraph.LedgerRelativePath)) - for i, e := range edges { - e.TS = "2026-07-02T00:00:0" + string(rune('0'+i)) + "Z" - if _, err := store.Append(e); err != nil { - t.Fatalf("append fixture edge %d in %s: %v", i, dir, err) - } - } - } - - for _, d := range []string{bare, builder, local} { - if err := os.MkdirAll(d, 0o755); err != nil { - t.Fatalf("mkdir %s: %v", d, err) - } - } - git(bare, "init", "-q", "--bare") - - // Builder seeds origin/main with the origin ledger and pushes it. - git(builder, "init", "-q") - writeLedger(builder, originEdges) - git(builder, "add", "--", provenancegraph.LedgerRelativePath) - git(builder, "commit", "-q", "-m", "origin ledger") - git(builder, "remote", "add", "origin", bare) - git(builder, "push", "-q", "origin", "HEAD:main") - - // Local is the operator's stale checkout: its working-tree ledger lags - // origin (localEdges), and it only fetches origin so origin/main resolves — - // mirroring the incident where the local ledger predated a pushed verdict. - git(local, "init", "-q") - writeLedger(local, localEdges) - git(local, "remote", "add", "origin", bare) - git(local, "fetch", "-q", "origin") - t.Chdir(local) -} - -// TestDone_OriginLedgerFallbackConfirms: the incident — a CONFIRMED verdict is -// already on origin/main but the local working-tree ledger predates it. ao done -// must consult origin/main and close exactly as if the verdict were local (same -// stamp), with NO manual `git merge --ff-only`. -func TestDone_OriginLedgerFallbackConfirms(t *testing.T) { - makeDoneOriginRepo(t, - nil, // stale local: no verdict for the commit - []provenancegraph.Edge{ - buildBeadCommitEdge("ag-orig.1", doneSHAConfirmed), - buildVerdictCommitEdge(pawlVerdict{ - BeadID: "ag-orig.1", HeadSHA: doneSHAConfirmed, Disposition: "CONFIRMED"}), - }) - resetDoneFlags(t) - argvFile := stubBr(t) - doneSHA = doneSHAConfirmed - - c, out := provTestCmd() - if err := runDone(c, []string{"ag-orig.1"}); err != nil { - t.Fatalf("ao done with CONFIRMED verdict only on origin/main: %v", err) - } - - wantArgv := []string{"close", "ag-orig.1", "-r", "Done [verdict:1a1a1a1:CONFIRMED]"} - got := readStubArgv(t, argvFile) - if len(got) != len(wantArgv) { - t.Fatalf("br argv = %q, want %q", got, wantArgv) - } - for i := range wantArgv { - if got[i] != wantArgv[i] { - t.Fatalf("br argv[%d] = %q, want %q", i, got[i], wantArgv[i]) - } - } - if !strings.Contains(out.String(), "closed ag-orig.1 at 1a1a1a1 [verdict:1a1a1a1:CONFIRMED]") { - t.Errorf("human output missing close line:\n%s", out.String()) - } -} - -// TestDone_OriginLedgerFallbackMissRefusesWithFetchHint: the verdict is in -// NEITHER the local ledger nor origin/main (origin holds only an unrelated -// verdict). ao done must still refuse — fail-closed — and the refusal must -// extend with the git-fetch hint so the operator knows a just-pushed verdict -// may not have reached this checkout's origin ref yet. -func TestDone_OriginLedgerFallbackMissRefusesWithFetchHint(t *testing.T) { - makeDoneOriginRepo(t, - nil, // local: nothing - []provenancegraph.Edge{ - // origin has a verdict, but for a DIFFERENT commit than we query. - buildBeadCommitEdge("ag-orig.2", doneSHARefuted), - buildVerdictCommitEdge(pawlVerdict{ - BeadID: "ag-orig.2", HeadSHA: doneSHARefuted, Disposition: "REFUTED"}), - }) - resetDoneFlags(t) - argvFile := stubBr(t) - doneSHA = doneSHANoVerdict // certified by neither local nor origin - - c, _ := provTestCmd() - err := runDone(c, []string{"ag-orig.2"}) - if err == nil { - t.Fatal("want refusal when the verdict is in neither ledger, got nil") - } - for _, want := range []string{ - "no verdict recorded for commit 3c3c3c3", - "no verdict = not done", - "git fetch origin && git merge --ff-only origin/main", - } { - if !strings.Contains(err.Error(), want) { - t.Errorf("refusal error missing %q:\n%s", want, err.Error()) - } - } - if _, statErr := os.Stat(argvFile); statErr == nil { - t.Error("br was invoked on a refused close — refusal must not close") - } -} - -// TestDone_LocalVerdictClosesUnchangedWithOrigin: existing behavior is -// untouched — when the LOCAL ledger already certifies the commit, ao done -// closes from it and never depends on origin/main (here origin carries no -// certifying verdict at all). -func TestDone_LocalVerdictClosesUnchangedWithOrigin(t *testing.T) { - makeDoneOriginRepo(t, - []provenancegraph.Edge{ // local already certifies - buildBeadCommitEdge("ag-loc.3", doneSHAConfirmed), - buildVerdictCommitEdge(pawlVerdict{ - BeadID: "ag-loc.3", HeadSHA: doneSHAConfirmed, Disposition: "CONFIRMED"}), - }, - []provenancegraph.Edge{ // origin: only a bead edge, no verdict - buildBeadCommitEdge("ag-loc.3", doneSHAConfirmed), - }) - resetDoneFlags(t) - argvFile := stubBr(t) - doneSHA = doneSHAConfirmed - - c, out := provTestCmd() - if err := runDone(c, []string{"ag-loc.3"}); err != nil { - t.Fatalf("ao done with a local CONFIRMED verdict: %v", err) - } - argv := readStubArgv(t, argvFile) - if argv[3] != "Done [verdict:1a1a1a1:CONFIRMED]" { - t.Fatalf("close reason = %q, want the local CONFIRMED stamp", argv[3]) - } - if !strings.Contains(out.String(), "closed ag-loc.3 at 1a1a1a1 [verdict:1a1a1a1:CONFIRMED]") { - t.Errorf("human output missing close line:\n%s", out.String()) +func TestDoneCommandCompositionRejectsInvalidSHAWithoutEffects(t *testing.T) { + command := doneModule.Command() + command.SetArgs([]string{"age-test", "--sha", "bad"}) + err := command.Execute() + if err == nil || !strings.Contains(err.Error(), "is not a commit sha") { + t.Fatalf("error = %v", err) } } diff --git a/cli/internal/commands/done/module.go b/cli/internal/commands/done/module.go new file mode 100644 index 000000000..00770764b --- /dev/null +++ b/cli/internal/commands/done/module.go @@ -0,0 +1,114 @@ +// Package done owns Cobra presentation for the done command family. +package done + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/spf13/cobra" + + "github.com/boshu2/agentops/cli/internal/clicontract" + doneapp "github.com/boshu2/agentops/cli/internal/done" +) + +type UseCases interface { + Execute(context.Context, doneapp.Request) (doneapp.Result, error) +} + +type Module struct{ useCases UseCases } + +func NewModule(useCases UseCases) Module { return Module{useCases: useCases} } + +func (Module) Contract() clicontract.CommandContract { + return clicontract.CommandContract{ + ID: "ao.done", + Profiles: clicontract.ProfileDefault | clicontract.ProfileFlywheel | + clicontract.ProfileLegacy | clicontract.ProfileCombined, + Args: clicontract.ArgsPolicy{Name: "exact-1", Validate: cobra.ExactArgs(1)}, + Output: clicontract.OutputNone, + Effects: clicontract.EffectFilesystem | clicontract.EffectProcess | clicontract.EffectTracker | clicontract.EffectEnvironment, + ExitClasses: map[int]clicontract.ExitClass{ + 0: clicontract.ExitSuccess, + 1: clicontract.ExitFailure, + }, + } +} + +type options struct { + sha, reason string + forceNoVerdict bool + jsonOutput bool +} + +func (module Module) Command() *cobra.Command { + options := options{reason: "Done"} + command := &cobra.Command{ + Use: "done ", + Short: "Close a bead with a verdict-referenced stamp — no verdict = not done", + Long: `Close a bead through the membrane's bookkeeping half: the close reason is +stamped with the ledger proof that the landed commit was actually reviewed. + +Resolution order for the target commit: --sha wins; otherwise the HEAD of the +git repository at the current working directory. The commit is looked up in +the committed provenance ledger (docs/provenance/ledger.jsonl): + + CONFIRMED verdict bound to the commit + -> close via br with "[verdict::CONFIRMED]" appended to the reason. + no verdict, and every changed file of the commit is under docs/provenance/ + -> the #trivial waiver class; close with "[verdict::waived-trivial]". + no verdict otherwise + -> REFUSE (non-zero exit) and name the command that produces one + (ao verify / ao pawl review). --force-no-verdict is the escape hatch: + it closes with an explicit, greppable "[verdict::UNVERIFIED]" + stamp instead of blocking forever. + +The close itself shells out to the br CLI on PATH with the environment as-is: +run 'ao done' where 'ao beads dir' resolves the live ledger (BEADS_DIR is +respected when already exported; in linked worktrees the resolution walks +git's common directory back to the canonical _beads — no path is hardcoded). + +Posture: warn-first. Nothing intercepts a raw 'br close'; the lever is this +recommended close path plus the warn-only verdict-close-rate gate. + +Examples: + ao done ag-x31t.4 # close against HEAD's verdict + ao done ag-x31t.4 --sha 4f2a91c # close against an explicit commit + ao done ag-x31t.4 --force-no-verdict # honest UNVERIFIED escape hatch`, + Args: cobra.ExactArgs(1), + } + flags := command.Flags() + flags.StringVar(&options.sha, "sha", "", "Commit sha (or >=7-char prefix) the bead landed as (default: HEAD at cwd)") + flags.StringVarP(&options.reason, "reason", "r", "Done", "Close reason prose (the verdict stamp is appended)") + flags.BoolVar(&options.forceNoVerdict, "force-no-verdict", false, "Close without a verdict, stamping an explicit UNVERIFIED marker") + flags.BoolVar(&options.jsonOutput, "json", false, "Emit machine-readable JSON (stdout-as-data)") + command.RunE = func(command *cobra.Command, args []string) error { + command.SilenceUsage = true + result, err := module.useCases.Execute(command.Context(), doneapp.Request{ + BeadID: args[0], SHA: options.sha, Reason: options.reason, ForceNoVerdict: options.forceNoVerdict, + }) + if err != nil { + return err + } + return render(command, result, options.jsonOutput) + } + return command +} + +func render(command *cobra.Command, result doneapp.Result, jsonOutput bool) error { + output := command.OutOrStdout() + if jsonOutput { + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(result) + } + if trackerOutput := strings.TrimSpace(result.TrackerOutput); trackerOutput != "" { + fmt.Fprintln(output, trackerOutput) + } + fmt.Fprintf(output, "closed %s at %s %s\n", result.BeadID, result.CommitSHA[:doneapp.MinimumSHAPrefix], result.Stamp) + if result.Disposition == doneapp.DispositionUnverified { + fmt.Fprintf(output, "note: UNVERIFIED close — run 'ao verify %s' next time so done carries proof\n", result.BeadID) + } + return nil +} diff --git a/cli/internal/commands/done/module_test.go b/cli/internal/commands/done/module_test.go new file mode 100644 index 000000000..fd3d270f9 --- /dev/null +++ b/cli/internal/commands/done/module_test.go @@ -0,0 +1,79 @@ +package done + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + + doneapp "github.com/boshu2/agentops/cli/internal/done" +) + +type fakeService struct { + request doneapp.Request + result doneapp.Result + err error +} + +func (service *fakeService) Execute(_ context.Context, request doneapp.Request) (doneapp.Result, error) { + service.request = request + return service.result, service.err +} + +func TestCommandOwnsFlagsAndDelegatesRequest(t *testing.T) { + service := &fakeService{result: doneapp.Result{BeadID: "age-1", CommitSHA: "abcdef0123456789", Disposition: doneapp.DispositionConfirmed, + Stamp: "[verdict:abcdef0:CONFIRMED]", Closed: true}} + command := NewModule(service).Command() + command.SetArgs([]string{"age-1", "--sha", "abcdef0", "--reason", "Shipped", "--force-no-verdict"}) + var stdout bytes.Buffer + command.SetOut(&stdout) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + want := doneapp.Request{BeadID: "age-1", SHA: "abcdef0", Reason: "Shipped", ForceNoVerdict: true} + if service.request != want { + t.Fatalf("request = %+v, want %+v", service.request, want) + } + if !strings.Contains(stdout.String(), "closed age-1 at abcdef0 [verdict:abcdef0:CONFIRMED]") { + t.Fatalf("stdout = %q", stdout.String()) + } + for _, flag := range []string{"sha", "reason", "force-no-verdict", "json"} { + if command.Flags().Lookup(flag) == nil { + t.Errorf("missing local flag %q", flag) + } + } +} + +func TestCommandJSONIsStructuredAndTrackerOutputIsNotData(t *testing.T) { + service := &fakeService{result: doneapp.Result{BeadID: "age-1", CommitSHA: "abcdef0123456789", Disposition: doneapp.DispositionUnverified, + Stamp: "[verdict:abcdef0:UNVERIFIED]", CloseReason: "Done [verdict:abcdef0:UNVERIFIED]", Closed: true, TrackerOutput: "tracker chatter"}} + command := NewModule(service).Command() + command.SetArgs([]string{"age-1", "--json"}) + var stdout bytes.Buffer + command.SetOut(&stdout) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + var result doneapp.Result + if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { + t.Fatalf("invalid JSON: %v\n%s", err, stdout.String()) + } + if result.BeadID != "age-1" || strings.Contains(stdout.String(), "tracker chatter") { + t.Fatalf("result=%+v stdout=%q", result, stdout.String()) + } +} + +func TestCommandRejectsWrongArityWithoutCallingService(t *testing.T) { + for _, args := range [][]string{nil, {"age-1", "extra"}} { + service := &fakeService{} + command := NewModule(service).Command() + command.SetArgs(args) + if err := command.Execute(); err == nil { + t.Fatalf("args %v accepted", args) + } + if service.request.BeadID != "" { + t.Fatalf("args %v called service", args) + } + } +} diff --git a/cli/internal/done/service_test.go b/cli/internal/done/service_test.go index 68d78e7c5..07d374d70 100644 --- a/cli/internal/done/service_test.go +++ b/cli/internal/done/service_test.go @@ -131,3 +131,37 @@ func TestProvenanceOnlyChangedFilesIsExactAndFailClosed(t *testing.T) { } } } + +func TestLookupVerdictsBindsExactBeadAndSHA(t *testing.T) { + edges := []Edge{ + verdictEdge("age-owner", testSHA, "REFUTED"), + verdictEdge("age-owner", testSHA, DispositionConfirmed), + verdictEdge("age-foreign", testSHA, DispositionConfirmed), + } + lookup := LookupVerdicts(edges, "age-owner", testSHA[:7]) + if !lookup.Confirmed || lookup.VerdictID != "age-owner@1a1a1a1" || len(lookup.Dispositions) != 2 { + t.Fatalf("lookup = %+v", lookup) + } + foreign := LookupVerdicts(edges, "age-other", testSHA) + if foreign.Confirmed || len(foreign.Dispositions) != 0 || len(foreign.ForeignBeads) != 3 { + t.Fatalf("foreign lookup = %+v", foreign) + } +} + +func TestSHABindsCommitUsesSevenHexCharacterFloor(t *testing.T) { + for _, test := range []struct { + query, commit string + want bool + }{ + {testSHA, testSHA, true}, + {testSHA[:7], testSHA, true}, + {strings.ToUpper(testSHA[:8]), testSHA, true}, + {testSHA[:6], testSHA, false}, + {"age-one", testSHA, false}, + {"2b2b2b2", testSHA, false}, + } { + if got := SHABindsCommit(test.query, test.commit); got != test.want { + t.Errorf("SHABindsCommit(%q, %q) = %v, want %v", test.query, test.commit, got, test.want) + } + } +} From e741222b21931c3f7fe98fb947bd74b9272a915a Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:14:52 -0400 Subject: [PATCH 085/108] test(done): seal accepted compatibility lineage (age-nw28h.2.8.2) --- cli/testdata/compatibility-baseline/families/done/lineage.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/testdata/compatibility-baseline/families/done/lineage.json b/cli/testdata/compatibility-baseline/families/done/lineage.json index b40a2f386..d2382e1cd 100644 --- a/cli/testdata/compatibility-baseline/families/done/lineage.json +++ b/cli/testdata/compatibility-baseline/families/done/lineage.json @@ -6,5 +6,6 @@ "fixture_sha256": "87d7b5a9e3aa9a253644ae334920532b5ca0aae4058f53984612e9bd8440194f", "ownership_sha256": "fecf63c7ba54c784cfa33c6b82f4821bf9cd98b8d64f1f66da7ff370f9a0ce7f", "scope_sha256": "d7c6731e492e229369af90cf48eaf0c429e5074faa9323139de01ab49a1f9eab", - "migration_state": "migrating" + "migration_state": "migrated", + "accepted_sha": "e3e75d0b95906ac411c214ae7436cb75b10562f4" } From 13167d4a028f3dab3b1da67f7616ad7dd517bc05 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:12:12 -0400 Subject: [PATCH 086/108] test(eval): freeze CLI compatibility before migration --- .../families/eval/case.json | 25 ++++++++++ .../families/eval/ownership.json | 50 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/eval/case.json create mode 100644 cli/testdata/compatibility-baseline/families/eval/ownership.json diff --git a/cli/testdata/compatibility-baseline/families/eval/case.json b/cli/testdata/compatibility-baseline/families/eval/case.json new file mode 100644 index 000000000..902fd805a --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/eval/case.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "family": "eval", + "checks": [ + { + "id": "eval-black-box", + "command": "tmp=$(mktemp -d /tmp/ao-eval.XXXXXX); (cd cli && go build -o \"$tmp/ao\" ./cmd/ao); \"$tmp/ao\" eval --help >\"$tmp/help\"; grep -Fq 'ao eval [command]' \"$tmp/help\"; for child in baseline baseline-audit bench chaos cleanup compare coverage outcomes run scenario scenario-ab scenario-moat scorecard session-outcome suite task; do grep -Eq \"^[[:space:]]+$child([[:space:]]|$)\" \"$tmp/help\"; done; \"$tmp/ao\" eval run --help >\"$tmp/run-help\"; for flag in baseline baseline-mode context-mode delta-out out run-id runtime; do grep -Fq -- \"--$flag\" \"$tmp/run-help\"; done; set +e; \"$tmp/ao\" eval run >\"$tmp/out\" 2>\"$tmp/err\"; code=$?; set -e; test \"$code\" -eq 1; grep -Fq 'accepts 1 arg(s), received 0' \"$tmp/err\"" + }, + { + "id": "eval-go-behavior", + "command": "cd cli && go test ./cmd/ao -run 'Test(Eval|RunEval|Scenario|RetrievalBench)' -count=1" + } + ], + "dimensions": { + "path": {"status":"evidence","evidence":["eval-black-box","eval-go-behavior"]}, + "aliases": {"status":"evidence","evidence":["eval-go-behavior"]}, + "accepted_args": {"status":"evidence","evidence":["eval-black-box","eval-go-behavior"]}, + "rejected_args": {"status":"evidence","evidence":["eval-black-box","eval-go-behavior"]}, + "stdout": {"status":"evidence","evidence":["eval-go-behavior"]}, + "stderr": {"status":"evidence","evidence":["eval-black-box","eval-go-behavior"]}, + "exit_classes": {"status":"evidence","evidence":["eval-black-box","eval-go-behavior"]}, + "tracker_selection": {"status":"not_applicable","reason":"The eval family does not select a repository issue tracker."}, + "ordered_effects": {"status":"evidence","evidence":["eval-go-behavior"]} + } +} diff --git a/cli/testdata/compatibility-baseline/families/eval/ownership.json b/cli/testdata/compatibility-baseline/families/eval/ownership.json new file mode 100644 index 000000000..f376a8481 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/eval/ownership.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "family": "eval", + "profiles": {"default":"present","flywheel":"present","legacy":"present","combined":"present"}, + "legacy_symbols": [ + "evalCmd", "evalRunCmd", "evalCompareCmd", "evalBaselineCmd", "evalBaselineAuditCmd", + "evalScorecardCmd", "evalCoverageCmd", "evalConfigured", "registerEvalCommand", "configureEvalCommand", + "evalRunOutput", "evalRunID", "evalRunRuntime", "evalRunBaseline", "evalRunBaselineMode", + "evalRunContextMode", "evalRunContextOffDir", "evalRunContextOnDir", "evalRunDeltaOut", + "evalCompareOutput", "evalCompareMaxAgg", "evalCompareMaxDim", "evalScorecardOutput", + "evalScorecardKind", "evalScorecardMaxCat", "evalBaselineOutput", "evalBaselineBy", + "evalBaselineReason", "evalBaselineAuditRoot", "evalBaselineAuditDir", "evalCoverageRoot", + "evalCoverageDomains", "evalCoverageEvidence", "evalCoverageDims", "evalCoverageRuntimes", + "evalTaskCmd", "evalTaskAddCmd", "evalTaskListCmd", "evalTaskShowCmd", "evalTaskRunCmd", + "runEvalTaskRun", "registerEvalTaskCmd", + "evalCleanupCmd", "runEvalCleanup", "registerEvalCleanupCmd", "evalCleanupDelete", + "evalCleanupTmpFiles", "evalCleanupAge", "evalCleanupDryRun", + "evalSuiteCmd", "evalSuiteVerdictCmd", "evalSuiteNRequiredCmd", "runEvalSuiteVerdict", + "runEvalSuiteNRequired", "registerEvalSuiteCmd", + "evalOutcomesCmd", "evalOutcomesCompileCmd", "runEvalOutcomesCompile", + "evalOutcomesIngestCmd", "runEvalOutcomesIngest", "evalOutcomesIngestExpectHash", + "evalOutcomesIngestBurnLedger", "evalOutcomesIngestManifestOut", "evalOutcomesIngestRunID", + "scenarioCmd", "scenarioInitCmd", "scenarioListCmd", "scenarioAddCmd", "scenarioValidateCmd", + "scenarioEvaluateCmd", "runScenarioAdd", "runScenarioEvaluate", "scenarioListStatus", + "evalScenarioABCmd", "evalScenarioMoatCmd", "evalSessionOutcomeCmd", "evalChaosCmd", "evalBenchCmd", + "evalScenarioABScenario", "evalScenarioABOutput", "evalScenarioABBudget", "evalScenarioABTimeout", + "evalScenarioABControlOnly", "evalScenarioMoatScorecards", "evalScenarioMoatOutput" + ], + "live_owner": "cli/internal/commands/eval", + "allowed_paths": [ + "cli/cmd/ao/eval*.go", + "cli/cmd/ao/scenario*.go", + "cli/cmd/ao/retrieval_bench*.go", + "cli/cmd/ao/root.go", + "cli/cmd/ao/cobra_commands_test.go", + "cli/cmd/ao/json_validity_test.go", + "cli/cmd/ao/default_spine_test.go", + "cli/internal/commands/eval/**", + "cli/internal/adapters/eval/**", + "cli/internal/eval/**", + "cli/internal/evalsubstrate/**", + "cli/internal/scenario/**", + "cli/internal/scenarioresults/**", + "cli/internal/bench/**", + "cli/internal/verdict/**", + "cli/internal/llm/**", + "cli/docs/COMMANDS.md", + "cli/testdata/compatibility-baseline/families/eval/**" + ] +} From 5b86f74476cb44d785b76d7ba03d12f8a3061659 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:12:29 -0400 Subject: [PATCH 087/108] chore(eval): bind migration lineage --- .../compatibility-baseline/families/eval/lineage.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/eval/lineage.json diff --git a/cli/testdata/compatibility-baseline/families/eval/lineage.json b/cli/testdata/compatibility-baseline/families/eval/lineage.json new file mode 100644 index 000000000..5096f7561 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/eval/lineage.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "family": "eval", + "old_implementation_sha": "e741222b21931c3f7fe98fb947bd74b9272a915a", + "freeze_sha": "13167d4a028f3dab3b1da67f7616ad7dd517bc05", + "fixture_sha256": "906273234e3fc94a9fec4e4f7762ce1ec7ea58485ccfeb080aa338931bbf0a1d", + "ownership_sha256": "e8eb03162ee4fded8e6aeb2c582ef265103b35cb5836c162ddae1a2803178565", + "scope_sha256": "42d515c799e9f1a50bcc5471d3df4837cd2c1b8a1b7c6d000dc02877ab8ce86e", + "migration_state": "migrating" +} From ebf5aa664ac63ac42da247f8f5a77f06e4fd6f38 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:18:15 -0400 Subject: [PATCH 088/108] refactor(eval): extract core use cases and command module --- cli/internal/adapters/eval/runtime.go | 47 ++++ cli/internal/adapters/eval/runtime_test.go | 26 ++ cli/internal/commands/eval/module.go | 303 +++++++++++++++++++++ cli/internal/commands/eval/module_test.go | 81 ++++++ cli/internal/eval/core_service.go | 242 ++++++++++++++++ cli/internal/eval/core_service_test.go | 74 +++++ 6 files changed, 773 insertions(+) create mode 100644 cli/internal/adapters/eval/runtime.go create mode 100644 cli/internal/adapters/eval/runtime_test.go create mode 100644 cli/internal/commands/eval/module.go create mode 100644 cli/internal/commands/eval/module_test.go create mode 100644 cli/internal/eval/core_service.go create mode 100644 cli/internal/eval/core_service_test.go diff --git a/cli/internal/adapters/eval/runtime.go b/cli/internal/adapters/eval/runtime.go new file mode 100644 index 000000000..9e13fa810 --- /dev/null +++ b/cli/internal/adapters/eval/runtime.go @@ -0,0 +1,47 @@ +package eval + +import ( + "os" + "path/filepath" + + aoeval "github.com/boshu2/agentops/cli/internal/eval" +) + +type Runtime struct{} + +func (Runtime) RunSuite(options aoeval.RunOptions) (*aoeval.RunRecord, error) { + return aoeval.RunSuite(options) +} +func (Runtime) RunBaselineAB(options aoeval.RunOptions) (aoeval.DeltaScorecard, *aoeval.RunRecord, *aoeval.RunRecord, error) { + return aoeval.RunBaselineAB(options) +} +func (Runtime) WriteDeltaScorecard(card aoeval.DeltaScorecard, path string) error { + return aoeval.WriteDeltaScorecard(card, path) +} +func (Runtime) RunContextAB(options aoeval.RunOptions, contextOptions aoeval.ContextABOptions) (aoeval.ContextDeltaScorecard, *aoeval.RunRecord, *aoeval.RunRecord, error) { + return aoeval.RunContextAB(options, contextOptions) +} +func (Runtime) WriteContextDeltaScorecard(card aoeval.ContextDeltaScorecard, path string) error { + return aoeval.WriteContextDeltaScorecard(card, path) +} +func (Runtime) LoadRun(path string) (*aoeval.RunRecord, error) { return aoeval.LoadRun(path) } +func (Runtime) CompareRuns(candidate, baseline *aoeval.RunRecord, options aoeval.CompareOptions) (*aoeval.RunRecord, error) { + return aoeval.CompareRuns(candidate, baseline, options) +} +func (Runtime) WorkDir() (string, error) { return os.Getwd() } +func (Runtime) Abs(path string) (string, error) { return filepath.Abs(path) } +func (Runtime) PromoteBaseline(run *aoeval.RunRecord, options aoeval.BaselineOptions) (*aoeval.RunRecord, error) { + return aoeval.PromoteBaseline(run, options) +} +func (Runtime) AuditBaselinePolicy(options aoeval.BaselineAuditOptions) (*aoeval.BaselineAuditReport, error) { + return aoeval.AuditBaselinePolicy(options) +} +func (Runtime) BuildScorecard(candidate, baseline *aoeval.RunRecord, options aoeval.ScorecardOptions) (*aoeval.Scorecard, error) { + return aoeval.BuildScorecard(candidate, baseline, options) +} +func (Runtime) WriteScorecard(path string, card *aoeval.Scorecard) error { + return aoeval.WriteScorecard(path, card) +} +func (Runtime) BuildCoverageReport(options aoeval.CoverageOptions) (*aoeval.CoverageReport, error) { + return aoeval.BuildCoverageReport(options) +} diff --git a/cli/internal/adapters/eval/runtime_test.go b/cli/internal/adapters/eval/runtime_test.go new file mode 100644 index 000000000..918840d1b --- /dev/null +++ b/cli/internal/adapters/eval/runtime_test.go @@ -0,0 +1,26 @@ +package eval + +import ( + "path/filepath" + "testing" + + aoeval "github.com/boshu2/agentops/cli/internal/eval" +) + +func TestRuntimeSatisfiesCorePortAndResolvesHostPaths(t *testing.T) { + var port aoeval.CoreRuntime = Runtime{} + workDir, err := port.WorkDir() + if err != nil { + t.Fatalf("WorkDir: %v", err) + } + if !filepath.IsAbs(workDir) { + t.Fatalf("WorkDir = %q, want absolute", workDir) + } + absolute, err := port.Abs("suite.json") + if err != nil { + t.Fatalf("Abs: %v", err) + } + if !filepath.IsAbs(absolute) { + t.Fatalf("Abs = %q, want absolute", absolute) + } +} diff --git a/cli/internal/commands/eval/module.go b/cli/internal/commands/eval/module.go new file mode 100644 index 000000000..523141bb3 --- /dev/null +++ b/cli/internal/commands/eval/module.go @@ -0,0 +1,303 @@ +// Package eval owns Cobra presentation for the eval command family. +package eval + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/spf13/cobra" + + "github.com/boshu2/agentops/cli/internal/clicontract" + aoeval "github.com/boshu2/agentops/cli/internal/eval" +) + +type CoreUseCases interface { + Run(context.Context, aoeval.CoreRunRequest) (aoeval.CoreRunResult, error) + Compare(context.Context, aoeval.CoreCompareRequest) (aoeval.CoreCompareResult, error) + PromoteBaseline(context.Context, aoeval.CoreBaselineRequest) (*aoeval.RunRecord, error) + AuditBaseline(context.Context, aoeval.CoreBaselineAuditRequest) (*aoeval.BaselineAuditReport, error) + Scorecard(context.Context, aoeval.CoreScorecardRequest) (*aoeval.Scorecard, error) + Coverage(context.Context, aoeval.CoreCoverageRequest) (*aoeval.CoverageReport, error) +} + +type HostOptions struct { + OutputMode func(*cobra.Command) string +} + +type Module struct { + core CoreUseCases + host HostOptions +} + +func NewModule(core CoreUseCases, host HostOptions) Module { return Module{core: core, host: host} } + +func (Module) Contract() clicontract.CommandContract { + return clicontract.CommandContract{ + ID: "ao.eval", + Profiles: clicontract.ProfileDefault | clicontract.ProfileFlywheel | + clicontract.ProfileLegacy | clicontract.ProfileCombined, + Args: clicontract.ArgsPolicy{Name: "no-args", Validate: cobra.NoArgs}, + Output: clicontract.OutputNone, + Effects: clicontract.EffectFilesystem | clicontract.EffectProcess | clicontract.EffectEnvironment | clicontract.EffectClock, + ExitClasses: map[int]clicontract.ExitClass{0: clicontract.ExitSuccess, 1: clicontract.ExitFailure}, + } +} + +type runOptions struct { + output, runID, runtime, baseline, baselineMode string + contextMode, contextOffDir, contextOnDir, deltaOut string +} +type compareOptions struct { + output string + maxAggregate, maxDimension float64 +} +type baselineOptions struct{ output, promotedBy, rationale string } +type baselineAuditOptions struct{ root, baselineDir string } +type scorecardOptions struct { + output, kind string + maxCategory float64 +} +type coverageOptions struct { + root string + domains, evidenceKinds, dimensions, runtimes []string +} + +func (module Module) Command() *cobra.Command { + command := &cobra.Command{ + Use: "eval", Short: "Run deterministic local evaluation suites", + Long: `Run deterministic AgentOps evaluation suites and compare run records. + +The eval surface intentionally supports only offline deterministic runs in this +release. Live Claude and Codex adapters are evaluated by a later runtime tier.`, + } + command.GroupID = "workflow" + command.AddCommand(module.runCommand(), module.compareCommand(), module.baselineCommand(), module.baselineAuditCommand(), module.scorecardCommand(), module.coverageCommand()) + return command +} + +func (module Module) outputMode(command *cobra.Command) string { + if module.host.OutputMode == nil { + return "" + } + return module.host.OutputMode(command) +} + +func (module Module) runCommand() *cobra.Command { + options := runOptions{baselineMode: string(aoeval.BaselineModeSkillOn), contextMode: string(aoeval.ContextModeNone)} + command := &cobra.Command{Use: "run ", Short: "Run a deterministic eval suite", Args: cobra.ExactArgs(1)} + flags := command.Flags() + flags.StringVar(&options.output, "out", "", "write eval run record to path") + flags.StringVar(&options.runID, "run-id", "", "stable run id to use in the run record") + flags.StringVar(&options.runtime, "runtime", "", "runtime override (static, mock, shell, claude, codex)") + flags.StringVar(&options.baseline, "baseline", "", "compare the run against a baseline run record") + flags.StringVar(&options.baselineMode, "baseline-mode", string(aoeval.BaselineModeSkillOn), "skill-on | skill-off | both — runs the suite once with skills loaded, once with hooks suppressed, or both for a delta scorecard") + flags.StringVar(&options.contextMode, "context-mode", string(aoeval.ContextModeNone), "none | ab — run context-off/context-on legs over isolated AO_AGENTS_DIR roots") + flags.StringVar(&options.contextOffDir, "context-off-agents-dir", "", "AO_AGENTS_DIR root for the context-off leg (defaults to suite fixtures)") + flags.StringVar(&options.contextOnDir, "context-on-agents-dir", "", "AO_AGENTS_DIR root for the context-on leg (defaults to suite fixtures)") + flags.StringVar(&options.deltaOut, "delta-out", "", "write delta scorecard JSON to path (with --baseline-mode=both or --context-mode=ab)") + _ = command.RegisterFlagCompletionFunc("runtime", staticCompletion("static", "mock", "shell", "claude", "codex")) + _ = command.RegisterFlagCompletionFunc("baseline-mode", staticCompletion(aoeval.AllBaselineModes()...)) + _ = command.RegisterFlagCompletionFunc("context-mode", staticCompletion(aoeval.AllContextModes()...)) + command.RunE = func(command *cobra.Command, args []string) error { + result, err := module.core.Run(command.Context(), aoeval.CoreRunRequest{SuitePath: args[0], RunID: options.runID, Runtime: options.runtime, OutputPath: options.output, BaselinePath: options.baseline, BaselineMode: options.baselineMode, ContextMode: options.contextMode, ContextOffDir: options.contextOffDir, ContextOnDir: options.contextOnDir, DeltaOut: options.deltaOut}) + if err != nil { + return err + } + return module.renderRun(command, result) + } + return command +} + +func (module Module) compareCommand() *cobra.Command { + var options compareOptions + command := &cobra.Command{Use: "compare ", Short: "Compare an eval run against a baseline", Args: cobra.ExactArgs(2)} + command.Flags().StringVar(&options.output, "out", "", "write compared eval run record to path") + command.Flags().Float64Var(&options.maxAggregate, "max-aggregate-regression", 0, "allowed aggregate regression before verdict becomes regression") + command.Flags().Float64Var(&options.maxDimension, "max-dimension-regression", 0, "allowed per-dimension regression before verdict becomes regression") + command.RunE = func(command *cobra.Command, args []string) error { + result, err := module.core.Compare(command.Context(), aoeval.CoreCompareRequest{CandidatePath: args[0], BaselinePath: args[1], OutputPath: options.output, MaxAggregateRegression: options.maxAggregate, MaxDimensionRegression: options.maxDimension}) + if err != nil { + return err + } + if module.outputMode(command) == "json" { + return writeJSON(command, result.Candidate) + } + delta := 0.0 + if result.Candidate.BaselineComparison != nil { + delta = result.Candidate.BaselineComparison.AggregateDelta + } + fmt.Fprintf(command.OutOrStdout(), "Eval compare %s vs %s: %s (aggregate delta %.4f)\n", result.Candidate.RunID, result.Baseline.RunID, result.Candidate.Verdict, delta) + if result.OutputPath != "" { + fmt.Fprintf(command.OutOrStdout(), "Comparison record: %s\n", result.OutputPath) + } + return nil + } + return command +} + +func (module Module) baselineCommand() *cobra.Command { + var options baselineOptions + command := &cobra.Command{Use: "baseline ", Short: "Promote an eval run record as a baseline", Args: cobra.ExactArgs(1)} + command.Flags().StringVar(&options.output, "out", "", "write promoted baseline run record to path") + command.Flags().StringVar(&options.promotedBy, "promoted-by", "", "identity promoting the baseline") + command.Flags().StringVar(&options.rationale, "rationale", "", "rationale for promoting the baseline") + command.RunE = func(command *cobra.Command, args []string) error { + result, err := module.core.PromoteBaseline(command.Context(), aoeval.CoreBaselineRequest{RunPath: args[0], OutputPath: options.output, PromotedBy: options.promotedBy, Rationale: options.rationale}) + if err != nil { + return err + } + if module.outputMode(command) == "json" { + return writeJSON(command, result) + } + path := "" + if result.Baseline != nil { + path = result.Baseline.BaselinePath + } + fmt.Fprintf(command.OutOrStdout(), "Eval baseline promoted: %s\n", path) + return nil + } + return command +} + +func (module Module) baselineAuditCommand() *cobra.Command { + options := baselineAuditOptions{root: "evals/agentops-core", baselineDir: ".agents/evals/baselines"} + command := &cobra.Command{Use: "baseline-audit [suite.json ...]", Short: "Audit eval suite baseline policy against promoted baselines", Args: cobra.ArbitraryArgs} + command.Flags().StringVar(&options.root, "root", options.root, "suite root to scan when no suite paths are provided") + command.Flags().StringVar(&options.baselineDir, "baseline-dir", options.baselineDir, "promoted baseline directory") + command.RunE = func(command *cobra.Command, args []string) error { + report, err := module.core.AuditBaseline(command.Context(), aoeval.CoreBaselineAuditRequest{SuitePaths: args, Root: options.root, BaselineDir: options.baselineDir}) + if err != nil { + return err + } + if module.outputMode(command) == "json" { + return writeJSON(command, report) + } + fmt.Fprintf(command.OutOrStdout(), "Eval baseline audit: %d suites, %d baselines, %d policy mismatches\n", report.SuiteCount, report.BaselineCount, report.PolicyMismatchCount) + if len(report.MissingCompareBaselines) > 0 { + fmt.Fprintf(command.OutOrStdout(), "Missing compare baselines: %d\n", len(report.MissingCompareBaselines)) + } + if len(report.UnexpectedBaselinesForNone) > 0 { + fmt.Fprintf(command.OutOrStdout(), "Unexpected baselines for none policy: %d\n", len(report.UnexpectedBaselinesForNone)) + } + if len(report.OrphanBaselines) > 0 { + fmt.Fprintf(command.OutOrStdout(), "Orphan baselines: %d\n", len(report.OrphanBaselines)) + } + if len(report.StaleSuiteHashes) > 0 { + fmt.Fprintf(command.OutOrStdout(), "Stale suite hashes: %d\n", len(report.StaleSuiteHashes)) + } + return nil + } + return command +} + +func (module Module) scorecardCommand() *cobra.Command { + options := scorecardOptions{kind: string(aoeval.ScorecardKindRPI)} + command := &cobra.Command{Use: "scorecard [baseline-run.json]", Short: "Build an eval scorecard from run records", Args: cobra.RangeArgs(1, 2)} + command.Flags().StringVar(&options.output, "out", "", "write scorecard JSON to path") + command.Flags().StringVar(&options.kind, "kind", options.kind, "scorecard kind (rpi, skill-change)") + command.Flags().Float64Var(&options.maxCategory, "max-category-regression", 0, "allowed per-category regression before verdict becomes regression") + _ = command.RegisterFlagCompletionFunc("kind", staticCompletion(string(aoeval.ScorecardKindRPI), string(aoeval.ScorecardKindSkillChange))) + command.RunE = func(command *cobra.Command, args []string) error { + baselinePath := "" + if len(args) == 2 { + baselinePath = args[1] + } + card, err := module.core.Scorecard(command.Context(), aoeval.CoreScorecardRequest{CandidatePath: args[0], BaselinePath: baselinePath, OutputPath: options.output, Kind: options.kind, MaxCategoryRegression: options.maxCategory}) + if err != nil { + return err + } + if module.outputMode(command) == "json" { + return writeJSON(command, card) + } + fmt.Fprintf(command.OutOrStdout(), "Eval scorecard %s: %s (%s, categories %d)\n", card.CandidateRunID, card.Verdict, card.Kind, len(card.Categories)) + if options.output != "" { + fmt.Fprintf(command.OutOrStdout(), "Scorecard: %s\n", options.output) + } + return nil + } + return command +} + +func (module Module) coverageCommand() *cobra.Command { + options := coverageOptions{root: "evals/agentops-core", domains: append([]string(nil), aoeval.DefaultCoverageDomains...), dimensions: append([]string(nil), aoeval.DefaultCoverageDimensions...), runtimes: append([]string(nil), aoeval.DefaultCoverageRuntimes...)} + command := &cobra.Command{Use: "coverage [suite.json ...]", Short: "Summarize eval suite coverage", Args: cobra.ArbitraryArgs} + command.Flags().StringVar(&options.root, "root", options.root, "suite root to scan when no suite paths are provided") + command.Flags().StringArrayVar(&options.domains, "require-domain", options.domains, "required product domain for missing-domain reporting") + command.Flags().StringArrayVar(&options.evidenceKinds, "require-evidence-kind", nil, "required evidence kind for missing-evidence-kind reporting") + command.Flags().StringArrayVar(&options.dimensions, "require-dimension", options.dimensions, "required score dimension for missing-dimension reporting") + command.Flags().StringArrayVar(&options.runtimes, "require-runtime", options.runtimes, "required deterministic runtime for missing-runtime reporting") + command.RunE = func(command *cobra.Command, args []string) error { + report, err := module.core.Coverage(command.Context(), aoeval.CoreCoverageRequest{SuitePaths: args, Root: options.root, RequiredDomains: options.domains, RequiredEvidenceKinds: options.evidenceKinds, RequiredDimensions: options.dimensions, RequiredRuntimes: options.runtimes}) + if err != nil { + return err + } + if module.outputMode(command) == "json" { + return writeJSON(command, report) + } + fmt.Fprintf(command.OutOrStdout(), "Eval coverage: %d suites, %d cases, %d critical cases\n", report.SuiteCount, report.CaseCount, report.CriticalCaseCount) + renderCoverage(command, "domains", report.MissingRequiredDomains, report.RequiredDomains) + renderCoverage(command, "evidence kinds", report.MissingRequiredEvidenceKinds, report.RequiredEvidenceKinds) + renderCoverage(command, "dimensions", report.MissingRequiredDimensions, report.RequiredDimensions) + renderCoverage(command, "runtimes", report.MissingRequiredRuntimes, report.RequiredRuntimes) + return nil + } + return command +} + +func (module Module) renderRun(command *cobra.Command, result aoeval.CoreRunResult) error { + var value any = result.Run + if result.Mode == aoeval.CoreRunBaselineAB { + value = result.Delta + } + if result.Mode == aoeval.CoreRunContextAB { + value = result.ContextDelta + } + if module.outputMode(command) == "json" { + return writeJSON(command, value) + } + switch result.Mode { + case aoeval.CoreRunContextAB: + card := result.ContextDelta + fmt.Fprintf(command.OutOrStdout(), "Eval context-AB %s: context-off=%.4f (%s) context-on=%.4f (%s) delta=%+.4f cases=%d\n", card.SuiteID, result.FirstRun.AggregateScore, result.FirstRun.Status, result.SecondRun.AggregateScore, result.SecondRun.Status, card.AggregateDelta, len(card.PerCase)) + if result.OutputPath != "" { + fmt.Fprintf(command.OutOrStdout(), "Context delta scorecard: %s\n", result.OutputPath) + } + case aoeval.CoreRunBaselineAB: + card := result.Delta + fmt.Fprintf(command.OutOrStdout(), "Eval baseline-AB %s: skill-on=%.4f (%s) skill-off=%.4f (%s) delta=%+.4f cases=%d\n", card.SuiteID, result.FirstRun.AggregateScore, result.FirstRun.Status, result.SecondRun.AggregateScore, result.SecondRun.Status, card.AggregateDelta, len(card.PerCase)) + if result.OutputPath != "" { + fmt.Fprintf(command.OutOrStdout(), "Delta scorecard: %s\n", result.OutputPath) + } + default: + run := result.Run + fmt.Fprintf(command.OutOrStdout(), "Eval %s: %s (aggregate %.4f, cases %d)\n", run.RunID, run.Status, run.AggregateScore, len(run.CaseResults)) + if result.OutputPath != "" { + fmt.Fprintf(command.OutOrStdout(), "Run record: %s\n", result.OutputPath) + } + } + return nil +} + +func renderCoverage(command *cobra.Command, label string, missing, required []string) { + if len(missing) > 0 { + fmt.Fprintf(command.OutOrStdout(), "Missing required %s: %s\n", label, strings.Join(missing, ", ")) + return + } + if len(required) > 0 { + fmt.Fprintf(command.OutOrStdout(), "Required %s covered\n", label) + } +} + +func writeJSON(command *cobra.Command, value any) error { + encoder := json.NewEncoder(command.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} + +func staticCompletion(values ...string) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return append([]string(nil), values...), cobra.ShellCompDirectiveNoFileComp + } +} diff --git a/cli/internal/commands/eval/module_test.go b/cli/internal/commands/eval/module_test.go new file mode 100644 index 000000000..5709863e7 --- /dev/null +++ b/cli/internal/commands/eval/module_test.go @@ -0,0 +1,81 @@ +package eval + +import ( + "context" + "strings" + "testing" + + "github.com/spf13/cobra" + + aoeval "github.com/boshu2/agentops/cli/internal/eval" +) + +type coreUseCasesSpy struct { + runRequest aoeval.CoreRunRequest +} + +func (useCases *coreUseCasesSpy) Run(_ context.Context, request aoeval.CoreRunRequest) (aoeval.CoreRunResult, error) { + useCases.runRequest = request + return aoeval.CoreRunResult{Mode: aoeval.CoreRunSingle, Run: &aoeval.RunRecord{RunID: "run-1", Status: aoeval.StatusPass}}, nil +} +func (*coreUseCasesSpy) Compare(context.Context, aoeval.CoreCompareRequest) (aoeval.CoreCompareResult, error) { + return aoeval.CoreCompareResult{}, nil +} +func (*coreUseCasesSpy) PromoteBaseline(context.Context, aoeval.CoreBaselineRequest) (*aoeval.RunRecord, error) { + return &aoeval.RunRecord{}, nil +} +func (*coreUseCasesSpy) AuditBaseline(context.Context, aoeval.CoreBaselineAuditRequest) (*aoeval.BaselineAuditReport, error) { + return &aoeval.BaselineAuditReport{}, nil +} +func (*coreUseCasesSpy) Scorecard(context.Context, aoeval.CoreScorecardRequest) (*aoeval.Scorecard, error) { + return &aoeval.Scorecard{}, nil +} +func (*coreUseCasesSpy) Coverage(context.Context, aoeval.CoreCoverageRequest) (*aoeval.CoverageReport, error) { + return &aoeval.CoverageReport{}, nil +} + +func TestModuleRunParsesClosureLocalFlagsAndDelegates(t *testing.T) { + useCases := &coreUseCasesSpy{} + command := NewModule(useCases, HostOptions{}).Command() + command.SetArgs([]string{"run", "suite.json", "--run-id", "run-1", "--runtime", "static", "--baseline-mode", "skill-on"}) + var output strings.Builder + command.SetOut(&output) + command.SetErr(&output) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if useCases.runRequest.SuitePath != "suite.json" || useCases.runRequest.RunID != "run-1" || useCases.runRequest.Runtime != "static" { + t.Fatalf("request = %#v", useCases.runRequest) + } + if !strings.Contains(output.String(), "Eval run-1: pass") { + t.Fatalf("output = %q", output.String()) + } +} + +func TestModuleCommandsDoNotShareFlagState(t *testing.T) { + first := NewModule(&coreUseCasesSpy{}, HostOptions{}).Command() + second := NewModule(&coreUseCasesSpy{}, HostOptions{}).Command() + firstRun, _, err := first.Find([]string{"run"}) + if err != nil { + t.Fatal(err) + } + if err := firstRun.Flags().Set("run-id", "changed"); err != nil { + t.Fatal(err) + } + secondRun, _, err := second.Find([]string{"run"}) + if err != nil { + t.Fatal(err) + } + if got := secondRun.Flag("run-id").Value.String(); got != "" { + t.Fatalf("second run-id = %q, want empty", got) + } +} + +func findChild(t *testing.T, command *cobra.Command, name string) *cobra.Command { + t.Helper() + child, _, err := command.Find([]string{name}) + if err != nil { + t.Fatal(err) + } + return child +} diff --git a/cli/internal/eval/core_service.go b/cli/internal/eval/core_service.go new file mode 100644 index 000000000..b89f217b1 --- /dev/null +++ b/cli/internal/eval/core_service.go @@ -0,0 +1,242 @@ +package eval + +import ( + "context" + "fmt" + "path/filepath" + "strings" +) + +type CoreRuntime interface { + RunSuite(RunOptions) (*RunRecord, error) + RunBaselineAB(RunOptions) (DeltaScorecard, *RunRecord, *RunRecord, error) + WriteDeltaScorecard(DeltaScorecard, string) error + RunContextAB(RunOptions, ContextABOptions) (ContextDeltaScorecard, *RunRecord, *RunRecord, error) + WriteContextDeltaScorecard(ContextDeltaScorecard, string) error + LoadRun(string) (*RunRecord, error) + CompareRuns(*RunRecord, *RunRecord, CompareOptions) (*RunRecord, error) + WorkDir() (string, error) + Abs(string) (string, error) + PromoteBaseline(*RunRecord, BaselineOptions) (*RunRecord, error) + AuditBaselinePolicy(BaselineAuditOptions) (*BaselineAuditReport, error) + BuildScorecard(*RunRecord, *RunRecord, ScorecardOptions) (*Scorecard, error) + WriteScorecard(string, *Scorecard) error + BuildCoverageReport(CoverageOptions) (*CoverageReport, error) +} + +type CoreService struct{ Runtime CoreRuntime } + +type CoreRunRequest struct { + SuitePath, RunID, Runtime, OutputPath, BaselinePath string + BaselineMode, ContextMode, ContextOffDir, ContextOnDir, DeltaOut string +} + +type CoreRunMode string + +const ( + CoreRunSingle CoreRunMode = "single" + CoreRunBaselineAB CoreRunMode = "baseline-ab" + CoreRunContextAB CoreRunMode = "context-ab" +) + +type CoreRunResult struct { + Mode CoreRunMode + Run *RunRecord + FirstRun *RunRecord + SecondRun *RunRecord + Delta *DeltaScorecard + ContextDelta *ContextDeltaScorecard + OutputPath string +} + +type CoreCompareRequest struct { + CandidatePath, BaselinePath, OutputPath string + MaxAggregateRegression, MaxDimensionRegression float64 +} + +type CoreCompareResult struct { + Candidate *RunRecord + Baseline *RunRecord + OutputPath string +} + +type CoreBaselineRequest struct{ RunPath, OutputPath, PromotedBy, Rationale string } +type CoreBaselineAuditRequest struct { + SuitePaths []string + Root, BaselineDir string +} +type CoreScorecardRequest struct { + CandidatePath, BaselinePath, OutputPath, Kind string + MaxCategoryRegression float64 +} +type CoreCoverageRequest struct { + SuitePaths []string + Root string + RequiredDomains, RequiredEvidenceKinds, RequiredDimensions, RequiredRuntimes []string +} + +func (service CoreService) Run(_ context.Context, request CoreRunRequest) (CoreRunResult, error) { + runtimeName, err := parseCoreRuntime(request.Runtime) + if err != nil { + return CoreRunResult{}, err + } + mode := ABBaselineMode(request.BaselineMode) + if !IsValidBaselineMode(string(mode)) { + return CoreRunResult{}, fmt.Errorf("invalid --baseline-mode %q (allowed: %s)", request.BaselineMode, strings.Join(AllBaselineModes(), ", ")) + } + contextMode := ContextMode(request.ContextMode) + if !IsValidContextMode(string(contextMode)) { + return CoreRunResult{}, fmt.Errorf("invalid --context-mode %q (allowed: %s)", request.ContextMode, strings.Join(AllContextModes(), ", ")) + } + options := RunOptions{SuitePath: request.SuitePath, RunID: request.RunID, Runtime: runtimeName, OutputPath: request.OutputPath, BaselinePath: request.BaselinePath} + if contextMode == ContextModeAB { + if mode != BaselineModeSkillOn { + return CoreRunResult{}, fmt.Errorf("--context-mode=ab cannot be combined with --baseline-mode=%s", mode) + } + contextOptions := service.contextOptions(request.SuitePath, request.ContextOffDir, request.ContextOnDir) + scorecard, offRun, onRun, err := service.Runtime.RunContextAB(options, contextOptions) + if err != nil { + return CoreRunResult{}, err + } + if err := service.Runtime.WriteContextDeltaScorecard(scorecard, request.DeltaOut); err != nil { + return CoreRunResult{}, err + } + return CoreRunResult{Mode: CoreRunContextAB, FirstRun: offRun, SecondRun: onRun, ContextDelta: &scorecard, OutputPath: request.DeltaOut}, nil + } + if mode == BaselineModeBoth { + scorecard, onRun, offRun, err := service.Runtime.RunBaselineAB(options) + if err != nil { + return CoreRunResult{}, err + } + if err := service.Runtime.WriteDeltaScorecard(scorecard, request.DeltaOut); err != nil { + return CoreRunResult{}, err + } + return CoreRunResult{Mode: CoreRunBaselineAB, FirstRun: onRun, SecondRun: offRun, Delta: &scorecard, OutputPath: request.DeltaOut}, nil + } + options.OverrideDisableHooks = mode == BaselineModeSkillOff + run, err := service.Runtime.RunSuite(options) + if err != nil { + return CoreRunResult{}, err + } + return CoreRunResult{Mode: CoreRunSingle, Run: run, OutputPath: request.OutputPath}, nil +} + +func (service CoreService) Compare(_ context.Context, request CoreCompareRequest) (CoreCompareResult, error) { + candidate, err := service.Runtime.LoadRun(request.CandidatePath) + if err != nil { + return CoreCompareResult{}, err + } + baseline, err := service.Runtime.LoadRun(request.BaselinePath) + if err != nil { + return CoreCompareResult{}, err + } + compared, err := service.Runtime.CompareRuns(candidate, baseline, CompareOptions{MaxAggregateRegression: request.MaxAggregateRegression, MaxDimensionRegression: request.MaxDimensionRegression, OutputPath: request.OutputPath}) + if err != nil { + return CoreCompareResult{}, err + } + return CoreCompareResult{Candidate: compared, Baseline: baseline, OutputPath: request.OutputPath}, nil +} + +func (service CoreService) PromoteBaseline(_ context.Context, request CoreBaselineRequest) (*RunRecord, error) { + run, err := service.Runtime.LoadRun(request.RunPath) + if err != nil { + return nil, err + } + workDir, err := service.Runtime.WorkDir() + if err != nil { + return nil, err + } + return service.Runtime.PromoteBaseline(run, BaselineOptions{OutputPath: request.OutputPath, PromotedBy: request.PromotedBy, Rationale: request.Rationale, WorkDir: workDir}) +} + +func (service CoreService) AuditBaseline(_ context.Context, request CoreBaselineAuditRequest) (*BaselineAuditReport, error) { + roots := []string{} + if len(request.SuitePaths) == 0 { + roots = append(roots, request.Root) + } + return service.Runtime.AuditBaselinePolicy(BaselineAuditOptions{SuitePaths: request.SuitePaths, Roots: roots, BaselineDir: request.BaselineDir}) +} + +func (service CoreService) Scorecard(_ context.Context, request CoreScorecardRequest) (*Scorecard, error) { + kind, err := parseCoreScorecardKind(request.Kind) + if err != nil { + return nil, err + } + candidate, err := service.Runtime.LoadRun(request.CandidatePath) + if err != nil { + return nil, err + } + var baseline *RunRecord + if request.BaselinePath != "" { + baseline, err = service.Runtime.LoadRun(request.BaselinePath) + if err != nil { + return nil, err + } + } + card, err := service.Runtime.BuildScorecard(candidate, baseline, ScorecardOptions{Kind: kind, MaxCategoryRegression: request.MaxCategoryRegression}) + if err != nil { + return nil, err + } + if request.OutputPath != "" { + if err := service.Runtime.WriteScorecard(request.OutputPath, card); err != nil { + return nil, err + } + } + return card, nil +} + +func (service CoreService) Coverage(_ context.Context, request CoreCoverageRequest) (*CoverageReport, error) { + roots := []string{} + if len(request.SuitePaths) == 0 { + roots = append(roots, request.Root) + } + return service.Runtime.BuildCoverageReport(CoverageOptions{SuitePaths: request.SuitePaths, Roots: roots, RequiredDomains: request.RequiredDomains, RequiredEvidenceKinds: request.RequiredEvidenceKinds, RequiredDimensions: request.RequiredDimensions, RequiredRuntimes: request.RequiredRuntimes}) +} + +func (service CoreService) contextOptions(suitePath, offDir, onDir string) ContextABOptions { + if offDir == "" { + offDir = defaultCoreContextDir(suitePath, "context-off") + } + if onDir == "" { + onDir = defaultCoreContextDir(suitePath, "context-on") + } + if absolute, err := service.Runtime.Abs(offDir); err == nil { + offDir = absolute + } + if absolute, err := service.Runtime.Abs(onDir); err == nil { + onDir = absolute + } + return ContextABOptions{ContextOffAgentsDir: offDir, ContextOnAgentsDir: onDir, ContextOffLabel: "context-off", ContextOnLabel: "context-on"} +} + +func defaultCoreContextDir(suitePath, leg string) string { + base := filepath.Base(suitePath) + name := strings.TrimSuffix(base, filepath.Ext(base)) + return filepath.Join(filepath.Dir(suitePath), "fixtures", name, leg, "agents") +} + +func parseCoreRuntime(value string) (Runtime, error) { + value = strings.TrimSpace(value) + if value == "" { + return "", nil + } + switch Runtime(value) { + case RuntimeStatic, RuntimeMock, RuntimeShell, RuntimeClaude, RuntimeCodex: + return Runtime(value), nil + default: + return "", fmt.Errorf("unknown runtime %q (use static, mock, shell, claude, or codex)", value) + } +} + +func parseCoreScorecardKind(value string) (ScorecardKind, error) { + value = strings.TrimSpace(value) + if value == "" { + return ScorecardKindRPI, nil + } + switch ScorecardKind(value) { + case ScorecardKindRPI, ScorecardKindSkillChange: + return ScorecardKind(value), nil + default: + return "", fmt.Errorf("unsupported scorecard kind %q (use rpi or skill-change)", value) + } +} diff --git a/cli/internal/eval/core_service_test.go b/cli/internal/eval/core_service_test.go new file mode 100644 index 000000000..8d40e7eed --- /dev/null +++ b/cli/internal/eval/core_service_test.go @@ -0,0 +1,74 @@ +package eval + +import ( + "context" + "strings" + "testing" +) + +type coreRuntimeSpy struct { + runCalls int +} + +func (runtime *coreRuntimeSpy) RunSuite(options RunOptions) (*RunRecord, error) { + runtime.runCalls++ + return &RunRecord{RunID: options.RunID, Status: StatusPass}, nil +} +func (*coreRuntimeSpy) RunBaselineAB(RunOptions) (DeltaScorecard, *RunRecord, *RunRecord, error) { + return DeltaScorecard{}, &RunRecord{}, &RunRecord{}, nil +} +func (*coreRuntimeSpy) WriteDeltaScorecard(DeltaScorecard, string) error { return nil } +func (*coreRuntimeSpy) RunContextAB(RunOptions, ContextABOptions) (ContextDeltaScorecard, *RunRecord, *RunRecord, error) { + return ContextDeltaScorecard{}, &RunRecord{}, &RunRecord{}, nil +} +func (*coreRuntimeSpy) WriteContextDeltaScorecard(ContextDeltaScorecard, string) error { return nil } +func (*coreRuntimeSpy) LoadRun(string) (*RunRecord, error) { return &RunRecord{}, nil } +func (*coreRuntimeSpy) CompareRuns(*RunRecord, *RunRecord, CompareOptions) (*RunRecord, error) { + return &RunRecord{}, nil +} +func (*coreRuntimeSpy) WorkDir() (string, error) { return "/work", nil } +func (*coreRuntimeSpy) Abs(path string) (string, error) { return "/abs/" + path, nil } +func (*coreRuntimeSpy) PromoteBaseline(*RunRecord, BaselineOptions) (*RunRecord, error) { + return &RunRecord{}, nil +} +func (*coreRuntimeSpy) AuditBaselinePolicy(BaselineAuditOptions) (*BaselineAuditReport, error) { + return &BaselineAuditReport{}, nil +} +func (*coreRuntimeSpy) BuildScorecard(*RunRecord, *RunRecord, ScorecardOptions) (*Scorecard, error) { + return &Scorecard{}, nil +} +func (*coreRuntimeSpy) WriteScorecard(string, *Scorecard) error { return nil } +func (*coreRuntimeSpy) BuildCoverageReport(CoverageOptions) (*CoverageReport, error) { + return &CoverageReport{}, nil +} + +func TestCoreServiceRunRejectsUnknownRuntimeBeforeEffects(t *testing.T) { + runtime := &coreRuntimeSpy{} + service := CoreService{Runtime: runtime} + _, err := service.Run(context.Background(), CoreRunRequest{ + SuitePath: "suite.json", Runtime: "bogus", BaselineMode: "skill-on", ContextMode: "none", + }) + if err == nil || !strings.Contains(err.Error(), "unknown runtime") { + t.Fatalf("Run error = %v, want unknown runtime", err) + } + if runtime.runCalls != 0 { + t.Fatalf("RunSuite calls = %d, want 0", runtime.runCalls) + } +} + +func TestCoreServiceRunDelegatesSingleRun(t *testing.T) { + runtime := &coreRuntimeSpy{} + service := CoreService{Runtime: runtime} + result, err := service.Run(context.Background(), CoreRunRequest{ + SuitePath: "suite.json", RunID: "run-1", Runtime: "static", BaselineMode: "skill-off", ContextMode: "none", + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if result.Mode != CoreRunSingle || result.Run == nil || result.Run.RunID != "run-1" { + t.Fatalf("result = %#v", result) + } + if runtime.runCalls != 1 { + t.Fatalf("RunSuite calls = %d, want 1", runtime.runCalls) + } +} From 5b558831c5aec7d8b837e45916523f76ecba6a0e Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:22:33 -0400 Subject: [PATCH 089/108] refactor(eval): extract cleanup service and command --- cli/internal/adapters/eval/runtime.go | 69 +++++++++++++ cli/internal/adapters/eval/runtime_test.go | 1 + cli/internal/commands/eval/module.go | 78 +++++++++++++-- cli/internal/commands/eval/module_test.go | 30 +++++- cli/internal/eval/cleanup_service.go | 107 +++++++++++++++++++++ cli/internal/eval/cleanup_service_test.go | 57 +++++++++++ 6 files changed, 330 insertions(+), 12 deletions(-) create mode 100644 cli/internal/eval/cleanup_service.go create mode 100644 cli/internal/eval/cleanup_service_test.go diff --git a/cli/internal/adapters/eval/runtime.go b/cli/internal/adapters/eval/runtime.go index 9e13fa810..b04d5ce5d 100644 --- a/cli/internal/adapters/eval/runtime.go +++ b/cli/internal/adapters/eval/runtime.go @@ -1,10 +1,16 @@ package eval import ( + "encoding/json" + "fmt" "os" "path/filepath" + "sort" + "strings" + "time" aoeval "github.com/boshu2/agentops/cli/internal/eval" + "github.com/boshu2/agentops/cli/internal/evalsubstrate" ) type Runtime struct{} @@ -45,3 +51,66 @@ func (Runtime) WriteScorecard(path string, card *aoeval.Scorecard) error { func (Runtime) BuildCoverageReport(options aoeval.CoverageOptions) (*aoeval.CoverageReport, error) { return aoeval.BuildCoverageReport(options) } + +func (Runtime) Root() string { + if root := strings.TrimSpace(os.Getenv("AGENTOPS_EVALS_ROOT")); root != "" { + return root + } + home, err := os.UserHomeDir() + if err != nil { + return ".agents/evals" + } + return filepath.Join(home, ".agents", "evals") +} + +func (Runtime) ListRunIDs(root string) ([]string, error) { + entries, err := os.ReadDir(filepath.Join(root, "runs")) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + ids := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + ids = append(ids, entry.Name()) + } + } + sort.Strings(ids) + return ids, nil +} + +func (Runtime) LoadManifest(root, id string) (*evalsubstrate.Manifest, error) { + return evalsubstrate.LoadManifest(evalsubstrate.ManifestPath(root, id)) +} + +func (Runtime) Transition(root, id string, next evalsubstrate.RunStatus, reason string, now time.Time) error { + path := evalsubstrate.ManifestPath(root, id) + manifest, err := evalsubstrate.LoadManifest(path) + if err != nil { + return fmt.Errorf("transitionStale: load: %w", err) + } + if !legalCleanupTransition(manifest.Status, next) { + return nil + } + manifest.Status = next + manifest.RetractionReason = reason + manifest.FinishedAt = now.UTC().Format(time.RFC3339) + manifest.FinishedAtUnixMs = now.UTC().UnixMilli() + data, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return fmt.Errorf("transitionStale: marshal: %w", err) + } + return evalsubstrate.WriteAtomic(path, append(data, '\n')) +} + +func legalCleanupTransition(current, next evalsubstrate.RunStatus) bool { + return current == evalsubstrate.StatusPending && next == evalsubstrate.StatusAborted || current == evalsubstrate.StatusRunning && next == evalsubstrate.StatusFailed +} + +func (Runtime) DeleteRun(root, id string) error { return os.RemoveAll(filepath.Join(root, "runs", id)) } +func (Runtime) SweepTempFiles(root string, age int64) ([]string, error) { + return evalsubstrate.SweepTempFiles(root, age) +} +func (Runtime) Now() time.Time { return time.Now().UTC() } diff --git a/cli/internal/adapters/eval/runtime_test.go b/cli/internal/adapters/eval/runtime_test.go index 918840d1b..296550c99 100644 --- a/cli/internal/adapters/eval/runtime_test.go +++ b/cli/internal/adapters/eval/runtime_test.go @@ -9,6 +9,7 @@ import ( func TestRuntimeSatisfiesCorePortAndResolvesHostPaths(t *testing.T) { var port aoeval.CoreRuntime = Runtime{} + var _ aoeval.CleanupRuntime = Runtime{} workDir, err := port.WorkDir() if err != nil { t.Fatalf("WorkDir: %v", err) diff --git a/cli/internal/commands/eval/module.go b/cli/internal/commands/eval/module.go index 523141bb3..fb5102974 100644 --- a/cli/internal/commands/eval/module.go +++ b/cli/internal/commands/eval/module.go @@ -22,16 +22,28 @@ type CoreUseCases interface { Coverage(context.Context, aoeval.CoreCoverageRequest) (*aoeval.CoverageReport, error) } +type CleanupUseCases interface { + Execute(context.Context, aoeval.CleanupRequest) (aoeval.CleanupReport, error) +} + +type UseCases struct { + Core CoreUseCases + Cleanup CleanupUseCases +} + type HostOptions struct { OutputMode func(*cobra.Command) string + Verbose func(*cobra.Command) bool } type Module struct { - core CoreUseCases - host HostOptions + useCases UseCases + host HostOptions } -func NewModule(core CoreUseCases, host HostOptions) Module { return Module{core: core, host: host} } +func NewModule(useCases UseCases, host HostOptions) Module { + return Module{useCases: useCases, host: host} +} func (Module) Contract() clicontract.CommandContract { return clicontract.CommandContract{ @@ -74,6 +86,9 @@ release. Live Claude and Codex adapters are evaluated by a later runtime tier.`, } command.GroupID = "workflow" command.AddCommand(module.runCommand(), module.compareCommand(), module.baselineCommand(), module.baselineAuditCommand(), module.scorecardCommand(), module.coverageCommand()) + if module.useCases.Cleanup != nil { + command.AddCommand(module.cleanupCommand()) + } return command } @@ -101,7 +116,7 @@ func (module Module) runCommand() *cobra.Command { _ = command.RegisterFlagCompletionFunc("baseline-mode", staticCompletion(aoeval.AllBaselineModes()...)) _ = command.RegisterFlagCompletionFunc("context-mode", staticCompletion(aoeval.AllContextModes()...)) command.RunE = func(command *cobra.Command, args []string) error { - result, err := module.core.Run(command.Context(), aoeval.CoreRunRequest{SuitePath: args[0], RunID: options.runID, Runtime: options.runtime, OutputPath: options.output, BaselinePath: options.baseline, BaselineMode: options.baselineMode, ContextMode: options.contextMode, ContextOffDir: options.contextOffDir, ContextOnDir: options.contextOnDir, DeltaOut: options.deltaOut}) + result, err := module.useCases.Core.Run(command.Context(), aoeval.CoreRunRequest{SuitePath: args[0], RunID: options.runID, Runtime: options.runtime, OutputPath: options.output, BaselinePath: options.baseline, BaselineMode: options.baselineMode, ContextMode: options.contextMode, ContextOffDir: options.contextOffDir, ContextOnDir: options.contextOnDir, DeltaOut: options.deltaOut}) if err != nil { return err } @@ -117,7 +132,7 @@ func (module Module) compareCommand() *cobra.Command { command.Flags().Float64Var(&options.maxAggregate, "max-aggregate-regression", 0, "allowed aggregate regression before verdict becomes regression") command.Flags().Float64Var(&options.maxDimension, "max-dimension-regression", 0, "allowed per-dimension regression before verdict becomes regression") command.RunE = func(command *cobra.Command, args []string) error { - result, err := module.core.Compare(command.Context(), aoeval.CoreCompareRequest{CandidatePath: args[0], BaselinePath: args[1], OutputPath: options.output, MaxAggregateRegression: options.maxAggregate, MaxDimensionRegression: options.maxDimension}) + result, err := module.useCases.Core.Compare(command.Context(), aoeval.CoreCompareRequest{CandidatePath: args[0], BaselinePath: args[1], OutputPath: options.output, MaxAggregateRegression: options.maxAggregate, MaxDimensionRegression: options.maxDimension}) if err != nil { return err } @@ -144,7 +159,7 @@ func (module Module) baselineCommand() *cobra.Command { command.Flags().StringVar(&options.promotedBy, "promoted-by", "", "identity promoting the baseline") command.Flags().StringVar(&options.rationale, "rationale", "", "rationale for promoting the baseline") command.RunE = func(command *cobra.Command, args []string) error { - result, err := module.core.PromoteBaseline(command.Context(), aoeval.CoreBaselineRequest{RunPath: args[0], OutputPath: options.output, PromotedBy: options.promotedBy, Rationale: options.rationale}) + result, err := module.useCases.Core.PromoteBaseline(command.Context(), aoeval.CoreBaselineRequest{RunPath: args[0], OutputPath: options.output, PromotedBy: options.promotedBy, Rationale: options.rationale}) if err != nil { return err } @@ -167,7 +182,7 @@ func (module Module) baselineAuditCommand() *cobra.Command { command.Flags().StringVar(&options.root, "root", options.root, "suite root to scan when no suite paths are provided") command.Flags().StringVar(&options.baselineDir, "baseline-dir", options.baselineDir, "promoted baseline directory") command.RunE = func(command *cobra.Command, args []string) error { - report, err := module.core.AuditBaseline(command.Context(), aoeval.CoreBaselineAuditRequest{SuitePaths: args, Root: options.root, BaselineDir: options.baselineDir}) + report, err := module.useCases.Core.AuditBaseline(command.Context(), aoeval.CoreBaselineAuditRequest{SuitePaths: args, Root: options.root, BaselineDir: options.baselineDir}) if err != nil { return err } @@ -204,7 +219,7 @@ func (module Module) scorecardCommand() *cobra.Command { if len(args) == 2 { baselinePath = args[1] } - card, err := module.core.Scorecard(command.Context(), aoeval.CoreScorecardRequest{CandidatePath: args[0], BaselinePath: baselinePath, OutputPath: options.output, Kind: options.kind, MaxCategoryRegression: options.maxCategory}) + card, err := module.useCases.Core.Scorecard(command.Context(), aoeval.CoreScorecardRequest{CandidatePath: args[0], BaselinePath: baselinePath, OutputPath: options.output, Kind: options.kind, MaxCategoryRegression: options.maxCategory}) if err != nil { return err } @@ -229,7 +244,7 @@ func (module Module) coverageCommand() *cobra.Command { command.Flags().StringArrayVar(&options.dimensions, "require-dimension", options.dimensions, "required score dimension for missing-dimension reporting") command.Flags().StringArrayVar(&options.runtimes, "require-runtime", options.runtimes, "required deterministic runtime for missing-runtime reporting") command.RunE = func(command *cobra.Command, args []string) error { - report, err := module.core.Coverage(command.Context(), aoeval.CoreCoverageRequest{SuitePaths: args, Root: options.root, RequiredDomains: options.domains, RequiredEvidenceKinds: options.evidenceKinds, RequiredDimensions: options.dimensions, RequiredRuntimes: options.runtimes}) + report, err := module.useCases.Core.Coverage(command.Context(), aoeval.CoreCoverageRequest{SuitePaths: args, Root: options.root, RequiredDomains: options.domains, RequiredEvidenceKinds: options.evidenceKinds, RequiredDimensions: options.dimensions, RequiredRuntimes: options.runtimes}) if err != nil { return err } @@ -246,6 +261,51 @@ func (module Module) coverageCommand() *cobra.Command { return command } +func (module Module) cleanupCommand() *cobra.Command { + options := aoeval.CleanupRequest{TmpAgeSeconds: 60} + command := &cobra.Command{ + Use: "cleanup", Short: "Recover stale Runs (state transitions, --delete, --tmp-files)", + Long: `Per SCHEMA.md §4 cleanup state-transition rule (rc2): + + Stale pending (no running transition within 60s) + -> aborted (retraction_reason=never_started) + Stale running (no heartbeat within 5min OR Inspect process not alive) + -> failed (retraction_reason=orphaned_process) + +After transitions: + --delete Remove Run dirs whose status is failed OR aborted (NEVER retracted). + --tmp-files Sweep orphan manifest.json.tmp left from rename-step crashes. + --dry-run Print what would be done; no mutations. + +The cleanup procedure honors the §4 atomic-write contract on every state +transition (temp + fsync + rename + fsync-parent-dir). Retracted Runs are +never auto-removed — retraction is an audit trail.`, + } + command.Flags().BoolVar(&options.Delete, "delete", false, "Remove Run directories whose status is failed or aborted (never retracted)") + command.Flags().BoolVar(&options.TmpFiles, "tmp-files", false, "Sweep orphan *.tmp files older than --tmp-age") + command.Flags().Int64Var(&options.TmpAgeSeconds, "tmp-age", 60, "Minimum tmp-file age in seconds before sweep (0 = sweep all)") + command.Flags().BoolVar(&options.DryRun, "dry-run", false, "Preview without mutations") + command.RunE = func(command *cobra.Command, _ []string) error { + report, err := module.useCases.Cleanup.Execute(command.Context(), options) + if err != nil { + return err + } + if module.outputMode(command) == "json" { + return writeJSON(command, report) + } + fmt.Fprintf(command.OutOrStdout(), "Eval cleanup:\n transitions->aborted: %d\n transitions->failed: %d\n runs deleted: %d\n tmp files swept: %d\n", report.TransitionsAborted, report.TransitionsFailed, report.RunsDeleted, report.TmpFilesSwept) + verbose := module.host.Verbose != nil && module.host.Verbose(command) + if len(report.Touched) > 0 && verbose { + fmt.Fprintln(command.OutOrStdout(), "Touched:") + for _, touched := range report.Touched { + fmt.Fprintf(command.OutOrStdout(), " %s\n", touched) + } + } + return nil + } + return command +} + func (module Module) renderRun(command *cobra.Command, result aoeval.CoreRunResult) error { var value any = result.Run if result.Mode == aoeval.CoreRunBaselineAB { diff --git a/cli/internal/commands/eval/module_test.go b/cli/internal/commands/eval/module_test.go index 5709863e7..2e2e3c750 100644 --- a/cli/internal/commands/eval/module_test.go +++ b/cli/internal/commands/eval/module_test.go @@ -14,6 +14,13 @@ type coreUseCasesSpy struct { runRequest aoeval.CoreRunRequest } +type cleanupUseCasesSpy struct{ request aoeval.CleanupRequest } + +func (useCases *cleanupUseCasesSpy) Execute(_ context.Context, request aoeval.CleanupRequest) (aoeval.CleanupReport, error) { + useCases.request = request + return aoeval.CleanupReport{TransitionsAborted: 1, Touched: []string{"run-1:pending->aborted"}}, nil +} + func (useCases *coreUseCasesSpy) Run(_ context.Context, request aoeval.CoreRunRequest) (aoeval.CoreRunResult, error) { useCases.runRequest = request return aoeval.CoreRunResult{Mode: aoeval.CoreRunSingle, Run: &aoeval.RunRecord{RunID: "run-1", Status: aoeval.StatusPass}}, nil @@ -36,7 +43,7 @@ func (*coreUseCasesSpy) Coverage(context.Context, aoeval.CoreCoverageRequest) (* func TestModuleRunParsesClosureLocalFlagsAndDelegates(t *testing.T) { useCases := &coreUseCasesSpy{} - command := NewModule(useCases, HostOptions{}).Command() + command := NewModule(UseCases{Core: useCases}, HostOptions{}).Command() command.SetArgs([]string{"run", "suite.json", "--run-id", "run-1", "--runtime", "static", "--baseline-mode", "skill-on"}) var output strings.Builder command.SetOut(&output) @@ -53,8 +60,8 @@ func TestModuleRunParsesClosureLocalFlagsAndDelegates(t *testing.T) { } func TestModuleCommandsDoNotShareFlagState(t *testing.T) { - first := NewModule(&coreUseCasesSpy{}, HostOptions{}).Command() - second := NewModule(&coreUseCasesSpy{}, HostOptions{}).Command() + first := NewModule(UseCases{Core: &coreUseCasesSpy{}}, HostOptions{}).Command() + second := NewModule(UseCases{Core: &coreUseCasesSpy{}}, HostOptions{}).Command() firstRun, _, err := first.Find([]string{"run"}) if err != nil { t.Fatal(err) @@ -71,6 +78,23 @@ func TestModuleCommandsDoNotShareFlagState(t *testing.T) { } } +func TestModuleCleanupDelegatesClosureLocalOptions(t *testing.T) { + cleanup := &cleanupUseCasesSpy{} + command := NewModule(UseCases{Core: &coreUseCasesSpy{}, Cleanup: cleanup}, HostOptions{Verbose: func(*cobra.Command) bool { return true }}).Command() + command.SetArgs([]string{"cleanup", "--delete", "--tmp-files", "--tmp-age", "7", "--dry-run"}) + var output strings.Builder + command.SetOut(&output) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if !cleanup.request.Delete || !cleanup.request.TmpFiles || !cleanup.request.DryRun || cleanup.request.TmpAgeSeconds != 7 { + t.Fatalf("request = %#v", cleanup.request) + } + if !strings.Contains(output.String(), "transitions->aborted: 1") || !strings.Contains(output.String(), "Touched:") { + t.Fatalf("output = %q", output.String()) + } +} + func findChild(t *testing.T, command *cobra.Command, name string) *cobra.Command { t.Helper() child, _, err := command.Find([]string{name}) diff --git a/cli/internal/eval/cleanup_service.go b/cli/internal/eval/cleanup_service.go new file mode 100644 index 000000000..4c4584e05 --- /dev/null +++ b/cli/internal/eval/cleanup_service.go @@ -0,0 +1,107 @@ +package eval + +import ( + "context" + "fmt" + "time" + + "github.com/boshu2/agentops/cli/internal/evalsubstrate" +) + +type CleanupRuntime interface { + Root() string + ListRunIDs(string) ([]string, error) + LoadManifest(string, string) (*evalsubstrate.Manifest, error) + Transition(string, string, evalsubstrate.RunStatus, string, time.Time) error + DeleteRun(string, string) error + SweepTempFiles(string, int64) ([]string, error) + Now() time.Time +} + +type CleanupService struct{ Runtime CleanupRuntime } + +type CleanupRequest struct { + Delete, TmpFiles, DryRun bool + TmpAgeSeconds int64 +} + +type CleanupReport struct { + TransitionsAborted int `json:"transitions_to_aborted"` + TransitionsFailed int `json:"transitions_to_failed"` + RunsDeleted int `json:"runs_deleted"` + TmpFilesSwept int `json:"tmp_files_swept"` + Touched []string `json:"touched"` +} + +func (service CleanupService) Execute(_ context.Context, request CleanupRequest) (CleanupReport, error) { + root := service.Runtime.Root() + report := CleanupReport{Touched: []string{}} + ids, err := service.Runtime.ListRunIDs(root) + if err != nil { + return report, fmt.Errorf("eval cleanup: read runs/: %w", err) + } + for _, id := range ids { + manifest, err := service.Runtime.LoadManifest(root, id) + if err != nil { + report.Touched = append(report.Touched, id+":unreadable") + continue + } + age := service.Runtime.Now().Sub(time.UnixMilli(manifest.StartedAtUnixMs).UTC()) + switch { + case manifest.Status == evalsubstrate.StatusPending && age >= time.Minute: + if err := service.Runtime.Transition(root, id, evalsubstrate.StatusAborted, "never_started", service.Runtime.Now()); err != nil { + return report, err + } + report.TransitionsAborted++ + report.Touched = append(report.Touched, id+":pending->aborted") + case manifest.Status == evalsubstrate.StatusRunning && age >= 5*time.Minute: + if err := service.Runtime.Transition(root, id, evalsubstrate.StatusFailed, "orphaned_process", service.Runtime.Now()); err != nil { + return report, err + } + report.TransitionsFailed++ + report.Touched = append(report.Touched, id+":running->failed") + } + } + if request.Delete { + if err := service.deleteRuns(root, ids, request.DryRun, &report); err != nil { + return report, err + } + } + if request.TmpFiles { + if request.DryRun { + report.Touched = append(report.Touched, "tmp-files: dry-run preview not implemented (sweep is a write op)") + } else { + swept, err := service.Runtime.SweepTempFiles(root, request.TmpAgeSeconds) + if err != nil { + return report, fmt.Errorf("eval cleanup: sweep tmp: %w", err) + } + report.TmpFilesSwept = len(swept) + for _, path := range swept { + report.Touched = append(report.Touched, "tmp:"+path) + } + } + } + return report, nil +} + +func (service CleanupService) deleteRuns(root string, ids []string, dryRun bool, report *CleanupReport) error { + for _, id := range ids { + manifest, err := service.Runtime.LoadManifest(root, id) + if err != nil { + continue + } + if manifest.Status != evalsubstrate.StatusFailed && manifest.Status != evalsubstrate.StatusAborted { + continue + } + if dryRun { + report.Touched = append(report.Touched, id+":would-delete") + continue + } + if err := service.Runtime.DeleteRun(root, id); err != nil { + return fmt.Errorf("eval cleanup: remove %q: %w", id, err) + } + report.RunsDeleted++ + report.Touched = append(report.Touched, id+":deleted") + } + return nil +} diff --git a/cli/internal/eval/cleanup_service_test.go b/cli/internal/eval/cleanup_service_test.go new file mode 100644 index 000000000..bec78177f --- /dev/null +++ b/cli/internal/eval/cleanup_service_test.go @@ -0,0 +1,57 @@ +package eval + +import ( + "context" + "testing" + "time" + + "github.com/boshu2/agentops/cli/internal/evalsubstrate" +) + +type cleanupRuntimeSpy struct { + manifests map[string]*evalsubstrate.Manifest + transitions []string + deletions []string +} + +func (*cleanupRuntimeSpy) Root() string { return "/evals" } +func (runtime *cleanupRuntimeSpy) ListRunIDs(string) ([]string, error) { + return []string{"pending", "running", "retracted"}, nil +} +func (runtime *cleanupRuntimeSpy) LoadManifest(_ string, id string) (*evalsubstrate.Manifest, error) { + return runtime.manifests[id], nil +} +func (runtime *cleanupRuntimeSpy) Transition(_ string, id string, next evalsubstrate.RunStatus, _ string, _ time.Time) error { + runtime.transitions = append(runtime.transitions, id+":"+string(next)) + runtime.manifests[id].Status = next + return nil +} +func (runtime *cleanupRuntimeSpy) DeleteRun(_ string, id string) error { + runtime.deletions = append(runtime.deletions, id) + return nil +} +func (*cleanupRuntimeSpy) SweepTempFiles(string, int64) ([]string, error) { + return []string{"one.tmp"}, nil +} +func (*cleanupRuntimeSpy) Now() time.Time { return time.Unix(1_000, 0).UTC() } + +func TestCleanupServiceTransitionsDeletesAndPreservesRetracted(t *testing.T) { + old := time.Unix(1_000, 0).Add(-10 * time.Minute).UnixMilli() + runtime := &cleanupRuntimeSpy{manifests: map[string]*evalsubstrate.Manifest{ + "pending": {Status: evalsubstrate.StatusPending, StartedAtUnixMs: old}, + "running": {Status: evalsubstrate.StatusRunning, StartedAtUnixMs: old}, + "retracted": {Status: evalsubstrate.StatusRetracted, StartedAtUnixMs: old}, + }} + report, err := (CleanupService{Runtime: runtime}).Execute(context.Background(), CleanupRequest{Delete: true, TmpFiles: true, TmpAgeSeconds: 60}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if report.TransitionsAborted != 1 || report.TransitionsFailed != 1 || report.RunsDeleted != 2 || report.TmpFilesSwept != 1 { + t.Fatalf("report = %#v", report) + } + for _, deleted := range runtime.deletions { + if deleted == "retracted" { + t.Fatal("retracted run was deleted") + } + } +} From 3cd9f726c196e060f661b8b345e3b04ec2d0896c Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:27:30 -0400 Subject: [PATCH 090/108] refactor(eval): extract task service and command tree --- cli/internal/adapters/eval/runtime.go | 33 +++ cli/internal/adapters/eval/runtime_test.go | 41 +++ cli/internal/commands/eval/module.go | 128 +++++++++ cli/internal/commands/eval/module_test.go | 34 +++ cli/internal/eval/task_service.go | 300 +++++++++++++++++++++ cli/internal/eval/task_service_test.go | 48 ++++ 6 files changed, 584 insertions(+) create mode 100644 cli/internal/eval/task_service.go create mode 100644 cli/internal/eval/task_service_test.go diff --git a/cli/internal/adapters/eval/runtime.go b/cli/internal/adapters/eval/runtime.go index b04d5ce5d..6cfc6bfb3 100644 --- a/cli/internal/adapters/eval/runtime.go +++ b/cli/internal/adapters/eval/runtime.go @@ -114,3 +114,36 @@ func (Runtime) SweepTempFiles(root string, age int64) ([]string, error) { return evalsubstrate.SweepTempFiles(root, age) } func (Runtime) Now() time.Time { return time.Now().UTC() } + +func (Runtime) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } +func (Runtime) WriteAtomic(path string, data []byte) error { + return evalsubstrate.WriteAtomic(path, data) +} +func (Runtime) ListDirectories(path string) ([]string, error) { + entries, err := os.ReadDir(path) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var names []string + for _, entry := range entries { + if entry.IsDir() { + names = append(names, entry.Name()) + } + } + sort.Strings(names) + return names, nil +} +func (Runtime) UserHome() (string, error) { return os.UserHomeDir() } +func (Runtime) SnapshotHarness(dir, ref, source string) (*evalsubstrate.Harness, *evalsubstrate.HarnessLock, error) { + return evalsubstrate.SnapshotHarness(dir, ref, source) +} +func (Runtime) LoadModelSpec(root, id string) (*evalsubstrate.ModelSpec, error) { + return evalsubstrate.LoadModelSpec(root, id) +} +func (Runtime) GenerateRunID(rigID string) string { return evalsubstrate.GenerateRunID(rigID) } +func (Runtime) OpenRun(root, id string, manifest evalsubstrate.Manifest) (aoeval.TaskRunWriter, error) { + return evalsubstrate.NewRunWriter(root, id, manifest) +} diff --git a/cli/internal/adapters/eval/runtime_test.go b/cli/internal/adapters/eval/runtime_test.go index 296550c99..e031b377c 100644 --- a/cli/internal/adapters/eval/runtime_test.go +++ b/cli/internal/adapters/eval/runtime_test.go @@ -1,15 +1,20 @@ package eval import ( + "context" + "os" "path/filepath" "testing" aoeval "github.com/boshu2/agentops/cli/internal/eval" + "github.com/boshu2/agentops/cli/internal/evalsubstrate" + "gopkg.in/yaml.v3" ) func TestRuntimeSatisfiesCorePortAndResolvesHostPaths(t *testing.T) { var port aoeval.CoreRuntime = Runtime{} var _ aoeval.CleanupRuntime = Runtime{} + var _ aoeval.TaskRuntime = Runtime{} workDir, err := port.WorkDir() if err != nil { t.Fatalf("WorkDir: %v", err) @@ -25,3 +30,39 @@ func TestRuntimeSatisfiesCorePortAndResolvesHostPaths(t *testing.T) { t.Fatalf("Abs = %q, want absolute", absolute) } } + +func TestRuntimeTaskServiceDryRunUsesRealFilesystemAdapter(t *testing.T) { + root := t.TempDir() + t.Setenv("AGENTOPS_EVALS_ROOT", root) + task := evalsubstrate.Task{SchemaVersion: 1, ID: "task-1", HarnessRef: "harness-1", Stats: evalsubstrate.TaskStat{MinNSamples: 3, DecisionRule: evalsubstrate.DecisionRule{Kind: "bootstrap_ci", Confidence: .95}}} + suite := evalsubstrate.Suite{SchemaVersion: 1, ID: "suite-1", Kind: "comparison", VariedAxis: evalsubstrate.VariedAxis{Kind: "model", Values: []string{"a", "b"}}, HeldConstant: evalsubstrate.HeldConstant{Task: "task-1", Harness: "harness-1", GroundTruthVersion: "gt-v1"}, SampleSplit: "dev", NSamples: 3, Stats: evalsubstrate.SuiteStat{DecisionRule: evalsubstrate.DecisionRule{Kind: "bootstrap_ci", Confidence: .95}}} + writeYAML := func(path string, value any) { + t.Helper() + data, err := yaml.Marshal(value) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + } + writeYAML(filepath.Join(root, "tasks", task.ID, "task.yaml"), task) + writeYAML(filepath.Join(root, "suites", suite.ID, "suite.yaml"), suite) + gtPath := filepath.Join(root, "ground-truth", "ground-truth.jsonl") + if err := os.MkdirAll(filepath.Dir(gtPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(gtPath, []byte("{\"id\":\"gt-1\",\"value\":\"ok\",\"source\":\"test\",\"confidence\":\"strong\",\"split\":\"dev\"}\n"), 0o644); err != nil { + t.Fatal(err) + } + result, err := (aoeval.TaskService{Runtime: Runtime{}}).Run(context.Background(), aoeval.TaskRunRequest{TaskID: task.ID, SuiteRef: suite.ID, Seeds: "1,2,3", HarnessRef: "harness-1", GroundTruthRef: "gt-1", DryRun: true}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !result.DryRun { + t.Fatalf("result = %#v", result) + } +} diff --git a/cli/internal/commands/eval/module.go b/cli/internal/commands/eval/module.go index fb5102974..6fde93f6a 100644 --- a/cli/internal/commands/eval/module.go +++ b/cli/internal/commands/eval/module.go @@ -4,6 +4,7 @@ package eval import ( "context" "encoding/json" + "errors" "fmt" "strings" @@ -11,6 +12,7 @@ import ( "github.com/boshu2/agentops/cli/internal/clicontract" aoeval "github.com/boshu2/agentops/cli/internal/eval" + "github.com/boshu2/agentops/cli/internal/evalsubstrate" ) type CoreUseCases interface { @@ -26,9 +28,17 @@ type CleanupUseCases interface { Execute(context.Context, aoeval.CleanupRequest) (aoeval.CleanupReport, error) } +type TaskUseCases interface { + Add(context.Context, aoeval.TaskAddRequest) (aoeval.TaskAddResult, error) + List(context.Context) (aoeval.TaskListResult, error) + Show(context.Context, string) (*evalsubstrate.Task, error) + Run(context.Context, aoeval.TaskRunRequest) (aoeval.TaskRunResult, error) +} + type UseCases struct { Core CoreUseCases Cleanup CleanupUseCases + Task TaskUseCases } type HostOptions struct { @@ -89,6 +99,9 @@ release. Live Claude and Codex adapters are evaluated by a later runtime tier.`, if module.useCases.Cleanup != nil { command.AddCommand(module.cleanupCommand()) } + if module.useCases.Task != nil { + command.AddCommand(module.taskCommand()) + } return command } @@ -306,6 +319,121 @@ never auto-removed — retraction is an audit trail.`, return command } +func (module Module) taskCommand() *cobra.Command { + command := &cobra.Command{Use: "task", Short: "Manage evaluation Tasks (add, list, show, run)", Long: `Operate on the §3 Task primitive of the eval substrate. + +Tasks live under $AGENTOPS_EVALS_ROOT/tasks//task.yaml and define the +input/output contract a Run will be evaluated against.`} + command.AddCommand(module.taskAddCommand(), module.taskListCommand(), module.taskShowCommand(), module.taskRunCommand()) + return command +} + +func (module Module) taskAddCommand() *cobra.Command { + command := &cobra.Command{Use: "add ", Short: "Register a Task by copying its yaml + samples into the substrate", Args: cobra.ExactArgs(1)} + command.RunE = func(command *cobra.Command, args []string) error { + result, err := module.useCases.Task.Add(command.Context(), aoeval.TaskAddRequest{SourcePath: args[0]}) + if err != nil { + return err + } + fmt.Fprintf(command.OutOrStdout(), "Task registered: %s\n path: %s\n min_n_samples: %d\n", result.Task.ID, result.Destination, result.Task.Stats.MinNSamples) + return nil + } + return command +} + +func (module Module) taskListCommand() *cobra.Command { + command := &cobra.Command{Use: "list", Short: "List registered Task ids", Args: cobra.NoArgs} + command.RunE = func(command *cobra.Command, _ []string) error { + result, err := module.useCases.Task.List(command.Context()) + if err != nil { + return err + } + ids := make([]string, 0, len(result.Tasks)) + for _, summary := range result.Tasks { + ids = append(ids, summary.ID) + } + if module.outputMode(command) == "json" { + return writeJSON(command, map[string]any{"tasks": ids, "root": result.Root}) + } + if len(result.Tasks) == 0 { + fmt.Fprintln(command.OutOrStdout(), "No tasks registered") + return nil + } + fmt.Fprintf(command.OutOrStdout(), "Tasks under %s:\n", result.Root) + for _, summary := range result.Tasks { + if summary.Error != "" { + fmt.Fprintf(command.OutOrStdout(), " %s\t\n", summary.ID, summary.Error) + continue + } + task := summary.Task + fmt.Fprintf(command.OutOrStdout(), " %s\tschema_version=%d min_n_samples=%d metric=%s\n", task.ID, task.SchemaVersion, task.Stats.MinNSamples, task.Stats.Metric) + } + return nil + } + return command +} + +func (module Module) taskShowCommand() *cobra.Command { + command := &cobra.Command{Use: "show ", Short: "Print a registered Task summary", Args: cobra.ExactArgs(1)} + command.RunE = func(command *cobra.Command, args []string) error { + task, err := module.useCases.Task.Show(command.Context(), args[0]) + if err != nil { + return err + } + if module.outputMode(command) == "json" { + return writeJSON(command, task) + } + fmt.Fprintf(command.OutOrStdout(), "Task: %s\n", task.ID) + fmt.Fprintf(command.OutOrStdout(), " schema_version: %d\n domain: %s\n description: %s\n harness_ref: %s\n", task.SchemaVersion, task.Domain, task.Description, task.HarnessRef) + fmt.Fprintf(command.OutOrStdout(), " stats.metric: %s\n stats.paired: %v\n stats.min_n_samples: %d\n stats.decision_rule: kind=%s confidence=%.2f\n", task.Stats.Metric, task.Stats.Paired, task.Stats.MinNSamples, task.Stats.DecisionRule.Kind, task.Stats.DecisionRule.Confidence) + return nil + } + return command +} + +func (module Module) taskRunCommand() *cobra.Command { + options := aoeval.TaskRunRequest{InspectVersion: "0.3.216"} + command := &cobra.Command{Use: "run ", Short: "Open a new Run manifest for ; refuses on gate failure", Args: cobra.ExactArgs(1)} + flags := command.Flags() + flags.StringVar(&options.SuiteRef, "suite", "", "Suite id or path to suite.yaml (required)") + flags.StringVar(&options.RigID, "rig-id", "", "Rig identifier stamped into the Run manifest") + flags.StringVar(&options.Seeds, "seeds", "", "Comma-separated seeds (>=3, per §4)") + flags.StringVar(&options.HarnessRef, "harness", "", "Harness id (recorded into manifest)") + flags.StringVar(&options.HarnessDir, "harness-dir", "", "Path to harness source dir for snapshot + gate #8") + flags.StringVar(&options.ModelSpecID, "model-spec", "", "ModelSpec id (already captured via ao eval models capture)") + flags.StringVar(&options.GroundTruthRef, "ground-truth", "", "Ground-truth row id (head of supersession chain)") + flags.StringVar(&options.SampleSplit, "sample-split", "", "Sample split (dev|holdout); default from suite") + flags.IntVar(&options.NSamples, "n-samples", 0, "Override Suite.n_samples") + flags.StringVar(&options.InspectVersion, "inspect-version", options.InspectVersion, "Inspect AI version stamped into manifest") + flags.StringVar(&options.InspectCommand, "inspect-command", "", "Inspect command recorded into the Run manifest (not executed yet)") + flags.BoolVar(&options.CrossSpec, "cross-spec", false, "Allow ModelSpec drift (gate #4)") + flags.BoolVar(&options.AllowWeak, "allow-weak-labels", false, "Allow runs against confidence=weak ground-truth rows (gate #7)") + flags.BoolVar(&options.QuickSession, "quick", false, "Mark Run as quick_session=true (excluded from --vs auto-baseline pool)") + flags.BoolVar(&options.DryRun, "dry-run", false, "Run gates and exit without writing a Run manifest") + command.RunE = func(command *cobra.Command, args []string) error { + options.TaskID = args[0] + result, err := module.useCases.Task.Run(command.Context(), options) + if err != nil { + var gateFailure *aoeval.TaskGateError + if errors.As(err, &gateFailure) { + fmt.Fprintln(command.ErrOrStderr(), gateFailure.Message) + } + return err + } + if result.DryRun { + fmt.Fprintln(command.OutOrStdout(), "Dry run: gates passed; no Run manifest written.") + return nil + } + if module.outputMode(command) == "json" { + return writeJSON(command, result.Manifest) + } + manifest := result.Manifest + fmt.Fprintf(command.OutOrStdout(), "Run opened: %s\n status: %s\n manifest: %s\n rig_id: %s\n task_ref: %s\n suite_ref: %s\n", manifest.ID, manifest.Status, result.Path, manifest.RigID, manifest.TaskRef, manifest.SuiteRef) + return nil + } + return command +} + func (module Module) renderRun(command *cobra.Command, result aoeval.CoreRunResult) error { var value any = result.Run if result.Mode == aoeval.CoreRunBaselineAB { diff --git a/cli/internal/commands/eval/module_test.go b/cli/internal/commands/eval/module_test.go index 2e2e3c750..a4b63ce6c 100644 --- a/cli/internal/commands/eval/module_test.go +++ b/cli/internal/commands/eval/module_test.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" aoeval "github.com/boshu2/agentops/cli/internal/eval" + "github.com/boshu2/agentops/cli/internal/evalsubstrate" ) type coreUseCasesSpy struct { @@ -16,6 +17,22 @@ type coreUseCasesSpy struct { type cleanupUseCasesSpy struct{ request aoeval.CleanupRequest } +type taskUseCasesSpy struct{ runRequest aoeval.TaskRunRequest } + +func (*taskUseCasesSpy) Add(context.Context, aoeval.TaskAddRequest) (aoeval.TaskAddResult, error) { + return aoeval.TaskAddResult{}, nil +} +func (*taskUseCasesSpy) List(context.Context) (aoeval.TaskListResult, error) { + return aoeval.TaskListResult{}, nil +} +func (*taskUseCasesSpy) Show(context.Context, string) (*evalsubstrate.Task, error) { + return &evalsubstrate.Task{}, nil +} +func (useCases *taskUseCasesSpy) Run(_ context.Context, request aoeval.TaskRunRequest) (aoeval.TaskRunResult, error) { + useCases.runRequest = request + return aoeval.TaskRunResult{DryRun: true}, nil +} + func (useCases *cleanupUseCasesSpy) Execute(_ context.Context, request aoeval.CleanupRequest) (aoeval.CleanupReport, error) { useCases.request = request return aoeval.CleanupReport{TransitionsAborted: 1, Touched: []string{"run-1:pending->aborted"}}, nil @@ -95,6 +112,23 @@ func TestModuleCleanupDelegatesClosureLocalOptions(t *testing.T) { } } +func TestModuleTaskRunDelegatesClosureLocalFlags(t *testing.T) { + task := &taskUseCasesSpy{} + command := NewModule(UseCases{Core: &coreUseCasesSpy{}, Task: task}, HostOptions{}).Command() + command.SetArgs([]string{"task", "run", "task-1", "--suite", "suite-1", "--seeds", "1,2,3", "--rig-id", "rig-1", "--dry-run"}) + var output strings.Builder + command.SetOut(&output) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if task.runRequest.TaskID != "task-1" || task.runRequest.SuiteRef != "suite-1" || task.runRequest.Seeds != "1,2,3" || !task.runRequest.DryRun { + t.Fatalf("request = %#v", task.runRequest) + } + if !strings.Contains(output.String(), "Dry run: gates passed") { + t.Fatalf("output = %q", output.String()) + } +} + func findChild(t *testing.T, command *cobra.Command, name string) *cobra.Command { t.Helper() child, _, err := command.Find([]string{name}) diff --git a/cli/internal/eval/task_service.go b/cli/internal/eval/task_service.go new file mode 100644 index 000000000..fd496de40 --- /dev/null +++ b/cli/internal/eval/task_service.go @@ -0,0 +1,300 @@ +package eval + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "path/filepath" + "sort" + "strings" + "time" + + "gopkg.in/yaml.v3" + + "github.com/boshu2/agentops/cli/internal/evalsubstrate" +) + +type TaskRunWriter interface { + Transition(evalsubstrate.RunStatus, func(*evalsubstrate.Manifest)) error + Manifest() evalsubstrate.Manifest + Path() string +} + +type TaskRuntime interface { + Root() string + ReadFile(string) ([]byte, error) + WriteAtomic(string, []byte) error + ListDirectories(string) ([]string, error) + UserHome() (string, error) + SnapshotHarness(string, string, string) (*evalsubstrate.Harness, *evalsubstrate.HarnessLock, error) + LoadModelSpec(string, string) (*evalsubstrate.ModelSpec, error) + GenerateRunID(string) string + OpenRun(string, string, evalsubstrate.Manifest) (TaskRunWriter, error) + Now() time.Time +} + +type TaskService struct{ Runtime TaskRuntime } +type TaskAddRequest struct{ SourcePath string } +type TaskAddResult struct { + Task evalsubstrate.Task + Destination string +} +type TaskSummary struct { + Task *evalsubstrate.Task `json:"task,omitempty"` + ID, Error string +} +type TaskListResult struct { + Tasks []TaskSummary `json:"tasks"` + Root string `json:"root"` +} +type TaskRunRequest struct { + TaskID, SuiteRef, RigID, Seeds, HarnessRef, HarnessDir, ModelSpecID, GroundTruthRef string + SampleSplit string + NSamples int + InspectVersion, InspectCommand string + CrossSpec, AllowWeak, QuickSession, DryRun bool +} +type TaskRunResult struct { + DryRun bool + Manifest evalsubstrate.Manifest + Path string +} + +func (service TaskService) Add(_ context.Context, request TaskAddRequest) (TaskAddResult, error) { + raw, err := service.Runtime.ReadFile(request.SourcePath) + if err != nil { + return TaskAddResult{}, fmt.Errorf("eval task add: read %q: %w", request.SourcePath, err) + } + var task evalsubstrate.Task + if err := yaml.Unmarshal(raw, &task); err != nil { + return TaskAddResult{}, fmt.Errorf("eval task add: parse: %w", err) + } + if task.ID == "" { + return TaskAddResult{}, fmt.Errorf("eval task add: missing required field id") + } + if task.Stats.MinNSamples <= 0 { + return TaskAddResult{}, fmt.Errorf("eval task add: task %q has no stats.min_n_samples (gate #6 cannot enforce)", task.ID) + } + canonical, err := evalsubstrate.CanonicalizeYAML(raw) + if err != nil { + return TaskAddResult{}, fmt.Errorf("eval task add: canonicalize: %w", err) + } + destination := filepath.Join(service.Runtime.Root(), "tasks", task.ID, "task.yaml") + if err := service.Runtime.WriteAtomic(destination, canonical); err != nil { + return TaskAddResult{}, fmt.Errorf("eval task add: write: %w", err) + } + return TaskAddResult{Task: task, Destination: destination}, nil +} + +func (service TaskService) List(_ context.Context) (TaskListResult, error) { + root := filepath.Join(service.Runtime.Root(), "tasks") + ids, err := service.Runtime.ListDirectories(root) + if err != nil { + return TaskListResult{}, fmt.Errorf("eval task list: %w", err) + } + sort.Strings(ids) + result := TaskListResult{Root: root, Tasks: make([]TaskSummary, 0, len(ids))} + for _, id := range ids { + task, err := service.loadTask(id) + if err != nil { + result.Tasks = append(result.Tasks, TaskSummary{ID: id, Error: err.Error()}) + continue + } + result.Tasks = append(result.Tasks, TaskSummary{ID: id, Task: task}) + } + return result, nil +} + +func (service TaskService) Show(_ context.Context, id string) (*evalsubstrate.Task, error) { + return service.loadTask(id) +} + +func (service TaskService) Run(_ context.Context, request TaskRunRequest) (TaskRunResult, error) { + task, err := service.loadTask(request.TaskID) + if err != nil { + return TaskRunResult{}, err + } + if request.SuiteRef == "" { + return TaskRunResult{}, fmt.Errorf("eval task run: --suite is required") + } + suite, err := service.loadSuite(request.SuiteRef) + if err != nil { + return TaskRunResult{}, err + } + seeds, err := parseTaskSeeds(request.Seeds) + if err != nil { + return TaskRunResult{}, err + } + if len(seeds) < 3 { + return TaskRunResult{}, fmt.Errorf("eval task run: --seeds requires >=3 values, got %d (per §4 manifest required field)", len(seeds)) + } + gateInputs := evalsubstrate.GateInputs{Suite: suite, Task: task, AllowWeak: request.AllowWeak, GTRequested: request.GroundTruthRef} + if request.HarnessDir != "" { + dir := request.HarnessDir + if strings.HasPrefix(dir, "~") { + if home, homeErr := service.Runtime.UserHome(); homeErr == nil { + dir = filepath.Join(home, strings.TrimPrefix(dir, "~")) + } + } + harness, lock, err := service.Runtime.SnapshotHarness(dir, request.HarnessRef, "ao eval task run") + if err != nil { + return TaskRunResult{}, fmt.Errorf("eval task run: harness snapshot: %w", err) + } + gateInputs.Harness, gateInputs.HarnessLock, gateInputs.HarnessDir = harness, lock, dir + } + var groundTruth []evalsubstrate.GroundTruthRow + if request.GroundTruthRef != "" { + groundTruth, _ = service.loadGroundTruth() + gateInputs.GroundTruth = groundTruth + } + refusals := evalsubstrate.RunGates(gateInputs) + if !refusals.Empty() { + return TaskRunResult{}, &TaskGateError{Message: refusals.Format(), Count: len(refusals)} + } + if request.DryRun { + return TaskRunResult{DryRun: true}, nil + } + rigID := request.RigID + if rigID == "" { + rigID = "unknown-rig" + } + manifest := service.taskManifest(request, task, suite, seeds, rigID, gateInputs, groundTruth) + runID := service.Runtime.GenerateRunID(rigID) + writer, err := service.Runtime.OpenRun(service.Runtime.Root(), runID, manifest) + if err != nil { + return TaskRunResult{}, fmt.Errorf("eval task run: open run: %w", err) + } + if err := writer.Transition(evalsubstrate.StatusRunning, func(manifest *evalsubstrate.Manifest) { + manifest.ValidityGatesPassed = []string{"held_constant_declared", "min_n_samples", "ground_truth_immutable", "harness_lock_match", "multi_comparison_correction"} + }); err != nil { + return TaskRunResult{}, fmt.Errorf("eval task run: transition->running: %w", err) + } + return TaskRunResult{Manifest: writer.Manifest(), Path: writer.Path()}, nil +} + +type TaskGateError struct { + Message string + Count int +} + +func (failure *TaskGateError) Error() string { + return fmt.Sprintf("eval task run: %d gate refusal(s)", failure.Count) +} + +func (service TaskService) loadTask(id string) (*evalsubstrate.Task, error) { + path := filepath.Join(service.Runtime.Root(), "tasks", id, "task.yaml") + raw, err := service.Runtime.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("loadTask %q: %w", id, err) + } + var task evalsubstrate.Task + if err := yaml.Unmarshal(raw, &task); err != nil { + return nil, fmt.Errorf("loadTask %q: parse: %w", id, err) + } + return &task, nil +} + +func (service TaskService) loadSuite(ref string) (*evalsubstrate.Suite, error) { + path := ref + if !strings.HasSuffix(ref, ".yaml") && !strings.HasSuffix(ref, ".yml") { + path = filepath.Join(service.Runtime.Root(), "suites", ref, "suite.yaml") + } + raw, err := service.Runtime.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("loadSuite %q: %w", ref, err) + } + var suite evalsubstrate.Suite + if err := yaml.Unmarshal(raw, &suite); err != nil { + return nil, fmt.Errorf("loadSuite %q: parse: %w", ref, err) + } + return &suite, nil +} + +func (service TaskService) loadGroundTruth() ([]evalsubstrate.GroundTruthRow, error) { + raw, err := service.Runtime.ReadFile(filepath.Join(service.Runtime.Root(), "ground-truth", "ground-truth.jsonl")) + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + var rows []evalsubstrate.GroundTruthRow + for decoder.More() { + var row evalsubstrate.GroundTruthRow + if err := decoder.Decode(&row); err != nil { + return nil, err + } + rows = append(rows, row) + } + return rows, nil +} + +func (service TaskService) taskManifest(request TaskRunRequest, task *evalsubstrate.Task, suite *evalsubstrate.Suite, seeds []int, rigID string, gates evalsubstrate.GateInputs, rows []evalsubstrate.GroundTruthRow) evalsubstrate.Manifest { + manifest := evalsubstrate.Manifest{TaskRef: task.ID, SuiteRef: suite.ID, HarnessRef: request.HarnessRef, ModelSpecRef: request.ModelSpecID, GroundTruthRef: request.GroundTruthRef, SampleSplit: pickTaskSplit(suite, request.SampleSplit), NSamples: pickTaskSamples(suite, request.NSamples), Seeds: seeds, RigID: rigID, InspectCommand: request.InspectCommand, InspectVersion: request.InspectVersion, QuickSession: request.QuickSession} + if gates.Harness != nil { + manifest.HarnessContentHash = gates.Harness.ContentHash + } + if request.ModelSpecID != "" { + if spec, err := service.Runtime.LoadModelSpec(service.Runtime.Root(), request.ModelSpecID); err == nil { + manifest.ModelSpecHash = spec.ContentHash + } + } + manifest.GroundTruthHash = taskGroundTruthHash(rows, request.GroundTruthRef) + if count := len(suite.VariedAxis.Values); count > 2 { + manifest.MultiComparisonMethod, manifest.ComparisonFamily, manifest.ReferenceArm, manifest.FamilySizeK = suite.Stats.MultiComparisonMethod, suite.Stats.ComparisonFamily, suite.Stats.ReferenceArm, evalsubstrate.FamilySizeK(suite.Stats.ComparisonFamily, count) + } + return manifest +} + +func parseTaskSeeds(value string) ([]int, error) { + if value == "" { + return nil, nil + } + parts := strings.Split(value, ",") + seeds := make([]int, 0, len(parts)) + for _, part := range parts { + var seed int + if _, err := fmt.Sscanf(strings.TrimSpace(part), "%d", &seed); err != nil { + return nil, fmt.Errorf("parseSeeds: bad seed %q: %w", part, err) + } + seeds = append(seeds, seed) + } + return seeds, nil +} +func pickTaskSplit(suite *evalsubstrate.Suite, override string) string { + if override != "" { + return override + } + if suite != nil && suite.SampleSplit != "" { + return suite.SampleSplit + } + return "dev" +} +func pickTaskSamples(suite *evalsubstrate.Suite, override int) int { + if override > 0 { + return override + } + if suite != nil { + return suite.NSamples + } + return 0 +} +func taskGroundTruthHash(rows []evalsubstrate.GroundTruthRow, ref string) string { + if ref == "" || len(rows) == 0 { + return "" + } + var data []byte + for _, row := range rows { + if row.ID == ref || row.Supersedes == ref { + encoded, _ := json.Marshal(row) + canonical, err := evalsubstrate.CanonicalizeJSON(encoded) + if err != nil { + canonical = encoded + } + data = append(data, canonical...) + } + } + if len(data) == 0 { + return "" + } + return evalsubstrate.ContentHash(data) +} diff --git a/cli/internal/eval/task_service_test.go b/cli/internal/eval/task_service_test.go new file mode 100644 index 000000000..31d58a0cf --- /dev/null +++ b/cli/internal/eval/task_service_test.go @@ -0,0 +1,48 @@ +package eval + +import ( + "context" + "testing" + "time" + + "github.com/boshu2/agentops/cli/internal/evalsubstrate" +) + +type taskRuntimeSpy struct { + files map[string][]byte + writes map[string][]byte +} + +func (*taskRuntimeSpy) Root() string { return "/evals" } +func (runtime *taskRuntimeSpy) ReadFile(path string) ([]byte, error) { return runtime.files[path], nil } +func (runtime *taskRuntimeSpy) WriteAtomic(path string, data []byte) error { + runtime.writes[path] = data + return nil +} +func (*taskRuntimeSpy) ListDirectories(string) ([]string, error) { return nil, nil } +func (*taskRuntimeSpy) UserHome() (string, error) { return "/home/test", nil } +func (*taskRuntimeSpy) SnapshotHarness(string, string, string) (*evalsubstrate.Harness, *evalsubstrate.HarnessLock, error) { + return nil, nil, nil +} +func (*taskRuntimeSpy) LoadModelSpec(string, string) (*evalsubstrate.ModelSpec, error) { + return nil, nil +} +func (*taskRuntimeSpy) GenerateRunID(string) string { return "run-1" } +func (*taskRuntimeSpy) OpenRun(string, string, evalsubstrate.Manifest) (TaskRunWriter, error) { + return nil, nil +} +func (*taskRuntimeSpy) Now() time.Time { return time.Unix(1_000, 0).UTC() } + +func TestTaskServiceAddCanonicalizesAndWritesOwnedDestination(t *testing.T) { + runtime := &taskRuntimeSpy{files: map[string][]byte{"task.yaml": []byte("id: task-1\nschema_version: 1\nstats:\n min_n_samples: 3\n")}, writes: map[string][]byte{}} + result, err := (TaskService{Runtime: runtime}).Add(context.Background(), TaskAddRequest{SourcePath: "task.yaml"}) + if err != nil { + t.Fatalf("Add: %v", err) + } + if result.Task.ID != "task-1" || result.Destination != "/evals/tasks/task-1/task.yaml" { + t.Fatalf("result = %#v", result) + } + if len(runtime.writes[result.Destination]) == 0 { + t.Fatal("canonical task was not written") + } +} From cc5a471a391583ca4d4b6c700a8ab3a94e7c5fc0 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:30:51 -0400 Subject: [PATCH 091/108] refactor(eval): extract suite service and commands --- cli/internal/adapters/eval/runtime.go | 35 +++++ cli/internal/adapters/eval/runtime_test.go | 1 + cli/internal/commands/eval/module.go | 63 +++++++++ cli/internal/commands/eval/module_test.go | 22 +++ cli/internal/eval/suite_service.go | 154 +++++++++++++++++++++ cli/internal/eval/suite_service_test.go | 36 +++++ 6 files changed, 311 insertions(+) create mode 100644 cli/internal/eval/suite_service.go create mode 100644 cli/internal/eval/suite_service_test.go diff --git a/cli/internal/adapters/eval/runtime.go b/cli/internal/adapters/eval/runtime.go index 6cfc6bfb3..e4eea6442 100644 --- a/cli/internal/adapters/eval/runtime.go +++ b/cli/internal/adapters/eval/runtime.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "os/exec" "path/filepath" "sort" "strings" @@ -147,3 +148,37 @@ func (Runtime) GenerateRunID(rigID string) string { return evalsubstrate.Generat func (Runtime) OpenRun(root, id string, manifest evalsubstrate.Manifest) (aoeval.TaskRunWriter, error) { return evalsubstrate.NewRunWriter(root, id, manifest) } + +func (runtime Runtime) RunStats(args []string) ([]byte, error) { + python := runtime.pythonBinary() + if python == "" { + return nil, fmt.Errorf("eval suite: substrate venv not found; provision via `python3 -m venv ~/.agents/evals/.venv && pip install numpy scipy`") + } + command := exec.Command(python, args...) + command.Env = append(os.Environ(), "PYTHONPATH="+runtime.Root()) + output, err := command.Output() + if err != nil { + stderr := "" + if exit, ok := err.(*exec.ExitError); ok { + stderr = string(exit.Stderr) + } + return nil, fmt.Errorf("eval suite: stats CLI failed: %w (stderr: %s)", err, stderr) + } + return output, nil +} + +func (runtime Runtime) pythonBinary() string { + if override := strings.TrimSpace(os.Getenv("AGENTOPS_EVALS_VENV")); override != "" { + return override + } + candidates := []string{filepath.Join(runtime.Root(), ".venv", "bin", "python"), filepath.Join(runtime.Root(), ".venv", "bin", "python3")} + if home, err := os.UserHomeDir(); err == nil { + candidates = append(candidates, filepath.Join(home, ".agents", "evals", ".venv", "bin", "python"), filepath.Join(home, ".agents", "evals", ".venv", "bin", "python3")) + } + for _, candidate := range candidates { + if _, err := os.Stat(candidate); err == nil { + return candidate + } + } + return "" +} diff --git a/cli/internal/adapters/eval/runtime_test.go b/cli/internal/adapters/eval/runtime_test.go index e031b377c..e314a5ae6 100644 --- a/cli/internal/adapters/eval/runtime_test.go +++ b/cli/internal/adapters/eval/runtime_test.go @@ -15,6 +15,7 @@ func TestRuntimeSatisfiesCorePortAndResolvesHostPaths(t *testing.T) { var port aoeval.CoreRuntime = Runtime{} var _ aoeval.CleanupRuntime = Runtime{} var _ aoeval.TaskRuntime = Runtime{} + var _ aoeval.SuiteRuntime = Runtime{} workDir, err := port.WorkDir() if err != nil { t.Fatalf("WorkDir: %v", err) diff --git a/cli/internal/commands/eval/module.go b/cli/internal/commands/eval/module.go index 6fde93f6a..dfc0259f6 100644 --- a/cli/internal/commands/eval/module.go +++ b/cli/internal/commands/eval/module.go @@ -35,10 +35,16 @@ type TaskUseCases interface { Run(context.Context, aoeval.TaskRunRequest) (aoeval.TaskRunResult, error) } +type SuiteUseCases interface { + Verdict(context.Context, aoeval.SuiteVerdictRequest) (aoeval.SuiteVerdictResult, error) + NRequired(context.Context, aoeval.SuiteNRequiredRequest) (aoeval.SuiteNRequiredResult, error) +} + type UseCases struct { Core CoreUseCases Cleanup CleanupUseCases Task TaskUseCases + Suite SuiteUseCases } type HostOptions struct { @@ -102,6 +108,9 @@ release. Live Claude and Codex adapters are evaluated by a later runtime tier.`, if module.useCases.Task != nil { command.AddCommand(module.taskCommand()) } + if module.useCases.Suite != nil { + command.AddCommand(module.suiteCommand()) + } return command } @@ -434,6 +443,60 @@ func (module Module) taskRunCommand() *cobra.Command { return command } +func (module Module) suiteCommand() *cobra.Command { + command := &cobra.Command{Use: "suite", Short: "Suite-level operations (verdict, n-required)", Long: "Suite-level operations against the §6.5 statistical contract."} + command.AddCommand(module.suiteVerdictCommand(), module.suiteNRequiredCommand()) + return command +} + +func (module Module) suiteVerdictCommand() *cobra.Command { + options := aoeval.SuiteVerdictRequest{BootstrapSamples: 10000} + command := &cobra.Command{Use: "verdict --arms a,b --inputs ", Short: "Compute the §6.5 paired cluster-bootstrap verdict", Args: cobra.ExactArgs(1)} + command.Flags().StringVar(&options.Arms, "arms", "", "Comma-separated arm ids (default: from suite varied_axis)") + command.Flags().StringVar(&options.InputsPath, "inputs", "", "Path to canonical bootstrap-inputs JSON (REQUIRED)") + command.Flags().Float64Var(&options.MDE, "mde", 0, "Minimum detectable effect (used for inconclusive_high_variance)") + command.Flags().IntVar(&options.BootstrapSamples, "B", 10000, "Bootstrap resamples") + command.Flags().IntVar(&options.NRequired, "n-required", 0, "Override n_required (default: derived from suite power block)") + command.RunE = func(command *cobra.Command, args []string) error { + options.SuiteID = args[0] + result, err := module.useCases.Suite.Verdict(command.Context(), options) + if err != nil { + return err + } + if module.outputMode(command) == "json" { + _, err = command.OutOrStdout().Write(append(append([]byte(nil), result.Raw...), '\n')) + return err + } + value := result.Values + fmt.Fprintf(command.OutOrStdout(), "Suite verdict: %v\n delta_point: %v\n ci: [%v, %v]\n n_clusters: %v\n n_required: %v\n bootstrap_seed: %v\n bootstrap_inputs_hash: %v\n", value["verdict"], value["delta_point"], value["ci_low"], value["ci_high"], value["n_clusters"], value["n_required"], value["bootstrap_seed"], value["bootstrap_inputs_hash"]) + return nil + } + return command +} + +func (module Module) suiteNRequiredCommand() *cobra.Command { + options := aoeval.SuiteNRequiredRequest{BaselineRate: .5, MDE: .05, Alpha: .05, Power: .80, Paired: true} + command := &cobra.Command{Use: "n-required", Short: "Compute power-derived n_required (gate #6 input on Day 3+)"} + command.Flags().Float64Var(&options.BaselineRate, "baseline-rate", .5, "Baseline rate (binomial worst-case fallback)") + command.Flags().Float64Var(&options.MDE, "mde", .05, "Minimum detectable effect") + command.Flags().Float64Var(&options.Alpha, "alpha", .05, "Type-I error rate") + command.Flags().Float64Var(&options.Power, "power", .80, "Statistical power (1-beta)") + command.Flags().BoolVar(&options.Paired, "paired", true, "Paired comparison") + command.RunE = func(command *cobra.Command, _ []string) error { + result, err := module.useCases.Suite.NRequired(command.Context(), options) + if err != nil { + return err + } + if module.outputMode(command) == "json" { + _, err = command.OutOrStdout().Write(append(append([]byte(nil), result.Raw...), '\n')) + return err + } + fmt.Fprintf(command.OutOrStdout(), "n_required: %d (baseline_rate=%g MDE=%g alpha=%g power=%g paired=%v)\n", result.NRequired, options.BaselineRate, options.MDE, options.Alpha, options.Power, options.Paired) + return nil + } + return command +} + func (module Module) renderRun(command *cobra.Command, result aoeval.CoreRunResult) error { var value any = result.Run if result.Mode == aoeval.CoreRunBaselineAB { diff --git a/cli/internal/commands/eval/module_test.go b/cli/internal/commands/eval/module_test.go index a4b63ce6c..1e0705219 100644 --- a/cli/internal/commands/eval/module_test.go +++ b/cli/internal/commands/eval/module_test.go @@ -19,6 +19,16 @@ type cleanupUseCasesSpy struct{ request aoeval.CleanupRequest } type taskUseCasesSpy struct{ runRequest aoeval.TaskRunRequest } +type suiteUseCasesSpy struct{ verdictRequest aoeval.SuiteVerdictRequest } + +func (useCases *suiteUseCasesSpy) Verdict(_ context.Context, request aoeval.SuiteVerdictRequest) (aoeval.SuiteVerdictResult, error) { + useCases.verdictRequest = request + return aoeval.SuiteVerdictResult{Values: map[string]any{"verdict": "improved"}}, nil +} +func (*suiteUseCasesSpy) NRequired(context.Context, aoeval.SuiteNRequiredRequest) (aoeval.SuiteNRequiredResult, error) { + return aoeval.SuiteNRequiredResult{NRequired: 42}, nil +} + func (*taskUseCasesSpy) Add(context.Context, aoeval.TaskAddRequest) (aoeval.TaskAddResult, error) { return aoeval.TaskAddResult{}, nil } @@ -129,6 +139,18 @@ func TestModuleTaskRunDelegatesClosureLocalFlags(t *testing.T) { } } +func TestModuleSuiteVerdictDelegatesFlags(t *testing.T) { + suite := &suiteUseCasesSpy{} + command := NewModule(UseCases{Core: &coreUseCasesSpy{}, Suite: suite}, HostOptions{}).Command() + command.SetArgs([]string{"suite", "verdict", "suite-1", "--arms", "a,b", "--inputs", "in.json", "--B", "99"}) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if suite.verdictRequest.SuiteID != "suite-1" || suite.verdictRequest.Arms != "a,b" || suite.verdictRequest.BootstrapSamples != 99 { + t.Fatalf("request = %#v", suite.verdictRequest) + } +} + func findChild(t *testing.T, command *cobra.Command, name string) *cobra.Command { t.Helper() child, _, err := command.Find([]string{name}) diff --git a/cli/internal/eval/suite_service.go b/cli/internal/eval/suite_service.go new file mode 100644 index 000000000..8c2f7ea01 --- /dev/null +++ b/cli/internal/eval/suite_service.go @@ -0,0 +1,154 @@ +package eval + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/boshu2/agentops/cli/internal/evalsubstrate" +) + +type SuiteRuntime interface { + Root() string + ReadFile(string) ([]byte, error) + RunStats([]string) ([]byte, error) +} + +type SuiteService struct{ Runtime SuiteRuntime } +type SuiteVerdictRequest struct { + SuiteID, Arms, InputsPath string + MDE float64 + BootstrapSamples, NRequired int +} +type SuiteVerdictResult struct { + Raw []byte + Values map[string]any +} +type SuiteNRequiredRequest struct { + BaselineRate, MDE, Alpha, Power float64 + Paired bool +} +type SuiteNRequiredResult struct { + Raw []byte + NRequired int +} + +func (service SuiteService) Verdict(_ context.Context, request SuiteVerdictRequest) (SuiteVerdictResult, error) { + if request.InputsPath == "" { + return SuiteVerdictResult{}, fmt.Errorf("eval suite verdict: --inputs is required") + } + suite, _ := service.loadSuite(request.SuiteID) + arms := strings.Join(suiteArmIDs(suite, request.Arms), ",") + if strings.TrimSpace(arms) == "" { + return SuiteVerdictResult{}, fmt.Errorf("eval suite verdict: --arms required when suite has no varied_axis on disk") + } + rule, err := suiteDecisionRuleJSON(suite) + if err != nil { + return SuiteVerdictResult{}, fmt.Errorf("eval suite verdict: %w", err) + } + nRequired := request.NRequired + if nRequired <= 0 { + nRequired = service.derivedNRequired(suite) + } + args := []string{"-m", "_stats.cli", "verdict", "--suite-id", request.SuiteID, "--arms", arms, "--inputs", request.InputsPath, "--decision-rule", rule, "--n-required", fmt.Sprintf("%d", nRequired), "--B", fmt.Sprintf("%d", request.BootstrapSamples)} + if request.MDE > 0 { + args = append(args, "--mde", fmt.Sprintf("%g", request.MDE)) + } + raw, err := service.Runtime.RunStats(args) + if err != nil { + return SuiteVerdictResult{}, err + } + values := map[string]any{} + if err := json.Unmarshal(raw, &values); err != nil { + return SuiteVerdictResult{}, fmt.Errorf("eval suite verdict: parse stats output: %w", err) + } + return SuiteVerdictResult{Raw: raw, Values: values}, nil +} + +func (service SuiteService) NRequired(_ context.Context, request SuiteNRequiredRequest) (SuiteNRequiredResult, error) { + args := []string{"-m", "_stats.cli", "n-required", "--baseline-rate", fmt.Sprintf("%g", request.BaselineRate), "--mde", fmt.Sprintf("%g", request.MDE), "--alpha", fmt.Sprintf("%g", request.Alpha), "--power", fmt.Sprintf("%g", request.Power), "--paired", fmt.Sprintf("%v", request.Paired)} + raw, err := service.Runtime.RunStats(args) + if err != nil { + return SuiteNRequiredResult{}, err + } + var parsed struct { + NRequired int `json:"n_required"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return SuiteNRequiredResult{}, fmt.Errorf("eval suite n-required: parse: %w", err) + } + return SuiteNRequiredResult{Raw: raw, NRequired: parsed.NRequired}, nil +} + +func (service SuiteService) loadSuite(id string) (*evalsubstrate.Suite, error) { + path := id + if !strings.HasSuffix(id, ".yaml") && !strings.HasSuffix(id, ".yml") { + path = filepath.Join(service.Runtime.Root(), "suites", id, "suite.yaml") + } + raw, err := service.Runtime.ReadFile(path) + if err != nil { + return nil, err + } + var suite evalsubstrate.Suite + if err := yaml.Unmarshal(raw, &suite); err != nil { + return nil, err + } + return &suite, nil +} + +func (service SuiteService) derivedNRequired(suite *evalsubstrate.Suite) int { + if suite == nil || suite.Stats.Power == nil { + return 0 + } + raw, err := service.Runtime.RunStats([]string{"-m", "_stats.cli", "n-required", "--baseline-rate", "0.5", "--mde", fmt.Sprintf("%g", suite.Stats.Power.MinimumDetectableEffect), "--alpha", fmt.Sprintf("%g", suite.Stats.Power.Alpha), "--power", "0.80", "--paired", "true"}) + if err != nil { + return 0 + } + var parsed struct { + NRequired int `json:"n_required"` + } + if json.Unmarshal(raw, &parsed) != nil { + return 0 + } + return parsed.NRequired +} + +func suiteArmIDs(suite *evalsubstrate.Suite, override string) []string { + if override != "" { + var ids []string + for _, value := range strings.Split(override, ",") { + if trimmed := strings.TrimSpace(value); trimmed != "" { + ids = append(ids, trimmed) + } + } + return ids + } + if suite != nil { + return suite.VariedAxis.Values + } + return nil +} + +func suiteDecisionRuleJSON(suite *evalsubstrate.Suite) (string, error) { + rule := map[string]any{"kind": "ci_excludes_zero", "confidence": .95} + if suite != nil { + if suite.Stats.DecisionRule.Kind != "" { + rule["kind"] = suite.Stats.DecisionRule.Kind + } + if suite.Stats.DecisionRule.Confidence > 0 { + rule["confidence"] = suite.Stats.DecisionRule.Confidence + } + if suite.Stats.DecisionRule.MinDelta > 0 { + rule["min_delta"] = suite.Stats.DecisionRule.MinDelta + } + } + data, err := json.Marshal(rule) + if err != nil { + return "", fmt.Errorf("marshal decision rule: %w", err) + } + return string(data), nil +} diff --git a/cli/internal/eval/suite_service_test.go b/cli/internal/eval/suite_service_test.go new file mode 100644 index 000000000..4e488f6cd --- /dev/null +++ b/cli/internal/eval/suite_service_test.go @@ -0,0 +1,36 @@ +package eval + +import ( + "context" + "strings" + "testing" +) + +type suiteRuntimeSpy struct { + args []string + output []byte +} + +func (*suiteRuntimeSpy) Root() string { return "/evals" } +func (*suiteRuntimeSpy) ReadFile(string) ([]byte, error) { return nil, errMissingSuite } +func (runtime *suiteRuntimeSpy) RunStats(args []string) ([]byte, error) { + runtime.args = args + return runtime.output, nil +} + +var errMissingSuite = &missingSuiteError{} + +type missingSuiteError struct{} + +func (*missingSuiteError) Error() string { return "missing suite" } + +func TestSuiteServiceVerdictBuildsDeterministicStatsInvocation(t *testing.T) { + runtime := &suiteRuntimeSpy{output: []byte(`{"verdict":"improved","n_required":3}`)} + result, err := (SuiteService{Runtime: runtime}).Verdict(context.Background(), SuiteVerdictRequest{SuiteID: "suite-1", Arms: "a,b", InputsPath: "inputs.json", BootstrapSamples: 10000}) + if err != nil { + t.Fatalf("Verdict: %v", err) + } + if result.Values["verdict"] != "improved" || !strings.Contains(strings.Join(runtime.args, " "), "--suite-id suite-1 --arms a,b") { + t.Fatalf("result=%#v args=%v", result, runtime.args) + } +} From 2d606cba7c9e23ba334c50e27f6411fed981269c Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:35:25 -0400 Subject: [PATCH 092/108] refactor(eval): extract outcomes service and commands --- cli/internal/adapters/eval/runtime.go | 44 +++++ cli/internal/adapters/eval/runtime_test.go | 1 + cli/internal/commands/eval/module.go | 61 +++++- cli/internal/commands/eval/module_test.go | 22 +++ cli/internal/eval/outcomes_service.go | 220 +++++++++++++++++++++ cli/internal/eval/outcomes_service_test.go | 78 ++++++++ 6 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 cli/internal/eval/outcomes_service.go create mode 100644 cli/internal/eval/outcomes_service_test.go diff --git a/cli/internal/adapters/eval/runtime.go b/cli/internal/adapters/eval/runtime.go index e4eea6442..70f5add5f 100644 --- a/cli/internal/adapters/eval/runtime.go +++ b/cli/internal/adapters/eval/runtime.go @@ -182,3 +182,47 @@ func (runtime Runtime) pythonBinary() string { } return "" } + +func (Runtime) LoadBurnLedger(path string) (evalsubstrate.HoldoutBurnLedger, error) { + var ledger evalsubstrate.HoldoutBurnLedger + raw, err := os.ReadFile(path) + if os.IsNotExist(err) { + return ledger, nil + } + if err != nil { + return ledger, fmt.Errorf("read burn ledger %s: %w", path, err) + } + if err := json.Unmarshal(raw, &ledger); err != nil { + return ledger, fmt.Errorf("parse burn ledger %s: %w", path, err) + } + return ledger, nil +} +func (Runtime) SaveBurnLedger(path string, ledger evalsubstrate.HoldoutBurnLedger) error { + data, err := json.MarshalIndent(ledger, "", " ") + if err != nil { + return fmt.Errorf("encode burn ledger: %w", err) + } + temporary := path + ".tmp" + if err := os.WriteFile(temporary, data, 0o644); err != nil { + return fmt.Errorf("write burn ledger temp: %w", err) + } + if err := os.Rename(temporary, path); err != nil { + return fmt.Errorf("commit burn ledger: %w", err) + } + return nil +} +func (Runtime) WriteOutcomesManifest(dir, runID string, record aoeval.RunRecord) (string, error) { + runDir := filepath.Join(dir, runID) + if err := os.MkdirAll(runDir, 0o750); err != nil { + return "", fmt.Errorf("create manifest dir %s: %w", runDir, err) + } + data, err := json.MarshalIndent(record, "", " ") + if err != nil { + return "", fmt.Errorf("encode eval-run manifest: %w", err) + } + path := filepath.Join(runDir, "manifest.json") + if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil { + return "", fmt.Errorf("write eval-run manifest %s: %w", path, err) + } + return path, nil +} diff --git a/cli/internal/adapters/eval/runtime_test.go b/cli/internal/adapters/eval/runtime_test.go index e314a5ae6..bfadff63f 100644 --- a/cli/internal/adapters/eval/runtime_test.go +++ b/cli/internal/adapters/eval/runtime_test.go @@ -16,6 +16,7 @@ func TestRuntimeSatisfiesCorePortAndResolvesHostPaths(t *testing.T) { var _ aoeval.CleanupRuntime = Runtime{} var _ aoeval.TaskRuntime = Runtime{} var _ aoeval.SuiteRuntime = Runtime{} + var _ aoeval.OutcomesRuntime = Runtime{} workDir, err := port.WorkDir() if err != nil { t.Fatalf("WorkDir: %v", err) diff --git a/cli/internal/commands/eval/module.go b/cli/internal/commands/eval/module.go index dfc0259f6..1e8d998d1 100644 --- a/cli/internal/commands/eval/module.go +++ b/cli/internal/commands/eval/module.go @@ -39,12 +39,17 @@ type SuiteUseCases interface { Verdict(context.Context, aoeval.SuiteVerdictRequest) (aoeval.SuiteVerdictResult, error) NRequired(context.Context, aoeval.SuiteNRequiredRequest) (aoeval.SuiteNRequiredResult, error) } +type OutcomesUseCases interface { + Compile(context.Context, string) (evalsubstrate.Rubric, error) + Ingest(context.Context, aoeval.OutcomesIngestRequest) (aoeval.OutcomesIngestResult, error) +} type UseCases struct { - Core CoreUseCases - Cleanup CleanupUseCases - Task TaskUseCases - Suite SuiteUseCases + Core CoreUseCases + Cleanup CleanupUseCases + Task TaskUseCases + Suite SuiteUseCases + Outcomes OutcomesUseCases } type HostOptions struct { @@ -111,6 +116,9 @@ release. Live Claude and Codex adapters are evaluated by a later runtime tier.`, if module.useCases.Suite != nil { command.AddCommand(module.suiteCommand()) } + if module.useCases.Outcomes != nil { + command.AddCommand(module.outcomesCommand()) + } return command } @@ -497,6 +505,51 @@ func (module Module) suiteNRequiredCommand() *cobra.Command { return command } +func (module Module) outcomesCommand() *cobra.Command { + command := &cobra.Command{Use: "outcomes", Short: "Project the locked eval substrate into Outcomes grading payloads (holdout-safe)", Long: "Outcomes is a derived projection of the locked eval substrate (SCHEMA.md), never an alternate authority."} + command.AddCommand(module.outcomesCompileCommand(), module.outcomesIngestCommand()) + return command +} + +func (module Module) outcomesCompileCommand() *cobra.Command { + command := &cobra.Command{Use: "compile ", Short: "Compile a holdout-safe Outcomes rubric payload from a locked Task + criteria", Args: cobra.ExactArgs(1)} + command.RunE = func(command *cobra.Command, args []string) error { + rubric, err := module.useCases.Outcomes.Compile(command.Context(), args[0]) + if err != nil { + return err + } + return writeJSON(command, rubric) + } + return command +} + +func (module Module) outcomesIngestCommand() *cobra.Command { + var options aoeval.OutcomesIngestRequest + command := &cobra.Command{Use: "ingest ", Short: "Ingest an Outcomes score payload into the one council verdict record", Args: cobra.ExactArgs(1)} + command.Flags().StringVar(&options.ExpectedJudgeHash, "expect-judge-hash", "", "refuse the ingest if the score's judge_content_hash does not match this value (gate #2 rubric-drift parity)") + command.Flags().StringVar(&options.BurnLedgerPath, "burn-ledger", "", "path to a JSON HoldoutBurnLedger; enforce and persist holdout burn quota") + command.Flags().StringVar(&options.ManifestDir, "manifest-out", "", "also write an eval-run.v1 manifest to //manifest.json") + command.Flags().StringVar(&options.RunID, "run-id", "", "run id for the --manifest-out manifest") + command.RunE = func(command *cobra.Command, args []string) error { + options.ScorePath = args[0] + result, err := module.useCases.Outcomes.Ingest(command.Context(), options) + if err != nil { + return err + } + if result.Warning != "" { + fmt.Fprintln(command.ErrOrStderr(), result.Warning) + } + if err := writeJSON(command, result.Verdict); err != nil { + return err + } + if result.ManifestPath != "" { + fmt.Fprintf(command.ErrOrStderr(), "wrote eval-run manifest: %s\n", result.ManifestPath) + } + return nil + } + return command +} + func (module Module) renderRun(command *cobra.Command, result aoeval.CoreRunResult) error { var value any = result.Run if result.Mode == aoeval.CoreRunBaselineAB { diff --git a/cli/internal/commands/eval/module_test.go b/cli/internal/commands/eval/module_test.go index 1e0705219..267b01f85 100644 --- a/cli/internal/commands/eval/module_test.go +++ b/cli/internal/commands/eval/module_test.go @@ -21,6 +21,16 @@ type taskUseCasesSpy struct{ runRequest aoeval.TaskRunRequest } type suiteUseCasesSpy struct{ verdictRequest aoeval.SuiteVerdictRequest } +type outcomesUseCasesSpy struct{ request aoeval.OutcomesIngestRequest } + +func (*outcomesUseCasesSpy) Compile(context.Context, string) (evalsubstrate.Rubric, error) { + return evalsubstrate.Rubric{}, nil +} +func (useCases *outcomesUseCasesSpy) Ingest(_ context.Context, request aoeval.OutcomesIngestRequest) (aoeval.OutcomesIngestResult, error) { + useCases.request = request + return aoeval.OutcomesIngestResult{Verdict: aoeval.OutcomesVerdict{Verdict: "PASS"}}, nil +} + func (useCases *suiteUseCasesSpy) Verdict(_ context.Context, request aoeval.SuiteVerdictRequest) (aoeval.SuiteVerdictResult, error) { useCases.verdictRequest = request return aoeval.SuiteVerdictResult{Values: map[string]any{"verdict": "improved"}}, nil @@ -151,6 +161,18 @@ func TestModuleSuiteVerdictDelegatesFlags(t *testing.T) { } } +func TestModuleOutcomesIngestDelegatesSafetyFlags(t *testing.T) { + outcomes := &outcomesUseCasesSpy{} + command := NewModule(UseCases{Core: &coreUseCasesSpy{}, Outcomes: outcomes}, HostOptions{}).Command() + command.SetArgs([]string{"outcomes", "ingest", "score.json", "--expect-judge-hash", "hash", "--burn-ledger", "burn.json", "--manifest-out", "runs"}) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if outcomes.request.ScorePath != "score.json" || outcomes.request.ExpectedJudgeHash != "hash" || outcomes.request.BurnLedgerPath != "burn.json" || outcomes.request.ManifestDir != "runs" { + t.Fatalf("request = %#v", outcomes.request) + } +} + func findChild(t *testing.T, command *cobra.Command, name string) *cobra.Command { t.Helper() child, _, err := command.Find([]string{name}) diff --git a/cli/internal/eval/outcomes_service.go b/cli/internal/eval/outcomes_service.go new file mode 100644 index 000000000..1cae29536 --- /dev/null +++ b/cli/internal/eval/outcomes_service.go @@ -0,0 +1,220 @@ +package eval + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strings" + "time" + + "github.com/boshu2/agentops/cli/internal/evalsubstrate" +) + +type OutcomesRuntime interface { + ReadFile(string) ([]byte, error) + LoadBurnLedger(string) (evalsubstrate.HoldoutBurnLedger, error) + SaveBurnLedger(string, evalsubstrate.HoldoutBurnLedger) error + WriteOutcomesManifest(string, string, RunRecord) (string, error) + Now() time.Time +} + +type OutcomesService struct{ Runtime OutcomesRuntime } + +type OutcomesCompileInput struct { + Task evalsubstrate.Task `json:"task"` + Criteria []evalsubstrate.Criterion `json:"criteria"` + JudgeContentHash string `json:"judge_content_hash"` + HoldoutValues []string `json:"holdout_values,omitempty"` +} + +type OutcomesScore struct { + SourceTaskID, JudgeContentHash string + Aggregate, Threshold float64 + CriterionScores map[string]float64 + Split, SuiteRef, GroundTruthVersion, RunID string +} + +func (score *OutcomesScore) UnmarshalJSON(data []byte) error { + type wire struct { + SourceTaskID string `json:"source_task_id"` + JudgeContentHash string `json:"judge_content_hash"` + Aggregate float64 `json:"aggregate"` + Threshold float64 `json:"threshold"` + CriterionScores map[string]float64 `json:"criterion_scores"` + Split string `json:"split"` + SuiteRef string `json:"suite_ref"` + GroundTruthVersion string `json:"ground_truth_version"` + RunID string `json:"run_id"` + } + var value wire + if err := json.Unmarshal(data, &value); err != nil { + return err + } + *score = OutcomesScore{SourceTaskID: value.SourceTaskID, JudgeContentHash: value.JudgeContentHash, Aggregate: value.Aggregate, Threshold: value.Threshold, CriterionScores: value.CriterionScores, Split: value.Split, SuiteRef: value.SuiteRef, GroundTruthVersion: value.GroundTruthVersion, RunID: value.RunID} + return nil +} + +type OutcomesVerdict struct { + Verdict string `json:"verdict"` + Confidence string `json:"confidence"` + KeyInsight string `json:"key_insight"` + Recommendation string `json:"recommendation"` + SchemaVersion int `json:"schema_version"` + SatisfactionScore *float64 `json:"satisfaction_score"` + SatisfactionBreakdown map[string]float64 `json:"satisfaction_breakdown"` + Findings []map[string]any `json:"findings"` +} + +type OutcomesIngestRequest struct{ ScorePath, ExpectedJudgeHash, BurnLedgerPath, ManifestDir, RunID string } +type OutcomesIngestResult struct { + Verdict OutcomesVerdict + Warning, ManifestPath string +} + +func (service OutcomesService) Compile(_ context.Context, path string) (evalsubstrate.Rubric, error) { + raw, err := service.Runtime.ReadFile(path) + if err != nil { + return evalsubstrate.Rubric{}, fmt.Errorf("read %s: %w", path, err) + } + var input OutcomesCompileInput + if err := json.Unmarshal(raw, &input); err != nil { + return evalsubstrate.Rubric{}, fmt.Errorf("parse %s: %w", path, err) + } + rubric := evalsubstrate.ProjectRubric(input.Task, input.Criteria, input.JudgeContentHash) + if hit, found := rubric.ContainsAny(input.HoldoutValues); found { + return evalsubstrate.Rubric{}, fmt.Errorf("outcomes compile: holdout value %q would leak into the rubric payload; refusing (Managed Agents are not ZDR)", hit) + } + if key, found := rubric.HasForbiddenKey(); found { + return evalsubstrate.Rubric{}, fmt.Errorf("outcomes compile: emitted payload carries holdout-leak key %q; refusing (Managed Agents are not ZDR)", key) + } + return rubric, nil +} + +func (service OutcomesService) Ingest(_ context.Context, request OutcomesIngestRequest) (OutcomesIngestResult, error) { + raw, err := service.Runtime.ReadFile(request.ScorePath) + if err != nil { + return OutcomesIngestResult{}, fmt.Errorf("read %s: %w", request.ScorePath, err) + } + var score OutcomesScore + if err := json.Unmarshal(raw, &score); err != nil { + return OutcomesIngestResult{}, fmt.Errorf("parse %s: %w", request.ScorePath, err) + } + warning := outcomesHashWarning(score.JudgeContentHash, request.ExpectedJudgeHash) + if request.ExpectedJudgeHash != "" && score.JudgeContentHash != request.ExpectedJudgeHash { + return OutcomesIngestResult{}, fmt.Errorf("outcomes ingest: judge_content_hash mismatch — score was graded against %q but the active rubric is %q; the rubric drifted and this score is stale (gate #2 parity, refused)", score.JudgeContentHash, request.ExpectedJudgeHash) + } + if request.BurnLedgerPath != "" && score.Split == "holdout" { + ledger, err := service.Runtime.LoadBurnLedger(request.BurnLedgerPath) + if err != nil { + return OutcomesIngestResult{}, err + } + updated, err := registerOutcomesBurn(ledger, score) + if err != nil { + return OutcomesIngestResult{}, err + } + if err := service.Runtime.SaveBurnLedger(request.BurnLedgerPath, updated); err != nil { + return OutcomesIngestResult{}, err + } + } + verdict := outcomesVerdict(score) + result := OutcomesIngestResult{Verdict: verdict, Warning: warning} + if request.ManifestDir != "" { + runID := resolveOutcomeRunID(request.RunID, score) + record := buildOutcomeManifest(score, verdict.Verdict, runID, service.Runtime.Now()) + path, err := service.Runtime.WriteOutcomesManifest(request.ManifestDir, runID, record) + if err != nil { + return OutcomesIngestResult{}, err + } + result.ManifestPath = path + } + return result, nil +} + +func outcomesHashWarning(scoreHash, expected string) string { + if expected != "" || scoreHash == "" { + return "" + } + return fmt.Sprintf("WARN outcomes ingest: score carries judge_content_hash %q but --expect-judge-hash was not provided; rubric-drift parity (gate #2) was NOT checked — pass --expect-judge-hash to enforce it", scoreHash) +} + +func registerOutcomesBurn(ledger evalsubstrate.HoldoutBurnLedger, score OutcomesScore) (evalsubstrate.HoldoutBurnLedger, error) { + if score.Split != "holdout" { + return ledger, nil + } + if ledger.Budget > 0 && ledger.Spent(score.SuiteRef, score.GroundTruthVersion) >= ledger.Budget { + return ledger, fmt.Errorf("outcomes ingest: holdout burn refused — quota for (suite=%q, gt_version=%q) is exhausted (%d/%d spent); the holdout split is statistically spent and may not be re-observed (gate #3, no escape)", score.SuiteRef, score.GroundTruthVersion, ledger.Spent(score.SuiteRef, score.GroundTruthVersion), ledger.Budget) + } + ledger.Records = append(ledger.Records, evalsubstrate.BurnRecord{SuiteRef: score.SuiteRef, GTVersion: score.GroundTruthVersion, RunID: score.RunID}) + return ledger, nil +} + +func outcomesVerdict(score OutcomesScore) OutcomesVerdict { + threshold := score.Threshold + if threshold <= 0 { + threshold = 1 + } + band := "WARN" + if score.Aggregate >= threshold { + band = "PASS" + } else if score.Aggregate < threshold*.7 { + band = "FAIL" + } + aggregate := score.Aggregate + return OutcomesVerdict{Verdict: band, Confidence: "HIGH", KeyInsight: fmt.Sprintf("Outcomes aggregate %.4f vs threshold %.4f for task %q", score.Aggregate, threshold, score.SourceTaskID), Recommendation: fmt.Sprintf("Outcomes grade ingested as %s; feeds the corpus via the eval-verdict pipeline.", band), SchemaVersion: 4, SatisfactionScore: &aggregate, SatisfactionBreakdown: score.CriterionScores, Findings: []map[string]any{}} +} + +var outcomeRunIDInvalid = regexp.MustCompile(`[^a-zA-Z0-9._:-]`) + +func resolveOutcomeRunID(explicit string, score OutcomesScore) string { + for _, candidate := range []string{explicit, score.RunID, score.SourceTaskID} { + cleaned := strings.TrimLeft(outcomeRunIDInvalid.ReplaceAllString(candidate, "-"), "-._:") + if cleaned != "" { + return cleaned + } + } + return "outcomes-run" +} + +func buildOutcomeManifest(score OutcomesScore, band, runID string, started time.Time) RunRecord { + dimensions := map[Dimension]float64{} + allowed := map[Dimension]bool{DimensionCorrectness: true, DimensionProcessAdherence: true, DimensionArtifactQuality: true, DimensionRuntimeCompatibility: true, DimensionEfficiency: true, DimensionSafety: true, DimensionLearningClosure: true} + for key, value := range score.CriterionScores { + dimension := Dimension(key) + if allowed[dimension] { + dimensions[dimension] = clampOutcomeScore(value) + } + } + if len(dimensions) == 0 { + dimensions[DimensionCorrectness] = clampOutcomeScore(score.Aggregate) + } + status, verdict := StatusInconclusive, VerdictAdvisory + if band == "PASS" { + status, verdict = StatusPass, VerdictPass + } else if band == "FAIL" { + status, verdict = StatusFail, VerdictFail + } + caseID := score.SourceTaskID + if caseID == "" { + caseID = "aggregate" + } + suiteID := score.SuiteRef + if suiteID == "" { + suiteID = "outcomes" + } + completed := started + visibility := VisibilityPublicCanary + if score.Split == "holdout" { + visibility = VisibilityPrivateHoldout + } + return RunRecord{SchemaVersion: 1, RunID: runID, Suite: SuiteRef{ID: suiteID, Path: "out-of-process", Visibility: visibility, Tier: TierLive}, StartedAt: started, CompletedAt: &completed, Status: status, Verdict: verdict, Git: GitRecord{CandidateRef: "out-of-process-outcomes-grade", CandidateSHA: "0000000"}, Runtime: RuntimeRecord{Name: RuntimeManual}, Environment: EnvironmentRecord{ScrubbedEnvPrefixes: []string{}, NetworkAccess: NetworkUnknown}, CaseResults: []CaseResult{{ID: caseID, Status: status, Score: clampOutcomeScore(score.Aggregate), DimensionScores: dimensions}}, AggregateScore: clampOutcomeScore(score.Aggregate), DimensionScores: dimensions, Notes: []string{fmt.Sprintf("Out-of-process Outcomes grade ingested by `ao eval outcomes ingest` (band=%s); git/runtime/environment are stubs — the grade was produced outside this harness.", band)}} +} +func clampOutcomeScore(value float64) float64 { + if value < 0 { + return 0 + } + if value > 1 { + return 1 + } + return value +} diff --git a/cli/internal/eval/outcomes_service_test.go b/cli/internal/eval/outcomes_service_test.go new file mode 100644 index 000000000..f01423643 --- /dev/null +++ b/cli/internal/eval/outcomes_service_test.go @@ -0,0 +1,78 @@ +package eval + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/boshu2/agentops/cli/internal/evalsubstrate" +) + +type outcomesRuntimeSpy struct { + files map[string][]byte + saves int + ledger evalsubstrate.HoldoutBurnLedger + manifest *RunRecord +} + +func (runtime *outcomesRuntimeSpy) ReadFile(path string) ([]byte, error) { + return runtime.files[path], nil +} +func (runtime *outcomesRuntimeSpy) LoadBurnLedger(string) (evalsubstrate.HoldoutBurnLedger, error) { + return runtime.ledger, nil +} +func (runtime *outcomesRuntimeSpy) SaveBurnLedger(string, evalsubstrate.HoldoutBurnLedger) error { + runtime.saves++ + return nil +} +func (runtime *outcomesRuntimeSpy) WriteOutcomesManifest(_ string, _ string, record RunRecord) (string, error) { + runtime.manifest = &record + return "manifest.json", nil +} + +func TestOutcomesServiceIngestPersistsHoldoutBurnAndManifest(t *testing.T) { + runtime := &outcomesRuntimeSpy{files: map[string][]byte{"score.json": []byte(`{"source_task_id":"task-1","judge_content_hash":"hash","aggregate":0.9,"threshold":0.8,"criterion_scores":{"correctness":0.9},"split":"holdout","suite_ref":"suite-1","ground_truth_version":"gt-1","run_id":"run-1"}`)}, ledger: evalsubstrate.HoldoutBurnLedger{Budget: 2}} + result, err := (OutcomesService{Runtime: runtime}).Ingest(context.Background(), OutcomesIngestRequest{ScorePath: "score.json", BurnLedgerPath: "burn.json", ManifestDir: "runs"}) + if err != nil { + t.Fatalf("Ingest: %v", err) + } + if result.Verdict.Verdict != "PASS" || result.Warning == "" || result.ManifestPath != "manifest.json" { + t.Fatalf("result = %#v", result) + } + if runtime.saves != 1 || runtime.manifest == nil || runtime.manifest.Suite.Visibility != VisibilityPrivateHoldout { + t.Fatalf("saves=%d manifest=%#v", runtime.saves, runtime.manifest) + } +} + +func TestOutcomesServiceIngestRefusesExhaustedHoldout(t *testing.T) { + ledger := evalsubstrate.HoldoutBurnLedger{Budget: 1, Records: []evalsubstrate.BurnRecord{{SuiteRef: "suite-1", GTVersion: "gt-1", RunID: "old"}}} + runtime := &outcomesRuntimeSpy{files: map[string][]byte{"score.json": []byte(`{"aggregate":1,"threshold":1,"split":"holdout","suite_ref":"suite-1","ground_truth_version":"gt-1","run_id":"new"}`)}, ledger: ledger} + _, err := (OutcomesService{Runtime: runtime}).Ingest(context.Background(), OutcomesIngestRequest{ScorePath: "score.json", BurnLedgerPath: "burn.json"}) + if err == nil || !strings.Contains(err.Error(), "quota") { + t.Fatalf("Ingest error = %v", err) + } + if runtime.saves != 0 { + t.Fatalf("saves = %d", runtime.saves) + } +} +func (*outcomesRuntimeSpy) Now() time.Time { return time.Unix(1_000, 0).UTC() } + +func TestOutcomesServiceCompileRefusesHoldoutLeak(t *testing.T) { + runtime := &outcomesRuntimeSpy{files: map[string][]byte{"input.json": []byte(`{"task":{"id":"task-1","stats":{"min_n_samples":3}},"criteria":[{"id":"c1","description":"secret answer","weight":1}],"holdout_values":["secret answer"]}`)}} + _, err := (OutcomesService{Runtime: runtime}).Compile(context.Background(), "input.json") + if err == nil || !strings.Contains(err.Error(), "would leak") { + t.Fatalf("Compile error = %v", err) + } +} + +func TestOutcomesServiceIngestRefusesJudgeHashMismatchBeforeBurn(t *testing.T) { + runtime := &outcomesRuntimeSpy{files: map[string][]byte{"score.json": []byte(`{"source_task_id":"task-1","judge_content_hash":"old","aggregate":1,"threshold":1,"split":"holdout"}`)}} + _, err := (OutcomesService{Runtime: runtime}).Ingest(context.Background(), OutcomesIngestRequest{ScorePath: "score.json", ExpectedJudgeHash: "new", BurnLedgerPath: "burn.json"}) + if err == nil || !strings.Contains(err.Error(), "judge_content_hash mismatch") { + t.Fatalf("Ingest error = %v", err) + } + if runtime.saves != 0 { + t.Fatalf("burn saves = %d, want 0", runtime.saves) + } +} From 4b07f72c65240b096bd6cf6633fb1a0851013644 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:44:37 -0400 Subject: [PATCH 093/108] refactor(eval): extract scenario service and commands --- cli/internal/adapters/eval/runtime.go | 67 ++++ cli/internal/adapters/eval/runtime_test.go | 1 + cli/internal/commands/eval/module.go | 153 ++++++++- cli/internal/commands/eval/module_test.go | 30 ++ cli/internal/eval/scenario_service.go | 351 +++++++++++++++++++++ cli/internal/eval/scenario_service_test.go | 103 ++++++ 6 files changed, 703 insertions(+), 2 deletions(-) create mode 100644 cli/internal/eval/scenario_service.go create mode 100644 cli/internal/eval/scenario_service_test.go diff --git a/cli/internal/adapters/eval/runtime.go b/cli/internal/adapters/eval/runtime.go index 70f5add5f..b75a426ce 100644 --- a/cli/internal/adapters/eval/runtime.go +++ b/cli/internal/adapters/eval/runtime.go @@ -12,6 +12,9 @@ import ( aoeval "github.com/boshu2/agentops/cli/internal/eval" "github.com/boshu2/agentops/cli/internal/evalsubstrate" + "github.com/boshu2/agentops/cli/internal/goals" + "github.com/boshu2/agentops/cli/internal/scenario" + "github.com/boshu2/agentops/cli/internal/scenarioresults" ) type Runtime struct{} @@ -226,3 +229,67 @@ func (Runtime) WriteOutcomesManifest(dir, runID string, record aoeval.RunRecord) } return path, nil } + +func (Runtime) Create(options scenario.CreateOptions) (*scenario.CreateResult, error) { + return scenario.Create(options) +} +func (Runtime) MkdirAll(path string, mode uint32) error { return os.MkdirAll(path, os.FileMode(mode)) } +func (Runtime) Exists(path string) (bool, error) { + _, err := os.Stat(path) + if os.IsNotExist(err) { + return false, nil + } + return err == nil, err +} +func (Runtime) ReadDir(path string) ([]aoeval.ScenarioFileEntry, error) { + entries, err := os.ReadDir(path) + if err != nil { + return nil, err + } + result := make([]aoeval.ScenarioFileEntry, 0, len(entries)) + for _, entry := range entries { + result = append(result, aoeval.ScenarioFileEntry{Name: entry.Name(), IsDir: entry.IsDir()}) + } + return result, nil +} +func (Runtime) WriteFile(path string, data []byte, mode uint32) error { + return os.WriteFile(path, data, os.FileMode(mode)) +} +func (Runtime) LoadDirectives(path, id string) ([]goals.ParsedDirective, error) { + patcher, _, err := goals.LoadGoalsPatcher(path) + if err != nil { + return nil, err + } + return goals.FilterDirectives(patcher.Directives(), 0, id), nil +} +func (Runtime) LoadGates(path string) (map[string]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + file, err := goals.ParseMarkdownGoals(data) + if err != nil { + return nil, err + } + gates := make(map[string]string, len(file.Goals)) + for _, goal := range file.Goals { + gates[goal.ID] = goal.Check + } + return gates, nil +} +func (Runtime) ScenarioDirs() []string { return goals.DefaultScenarioDirs() } +func (Runtime) Measure(command string, timeout time.Duration) (string, string) { + measurement := goals.MeasureOne(goals.Goal{ID: "scenario-check", Check: command}, timeout) + return measurement.Result, measurement.Output +} +func (Runtime) CurrentIteration(root string) int { + loaded, err := scenarioresults.Load(root, false) + if err == nil && loaded.Artifact != nil { + return loaded.Artifact.Iteration + 1 + } + return 1 +} +func (Runtime) AppendResults(root, runID string, iteration int, results []scenarioresults.ScenarioResult, now time.Time) error { + _, err := (scenarioresults.Writer{Now: func() time.Time { return now }}).Append(root, runID, iteration, results) + return err +} diff --git a/cli/internal/adapters/eval/runtime_test.go b/cli/internal/adapters/eval/runtime_test.go index bfadff63f..457fac206 100644 --- a/cli/internal/adapters/eval/runtime_test.go +++ b/cli/internal/adapters/eval/runtime_test.go @@ -17,6 +17,7 @@ func TestRuntimeSatisfiesCorePortAndResolvesHostPaths(t *testing.T) { var _ aoeval.TaskRuntime = Runtime{} var _ aoeval.SuiteRuntime = Runtime{} var _ aoeval.OutcomesRuntime = Runtime{} + var _ aoeval.ScenarioRuntime = Runtime{} workDir, err := port.WorkDir() if err != nil { t.Fatalf("WorkDir: %v", err) diff --git a/cli/internal/commands/eval/module.go b/cli/internal/commands/eval/module.go index 1e8d998d1..b309506ec 100644 --- a/cli/internal/commands/eval/module.go +++ b/cli/internal/commands/eval/module.go @@ -7,12 +7,14 @@ import ( "errors" "fmt" "strings" + "time" "github.com/spf13/cobra" "github.com/boshu2/agentops/cli/internal/clicontract" aoeval "github.com/boshu2/agentops/cli/internal/eval" "github.com/boshu2/agentops/cli/internal/evalsubstrate" + scenarioapp "github.com/boshu2/agentops/cli/internal/scenario" ) type CoreUseCases interface { @@ -43,6 +45,13 @@ type OutcomesUseCases interface { Compile(context.Context, string) (evalsubstrate.Rubric, error) Ingest(context.Context, aoeval.OutcomesIngestRequest) (aoeval.OutcomesIngestResult, error) } +type ScenarioUseCases interface { + Add(context.Context, aoeval.ScenarioAddRequest) (*scenarioapp.CreateResult, error) + Init(context.Context) (string, error) + List(context.Context, string) (aoeval.ScenarioListResult, error) + Validate(context.Context) (aoeval.ScenarioValidationResult, error) + Evaluate(context.Context, aoeval.ScenarioEvaluateRequest) (*aoeval.ScenarioEvaluateReport, error) +} type UseCases struct { Core CoreUseCases @@ -50,11 +59,14 @@ type UseCases struct { Task TaskUseCases Suite SuiteUseCases Outcomes OutcomesUseCases + Scenario ScenarioUseCases } type HostOptions struct { - OutputMode func(*cobra.Command) string - Verbose func(*cobra.Command) bool + OutputMode func(*cobra.Command) string + Verbose func(*cobra.Command) bool + ProjectRoot func() string + GoalsPath func() string } type Module struct { @@ -119,6 +131,9 @@ release. Live Claude and Codex adapters are evaluated by a later runtime tier.`, if module.useCases.Outcomes != nil { command.AddCommand(module.outcomesCommand()) } + if module.useCases.Scenario != nil { + command.AddCommand(module.scenarioCommand()) + } return command } @@ -550,6 +565,140 @@ func (module Module) outcomesIngestCommand() *cobra.Command { return command } +func (module Module) scenarioCommand() *cobra.Command { + command := &cobra.Command{Use: "scenario", Short: "Manage holdout scenarios for behavioral validation", Long: "Create, list, validate, and evaluate holdout scenarios stored in .agents/holdout/."} + command.AddCommand(module.scenarioInitCommand(), module.scenarioAddCommand(), module.scenarioListCommand(), module.scenarioValidateCommand(), module.scenarioEvaluateCommand()) + return command +} + +func (module Module) scenarioInitCommand() *cobra.Command { + command := &cobra.Command{Use: "init", Short: "Initialize .agents/holdout/ directory for scenario storage"} + command.RunE = func(command *cobra.Command, _ []string) error { + path, err := module.useCases.Scenario.Init(command.Context()) + if err == nil { + fmt.Fprintf(command.OutOrStdout(), "Initialized holdout directory at %s\n", path) + } + return err + } + return command +} + +func (module Module) scenarioAddCommand() *cobra.Command { + options := aoeval.ScenarioAddRequest{Threshold: .8, Status: "draft", Source: "human"} + command := &cobra.Command{Use: "add ", Short: "Author a holdout scenario from a goal description", Args: cobra.ExactArgs(1)} + command.Flags().StringVar(&options.Narrative, "narrative", "", "Narrative description (default: inferred from goal)") + command.Flags().StringVar(&options.ExpectedOutcome, "expected-outcome", "", "Expected observable outcome (default: inferred from goal)") + command.Flags().Float64Var(&options.Threshold, "threshold", .8, "Satisfaction threshold in [0,1]") + command.Flags().StringVar(&options.Status, "status", "draft", "Scenario status (active, draft, retired)") + command.Flags().StringVar(&options.Source, "source", "human", "Scenario source (human, agent, prod-telemetry)") + _ = command.RegisterFlagCompletionFunc("status", staticCompletion("active", "draft", "retired")) + _ = command.RegisterFlagCompletionFunc("source", staticCompletion("human", "agent", "prod-telemetry")) + command.RunE = func(command *cobra.Command, args []string) error { + options.Goal = args[0] + result, err := module.useCases.Scenario.Add(command.Context(), options) + if err != nil { + return err + } + if module.outputMode(command) == "json" { + return writeJSON(command, result.Scenario) + } + fmt.Fprintf(command.OutOrStdout(), "Created scenario %s at %s\n", result.Scenario.ID, result.Path) + return nil + } + return command +} + +func (module Module) scenarioListCommand() *cobra.Command { + var status string + command := &cobra.Command{Use: "list", Short: "List holdout scenarios"} + command.Flags().StringVar(&status, "status", "", "Filter by status (active, draft, retired)") + command.RunE = func(command *cobra.Command, _ []string) error { + result, err := module.useCases.Scenario.List(command.Context(), status) + if err != nil { + return err + } + if result.MissingDirectory { + fmt.Fprintln(command.OutOrStdout(), "No holdout directory found. Run 'ao scenario init' first.") + return nil + } + if len(result.Scenarios) == 0 { + fmt.Fprintln(command.OutOrStdout(), "No scenarios found.") + return nil + } + return writeJSON(command, result.Scenarios) + } + return command +} + +func (module Module) scenarioValidateCommand() *cobra.Command { + command := &cobra.Command{Use: "validate", Short: "Validate holdout scenarios against schema"} + command.RunE = func(command *cobra.Command, _ []string) error { + result, err := module.useCases.Scenario.Validate(command.Context()) + if err != nil { + return err + } + if result.MissingDirectory { + fmt.Fprintln(command.OutOrStdout(), "No holdout directory found. Run 'ao scenario init' first.") + return nil + } + if len(result.Errors) > 0 { + fmt.Fprintf(command.ErrOrStderr(), "Validation errors:\n%s\n", strings.Join(result.Errors, "\n")) + return fmt.Errorf("%d validation error(s) found", len(result.Errors)) + } + if result.Validated == 0 { + fmt.Fprintln(command.OutOrStdout(), "No scenario files found to validate.") + } else { + fmt.Fprintf(command.OutOrStdout(), "Validated %d scenario(s): all pass\n", result.Validated) + } + return nil + } + return command +} + +func (module Module) scenarioEvaluateCommand() *cobra.Command { + options := aoeval.ScenarioEvaluateRequest{Timeout: 2 * time.Minute, RunID: "ao-scenario-evaluate"} + var jsonOutput bool + command := &cobra.Command{Use: "evaluate", Short: "Evaluate directive-linked scenarios and record satisfaction results", Args: cobra.NoArgs} + command.Flags().StringVar(&options.DirectiveID, "directive", "", "Evaluate only the directive with this stable Directive ID") + command.Flags().BoolVar(&options.All, "all", false, "Evaluate every directive's linked scenarios") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit the machine-readable evaluation report") + command.Flags().DurationVar(&options.Timeout, "timeout", 2*time.Minute, "Per-check execution timeout") + command.Flags().StringVar(&options.RunID, "run-id", "ao-scenario-evaluate", "run_id recorded in the results artifact") + command.RunE = func(command *cobra.Command, _ []string) error { + if module.host.ProjectRoot != nil { + options.ProjectRoot = module.host.ProjectRoot() + } + if module.host.GoalsPath != nil { + options.GoalsPath = module.host.GoalsPath() + } + report, err := module.useCases.Scenario.Evaluate(command.Context(), options) + if err != nil { + return err + } + if jsonOutput || module.outputMode(command) == "json" { + return writeJSON(command, report) + } + return renderScenarioEvaluate(command, report) + } + return command +} + +func renderScenarioEvaluate(command *cobra.Command, report *aoeval.ScenarioEvaluateReport) error { + for _, evaluation := range report.Evaluations { + if !evaluation.Recorded { + fmt.Fprintf(command.OutOrStdout(), "%-20s %-10s not recorded — %s\n", evaluation.ScenarioID, "-", evaluation.Note) + } else { + fmt.Fprintf(command.OutOrStdout(), "%-20s %-10s shape=%s score=%.2f threshold=%.2f\n", evaluation.ScenarioID, evaluation.Verdict, evaluation.Shape, evaluation.Score, evaluation.Threshold) + } + } + if report.Written == 0 { + fmt.Fprintln(command.OutOrStdout(), "No results recorded; artifact untouched.") + return nil + } + fmt.Fprintf(command.OutOrStdout(), "Wrote %d result(s) to %s (run %s, iteration %d)\n", report.Written, report.Artifact, report.RunID, report.Iteration) + return nil +} + func (module Module) renderRun(command *cobra.Command, result aoeval.CoreRunResult) error { var value any = result.Run if result.Mode == aoeval.CoreRunBaselineAB { diff --git a/cli/internal/commands/eval/module_test.go b/cli/internal/commands/eval/module_test.go index 267b01f85..f5425c38d 100644 --- a/cli/internal/commands/eval/module_test.go +++ b/cli/internal/commands/eval/module_test.go @@ -9,6 +9,7 @@ import ( aoeval "github.com/boshu2/agentops/cli/internal/eval" "github.com/boshu2/agentops/cli/internal/evalsubstrate" + scenarioapp "github.com/boshu2/agentops/cli/internal/scenario" ) type coreUseCasesSpy struct { @@ -23,6 +24,23 @@ type suiteUseCasesSpy struct{ verdictRequest aoeval.SuiteVerdictRequest } type outcomesUseCasesSpy struct{ request aoeval.OutcomesIngestRequest } +type scenarioUseCasesSpy struct{ addRequest aoeval.ScenarioAddRequest } + +func (useCases *scenarioUseCasesSpy) Add(_ context.Context, request aoeval.ScenarioAddRequest) (*scenarioapp.CreateResult, error) { + useCases.addRequest = request + return &scenarioapp.CreateResult{Scenario: scenarioapp.Scenario{ID: "s-2026-01-01-001"}, Path: "path"}, nil +} +func (*scenarioUseCasesSpy) Init(context.Context) (string, error) { return ".agents/holdout", nil } +func (*scenarioUseCasesSpy) List(context.Context, string) (aoeval.ScenarioListResult, error) { + return aoeval.ScenarioListResult{}, nil +} +func (*scenarioUseCasesSpy) Validate(context.Context) (aoeval.ScenarioValidationResult, error) { + return aoeval.ScenarioValidationResult{}, nil +} +func (*scenarioUseCasesSpy) Evaluate(context.Context, aoeval.ScenarioEvaluateRequest) (*aoeval.ScenarioEvaluateReport, error) { + return &aoeval.ScenarioEvaluateReport{}, nil +} + func (*outcomesUseCasesSpy) Compile(context.Context, string) (evalsubstrate.Rubric, error) { return evalsubstrate.Rubric{}, nil } @@ -173,6 +191,18 @@ func TestModuleOutcomesIngestDelegatesSafetyFlags(t *testing.T) { } } +func TestModuleScenarioAddDelegatesClosureLocalFlags(t *testing.T) { + useCases := &scenarioUseCasesSpy{} + command := NewModule(UseCases{Core: &coreUseCasesSpy{}, Scenario: useCases}, HostOptions{}).Command() + command.SetArgs([]string{"scenario", "add", "goal", "--threshold", "0.7", "--status", "active"}) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if useCases.addRequest.Goal != "goal" || useCases.addRequest.Threshold != .7 || useCases.addRequest.Status != "active" { + t.Fatalf("request=%#v", useCases.addRequest) + } +} + func findChild(t *testing.T, command *cobra.Command, name string) *cobra.Command { t.Helper() child, _, err := command.Find([]string{name}) diff --git a/cli/internal/eval/scenario_service.go b/cli/internal/eval/scenario_service.go new file mode 100644 index 000000000..9bb716f74 --- /dev/null +++ b/cli/internal/eval/scenario_service.go @@ -0,0 +1,351 @@ +package eval + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/boshu2/agentops/cli/internal/goals" + "github.com/boshu2/agentops/cli/internal/goalsfitness" + "github.com/boshu2/agentops/cli/internal/scenario" + "github.com/boshu2/agentops/cli/internal/scenarioresults" +) + +const ScenarioHoldoutDir = ".agents/holdout" + +type ScenarioFileEntry struct { + Name string + IsDir bool +} + +type ScenarioRuntime interface { + Create(scenario.CreateOptions) (*scenario.CreateResult, error) + MkdirAll(string, uint32) error + Exists(string) (bool, error) + ReadDir(string) ([]ScenarioFileEntry, error) + ReadFile(string) ([]byte, error) + WriteFile(string, []byte, uint32) error + LoadDirectives(string, string) ([]goals.ParsedDirective, error) + LoadGates(string) (map[string]string, error) + ScenarioDirs() []string + Measure(string, time.Duration) (string, string) + CurrentIteration(string) int + AppendResults(string, string, int, []scenarioresults.ScenarioResult, time.Time) error + Now() time.Time +} + +type ScenarioService struct{ Runtime ScenarioRuntime } +type ScenarioAddRequest struct { + Goal, Narrative, ExpectedOutcome string + Threshold float64 + Status, Source string +} +type ScenarioSummary struct { + ID string `json:"id"` + Goal string `json:"goal"` + Status string `json:"status"` + Date string `json:"date"` +} +type ScenarioListResult struct { + Scenarios []ScenarioSummary `json:"scenarios"` + MissingDirectory bool `json:"-"` +} +type ScenarioValidationResult struct { + Validated int + Errors []string + MissingDirectory bool +} +type ScenarioEvaluateRequest struct { + ProjectRoot, GoalsPath, DirectiveID string + All bool + Timeout time.Duration + RunID string +} +type ScenarioEvaluation struct { + ScenarioID string `json:"scenario_id"` + DirectiveID string `json:"directive_id"` + Shape string `json:"shape,omitempty"` + Verdict string `json:"verdict,omitempty"` + Score float64 `json:"score"` + Threshold float64 `json:"threshold"` + Evidence []string `json:"evidence,omitempty"` + Recorded bool `json:"recorded"` + Note string `json:"note,omitempty"` +} +type ScenarioEvaluateReport struct { + RunID string `json:"run_id"` + Iteration int `json:"iteration"` + Artifact string `json:"artifact"` + Written int `json:"written"` + Evaluations []ScenarioEvaluation `json:"evaluations"` +} + +func (service ScenarioService) Add(_ context.Context, request ScenarioAddRequest) (*scenario.CreateResult, error) { + return service.Runtime.Create(scenario.CreateOptions{Goal: request.Goal, Narrative: request.Narrative, ExpectedOutcome: request.ExpectedOutcome, Threshold: request.Threshold, Status: request.Status, Source: request.Source, Dir: filepath.FromSlash(ScenarioHoldoutDir), Now: service.Runtime.Now}) +} + +func (service ScenarioService) Init(_ context.Context) (string, error) { + dir := filepath.FromSlash(ScenarioHoldoutDir) + if err := service.Runtime.MkdirAll(dir, 0o755); err != nil { + return "", fmt.Errorf("creating holdout directory: %w", err) + } + readme := filepath.Join(dir, "README.md") + exists, err := service.Runtime.Exists(readme) + if err != nil { + return "", err + } + if !exists { + body := []byte("# Holdout Scenarios\n\nThis directory contains behavioral validation scenarios that implementing agents cannot see.\n") + if err := service.Runtime.WriteFile(readme, body, 0o644); err != nil { + return "", fmt.Errorf("writing README: %w", err) + } + } + return dir, nil +} + +func (service ScenarioService) List(_ context.Context, status string) (ScenarioListResult, error) { + entries, err := service.Runtime.ReadDir(filepath.FromSlash(ScenarioHoldoutDir)) + if err != nil { + if isMissingScenarioError(err) { + return ScenarioListResult{MissingDirectory: true}, nil + } + return ScenarioListResult{}, fmt.Errorf("reading holdout directory: %w", err) + } + result := ScenarioListResult{Scenarios: []ScenarioSummary{}} + for _, entry := range entries { + if entry.IsDir || filepath.Ext(entry.Name) != ".json" { + continue + } + raw, err := service.Runtime.ReadFile(filepath.Join(filepath.FromSlash(ScenarioHoldoutDir), entry.Name)) + if err != nil { + continue + } + var summary ScenarioSummary + if json.Unmarshal(raw, &summary) != nil || status != "" && summary.Status != status { + continue + } + result.Scenarios = append(result.Scenarios, summary) + } + sort.Slice(result.Scenarios, func(i, j int) bool { return result.Scenarios[i].ID < result.Scenarios[j].ID }) + return result, nil +} + +func (service ScenarioService) Validate(_ context.Context) (ScenarioValidationResult, error) { + entries, err := service.Runtime.ReadDir(filepath.FromSlash(ScenarioHoldoutDir)) + if err != nil { + if isMissingScenarioError(err) { + return ScenarioValidationResult{MissingDirectory: true}, nil + } + return ScenarioValidationResult{}, fmt.Errorf("reading holdout directory: %w", err) + } + result := ScenarioValidationResult{} + for _, entry := range entries { + if entry.IsDir || filepath.Ext(entry.Name) != ".json" { + continue + } + raw, err := service.Runtime.ReadFile(filepath.Join(filepath.FromSlash(ScenarioHoldoutDir), entry.Name)) + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("%s: read error: %v", entry.Name, err)) + continue + } + var object map[string]any + if err := json.Unmarshal(raw, &object); err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("%s: invalid JSON: %v", entry.Name, err)) + continue + } + for _, field := range []string{"id", "version", "date", "goal", "narrative", "expected_outcome", "satisfaction_threshold", "status"} { + if _, ok := object[field]; !ok { + result.Errors = append(result.Errors, fmt.Sprintf("%s: missing required field '%s'", entry.Name, field)) + } + } + if id, ok := object["id"].(string); ok && !scenario.IDPattern.MatchString(id) { + result.Errors = append(result.Errors, fmt.Sprintf("%s: id '%s' does not match pattern s-YYYY-MM-DD-NNN or auto-*", entry.Name, id)) + } + if value, ok := object["status"].(string); ok && !scenario.ValidStatus(value) { + result.Errors = append(result.Errors, fmt.Sprintf("%s: invalid status '%s' (must be active, draft, or retired)", entry.Name, value)) + } + if value, ok := object["source"].(string); ok && !scenario.ValidSource(value) { + result.Errors = append(result.Errors, fmt.Sprintf("%s: invalid source '%s' (must be human, agent, or prod-telemetry)", entry.Name, value)) + } + if value, ok := object["satisfaction_threshold"].(float64); ok && (value < 0 || value > 1) { + result.Errors = append(result.Errors, fmt.Sprintf("%s: satisfaction_threshold %.2f out of range [0, 1]", entry.Name, value)) + } + result.Validated++ + } + return result, nil +} + +func (service ScenarioService) Evaluate(_ context.Context, request ScenarioEvaluateRequest) (*ScenarioEvaluateReport, error) { + if !request.All && request.DirectiveID == "" { + return nil, fmt.Errorf("choose a scope: --all for every directive, or --directive ") + } + directives, err := service.Runtime.LoadDirectives(request.GoalsPath, request.DirectiveID) + if err != nil { + return nil, fmt.Errorf("loading directives: %w", err) + } + if len(directives) == 0 { + return nil, fmt.Errorf("no directive matches --directive %q", request.DirectiveID) + } + gates, err := service.Runtime.LoadGates(request.GoalsPath) + if err != nil { + return nil, fmt.Errorf("loading gates table: %w", err) + } + var evaluations []ScenarioEvaluation + for _, directive := range directives { + for _, id := range directive.Scenarios { + evaluations = append(evaluations, service.evaluateOne(directive, id, gates, request.Timeout)) + } + } + report := &ScenarioEvaluateReport{RunID: request.RunID, Artifact: scenarioresults.ArtifactRelPath, Evaluations: evaluations} + if report.Evaluations == nil { + report.Evaluations = []ScenarioEvaluation{} + } + results := service.persistable(evaluations) + report.Written = len(results) + if len(results) > 0 { + report.Iteration = service.Runtime.CurrentIteration(request.ProjectRoot) + if err := service.Runtime.AppendResults(request.ProjectRoot, request.RunID, report.Iteration, results, service.Runtime.Now()); err != nil { + return nil, fmt.Errorf("writing scenario results: %w", err) + } + } + return report, nil +} + +func (service ScenarioService) evaluateOne(directive goals.ParsedDirective, id string, gates map[string]string, timeout time.Duration) ScenarioEvaluation { + evaluation := ScenarioEvaluation{ScenarioID: id, DirectiveID: directive.StableID} + spec, err := service.loadSpec(id) + if err != nil { + evaluation.Note = fmt.Sprintf("unreadable scenario spec: %v", err) + return evaluation + } + if spec == nil { + evaluation.Note = "no scenario file found on the search path" + return evaluation + } + if spec.Status == "retired" { + evaluation.Note = "scenario is retired; not evaluated" + return evaluation + } + if evaluation.DirectiveID == "" { + evaluation.DirectiveID = spec.DirectiveID + } + if !scenarioresults.ValidDirectiveID(evaluation.DirectiveID) || !scenarioresults.ValidScenarioID(id) { + evaluation.Note = "no valid stable directive/scenario ID; result would not validate" + return evaluation + } + evaluation.Threshold = normalizeScenarioThreshold(spec.SatisfactionThreshold) + if spec.SatisfactionThreshold < 0 || spec.SatisfactionThreshold > 1 { + evaluation.Shape = "judgment" + evaluation.Verdict = scenarioresults.VerdictSkip + evaluation.Evidence = []string{fmt.Sprintf("invalid satisfaction_threshold %g (must be in (0,1]); spec must be fixed before this scenario can certify", spec.SatisfactionThreshold)} + evaluation.Recorded = true + return evaluation + } + checks := mechanicalScenarioChecks(spec.AcceptanceVectors) + if len(checks) == 0 { + evaluation.Shape = "judgment" + evaluation.Verdict = scenarioresults.VerdictSkip + evaluation.Evidence = []string{"attestation-needed: no mechanical acceptance-vector check; requires judge/human attestation"} + evaluation.Recorded = true + return evaluation + } + verdict, score, evidence := service.runChecks(checks, gates, evaluation.Threshold, timeout) + evaluation.Score, evaluation.Evidence, evaluation.Recorded = score, evidence, true + if len(checks) < len(spec.AcceptanceVectors) { + evaluation.Shape = "judgment" + evaluation.Verdict = scenarioresults.VerdictSkip + evaluation.Evidence = append([]string{fmt.Sprintf("attestation-needed: %d of %d acceptance vectors carry no mechanical check; mechanical subset ran for evidence only", len(spec.AcceptanceVectors)-len(checks), len(spec.AcceptanceVectors))}, evidence...) + return evaluation + } + evaluation.Shape, evaluation.Verdict = "gate", verdict + return evaluation +} + +func (service ScenarioService) loadSpec(id string) (*scenario.Scenario, error) { + for _, dir := range service.Runtime.ScenarioDirs() { + raw, err := service.Runtime.ReadFile(filepath.Join(dir, id+".json")) + if err != nil { + if isMissingScenarioError(err) { + continue + } + return nil, err + } + var spec scenario.Scenario + if err := json.Unmarshal(raw, &spec); err != nil { + return nil, err + } + return &spec, nil + } + return nil, nil +} + +func (service ScenarioService) runChecks(checks []scenario.AcceptanceVector, gates map[string]string, threshold float64, timeout time.Duration) (string, float64, []string) { + passed, skipped := 0, 0 + evidence := make([]string, 0, len(checks)) + for _, vector := range checks { + command := strings.TrimSpace(vector.Check) + if id, ok := strings.CutPrefix(command, "gate:"); ok { + resolved, found := gates[strings.TrimSpace(id)] + if !found { + skipped++ + evidence = append(evidence, fmt.Sprintf("%s: skip — unresolvable gate reference %q (not in the GOALS.md Gates table)", vector.Dimension, command)) + continue + } + command = resolved + } + outcome, detail := service.Runtime.Measure(command, timeout) + if outcome == "pass" { + passed++ + } else if outcome == "skip" { + skipped++ + } + text := fmt.Sprintf("`%s` -> %s", command, outcome) + if outcome != "pass" && detail != "" { + text += ": " + detail + } + evidence = append(evidence, fmt.Sprintf("%s: %s — %s", vector.Dimension, outcome, text)) + } + score := float64(passed) / float64(len(checks)) + if skipped > 0 { + return scenarioresults.VerdictSkip, score, evidence + } + if score >= threshold { + return scenarioresults.VerdictPass, score, evidence + } + return scenarioresults.VerdictFail, score, evidence +} + +func (service ScenarioService) persistable(evaluations []ScenarioEvaluation) []scenarioresults.ScenarioResult { + judged := service.Runtime.Now().UTC().Format(time.RFC3339) + var results []scenarioresults.ScenarioResult + for _, evaluation := range evaluations { + if evaluation.Recorded { + results = append(results, scenarioresults.ScenarioResult{ScenarioID: evaluation.ScenarioID, DirectiveID: evaluation.DirectiveID, Score: evaluation.Score, Threshold: evaluation.Threshold, Verdict: evaluation.Verdict, JudgedAt: judged, Evidence: evaluation.Evidence}) + } + } + return results +} + +func mechanicalScenarioChecks(vectors []scenario.AcceptanceVector) []scenario.AcceptanceVector { + var checks []scenario.AcceptanceVector + for _, vector := range vectors { + if strings.TrimSpace(vector.Check) != "" { + checks = append(checks, vector) + } + } + return checks +} +func normalizeScenarioThreshold(value float64) float64 { + if value <= 0 || value > 1 { + return goalsfitness.DefaultScenarioThreshold + } + return value +} + +func isMissingScenarioError(err error) bool { return errors.Is(err, os.ErrNotExist) } diff --git a/cli/internal/eval/scenario_service_test.go b/cli/internal/eval/scenario_service_test.go new file mode 100644 index 000000000..4a485657a --- /dev/null +++ b/cli/internal/eval/scenario_service_test.go @@ -0,0 +1,103 @@ +package eval + +import ( + "context" + "testing" + "time" + + "github.com/boshu2/agentops/cli/internal/goals" + "github.com/boshu2/agentops/cli/internal/scenario" + "github.com/boshu2/agentops/cli/internal/scenarioresults" +) + +type scenarioRuntimeSpy struct { + created scenario.CreateOptions + files map[string][]byte + directives []goals.ParsedDirective + dirs []string + appended []scenarioresults.ScenarioResult + entries []ScenarioFileEntry +} + +func (runtime *scenarioRuntimeSpy) Create(options scenario.CreateOptions) (*scenario.CreateResult, error) { + runtime.created = options + return &scenario.CreateResult{Scenario: scenario.Scenario{ID: "s-2026-01-01-001"}, Path: ".agents/holdout/s-2026-01-01-001.json"}, nil +} +func (*scenarioRuntimeSpy) MkdirAll(string, uint32) error { return nil } +func (*scenarioRuntimeSpy) Exists(string) (bool, error) { return false, nil } +func (runtime *scenarioRuntimeSpy) ReadDir(string) ([]ScenarioFileEntry, error) { + return runtime.entries, nil +} +func (runtime *scenarioRuntimeSpy) ReadFile(path string) ([]byte, error) { + return runtime.files[path], nil +} +func (*scenarioRuntimeSpy) WriteFile(string, []byte, uint32) error { return nil } +func (runtime *scenarioRuntimeSpy) LoadDirectives(string, string) ([]goals.ParsedDirective, error) { + return runtime.directives, nil +} +func (*scenarioRuntimeSpy) LoadGates(string) (map[string]string, error) { return nil, nil } +func (runtime *scenarioRuntimeSpy) ScenarioDirs() []string { return runtime.dirs } +func (*scenarioRuntimeSpy) Measure(string, time.Duration) (string, string) { return "pass", "" } +func (*scenarioRuntimeSpy) CurrentIteration(string) int { return 1 } +func (runtime *scenarioRuntimeSpy) AppendResults(_ string, _ string, _ int, results []scenarioresults.ScenarioResult, _ time.Time) error { + runtime.appended = results + return nil +} +func (*scenarioRuntimeSpy) Now() time.Time { return time.Unix(1_000, 0).UTC() } + +func TestScenarioServiceAddDelegatesValidatedDomainCreation(t *testing.T) { + runtime := &scenarioRuntimeSpy{} + result, err := (ScenarioService{Runtime: runtime}).Add(context.Background(), ScenarioAddRequest{Goal: "ship behavior", Threshold: .8, Status: "draft", Source: "human"}) + if err != nil { + t.Fatalf("Add: %v", err) + } + if runtime.created.Goal != "ship behavior" || runtime.created.Dir != ".agents/holdout" || result.Scenario.ID == "" { + t.Fatalf("created=%#v result=%#v", runtime.created, result) + } +} + +func TestScenarioServiceEvaluateRecordsMechanicalPass(t *testing.T) { + id := "s-2026-01-01-001" + runtime := &scenarioRuntimeSpy{ + dirs: []string{"spec"}, + directives: []goals.ParsedDirective{{StableID: "d-one", Scenarios: []string{id}}}, + files: map[string][]byte{"spec/" + id + ".json": []byte(`{"id":"s-2026-01-01-001","status":"active","satisfaction_threshold":0.8,"acceptance_vectors":[{"dimension":"works","check":"true"}]}`)}, + } + report, err := (ScenarioService{Runtime: runtime}).Evaluate(context.Background(), ScenarioEvaluateRequest{ProjectRoot: ".", GoalsPath: "GOALS.md", All: true, Timeout: time.Second, RunID: "run-1"}) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + if report.Written != 1 || report.Evaluations[0].Verdict != scenarioresults.VerdictPass || len(runtime.appended) != 1 { + t.Fatalf("report=%#v appended=%#v", report, runtime.appended) + } +} + +func TestScenarioServiceEvaluateRequiresExplicitScope(t *testing.T) { + _, err := (ScenarioService{Runtime: &scenarioRuntimeSpy{}}).Evaluate(context.Background(), ScenarioEvaluateRequest{}) + if err == nil { + t.Fatal("Evaluate accepted missing scope") + } +} + +func TestScenarioServiceEvaluateMixedVectorsNeverCertifiesPass(t *testing.T) { + id := "s-2026-01-01-002" + runtime := &scenarioRuntimeSpy{dirs: []string{"spec"}, directives: []goals.ParsedDirective{{StableID: "d-one", Scenarios: []string{id}}}, files: map[string][]byte{"spec/" + id + ".json": []byte(`{"id":"s-2026-01-01-002","status":"active","satisfaction_threshold":0.5,"acceptance_vectors":[{"dimension":"mechanical","check":"true"},{"dimension":"judgment"}]}`)}} + report, err := (ScenarioService{Runtime: runtime}).Evaluate(context.Background(), ScenarioEvaluateRequest{All: true, Timeout: time.Second, RunID: "run"}) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + if report.Evaluations[0].Verdict != scenarioresults.VerdictSkip || report.Evaluations[0].Shape != "judgment" { + t.Fatalf("evaluation=%#v", report.Evaluations[0]) + } +} + +func TestScenarioServiceValidateReportsMalformedScenario(t *testing.T) { + runtime := &scenarioRuntimeSpy{entries: []ScenarioFileEntry{{Name: "bad.json"}}, files: map[string][]byte{".agents/holdout/bad.json": []byte(`{"id":"bad","status":"bogus","satisfaction_threshold":2}`)}} + result, err := (ScenarioService{Runtime: runtime}).Validate(context.Background()) + if err != nil { + t.Fatalf("Validate: %v", err) + } + if result.Validated != 1 || len(result.Errors) < 2 { + t.Fatalf("result=%#v", result) + } +} From 10ac18515860a22247245666afeee26bd9b05f82 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:48:36 -0400 Subject: [PATCH 094/108] refactor(eval): extract scenario AB and moat service --- cli/internal/commands/eval/module.go | 93 +++++++++++++++++-- cli/internal/commands/eval/module_test.go | 22 +++++ cli/internal/eval/scenario_ab_service.go | 92 ++++++++++++++++++ cli/internal/eval/scenario_ab_service_test.go | 74 +++++++++++++++ 4 files changed, 275 insertions(+), 6 deletions(-) create mode 100644 cli/internal/eval/scenario_ab_service.go create mode 100644 cli/internal/eval/scenario_ab_service_test.go diff --git a/cli/internal/commands/eval/module.go b/cli/internal/commands/eval/module.go index b309506ec..bcda17e57 100644 --- a/cli/internal/commands/eval/module.go +++ b/cli/internal/commands/eval/module.go @@ -52,14 +52,19 @@ type ScenarioUseCases interface { Validate(context.Context) (aoeval.ScenarioValidationResult, error) Evaluate(context.Context, aoeval.ScenarioEvaluateRequest) (*aoeval.ScenarioEvaluateReport, error) } +type ScenarioABUseCases interface { + Run(context.Context, aoeval.ScenarioABRequest) (aoeval.ScenarioABResult, error) + Moat(context.Context, aoeval.ScenarioMoatRequest) (aoeval.MoatClaimResult, error) +} type UseCases struct { - Core CoreUseCases - Cleanup CleanupUseCases - Task TaskUseCases - Suite SuiteUseCases - Outcomes OutcomesUseCases - Scenario ScenarioUseCases + Core CoreUseCases + Cleanup CleanupUseCases + Task TaskUseCases + Suite SuiteUseCases + Outcomes OutcomesUseCases + Scenario ScenarioUseCases + ScenarioAB ScenarioABUseCases } type HostOptions struct { @@ -134,6 +139,9 @@ release. Live Claude and Codex adapters are evaluated by a later runtime tier.`, if module.useCases.Scenario != nil { command.AddCommand(module.scenarioCommand()) } + if module.useCases.ScenarioAB != nil { + command.AddCommand(module.scenarioABCommand(), module.scenarioMoatCommand()) + } return command } @@ -699,6 +707,79 @@ func renderScenarioEvaluate(command *cobra.Command, report *aoeval.ScenarioEvalu return nil } +func (module Module) scenarioABCommand() *cobra.Command { + var options aoeval.ScenarioABRequest + command := &cobra.Command{Use: "scenario-ab", Short: "Run a knowledge-reuse holdout scenario with vs. without the gold pull (the discriminating A/B)", Args: cobra.NoArgs, SilenceUsage: true} + command.Flags().StringVar(&options.ScenarioPath, "scenario", "", "Path to the scenario.v1 JSON file (required)") + command.Flags().StringVar(&options.OutputPath, "output", "", "Write the ScenarioDeltaScorecard JSON to this path") + command.Flags().IntVar(&options.TokenBudget, "token-budget", 0, "Fail the gate if summed arm token cost exceeds this (0 = default 200000)") + command.Flags().DurationVar(&options.Timeout, "timeout", 0, "Per-arm timeout (0 = default 5m)") + command.Flags().BoolVar(&options.ControlOnly, "control-only", false, "Run only the without-gold control arm and fail on ceiling/no-headroom") + command.RunE = func(command *cobra.Command, _ []string) error { + result, err := module.useCases.ScenarioAB.Run(command.Context(), options) + if result.Card.ScenarioID != "" { + renderScenarioAB(command, result.Card) + } + if err != nil { + var gate *aoeval.ScenarioABGateError + if errors.As(err, &gate) { + for _, reason := range gate.Reasons { + fmt.Fprintf(command.ErrOrStderr(), " FAIL: %s\n", reason) + } + } + } + return err + } + return command +} +func renderScenarioAB(command *cobra.Command, card aoeval.ScenarioDeltaScorecard) { + if card.CeilingViolation { + fmt.Fprintf(command.OutOrStdout(), "scenario-ab %s [%s]: CEILING VIOLATION (without=%.4f >= threshold=%.4f) — invalid scenario, no delta emitted\n", card.ScenarioID, card.VerdictClass, card.Without.Score, card.SatisfactionThreshold) + } else if card.ControlOnly { + fmt.Fprintf(command.OutOrStdout(), "scenario-ab %s [%s]: control-only headroom=PASS (without=%.4f < threshold=%.4f) tokens=%d gate=%s\n", card.ScenarioID, card.VerdictClass, card.Without.Score, card.SatisfactionThreshold, card.Without.TokenCost, passLabel(card.Gate.Pass)) + } else { + fmt.Fprintf(command.OutOrStdout(), "scenario-ab %s [%s]: delta=%.4f (with=%.4f without=%.4f) tokens=%d gate=%s %s\n", card.ScenarioID, card.VerdictClass, card.AggregateDelta, card.With.Score, card.Without.Score, card.With.TokenCost+card.Without.TokenCost, passLabel(card.Gate.Pass), moatLabel(card.MoatEligible)) + } +} +func passLabel(pass bool) string { + if pass { + return "PASS" + } + return "FAIL" +} +func moatLabel(eligible bool) string { + if eligible { + return "moat-eligible" + } + return "NOT-moat-evidence(plumbing)" +} + +func (module Module) scenarioMoatCommand() *cobra.Command { + var options aoeval.ScenarioMoatRequest + command := &cobra.Command{Use: "scenario-moat", Short: "Aggregate moat-eligible scenario A/B scorecards into a publication verdict", Args: cobra.NoArgs, SilenceUsage: true} + command.Flags().StringArrayVar(&options.ScorecardPaths, "scorecard", nil, "Path to a ScenarioDeltaScorecard JSON (repeatable)") + command.Flags().StringVar(&options.OutputPath, "output", "", "Write the MoatClaimResult JSON to this path") + command.RunE = func(command *cobra.Command, _ []string) error { + result, err := module.useCases.ScenarioAB.Moat(command.Context(), options) + if err != nil { + var ineligible aoeval.ErrMoatIneligibleScorecard + if errors.As(err, &ineligible) { + fmt.Fprintf(command.ErrOrStderr(), "REJECTED: %s\n", err) + } + return err + } + if module.outputMode(command) == "json" { + return writeJSON(command, result) + } + fmt.Fprintf(command.OutOrStdout(), "scenario-moat: verdict=%s scenarios=%d mean_delta=%.4f\n %s\n", result.Verdict, result.ScenarioCount, result.MeanAggregateDelta, result.Reason) + if options.OutputPath != "" { + fmt.Fprintf(command.OutOrStdout(), "Moat claim result: %s\n", options.OutputPath) + } + return nil + } + return command +} + func (module Module) renderRun(command *cobra.Command, result aoeval.CoreRunResult) error { var value any = result.Run if result.Mode == aoeval.CoreRunBaselineAB { diff --git a/cli/internal/commands/eval/module_test.go b/cli/internal/commands/eval/module_test.go index f5425c38d..4f2b5e479 100644 --- a/cli/internal/commands/eval/module_test.go +++ b/cli/internal/commands/eval/module_test.go @@ -26,6 +26,16 @@ type outcomesUseCasesSpy struct{ request aoeval.OutcomesIngestRequest } type scenarioUseCasesSpy struct{ addRequest aoeval.ScenarioAddRequest } +type scenarioABUseCasesSpy struct{ request aoeval.ScenarioABRequest } + +func (useCases *scenarioABUseCasesSpy) Run(_ context.Context, request aoeval.ScenarioABRequest) (aoeval.ScenarioABResult, error) { + useCases.request = request + return aoeval.ScenarioABResult{Card: aoeval.ScenarioDeltaScorecard{ScenarioID: "s-1", Gate: aoeval.ScenarioGate{Pass: true}}}, nil +} +func (*scenarioABUseCasesSpy) Moat(context.Context, aoeval.ScenarioMoatRequest) (aoeval.MoatClaimResult, error) { + return aoeval.MoatClaimResult{}, nil +} + func (useCases *scenarioUseCasesSpy) Add(_ context.Context, request aoeval.ScenarioAddRequest) (*scenarioapp.CreateResult, error) { useCases.addRequest = request return &scenarioapp.CreateResult{Scenario: scenarioapp.Scenario{ID: "s-2026-01-01-001"}, Path: "path"}, nil @@ -203,6 +213,18 @@ func TestModuleScenarioAddDelegatesClosureLocalFlags(t *testing.T) { } } +func TestModuleScenarioABDelegatesFlags(t *testing.T) { + useCases := &scenarioABUseCasesSpy{} + command := NewModule(UseCases{Core: &coreUseCasesSpy{}, ScenarioAB: useCases}, HostOptions{}).Command() + command.SetArgs([]string{"scenario-ab", "--scenario", "s.json", "--token-budget", "9", "--control-only"}) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if useCases.request.ScenarioPath != "s.json" || useCases.request.TokenBudget != 9 || !useCases.request.ControlOnly { + t.Fatalf("request=%#v", useCases.request) + } +} + func findChild(t *testing.T, command *cobra.Command, name string) *cobra.Command { t.Helper() child, _, err := command.Find([]string{name}) diff --git a/cli/internal/eval/scenario_ab_service.go b/cli/internal/eval/scenario_ab_service.go new file mode 100644 index 000000000..7b2dc2e09 --- /dev/null +++ b/cli/internal/eval/scenario_ab_service.go @@ -0,0 +1,92 @@ +package eval + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/boshu2/agentops/cli/internal/scenario" +) + +type ScenarioABRuntime interface { + LoadScenario(string) (*scenario.Scenario, error) + Runner(scenario.Scenario) ScenarioRunner + Judge(scenario.Scenario) ScenarioJudge + WriteScenarioCard(ScenarioDeltaScorecard, string) error + LoadScenarioCard(string) (ScenarioDeltaScorecard, error) + WriteMoatResult(string, MoatClaimResult) error +} + +type ScenarioABService struct{ Runtime ScenarioABRuntime } +type ScenarioABRequest struct { + ScenarioPath, OutputPath string + TokenBudget int + Timeout time.Duration + ControlOnly bool +} +type ScenarioABResult struct{ Card ScenarioDeltaScorecard } +type ScenarioMoatRequest struct { + ScorecardPaths []string + OutputPath string +} + +type ScenarioABGateError struct { + ScenarioID string + Reasons []string +} + +func (failure *ScenarioABGateError) Error() string { + return fmt.Sprintf("scenario-ab gate failed for %s", failure.ScenarioID) +} + +func (service ScenarioABService) Run(ctx context.Context, request ScenarioABRequest) (ScenarioABResult, error) { + if strings.TrimSpace(request.ScenarioPath) == "" { + return ScenarioABResult{}, fmt.Errorf("--scenario is required") + } + spec, err := service.Runtime.LoadScenario(request.ScenarioPath) + if err != nil { + return ScenarioABResult{}, err + } + judge := service.Runtime.Judge(*spec) + if strings.TrimSpace(spec.AnswerKey) != "" { + judge = AnswerKeyJudge{} + } + card, err := RunScenarioAB(ctx, ScenarioABOptions{Scenario: *spec, ScenarioPath: request.ScenarioPath, Runner: service.Runtime.Runner(*spec), Judge: judge, Timeout: request.Timeout, TokenBudget: request.TokenBudget, ControlOnly: request.ControlOnly}) + if err != nil { + return ScenarioABResult{}, err + } + if err := service.Runtime.WriteScenarioCard(card, request.OutputPath); err != nil { + return ScenarioABResult{}, err + } + result := ScenarioABResult{Card: card} + if !card.Gate.Pass { + return result, &ScenarioABGateError{ScenarioID: card.ScenarioID, Reasons: append([]string(nil), card.Gate.Reasons...)} + } + return result, nil +} + +func (service ScenarioABService) Moat(_ context.Context, request ScenarioMoatRequest) (MoatClaimResult, error) { + if len(request.ScorecardPaths) == 0 { + return MoatClaimResult{}, fmt.Errorf("at least one --scorecard path is required") + } + cards := make([]ScenarioDeltaScorecard, 0, len(request.ScorecardPaths)) + for _, path := range request.ScorecardPaths { + card, err := service.Runtime.LoadScenarioCard(path) + if err != nil { + return MoatClaimResult{}, err + } + cards = append(cards, card) + } + result, err := AggregateMoatClaim(cards) + if err != nil { + return MoatClaimResult{}, err + } + if err := service.Runtime.WriteMoatResult(request.OutputPath, result); err != nil { + return MoatClaimResult{}, err + } + if result.Verdict == MoatVerdictInconclusive { + return result, fmt.Errorf("moat claim inconclusive — cannot publish positive or honest null") + } + return result, nil +} diff --git a/cli/internal/eval/scenario_ab_service_test.go b/cli/internal/eval/scenario_ab_service_test.go new file mode 100644 index 000000000..4cb6a142b --- /dev/null +++ b/cli/internal/eval/scenario_ab_service_test.go @@ -0,0 +1,74 @@ +package eval + +import ( + "context" + "testing" + + "github.com/boshu2/agentops/cli/internal/scenario" +) + +type scenarioABRuntimeSpy struct { + scenario scenario.Scenario + runner ScenarioRunner + judge ScenarioJudge + wrote bool + card ScenarioDeltaScorecard +} + +func (runtime *scenarioABRuntimeSpy) LoadScenario(string) (*scenario.Scenario, error) { + return &runtime.scenario, nil +} +func (runtime *scenarioABRuntimeSpy) Runner(scenario.Scenario) ScenarioRunner { return runtime.runner } +func (runtime *scenarioABRuntimeSpy) Judge(scenario.Scenario) ScenarioJudge { return runtime.judge } +func (runtime *scenarioABRuntimeSpy) WriteScenarioCard(ScenarioDeltaScorecard, string) error { + runtime.wrote = true + return nil +} +func (*scenarioABRuntimeSpy) LoadScenarioCard(string) (ScenarioDeltaScorecard, error) { + return ScenarioDeltaScorecard{}, nil +} + +func TestScenarioABServiceMoatRejectsPlumbingCard(t *testing.T) { + runtime := &scenarioABRuntimeSpy{} + runtime.card = ScenarioDeltaScorecard{ScenarioID: "s-1", MoatEligible: false, VerdictClass: "fact-recall"} + // Override through a tiny runtime wrapper so the service receives the card. + _, err := (ScenarioABService{Runtime: scenarioABCardRuntime{scenarioABRuntimeSpy: runtime}}).Moat(context.Background(), ScenarioMoatRequest{ScorecardPaths: []string{"card.json"}}) + if err == nil { + t.Fatal("Moat accepted moat_eligible=false card") + } +} + +type scenarioABCardRuntime struct{ *scenarioABRuntimeSpy } + +func (runtime scenarioABCardRuntime) LoadScenarioCard(string) (ScenarioDeltaScorecard, error) { + return runtime.card, nil +} +func (*scenarioABRuntimeSpy) WriteMoatResult(string, MoatClaimResult) error { return nil } + +type fixedArmRunner struct{} + +func (fixedArmRunner) RunArm(_ context.Context, _ scenario.Scenario, withGold bool) (ArmOutcome, error) { + output := "miss" + if withGold { + output = "answer" + } + return ArmOutcome{Output: output, TokenCost: 1}, nil +} + +func TestScenarioABServiceRequiresScenarioPath(t *testing.T) { + _, err := (ScenarioABService{Runtime: &scenarioABRuntimeSpy{}}).Run(context.Background(), ScenarioABRequest{}) + if err == nil { + t.Fatal("Run accepted missing scenario path") + } +} + +func TestScenarioABServiceUsesAnswerKeyJudgeAndPersistsCard(t *testing.T) { + runtime := &scenarioABRuntimeSpy{scenario: scenario.Scenario{ID: "s-1", Goal: "g", Narrative: "n", AnswerKey: "answer", SatisfactionThreshold: .5}, runner: fixedArmRunner{}} + result, err := (ScenarioABService{Runtime: runtime}).Run(context.Background(), ScenarioABRequest{ScenarioPath: "scenario.json", OutputPath: "out.json"}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !runtime.wrote || !result.Card.Gate.Pass { + t.Fatalf("wrote=%v result=%#v", runtime.wrote, result) + } +} From 2610e190fde9f35c7fbf628d90f01443e0406bbe Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:52:11 -0400 Subject: [PATCH 095/108] refactor(eval): move scenario AB runtime adapter --- cli/internal/adapters/eval/runtime_test.go | 1 + .../adapters/eval/scenario_ab_agentic.go | 195 ++++++++++++++ .../adapters/eval/scenario_ab_agentic_test.go | 88 +++++++ .../adapters/eval/scenario_ab_runtime.go | 173 +++++++++++++ .../adapters/eval/scenario_ab_runtime_test.go | 24 ++ .../adapters/eval/scenario_ab_sandbox.go | 196 ++++++++++++++ .../adapters/eval/scenario_ab_sandbox_test.go | 243 ++++++++++++++++++ 7 files changed, 920 insertions(+) create mode 100644 cli/internal/adapters/eval/scenario_ab_agentic.go create mode 100644 cli/internal/adapters/eval/scenario_ab_agentic_test.go create mode 100644 cli/internal/adapters/eval/scenario_ab_runtime.go create mode 100644 cli/internal/adapters/eval/scenario_ab_runtime_test.go create mode 100644 cli/internal/adapters/eval/scenario_ab_sandbox.go create mode 100644 cli/internal/adapters/eval/scenario_ab_sandbox_test.go diff --git a/cli/internal/adapters/eval/runtime_test.go b/cli/internal/adapters/eval/runtime_test.go index 457fac206..81ed594ac 100644 --- a/cli/internal/adapters/eval/runtime_test.go +++ b/cli/internal/adapters/eval/runtime_test.go @@ -18,6 +18,7 @@ func TestRuntimeSatisfiesCorePortAndResolvesHostPaths(t *testing.T) { var _ aoeval.SuiteRuntime = Runtime{} var _ aoeval.OutcomesRuntime = Runtime{} var _ aoeval.ScenarioRuntime = Runtime{} + var _ aoeval.ScenarioABRuntime = Runtime{} workDir, err := port.WorkDir() if err != nil { t.Fatalf("WorkDir: %v", err) diff --git a/cli/internal/adapters/eval/scenario_ab_agentic.go b/cli/internal/adapters/eval/scenario_ab_agentic.go new file mode 100644 index 000000000..98ebcd9eb --- /dev/null +++ b/cli/internal/adapters/eval/scenario_ab_agentic.go @@ -0,0 +1,195 @@ +// practices: [llm-eval-harness] +package eval + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + aoeval "github.com/boshu2/agentops/cli/internal/eval" + "github.com/boshu2/agentops/cli/internal/scenario" +) + +const ( + agenticMaxTurns = 8 + agenticStepSchemaJSON = `{"type":"object","additionalProperties":false,"properties":{"commands":{"type":"array","items":{"type":"string"}},"done":{"type":"boolean"},"summary":{"type":"string"}},"required":["commands","done","summary"]}` +) + +// agenticStep is one structured turn from the worker model. +type agenticStep struct { + Commands []string `json:"commands"` + Done bool `json:"done"` + Summary string `json:"summary"` +} + +// workspaceCommandRunner executes shell commands inside the isolated workspace. +type workspaceCommandRunner func(ctx context.Context, workDir, command string) (stdout string, exitCode int, err error) + +// agenticRunnerHooks is the injectable seam for tests (no live codex). +var agenticRunnerHooks = struct { + runCodex func(ctx context.Context, prompt, schemaPath string) (string, int, error) + runCmd workspaceCommandRunner +}{ + runCodex: runCodexExec, + runCmd: defaultWorkspaceCommandRunner, +} + +type agenticScenarioRunner struct{} + +func (agenticScenarioRunner) RunArm(ctx context.Context, sc scenario.Scenario, withGold bool) (aoeval.ArmOutcome, error) { + workDir, err := os.MkdirTemp("", "scenario-ab-agentic-*") + if err != nil { + return aoeval.ArmOutcome{}, fmt.Errorf("agentic workspace: %w", err) + } + defer func() { _ = os.RemoveAll(workDir) }() + + schemaPath, cleanup, err := writeAgenticStepSchema() + if err != nil { + return aoeval.ArmOutcome{}, err + } + defer cleanup() + + var history strings.Builder + totalTokens := 0 + for turn := 0; turn < agenticMaxTurns; turn++ { + prompt := buildAgenticPrompt(ctx, sc, withGold, workDir, history.String()) + out, tokens, err := agenticRunnerHooks.runCodex(ctx, prompt, schemaPath) + totalTokens += tokens + if err != nil { + return aoeval.ArmOutcome{}, fmt.Errorf("agentic turn %d: %w", turn+1, err) + } + step, err := parseAgenticStep(out) + if err != nil { + return aoeval.ArmOutcome{}, fmt.Errorf("agentic turn %d parse: %w", turn+1, err) + } + for _, command := range step.Commands { + command = strings.TrimSpace(command) + if command == "" { + continue + } + stdout, exitCode, runErr := agenticRunnerHooks.runCmd(ctx, workDir, command) + if runErr != nil { + return aoeval.ArmOutcome{}, fmt.Errorf("agentic command %q: %w", command, runErr) + } + fmt.Fprintf(&history, "$ %s\nexit=%d\n%s\n\n", command, exitCode, strings.TrimSpace(stdout)) + } + if step.Done { + summary := strings.TrimSpace(step.Summary) + if summary == "" { + summary = strings.TrimSpace(history.String()) + } + artifactSummary, _ := summarizeWorkspace(workDir) + if artifactSummary != "" { + summary = summary + "\n\nWorkspace artifacts:\n" + artifactSummary + } + return aoeval.ArmOutcome{Output: summary, TokenCost: totalTokens}, nil + } + } + return aoeval.ArmOutcome{}, fmt.Errorf("agentic runner exceeded %d turns without done=true", agenticMaxTurns) +} + +func buildAgenticPrompt(ctx context.Context, sc scenario.Scenario, withGold bool, workDir, history string) string { + var p strings.Builder + p.WriteString("You are an agentic worker in an isolated workspace. Execute the task by running shell commands in the workspace directory.\n") + p.WriteString("Return ONLY strict JSON matching the schema: commands (shell lines to run now), done (true when finished), summary (final result for grading when done=true).\n\n") + if withGold { + if pointers := goldPointers(ctx, sc.Goal); pointers != "" { + p.WriteString("Relevant prior knowledge (gold corpus pointers):\n") + p.WriteString(pointers) + p.WriteString("\n\n") + } + } + p.WriteString(sc.Narrative) + p.WriteString("\n\nGoal: ") + p.WriteString(sc.Goal) + if strings.TrimSpace(sc.ExpectedOutcome) != "" { + p.WriteString("\nExpected outcome: ") + p.WriteString(sc.ExpectedOutcome) + } + p.WriteString("\n\nWorkspace: ") + p.WriteString(workDir) + p.WriteString("\nRun commands relative to this directory. Do not read outside the workspace.\n") + if history != "" { + p.WriteString("\nPrior command history:\n") + p.WriteString(history) + } + return p.String() +} + +func writeAgenticStepSchema() (string, func(), error) { + f, err := os.CreateTemp("", "scenario-ab-agentic-schema-*.json") + if err != nil { + return "", func() {}, fmt.Errorf("agentic schema temp: %w", err) + } + name := f.Name() + cleanup := func() { _ = os.Remove(name) } + if _, err := f.WriteString(agenticStepSchemaJSON); err != nil { + _ = f.Close() + cleanup() + return "", func() {}, fmt.Errorf("write agentic schema: %w", err) + } + if err := f.Close(); err != nil { + cleanup() + return "", func() {}, fmt.Errorf("close agentic schema: %w", err) + } + return name, cleanup, nil +} + +func parseAgenticStep(text string) (agenticStep, error) { + start := strings.Index(text, "{") + end := strings.LastIndex(text, "}") + if start < 0 || end <= start { + return agenticStep{}, fmt.Errorf("no JSON object found") + } + var step agenticStep + if err := json.Unmarshal([]byte(text[start:end+1]), &step); err != nil { + return agenticStep{}, err + } + return step, nil +} + +func defaultWorkspaceCommandRunner(ctx context.Context, workDir, command string) (string, int, error) { + cmd := exec.CommandContext(ctx, "bash", "-lc", command) + cmd.Dir = workDir + out, err := cmd.CombinedOutput() + exitCode := 0 + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + exitCode = ee.ExitCode() + } else { + return "", 0, err + } + } + return string(out), exitCode, nil +} + +func summarizeWorkspace(workDir string) (string, error) { + var lines []string + _ = filepath.WalkDir(workDir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + rel, relErr := filepath.Rel(workDir, path) + if relErr != nil || strings.HasPrefix(rel, ".") { + return nil + } + info, statErr := os.Stat(path) + if statErr != nil { + return nil + } + lines = append(lines, fmt.Sprintf("- %s (%d bytes)", rel, info.Size())) + return nil + }) + return strings.Join(lines, "\n"), nil +} + +func selectScenarioRunner(sc scenario.Scenario) aoeval.ScenarioRunner { + if sc.EffectiveRunnerMode() == scenario.RunnerModeAgentic { + return agenticScenarioRunner{} + } + return codexScenarioRunner{} +} diff --git a/cli/internal/adapters/eval/scenario_ab_agentic_test.go b/cli/internal/adapters/eval/scenario_ab_agentic_test.go new file mode 100644 index 000000000..04fcd73cd --- /dev/null +++ b/cli/internal/adapters/eval/scenario_ab_agentic_test.go @@ -0,0 +1,88 @@ +package eval + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/boshu2/agentops/cli/internal/scenario" +) + +func withAgenticHooks(t *testing.T, steps []agenticStep) { + t.Helper() + orig := agenticRunnerHooks + call := 0 + agenticRunnerHooks.runCodex = func(_ context.Context, _ string, _ string) (string, int, error) { + idx := call + if len(steps) == 0 { + return "", 0, context.Canceled + } + if idx >= len(steps) { + idx = len(steps) - 1 + } + step := steps[idx] + call++ + data, err := json.Marshal(step) + if err != nil { + return "", 0, err + } + return string(data), 10, nil + } + agenticRunnerHooks.runCmd = func(_ context.Context, workDir, command string) (string, int, error) { + if strings.Contains(command, "touch") { + name := strings.TrimSpace(strings.TrimPrefix(command, "touch")) + if err := os.WriteFile(filepath.Join(workDir, name), []byte("ok\n"), 0o644); err != nil { + return "", 1, err + } + return "", 0, nil + } + return "", 0, nil + } + t.Cleanup(func() { + agenticRunnerHooks = orig + }) +} + +func TestAgenticScenarioRunnerProducesWorkspaceArtifact(t *testing.T) { + withAgenticHooks(t, []agenticStep{ + {Commands: []string{"touch result.txt"}, Done: true, Summary: "created result.txt in workspace"}, + }) + sc := scenario.Scenario{ + ID: "s-agentic-test", + Goal: "create a file named result.txt", + Narrative: "Ship a file in the workspace.", + } + out, err := agenticScenarioRunner{}.RunArm(context.Background(), sc, false) + if err != nil { + t.Fatalf("RunArm: %v", err) + } + if !strings.Contains(out.Output, "result.txt") { + t.Errorf("output = %q, want result.txt mention", out.Output) + } +} + +func TestSelectScenarioRunnerAgentic(t *testing.T) { + sc := scenario.Scenario{RunnerMode: "agentic"} + if _, ok := selectScenarioRunner(sc).(agenticScenarioRunner); !ok { + t.Fatalf("selectScenarioRunner(agentic) = %T, want agenticScenarioRunner", selectScenarioRunner(sc)) + } +} + +func TestSelectScenarioRunnerOneShotDefault(t *testing.T) { + sc := scenario.Scenario{} + if _, ok := selectScenarioRunner(sc).(codexScenarioRunner); !ok { + t.Fatalf("selectScenarioRunner(default) = %T, want codexScenarioRunner", selectScenarioRunner(sc)) + } +} + +func TestScenarioEffectiveRunnerMode(t *testing.T) { + if got := (scenario.Scenario{}).EffectiveRunnerMode(); got != scenario.RunnerModeOneShot { + t.Errorf("default = %q", got) + } + if got := (scenario.Scenario{RunnerMode: "agentic"}).EffectiveRunnerMode(); got != scenario.RunnerModeAgentic { + t.Errorf("agentic = %q", got) + } +} diff --git a/cli/internal/adapters/eval/scenario_ab_runtime.go b/cli/internal/adapters/eval/scenario_ab_runtime.go new file mode 100644 index 000000000..baef02b68 --- /dev/null +++ b/cli/internal/adapters/eval/scenario_ab_runtime.go @@ -0,0 +1,173 @@ +package eval + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + + aoeval "github.com/boshu2/agentops/cli/internal/eval" + "github.com/boshu2/agentops/cli/internal/scenario" +) + +func (Runtime) LoadScenario(path string) (*scenario.Scenario, error) { return scenario.Load(path) } +func (Runtime) Runner(spec scenario.Scenario) aoeval.ScenarioRunner { + return selectScenarioRunner(spec) +} +func (Runtime) Judge(scenario.Scenario) aoeval.ScenarioJudge { return codexScenarioJudge{} } +func (Runtime) WriteScenarioCard(card aoeval.ScenarioDeltaScorecard, path string) error { + return aoeval.WriteScenarioDeltaScorecard(card, path) +} +func (Runtime) LoadScenarioCard(path string) (aoeval.ScenarioDeltaScorecard, error) { + return aoeval.LoadScenarioDeltaScorecard(path) +} +func (Runtime) WriteMoatResult(path string, result aoeval.MoatClaimResult) error { + return aoeval.WriteMoatClaimResult(path, result) +} + +type codexScenarioRunner struct{} + +func (codexScenarioRunner) RunArm(ctx context.Context, spec scenario.Scenario, withGold bool) (aoeval.ArmOutcome, error) { + var prompt strings.Builder + if withGold { + if pointers := goldPointers(ctx, spec.Goal); pointers != "" { + prompt.WriteString("Relevant prior knowledge (gold corpus pointers):\n") + prompt.WriteString(pointers) + prompt.WriteString("\n\n") + } + } + prompt.WriteString(spec.Narrative) + prompt.WriteString("\n\nGoal: ") + prompt.WriteString(spec.Goal) + prompt.WriteString("\n\nProduce your best attempt. Output only the result, no preamble.") + output, tokens, err := runCodexExec(ctx, prompt.String(), "") + if err != nil { + return aoeval.ArmOutcome{}, err + } + return aoeval.ArmOutcome{Output: output, TokenCost: tokens}, nil +} + +type codexScenarioJudge struct{} + +func (codexScenarioJudge) Judge(ctx context.Context, spec scenario.Scenario, arm aoeval.ScenarioArm, outcome aoeval.ArmOutcome) (aoeval.JudgeVerdict, error) { + schema, cleanup, err := writeJudgeSchema() + if err != nil { + return aoeval.JudgeVerdict{}, err + } + defer cleanup() + output, _, err := runCodexExec(ctx, taskSuccessJudgePrompt(spec, outcome.Output), schema) + if err != nil { + return aoeval.JudgeVerdict{}, err + } + verdict, err := parseJudgeJSON(output) + if err != nil { + return aoeval.JudgeVerdict{}, fmt.Errorf("judge (%s) returned unparseable output: %w", arm, err) + } + return verdict, nil +} + +func taskSuccessJudgePrompt(spec scenario.Scenario, output string) string { + var prompt strings.Builder + prompt.WriteString("You are a strict, fair judge. Grade ONLY whether the OUTPUT below accomplishes the task — score 0.0 to 1.0 (1.0 = the goal is fully achieved and the expected outcome met; 0.0 = not at all). Judge this OUTPUT on its own merits; do NOT compare it to any other answer.\n\n") + prompt.WriteString("Goal: " + spec.Goal + "\n") + if strings.TrimSpace(spec.ExpectedOutcome) != "" { + prompt.WriteString("Expected outcome: " + spec.ExpectedOutcome + "\n") + } + prompt.WriteString("\nOUTPUT:\n" + output + "\n\n") + prompt.WriteString(`Return ONLY strict JSON, no prose: {"vectors":[{"dimension":"task-success","pass":true,"score":0.0}],"aggregate_score":0.0}. Set pass=true iff score>=0.5; aggregate_score in [0,1] is your task-success grade.`) + return prompt.String() +} + +const judgeOutputSchema = `{"type":"object","additionalProperties":false,"properties":{"vectors":{"type":"array","items":{"type":"object","additionalProperties":false,"properties":{"dimension":{"type":"string"},"pass":{"type":"boolean"},"score":{"type":"number"}},"required":["dimension","pass","score"]}},"aggregate_score":{"type":"number"}},"required":["vectors","aggregate_score"]}` + +func writeJudgeSchema() (string, func(), error) { + file, err := os.CreateTemp("", "scenario-ab-judge-schema-*.json") + if err != nil { + return "", func() {}, fmt.Errorf("judge schema temp: %w", err) + } + name := file.Name() + cleanup := func() { _ = os.Remove(name) } + if _, err := file.WriteString(judgeOutputSchema); err != nil { + _ = file.Close() + cleanup() + return "", func() {}, fmt.Errorf("write judge schema: %w", err) + } + if err := file.Close(); err != nil { + cleanup() + return "", func() {}, fmt.Errorf("close judge schema: %w", err) + } + return name, cleanup, nil +} + +func goldPointers(ctx context.Context, query string) string { + if _, err := os.Stat(".ao/wiki"); err != nil { + return "" + } + command := exec.CommandContext(ctx, "ao", "lookup", "--query", query, "--gold", "--pointers", "--limit", "3") + output, err := command.Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(output)) +} + +var codexTokensRe = regexp.MustCompile(`(?i)tokens used[^\d]*([\d,]+)`) + +func runCodexExec(ctx context.Context, prompt, outputSchemaPath string) (string, int, error) { + messageFile, err := os.CreateTemp("", "scenario-ab-codex-msg-*.txt") + if err != nil { + return "", 0, fmt.Errorf("codex last-message temp: %w", err) + } + messagePath := messageFile.Name() + _ = messageFile.Close() + defer func() { _ = os.Remove(messagePath) }() + args := codexExecArgs(messagePath, outputSchemaPath, prompt) + cwd, err := os.Getwd() + if err != nil { + return "", 0, fmt.Errorf("resolve cwd for arm isolation: %w", err) + } + command, err := sandboxedCodexCmd(ctx, corpusDenyPaths(cwd), args) + if err != nil { + return "", 0, err + } + command.Stdin = strings.NewReader("") + combined, err := command.CombinedOutput() + if err != nil { + return "", 0, fmt.Errorf("codex exec: %w", err) + } + tokens := parseCodexTokens(string(combined)) + if message, readErr := os.ReadFile(messagePath); readErr == nil && strings.TrimSpace(string(message)) != "" { + return strings.TrimSpace(string(message)), tokens, nil + } + return string(combined), tokens, nil +} +func codexExecArgs(messagePath, outputSchemaPath, prompt string) []string { + args := []string{"exec", "--dangerously-bypass-approvals-and-sandbox", "--skip-git-repo-check", "--output-last-message", messagePath} + if outputSchemaPath != "" { + args = append(args, "--output-schema", outputSchemaPath) + } + return append(args, prompt) +} +func parseJudgeJSON(text string) (aoeval.JudgeVerdict, error) { + start, end := strings.Index(text, "{"), strings.LastIndex(text, "}") + if start < 0 || end <= start { + return aoeval.JudgeVerdict{}, fmt.Errorf("no JSON object found") + } + var verdict aoeval.JudgeVerdict + if err := json.Unmarshal([]byte(text[start:end+1]), &verdict); err != nil { + return aoeval.JudgeVerdict{}, err + } + return verdict, nil +} +func parseCodexTokens(text string) int { + if match := codexTokensRe.FindStringSubmatch(text); match != nil { + if tokens, err := strconv.Atoi(strings.ReplaceAll(match[1], ",", "")); err == nil { + return tokens + } + } + return len(text) / 4 +} diff --git a/cli/internal/adapters/eval/scenario_ab_runtime_test.go b/cli/internal/adapters/eval/scenario_ab_runtime_test.go new file mode 100644 index 000000000..c535b70f9 --- /dev/null +++ b/cli/internal/adapters/eval/scenario_ab_runtime_test.go @@ -0,0 +1,24 @@ +package eval + +import ( + "strings" + "testing" +) + +func TestParseJudgeJSONAndCodexTokens(t *testing.T) { + verdict, err := parseJudgeJSON(`prefix {"vectors":[{"dimension":"task-success","pass":true,"score":0.9}],"aggregate_score":0.9} suffix`) + if err != nil || verdict.AggregateScore != .9 { + t.Fatalf("verdict=%#v err=%v", verdict, err) + } + if tokens := parseCodexTokens("tokens used: 1,234"); tokens != 1234 { + t.Fatalf("tokens=%d", tokens) + } +} + +func TestScenarioRunnerSourceUsesCodexExecAndNeverClaudePrintMode(t *testing.T) { + args := codexExecArgs("message.txt", "schema.json", "prompt") + joined := strings.Join(args, " ") + if args[0] != "exec" || !strings.Contains(joined, "--output-last-message message.txt") || strings.Contains(joined, "claude") { + t.Fatalf("codex args=%q", joined) + } +} diff --git a/cli/internal/adapters/eval/scenario_ab_sandbox.go b/cli/internal/adapters/eval/scenario_ab_sandbox.go new file mode 100644 index 000000000..a5b9f983c --- /dev/null +++ b/cli/internal/adapters/eval/scenario_ab_sandbox.go @@ -0,0 +1,196 @@ +package eval + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/boshu2/agentops/cli/internal/config" + "github.com/boshu2/agentops/cli/internal/wiki" +) + +// corpusMarkers are the dir names that mark a corpus root (and the per-root dirs +// an arm must not read). +var corpusMarkers = []string{".agents", ".ao"} + +// corpusRoot walks up from startDir to the nearest ancestor containing a corpus +// marker (.agents or .ao), so the deny set is correct no matter which subdir the +// command was invoked from (refuter: os.Getwd() under cli/ left /.agents +// readable). Falls back to startDir. +func corpusRoot(startDir string) string { + dir := startDir + for { + for _, m := range corpusMarkers { + if fi, err := os.Stat(filepath.Join(dir, m)); err == nil && fi.IsDir() { + return dir + } + } + parent := filepath.Dir(dir) + if parent == dir { + return startDir + } + dir = parent + } +} + +// corpusDenyPaths returns the absolute corpus directories a scenario-ab arm must +// NOT read: the repo corpus (/.agents, /.ao — resolved from any subdir) +// AND the global corpus (~/.agents), each plus its symlink-resolved canonical path +// (Seatbelt matches the real path). A leak in ANY of these voids the A/B. +func corpusDenyPaths(startDir string) []string { + root := corpusRoot(startDir) + cand := []string{filepath.Join(root, ".agents"), filepath.Join(root, ".ao")} + if home, err := os.UserHomeDir(); err == nil && home != "" { + cand = append(cand, filepath.Join(home, ".agents")) + } + // Deny the ENV-resolved corpus dir too. AO_AGENTS_DIR / AO_HOME override where + // `ao` reads the corpus (wiki.AgentsDirIn — the authoritative resolver); a + // NON-DEFAULT override points OUTSIDE /.agents, leaving it readable by the + // control arm (age-58r). Resolving the deny path through the same resolver `ao` + // uses (not a literal ".agents" join) closes that leak under any env config. + cand = append(cand, envCorpusDenyPaths(root)...) + // Deny the CONFIG-resolved global AND local corpus roots too — they may be + // configured OUTSIDE ~/.agents / /.agents, and `ao lookup` reads exactly + // these (refuter r2; age-58r for the local LearningsDir/PatternsDir). Sourcing + // the deny set from config (the authoritative resolver) — not a hand-enumerated + // list — is what closes the whack-a-mole of "another corpus path". + if cfg, err := config.Load(nil); err == nil && cfg != nil { + cand = append(cand, configCorpusDenyPaths(cfg.Paths.GlobalLearningsDir, cfg.Paths.GlobalPatternsDir)...) + cand = append(cand, localCorpusDenyPaths(root, cfg.Paths.LearningsDir, cfg.Paths.PatternsDir)...) + } + seen := map[string]bool{} + var out []string + for _, p := range cand { + for _, q := range []string{p, resolveSymlink(p)} { + if q != "" && !seen[q] { + seen[q] = true + out = append(out, q) + } + } + } + return out +} + +// configCorpusDenyPaths returns the global corpus directories `ao lookup` resolves +// from config — the global learnings dir, the DERIVED global findings dir +// (Dir(learnings)/findings, exactly as lookup computes it), and the global patterns +// dir. Denying precisely these (sourced from config, not hand-enumerated) closes the +// gap where a global corpus configured outside ~/.agents stayed readable (refuter r2). +// Pure (config values passed in) so it is unit-testable without loading from disk. +func configCorpusDenyPaths(globalLearningsDir, globalPatternsDir string) []string { + var out []string + if d := strings.TrimSpace(globalLearningsDir); d != "" { + out = append(out, d, filepath.Join(filepath.Dir(d), "findings")) + } + if d := strings.TrimSpace(globalPatternsDir); d != "" { + out = append(out, d) + } + return out +} + +// envCorpusDenyPaths returns the corpus dir an AO_AGENTS_DIR / AO_HOME env override +// redirects `ao` to read from (resolved via wiki.AgentsDirIn — the same resolver the +// real code uses). It returns the override only when it resolves to something OTHER +// than the default /.agents (which is already denied): a non-default override is +// the age-58r leak path. With no env override set the override equals the default and +// nothing is added. +func envCorpusDenyPaths(root string) []string { + resolved := wiki.AgentsDirIn(root) + if resolved == "" || resolved == filepath.Join(root, ".agents") { + return nil + } + return []string{resolved} +} + +// localCorpusDenyPaths returns the config-resolved LOCAL (per-repo) corpus dirs — +// cfg.Paths.LearningsDir / PatternsDir — that fall OUTSIDE the already-denied +// /.agents subtree. These default to ".agents/learnings" / ".agents/patterns" +// (relative, under the denied .agents root), but a config can point them at an +// ABSOLUTE path elsewhere; that absolute local corpus stayed readable by the control +// arm (age-58r). A relative dir resolves under root/.agents and is dropped (already +// covered); an absolute dir not under /.agents is denied. Pure (config values +// passed in) so it is unit-testable without loading from disk. +func localCorpusDenyPaths(root, learningsDir, patternsDir string) []string { + var out []string + for _, d := range []string{learningsDir, patternsDir} { + d = strings.TrimSpace(d) + if d == "" || !filepath.IsAbs(d) { + // Relative dirs resolve under /.agents (already denied). + continue + } + if within(d, filepath.Join(root, ".agents")) { + continue + } + out = append(out, d) + } + return out +} + +// within reports whether path p is the same as, or nested under, dir. +func within(p, dir string) bool { + rel, err := filepath.Rel(dir, p) + if err != nil { + return false + } + return rel == "." || (!strings.HasPrefix(rel, ".."+string(filepath.Separator)) && rel != "..") +} + +func resolveSymlink(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return "" +} + +// sandboxExecDenyProfile builds a macOS Seatbelt profile that allows everything +// EXCEPT reads of the given corpus dirs. A denylist (not allowlist) keeps codex +// functional (auth, binary, system libs, temp writes) while the corpus is +// unreadable (age-9a9). All paths must be absolute. +func sandboxExecDenyProfile(denyPaths []string) string { + var b strings.Builder + b.WriteString("(version 1)(allow default)(deny file-read*") + for _, p := range denyPaths { + fmt.Fprintf(&b, " (subpath %q)", p) + } + b.WriteString(")") + return b.String() +} + +// sandboxedCodexArgs returns the argv for sandbox-exec wrapping `codex ` +// with the given profile. Pure (no exec) so the wrapping is unit-testable. +func sandboxedCodexArgs(profile string, codexArgs []string) []string { + return append([]string{"-p", profile, "codex"}, codexArgs...) +} + +// macOSSandboxExec is the Seatbelt binary. +const macOSSandboxExec = "/usr/bin/sandbox-exec" + +// sandboxedCodexCmd builds the *exec.Cmd that runs codex CONFINED so it cannot read +// any of denyPaths. The inner codex runs --dangerously-bypass-approvals-and-sandbox +// (documented for externally-sandboxed environments — the OUTER sandbox-exec is the +// real confinement, and macOS Seatbelt is inherited by child processes). +// +// FAIL-CLOSED: returns an error — never a bare, unconfined codex command — when the +// platform isolation is unavailable or no corpus path was resolved. A control arm +// that silently reads the corpus is precisely the failure this guards. +func sandboxedCodexCmd(ctx context.Context, denyPaths []string, codexArgs []string) (*exec.Cmd, error) { + if len(denyPaths) == 0 { + return nil, fmt.Errorf("arm isolation resolved no corpus paths to deny; refusing to run a potentially-unisolated control arm") + } + switch runtime.GOOS { + case "darwin": + if _, err := os.Stat(macOSSandboxExec); err != nil { + return nil, fmt.Errorf("arm isolation requires %s (macOS Seatbelt); refusing to run an unisolated control arm: %w", macOSSandboxExec, err) + } + profile := sandboxExecDenyProfile(denyPaths) + return exec.CommandContext(ctx, macOSSandboxExec, sandboxedCodexArgs(profile, codexArgs)...), nil + default: + // Linux/WSL bwrap path is age-9a9 S2. Until it lands, fail closed rather + // than run an unisolated (corpus-readable) control arm. + return nil, fmt.Errorf("scenario-ab arm isolation is not yet implemented for GOOS=%q (age-9a9 S2: bubblewrap); refusing to run an unisolated control arm", runtime.GOOS) + } +} diff --git a/cli/internal/adapters/eval/scenario_ab_sandbox_test.go b/cli/internal/adapters/eval/scenario_ab_sandbox_test.go new file mode 100644 index 000000000..d5accbf49 --- /dev/null +++ b/cli/internal/adapters/eval/scenario_ab_sandbox_test.go @@ -0,0 +1,243 @@ +package eval + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// TestCorpusRoot_WalksUpFromSubdir: the corpus root is found by walking UP to the +// nearest .agents/.ao ancestor, so isolation is correct no matter which subdir the +// command ran from (refuter: os.Getwd() under cli/ left /.agents readable). +func TestCorpusRoot_WalksUpFromSubdir(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, ".agents"), 0o755); err != nil { + t.Fatal(err) + } + sub := filepath.Join(root, "cli", "cmd", "ao") + if err := os.MkdirAll(sub, 0o755); err != nil { + t.Fatal(err) + } + if got := corpusRoot(sub); got != root { + t.Errorf("corpusRoot(%q) = %q, want %q (must walk up to the .agents root)", sub, got, root) + } +} + +// TestCorpusDenyPaths_RepoAndGlobal: the deny set covers the repo corpus (both +// .agents and .ao under the resolved root) AND the global ~/.agents. +func TestCorpusDenyPaths_RepoAndGlobal(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, ".agents"), 0o755); err != nil { + t.Fatal(err) + } + joined := strings.Join(corpusDenyPaths(root), "|") + for _, want := range []string{filepath.Join(root, ".agents"), filepath.Join(root, ".ao")} { + if !strings.Contains(joined, want) { + t.Errorf("deny set must include repo %q; got %s", want, joined) + } + } + if home, err := os.UserHomeDir(); err == nil && home != "" { + if !strings.Contains(joined, filepath.Join(home, ".agents")) { + t.Errorf("deny set must include the global ~/.agents; got %s", joined) + } + } +} + +// TestConfigCorpusDenyPaths_CustomGlobalRoots: a global corpus configured OUTSIDE +// ~/.agents (refuter r2) is denied, including the derived global findings dir that +// `ao lookup` computes as Dir(globalLearnings)/findings. +func TestConfigCorpusDenyPaths_CustomGlobalRoots(t *testing.T) { + got := configCorpusDenyPaths("/data/corpus/learnings", "/opt/patterns") + joined := strings.Join(got, "|") + for _, want := range []string{"/data/corpus/learnings", filepath.Join("/data/corpus", "findings"), "/opt/patterns"} { + if !strings.Contains(joined, want) { + t.Errorf("must deny configured global corpus root %q; got %v", want, got) + } + } + if len(configCorpusDenyPaths("", "")) != 0 { + t.Error("empty config global dirs must add no deny paths") + } +} + +// TestEnvCorpusDenyPaths_NonDefaultOverride: a NON-DEFAULT AO_AGENTS_DIR (or AO_HOME) +// redirects where `ao` reads the corpus OUTSIDE /.agents, so the resolved +// override must enter the deny set; with no override (or the default), nothing is +// added (it equals the already-denied /.agents). age-58r. +func TestEnvCorpusDenyPaths_NonDefaultOverride(t *testing.T) { + root := t.TempDir() + custom := filepath.Join(t.TempDir(), "elsewhere-corpus") + + // AO_AGENTS_DIR override → denied verbatim (the path `ao` actually reads). + t.Setenv("AO_HOME", "") + t.Setenv("AO_AGENTS_DIR", custom) + got := envCorpusDenyPaths(root) + if len(got) != 1 || got[0] != custom { + t.Errorf("AO_AGENTS_DIR override must be denied; got %v want [%q]", got, custom) + } + + // AO_HOME override (no AO_AGENTS_DIR) → also denied. + t.Setenv("AO_AGENTS_DIR", "") + t.Setenv("AO_HOME", custom) + if got := envCorpusDenyPaths(root); len(got) != 1 || got[0] != custom { + t.Errorf("AO_HOME override must be denied; got %v want [%q]", got, custom) + } + + // No override → resolves to the default /.agents (already denied) → nothing added. + t.Setenv("AO_AGENTS_DIR", "") + t.Setenv("AO_HOME", "") + if got := envCorpusDenyPaths(root); len(got) != 0 { + t.Errorf("no env override must add no deny path; got %v", got) + } +} + +// TestEnvCorpusDenyPaths_FlowsIntoCorpusDenyPaths: end-to-end, a non-default +// AO_AGENTS_DIR appears in the full deny set corpusDenyPaths builds (not just the +// helper). The control arm cannot read the override corpus. age-58r. +func TestEnvCorpusDenyPaths_FlowsIntoCorpusDenyPaths(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, ".agents"), 0o755); err != nil { + t.Fatal(err) + } + custom := filepath.Join(t.TempDir(), "elsewhere-corpus") + t.Setenv("AO_HOME", "") + t.Setenv("AO_AGENTS_DIR", custom) + joined := strings.Join(corpusDenyPaths(root), "|") + if !strings.Contains(joined, custom) { + t.Errorf("full deny set must include the AO_AGENTS_DIR override %q; got %s", custom, joined) + } +} + +// TestLocalCorpusDenyPaths_AbsoluteOutsideRoot: a config-local LearningsDir/PatternsDir +// pointed at an ABSOLUTE path outside /.agents is denied; a relative dir (the +// default ".agents/learnings", resolving under the already-denied .agents) and an +// absolute dir nested inside /.agents are NOT re-added. age-58r. +func TestLocalCorpusDenyPaths_AbsoluteOutsideRoot(t *testing.T) { + root := t.TempDir() + + // Absolute, outside /.agents → denied. + got := localCorpusDenyPaths(root, "/custom/learnings", "/opt/patterns") + joined := strings.Join(got, "|") + for _, want := range []string{"/custom/learnings", "/opt/patterns"} { + if !strings.Contains(joined, want) { + t.Errorf("must deny absolute local corpus root %q; got %v", want, got) + } + } + + // Relative dirs resolve under /.agents (already denied) → not re-added. + if out := localCorpusDenyPaths(root, ".agents/learnings", ".agents/patterns"); len(out) != 0 { + t.Errorf("relative local dirs must add no deny path (covered by .agents); got %v", out) + } + + // Absolute but nested inside /.agents → already covered, not re-added. + inside := filepath.Join(root, ".agents", "learnings") + if out := localCorpusDenyPaths(root, inside, ""); len(out) != 0 { + t.Errorf("absolute dir inside /.agents must not be re-added; got %v", out) + } + + // Empty → nothing. + if out := localCorpusDenyPaths(root, "", ""); len(out) != 0 { + t.Errorf("empty local dirs must add no deny path; got %v", out) + } +} + +// TestSandboxExecDenyProfile_DeniesAllPaths: every deny path appears in a +// (deny file-read* (subpath ...)) clause, and non-corpus reads remain allowed. +func TestSandboxExecDenyProfile_DeniesAllPaths(t *testing.T) { + p := sandboxExecDenyProfile([]string{"/r/.agents", "/r/.ao", "/home/u/.agents"}) + if !strings.Contains(p, "(allow default)") || !strings.Contains(p, "deny file-read*") { + t.Errorf("profile must allow-default then deny corpus reads; got %s", p) + } + for _, d := range []string{"/r/.agents", "/r/.ao", "/home/u/.agents"} { + if !strings.Contains(p, `(subpath "`+d+`")`) { + t.Errorf("profile must deny subpath %q; got %s", d, p) + } + } +} + +// TestSandboxedCodexArgs_WrapsCodex: codex runs under sandbox-exec with the profile. +func TestSandboxedCodexArgs_WrapsCodex(t *testing.T) { + got := sandboxedCodexArgs("PROFILE", []string{"exec", "--foo", "the prompt"}) + want := []string{"-p", "PROFILE", "codex", "exec", "--foo", "the prompt"} + if len(got) != len(want) { + t.Fatalf("argv = %v (len %d), want len %d", got, len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("argv[%d] = %q, want %q", i, got[i], want[i]) + } + } +} + +// TestSandboxedCodexCmd_FailClosed: a sandboxed command on macOS; an ERROR (never a +// bare codex) when isolation is unavailable — empty deny set, or a non-darwin GOOS. +func TestSandboxedCodexCmd_FailClosed(t *testing.T) { + // empty deny set → fail closed on every platform + if cmd, err := sandboxedCodexCmd(context.Background(), nil, []string{"exec"}); err == nil || cmd != nil { + t.Errorf("empty deny set must fail closed; got cmd=%v err=%v", cmd, err) + } + + cmd, err := sandboxedCodexCmd(context.Background(), []string{"/r/.agents", "/r/.ao"}, []string{"exec", "--output-last-message", "/tmp/x"}) + switch runtime.GOOS { + case "darwin": + if err != nil || cmd == nil { + t.Fatalf("darwin: expected a sandboxed command; got cmd=%v err=%v", cmd, err) + } + if !strings.Contains(cmd.Path, "sandbox-exec") { + t.Errorf("darwin: must wrap sandbox-exec; got path %q", cmd.Path) + } + joined := strings.Join(cmd.Args, " ") + if !strings.Contains(joined, "deny file-read*") || !strings.Contains(joined, "codex") { + t.Errorf("darwin: args must carry the deny profile + codex; got %q", joined) + } + default: + if err == nil || cmd != nil { + t.Errorf("non-darwin must fail closed (no unisolated arm); got cmd=%v err=%v", cmd, err) + } + } +} + +// TestSandboxExec_Integration_DeniesCorpusRead (age-wp1): on darwin, sandbox-exec +// with the scenario-ab deny profile must block reads under .agents while allowing +// reads outside the denied corpus paths. String-only profile tests are insufficient. +func TestSandboxExec_Integration_DeniesCorpusRead(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("darwin-only: macOS Seatbelt sandbox-exec integration") + } + if _, err := os.Stat(macOSSandboxExec); err != nil { + t.Skipf("sandbox-exec unavailable: %v", err) + } + + root := t.TempDir() + agents := filepath.Join(root, ".agents") + if err := os.MkdirAll(agents, 0o755); err != nil { + t.Fatal(err) + } + secret := filepath.Join(agents, "sentinel.txt") + if err := os.WriteFile(secret, []byte("CORPUS-SECRET-MUST-NOT-READ"), 0o644); err != nil { + t.Fatal(err) + } + allowed := filepath.Join(root, "allowed.txt") + if err := os.WriteFile(allowed, []byte("allowed-ok"), 0o644); err != nil { + t.Fatal(err) + } + + profile := sandboxExecDenyProfile(corpusDenyPaths(root)) + + denyCmd := exec.Command(macOSSandboxExec, "-p", profile, "/bin/cat", secret) + if err := denyCmd.Run(); err == nil { + t.Fatal("expected sandbox-exec to deny read of corpus file under .agents") + } + + allowCmd := exec.Command(macOSSandboxExec, "-p", profile, "/bin/cat", allowed) + out, err := allowCmd.CombinedOutput() + if err != nil { + t.Fatalf("expected read outside deny set to succeed: %v output=%q", err, out) + } + if !strings.Contains(string(out), "allowed-ok") { + t.Errorf("allowed read output = %q, want allowed-ok", out) + } +} From a1dff4ae1eab8e1fb7328c09836731da07d4b498 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:56:38 -0400 Subject: [PATCH 096/108] refactor(eval): extract session and chaos alias ports --- cli/cmd/ao/eval_alias_adapter.go | 47 +++++++++++++++++++ cli/cmd/ao/eval_alias_adapter_test.go | 34 ++++++++++++++ cli/internal/commands/eval/module.go | 56 +++++++++++++++++++++++ cli/internal/commands/eval/module_test.go | 35 ++++++++++++++ cli/internal/eval/alias_service.go | 31 +++++++++++++ 5 files changed, 203 insertions(+) create mode 100644 cli/cmd/ao/eval_alias_adapter.go create mode 100644 cli/cmd/ao/eval_alias_adapter_test.go create mode 100644 cli/internal/eval/alias_service.go diff --git a/cli/cmd/ao/eval_alias_adapter.go b/cli/cmd/ao/eval_alias_adapter.go new file mode 100644 index 000000000..bfa5d2fe8 --- /dev/null +++ b/cli/cmd/ao/eval_alias_adapter.go @@ -0,0 +1,47 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + + aoeval "github.com/boshu2/agentops/cli/internal/eval" +) + +type evalAliasAdapter struct{} + +func (evalAliasAdapter) SessionOutcome(_ context.Context, request aoeval.SessionOutcomeRequest) (aoeval.SessionOutcomeResult, error) { + path := request.TranscriptPath + if path == "" { + home, _ := os.UserHomeDir() + path = findMostRecentTranscript(filepath.Join(home, ".claude", "projects")) + if path == "" { + return aoeval.SessionOutcomeResult{}, fmt.Errorf("no transcript found; specify path as argument") + } + } + if request.DryRun { + return aoeval.SessionOutcomeResult{Transcript: path, DryRun: true}, nil + } + outcome, err := analyzeTranscript(path, request.SessionID) + if err != nil { + return aoeval.SessionOutcomeResult{}, fmt.Errorf("analyze transcript: %w", err) + } + result := aoeval.SessionOutcomeResult{SessionID: outcome.SessionID, Reward: outcome.Reward, AnalyzedAt: outcome.AnalyzedAt, Transcript: outcome.Transcript, TotalLines: outcome.TotalLines, Signals: make([]aoeval.SessionSignal, 0, len(outcome.Signals))} + for _, signal := range outcome.Signals { + result.Signals = append(result.Signals, aoeval.SessionSignal{Name: signal.Name, Value: signal.Value, Weight: signal.Weight}) + } + return result, nil +} + +func (evalAliasAdapter) Chaos(_ context.Context) (aoeval.AliasOutput, error) { + workDir, err := os.Getwd() + if err != nil { + workDir = "." + } + var stdout, stderr bytes.Buffer + err = tickSmoke(tickRuntime{workDir: workDir, stdin: strings.NewReader(""), stdout: &stdout, stderr: &stderr}) + return aoeval.AliasOutput{Stdout: stdout.String(), Stderr: stderr.String()}, err +} diff --git a/cli/cmd/ao/eval_alias_adapter_test.go b/cli/cmd/ao/eval_alias_adapter_test.go new file mode 100644 index 000000000..fc6d34f1d --- /dev/null +++ b/cli/cmd/ao/eval_alias_adapter_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + aoeval "github.com/boshu2/agentops/cli/internal/eval" +) + +func TestEvalAliasAdapterSessionOutcomeMapsSharedAnalyzer(t *testing.T) { + path := filepath.Join(t.TempDir(), "transcript.jsonl") + if err := os.WriteFile(path, []byte("tests passed\ngit commit\n"), 0o644); err != nil { + t.Fatal(err) + } + result, err := (evalAliasAdapter{}).SessionOutcome(context.Background(), aoeval.SessionOutcomeRequest{TranscriptPath: path, SessionID: "session-1"}) + if err != nil { + t.Fatalf("SessionOutcome: %v", err) + } + if result.SessionID != "session-1" || result.Reward <= 0 || len(result.Signals) == 0 { + t.Fatalf("result=%#v", result) + } +} + +func TestEvalAliasAdapterSessionOutcomeDryRunDoesNotAnalyze(t *testing.T) { + result, err := (evalAliasAdapter{}).SessionOutcome(context.Background(), aoeval.SessionOutcomeRequest{TranscriptPath: "missing", DryRun: true}) + if err != nil { + t.Fatalf("SessionOutcome: %v", err) + } + if !result.DryRun || result.Transcript != "missing" { + t.Fatalf("result=%#v", result) + } +} diff --git a/cli/internal/commands/eval/module.go b/cli/internal/commands/eval/module.go index bcda17e57..f2921a93a 100644 --- a/cli/internal/commands/eval/module.go +++ b/cli/internal/commands/eval/module.go @@ -65,6 +65,7 @@ type UseCases struct { Outcomes OutcomesUseCases Scenario ScenarioUseCases ScenarioAB ScenarioABUseCases + Aliases aoeval.AliasUseCases } type HostOptions struct { @@ -72,6 +73,7 @@ type HostOptions struct { Verbose func(*cobra.Command) bool ProjectRoot func() string GoalsPath func() string + DryRun func(*cobra.Command) bool } type Module struct { @@ -142,6 +144,9 @@ release. Live Claude and Codex adapters are evaluated by a later runtime tier.`, if module.useCases.ScenarioAB != nil { command.AddCommand(module.scenarioABCommand(), module.scenarioMoatCommand()) } + if module.useCases.Aliases != nil { + command.AddCommand(module.sessionOutcomeCommand(), module.chaosCommand()) + } return command } @@ -780,6 +785,57 @@ func (module Module) scenarioMoatCommand() *cobra.Command { return command } +func (module Module) sessionOutcomeCommand() *cobra.Command { + var sessionID, format string + command := &cobra.Command{Use: "session-outcome [transcript-path]", Short: "Analyze session transcript to derive reward signal", Args: cobra.MaximumNArgs(1)} + command.Flags().StringVar(&sessionID, "session", "", "Session ID (extracted from transcript if not provided)") + command.Flags().StringVar(&format, "output", "text", "Output format: text, json") + command.RunE = func(command *cobra.Command, args []string) error { + path := "" + if len(args) > 0 { + path = args[0] + } + dryRun := module.host.DryRun != nil && module.host.DryRun(command) + result, err := module.useCases.Aliases.SessionOutcome(command.Context(), aoeval.SessionOutcomeRequest{TranscriptPath: path, SessionID: sessionID, DryRun: dryRun}) + if err != nil { + return err + } + if result.DryRun { + fmt.Fprintf(command.OutOrStdout(), "[dry-run] Would analyze transcript: %s\n", result.Transcript) + return nil + } + if format == "json" || module.outputMode(command) == "json" { + return writeJSON(command, result) + } + fmt.Fprintln(command.OutOrStdout(), "Session Outcome Analysis\n========================") + fmt.Fprintf(command.OutOrStdout(), "Session ID: %s\nReward: %.2f\nLines: %d\n\nSignals detected:\n", result.SessionID, result.Reward, result.TotalLines) + for _, signal := range result.Signals { + status := "✗" + if signal.Value { + status = "✓" + } + fmt.Fprintf(command.OutOrStdout(), " %s %-20s (weight: %+.2f)\n", status, signal.Name, signal.Weight) + } + return nil + } + return command +} + +func (module Module) chaosCommand() *cobra.Command { + command := &cobra.Command{Use: "chaos", Short: "Run a read-only smoke test of the tick membrane", Args: cobra.NoArgs} + command.RunE = func(command *cobra.Command, _ []string) error { + output, err := module.useCases.Aliases.Chaos(command.Context()) + if output.Stdout != "" { + fmt.Fprint(command.OutOrStdout(), output.Stdout) + } + if output.Stderr != "" { + fmt.Fprint(command.ErrOrStderr(), output.Stderr) + } + return err + } + return command +} + func (module Module) renderRun(command *cobra.Command, result aoeval.CoreRunResult) error { var value any = result.Run if result.Mode == aoeval.CoreRunBaselineAB { diff --git a/cli/internal/commands/eval/module_test.go b/cli/internal/commands/eval/module_test.go index 4f2b5e479..1e4416abc 100644 --- a/cli/internal/commands/eval/module_test.go +++ b/cli/internal/commands/eval/module_test.go @@ -28,6 +28,16 @@ type scenarioUseCasesSpy struct{ addRequest aoeval.ScenarioAddRequest } type scenarioABUseCasesSpy struct{ request aoeval.ScenarioABRequest } +type aliasUseCasesSpy struct{ request aoeval.SessionOutcomeRequest } + +func (useCases *aliasUseCasesSpy) SessionOutcome(_ context.Context, request aoeval.SessionOutcomeRequest) (aoeval.SessionOutcomeResult, error) { + useCases.request = request + return aoeval.SessionOutcomeResult{SessionID: "s", Reward: .5, Signals: []aoeval.SessionSignal{}}, nil +} +func (*aliasUseCasesSpy) Chaos(context.Context) (aoeval.AliasOutput, error) { + return aoeval.AliasOutput{Stdout: "PASS smoke\n"}, nil +} + func (useCases *scenarioABUseCasesSpy) Run(_ context.Context, request aoeval.ScenarioABRequest) (aoeval.ScenarioABResult, error) { useCases.request = request return aoeval.ScenarioABResult{Card: aoeval.ScenarioDeltaScorecard{ScenarioID: "s-1", Gate: aoeval.ScenarioGate{Pass: true}}}, nil @@ -225,6 +235,31 @@ func TestModuleScenarioABDelegatesFlags(t *testing.T) { } } +func TestModuleSessionOutcomeDelegatesFlags(t *testing.T) { + useCases := &aliasUseCasesSpy{} + command := NewModule(UseCases{Core: &coreUseCasesSpy{}, Aliases: useCases}, HostOptions{}).Command() + command.SetArgs([]string{"session-outcome", "transcript.jsonl", "--session", "s-1"}) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if useCases.request.TranscriptPath != "transcript.jsonl" || useCases.request.SessionID != "s-1" { + t.Fatalf("request=%#v", useCases.request) + } +} + +func TestModuleChaosRendersAdapterStreams(t *testing.T) { + command := NewModule(UseCases{Core: &coreUseCasesSpy{}, Aliases: &aliasUseCasesSpy{}}, HostOptions{}).Command() + command.SetArgs([]string{"chaos"}) + var stdout strings.Builder + command.SetOut(&stdout) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if !strings.Contains(stdout.String(), "PASS smoke") { + t.Fatalf("stdout=%q", stdout.String()) + } +} + func findChild(t *testing.T, command *cobra.Command, name string) *cobra.Command { t.Helper() child, _, err := command.Find([]string{name}) diff --git a/cli/internal/eval/alias_service.go b/cli/internal/eval/alias_service.go new file mode 100644 index 000000000..7820dfbf5 --- /dev/null +++ b/cli/internal/eval/alias_service.go @@ -0,0 +1,31 @@ +package eval + +import ( + "context" + "time" +) + +type SessionOutcomeRequest struct { + TranscriptPath, SessionID string + DryRun bool +} +type SessionSignal struct { + Name string `json:"name"` + Value bool `json:"value"` + Weight float64 `json:"weight"` +} +type SessionOutcomeResult struct { + SessionID string `json:"session_id"` + Reward float64 `json:"reward"` + Signals []SessionSignal `json:"signals"` + AnalyzedAt time.Time `json:"analyzed_at"` + Transcript string `json:"transcript,omitempty"` + TotalLines int `json:"total_lines,omitempty"` + DryRun bool `json:"-"` +} +type AliasOutput struct{ Stdout, Stderr string } + +type AliasUseCases interface { + SessionOutcome(context.Context, SessionOutcomeRequest) (SessionOutcomeResult, error) + Chaos(context.Context) (AliasOutput, error) +} From f9e066a362f9b4b5ba728a93ad9fa33c933751bc Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:00:41 -0400 Subject: [PATCH 097/108] refactor(eval): extract retrieval bench command adapter --- cli/cmd/ao/eval_bench_adapter.go | 174 ++++++++++++++++++++++ cli/cmd/ao/eval_bench_adapter_test.go | 21 +++ cli/internal/commands/eval/module.go | 30 ++++ cli/internal/commands/eval/module_test.go | 19 +++ cli/internal/eval/alias_service.go | 12 ++ 5 files changed, 256 insertions(+) create mode 100644 cli/cmd/ao/eval_bench_adapter.go create mode 100644 cli/cmd/ao/eval_bench_adapter_test.go diff --git a/cli/cmd/ao/eval_bench_adapter.go b/cli/cmd/ao/eval_bench_adapter.go new file mode 100644 index 000000000..0592cb77e --- /dev/null +++ b/cli/cmd/ao/eval_bench_adapter.go @@ -0,0 +1,174 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + aoeval "github.com/boshu2/agentops/cli/internal/eval" +) + +type evalBenchAdapter struct{} + +func (evalBenchAdapter) Bench(_ context.Context, request aoeval.BenchRequest) (aoeval.AliasOutput, error) { + if strings.TrimSpace(request.SearchEval) != "" { + return evalSearchBench(request) + } + if request.Live { + return evalLiveBench(request) + } + cwd, err := os.Getwd() + if err != nil { + return aoeval.AliasOutput{}, fmt.Errorf("get working directory: %w", err) + } + corpus, err := resolveRetrievalBenchCorpus(cwd, request.Corpus) + if err != nil { + return aoeval.AliasOutput{}, err + } + report, err := buildBenchReport(corpus, corpus, request.K) + if err != nil { + return aoeval.AliasOutput{}, err + } + if request.JSON { + return jsonAliasOutput(report) + } + var out bytes.Buffer + fmt.Fprintln(&out, "Retrieval Quality Report\n========================") + fmt.Fprintf(&out, "Queries: %d\nPrecision@%d: %.2f (target: %.2f)\nMRR: %.2f (target: %.2f)\n", report.Queries, request.K, report.AvgPAtK, report.TargetPAtK, report.AvgMRR, report.TargetMRR) + if len(report.Splits) > 0 { + fmt.Fprintln(&out, "Splits:") + names := make([]string, 0, len(report.Splits)) + for name := range report.Splits { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + summary := report.Splits[name] + fmt.Fprintf(&out, " %-10s cases=%d P@%d=%.2f MRR=%.2f", name, summary.Cases, request.K, summary.AvgPAtK, summary.AvgMRR) + if summary.SectionCases > 0 { + fmt.Fprintf(&out, " section-MRR=%.2f", summary.AvgSectionMRR) + } + fmt.Fprintln(&out) + } + } + fmt.Fprintln(&out, "\nPer-query breakdown:") + for _, result := range report.Results { + status := "PASS" + if !result.Pass { + status = "WARN (below target)" + } + fmt.Fprintf(&out, " %-30s P@%d=%.2f MRR=%.2f", fmt.Sprintf("%q", result.Query), request.K, result.PAtK, result.MRR) + if result.ExpectedSection != "" { + fmt.Fprintf(&out, " section-MRR=%.2f", result.SectionMRR) + } + fmt.Fprintf(&out, " %s\n", status) + } + return aoeval.AliasOutput{Stdout: out.String()}, nil +} + +func evalLiveBench(request aoeval.BenchRequest) (aoeval.AliasOutput, error) { + cwd, err := os.Getwd() + if err != nil { + return aoeval.AliasOutput{}, fmt.Errorf("getting working directory: %w", err) + } + globalDir, mode, storeLabel, modeLabel, benchCwd := "", "live-local", ".agents/learnings/", "Live (local)", cwd + if request.Corpus != "" { + if _, err := os.Stat(request.Corpus); err != nil { + return aoeval.AliasOutput{}, fmt.Errorf("live benchmark corpus not found at %s", request.Corpus) + } + tmp, err := os.MkdirTemp("", "retrieval-live-*") + if err != nil { + return aoeval.AliasOutput{}, err + } + defer func() { _ = os.RemoveAll(tmp) }() + benchCwd, globalDir, mode, storeLabel, modeLabel = tmp, request.Corpus, "live-corpus", request.Corpus, "Live (fixture)" + } else if request.Global { + home, err := os.UserHomeDir() + if err != nil { + return aoeval.AliasOutput{}, err + } + globalDir = filepath.Join(home, ".agents", "learnings") + if _, err := os.Stat(globalDir); err != nil { + return aoeval.AliasOutput{}, fmt.Errorf("global knowledge store not found at %s — run 'ao harvest' first", globalDir) + } + mode, storeLabel, modeLabel = "live-global", "~/.agents/learnings/ (cross-rig)", "Live (global)" + } + report, err := buildLiveReport(benchCwd, globalDir, mode, request.K) + if err != nil { + return aoeval.AliasOutput{}, err + } + if request.JSON { + return jsonAliasOutput(report) + } + var out bytes.Buffer + fmt.Fprintf(&out, "Retrieval Quality Report (%s)\n================================\nCorpus: %d learnings in %s\nQueries: %d\nK: %d\nCoverage: %.0f%% (%d/%d queries with hits)\n\n", modeLabel, report.TotalLearnings, storeLabel, report.Queries, request.K, report.Coverage*100, report.QueriesWithHits, report.Queries) + if report.TotalLearnings == 0 { + fmt.Fprintln(&out, "No learnings found. Run /retro or /post-mortem to populate the knowledge base.") + return aoeval.AliasOutput{Stdout: out.String()}, nil + } + fmt.Fprintln(&out, "Per-query breakdown:") + for _, result := range report.Results { + if result.Count == 0 { + fmt.Fprintf(&out, " %-25s hits=0 (no matches)\n", fmt.Sprintf("%q", result.Query)) + } else { + fmt.Fprintf(&out, " %-25s hits=%-3d score=[%.2f, %.2f] mean=%.2f top=%v\n", fmt.Sprintf("%q", result.Query), result.Count, result.MinScore, result.MaxScore, result.MeanScore, result.TopIDs) + } + } + return aoeval.AliasOutput{Stdout: out.String()}, nil +} + +func evalSearchBench(request aoeval.BenchRequest) (aoeval.AliasOutput, error) { + k := request.K + if !request.KChanged { + k = defaultSearchEvalK + } + backends, compare, err := resolveSearchEvalRunBackends(request.SearchBackend, request.CompareBackends) + if err != nil { + return aoeval.AliasOutput{}, err + } + if compare { + report, err := buildSearchEvalComparisonReport(request.SearchRoot, request.SearchEval, k, backends) + if err != nil { + return aoeval.AliasOutput{}, err + } + if request.JSON { + return jsonAliasOutput(report) + } + var out bytes.Buffer + fmt.Fprintf(&out, "AO Search Retrieval Eval Comparison\n===================================\nEval set: %s\nQueries: %d\nK: %d\n", report.ID, report.Queries, report.K) + for _, backend := range report.Backends { + fmt.Fprintf(&out, " %-20s any@k=%.0f%% precision=%.2f MRR=%.2f\n", backend.Backend, backend.AnyRelevantAtK*100, backend.AvgPrecisionAtK, backend.MeanReciprocalRank) + } + return aoeval.AliasOutput{Stdout: out.String()}, nil + } + report, err := buildSearchEvalReportForBackend(request.SearchRoot, request.SearchEval, k, backends[0]) + if err != nil { + return aoeval.AliasOutput{}, err + } + if request.JSON { + return jsonAliasOutput(report) + } + var out bytes.Buffer + fmt.Fprintln(&out, "AO Search Retrieval Eval\n========================") + fmt.Fprintf(&out, "Eval set: %s\nBackend: %s\nManifest: %s\nSearch root: %s\nQueries: %d\nK: %d\n", report.ID, report.Backend, report.ManifestPath, report.SearchRoot, report.Queries, report.K) + if report.MissingGroundTruth > 0 { + fmt.Fprintf(&out, "Missing labels: %d ground-truth path(s)\n", report.MissingGroundTruth) + } + fmt.Fprintf(&out, "Any-relevant@%d: %.0f%% (%d/%d)\nAvg precision@%d: %.2f\nMRR: %.2f\n", report.K, report.AnyRelevantAtK*100, report.Hits, report.Queries, report.K, report.AvgPrecisionAtK, report.MeanReciprocalRank) + return aoeval.AliasOutput{Stdout: out.String()}, nil +} + +func jsonAliasOutput(value any) (aoeval.AliasOutput, error) { + var out bytes.Buffer + encoder := json.NewEncoder(&out) + encoder.SetIndent("", " ") + if err := encoder.Encode(value); err != nil { + return aoeval.AliasOutput{}, err + } + return aoeval.AliasOutput{Stdout: out.String()}, nil +} diff --git a/cli/cmd/ao/eval_bench_adapter_test.go b/cli/cmd/ao/eval_bench_adapter_test.go new file mode 100644 index 000000000..2fa3b430c --- /dev/null +++ b/cli/cmd/ao/eval_bench_adapter_test.go @@ -0,0 +1,21 @@ +package main + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" + + aoeval "github.com/boshu2/agentops/cli/internal/eval" +) + +func TestEvalBenchAdapterStandardJSON(t *testing.T) { + corpus := filepath.Join("testdata", "retrieval-bench") + output, err := (evalBenchAdapter{}).Bench(context.Background(), aoeval.BenchRequest{Corpus: corpus, JSON: true, K: 3}) + if err != nil { + t.Fatalf("Bench: %v", err) + } + if !json.Valid([]byte(output.Stdout)) { + t.Fatalf("stdout is not JSON: %q", output.Stdout) + } +} diff --git a/cli/internal/commands/eval/module.go b/cli/internal/commands/eval/module.go index f2921a93a..dea12e0df 100644 --- a/cli/internal/commands/eval/module.go +++ b/cli/internal/commands/eval/module.go @@ -66,6 +66,7 @@ type UseCases struct { Scenario ScenarioUseCases ScenarioAB ScenarioABUseCases Aliases aoeval.AliasUseCases + Bench aoeval.BenchUseCases } type HostOptions struct { @@ -147,6 +148,9 @@ release. Live Claude and Codex adapters are evaluated by a later runtime tier.`, if module.useCases.Aliases != nil { command.AddCommand(module.sessionOutcomeCommand(), module.chaosCommand()) } + if module.useCases.Bench != nil { + command.AddCommand(module.benchCommand()) + } return command } @@ -836,6 +840,32 @@ func (module Module) chaosCommand() *cobra.Command { return command } +func (module Module) benchCommand() *cobra.Command { + options := aoeval.BenchRequest{K: 3, SearchBackend: "local-lexical"} + command := &cobra.Command{Use: "bench", Short: "Run retrieval quality benchmarks"} + command.Flags().StringVar(&options.Corpus, "corpus", "", "Path to benchmark corpus directory") + command.Flags().BoolVar(&options.JSON, "json", false, "JSON output") + command.Flags().IntVar(&options.K, "k", 3, "K for Precision@K") + command.Flags().BoolVar(&options.Live, "live", false, "Benchmark against real .agents/learnings/ instead of synthetic corpus") + command.Flags().BoolVar(&options.Global, "global", false, "Include ~/.agents/learnings/ (cross-rig aggregated store, requires --live)") + command.Flags().StringVar(&options.SearchEval, "search-eval", "", "Path to an ao-search eval manifest with queries and ground_truth paths") + command.Flags().StringVar(&options.SearchRoot, "search-root", "", "Repo root to search for --search-eval (defaults to current directory)") + command.Flags().StringVar(&options.SearchBackend, "search-backend", "local-lexical", "Search backend for --search-eval") + command.Flags().StringVar(&options.CompareBackends, "search-compare-backends", "", "Comma-separated search backends to compare for --search-eval") + command.RunE = func(command *cobra.Command, _ []string) error { + options.KChanged = command.Flags().Changed("k") + output, err := module.useCases.Bench.Bench(command.Context(), options) + if output.Stdout != "" { + fmt.Fprint(command.OutOrStdout(), output.Stdout) + } + if output.Stderr != "" { + fmt.Fprint(command.ErrOrStderr(), output.Stderr) + } + return err + } + return command +} + func (module Module) renderRun(command *cobra.Command, result aoeval.CoreRunResult) error { var value any = result.Run if result.Mode == aoeval.CoreRunBaselineAB { diff --git a/cli/internal/commands/eval/module_test.go b/cli/internal/commands/eval/module_test.go index 1e4416abc..05f050f86 100644 --- a/cli/internal/commands/eval/module_test.go +++ b/cli/internal/commands/eval/module_test.go @@ -30,6 +30,13 @@ type scenarioABUseCasesSpy struct{ request aoeval.ScenarioABRequest } type aliasUseCasesSpy struct{ request aoeval.SessionOutcomeRequest } +type benchUseCasesSpy struct{ request aoeval.BenchRequest } + +func (useCases *benchUseCasesSpy) Bench(_ context.Context, request aoeval.BenchRequest) (aoeval.AliasOutput, error) { + useCases.request = request + return aoeval.AliasOutput{Stdout: "bench\n"}, nil +} + func (useCases *aliasUseCasesSpy) SessionOutcome(_ context.Context, request aoeval.SessionOutcomeRequest) (aoeval.SessionOutcomeResult, error) { useCases.request = request return aoeval.SessionOutcomeResult{SessionID: "s", Reward: .5, Signals: []aoeval.SessionSignal{}}, nil @@ -260,6 +267,18 @@ func TestModuleChaosRendersAdapterStreams(t *testing.T) { } } +func TestModuleBenchDelegatesClosureLocalFlags(t *testing.T) { + useCases := &benchUseCasesSpy{} + command := NewModule(UseCases{Core: &coreUseCasesSpy{}, Bench: useCases}, HostOptions{}).Command() + command.SetArgs([]string{"bench", "--corpus", "fixture", "--k", "7", "--json"}) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if useCases.request.Corpus != "fixture" || useCases.request.K != 7 || !useCases.request.JSON || !useCases.request.KChanged { + t.Fatalf("request=%#v", useCases.request) + } +} + func findChild(t *testing.T, command *cobra.Command, name string) *cobra.Command { t.Helper() child, _, err := command.Find([]string{name}) diff --git a/cli/internal/eval/alias_service.go b/cli/internal/eval/alias_service.go index 7820dfbf5..585d030cd 100644 --- a/cli/internal/eval/alias_service.go +++ b/cli/internal/eval/alias_service.go @@ -29,3 +29,15 @@ type AliasUseCases interface { SessionOutcome(context.Context, SessionOutcomeRequest) (SessionOutcomeResult, error) Chaos(context.Context) (AliasOutput, error) } + +type BenchRequest struct { + Corpus string + JSON bool + K int + Live, Global bool + SearchEval, SearchRoot, SearchBackend, CompareBackends string + KChanged bool +} +type BenchUseCases interface { + Bench(context.Context, BenchRequest) (AliasOutput, error) +} From 39e7a6885a01ee459d702a386775fdffd20f411b Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:26:33 -0400 Subject: [PATCH 098/108] refactor(eval): cut over to internal command module --- cli/cmd/ao/cobra_commands_test.go | 20 - cli/cmd/ao/eval.go | 469 -------- cli/cmd/ao/eval_chaos.go | 21 - cli/cmd/ao/eval_cleanup.go | 240 ---- cli/cmd/ao/eval_cleanup_test.go | 142 --- cli/cmd/ao/eval_codex_exec_bridge.go | 13 + cli/cmd/ao/eval_composition.go | 32 + cli/cmd/ao/eval_composition_test.go | 14 + cli/cmd/ao/eval_fold_surface_test.go | 88 -- cli/cmd/ao/eval_outcomes.go | 84 -- cli/cmd/ao/eval_outcomes_burn_test.go | 137 --- cli/cmd/ao/eval_outcomes_burn_wire_test.go | 174 --- cli/cmd/ao/eval_outcomes_codex.go | 37 - cli/cmd/ao/eval_outcomes_codex_test.go | 66 -- cli/cmd/ao/eval_outcomes_gate2_warn_test.go | 89 -- cli/cmd/ao/eval_outcomes_hash_parity_test.go | 80 -- cli/cmd/ao/eval_outcomes_ingest.go | 265 ----- cli/cmd/ao/eval_outcomes_ingest_test.go | 53 - cli/cmd/ao/eval_outcomes_manifest.go | 227 ---- cli/cmd/ao/eval_outcomes_manifest_test.go | 254 ---- cli/cmd/ao/eval_outcomes_test.go | 64 - cli/cmd/ao/eval_scenario_ab.go | 320 ----- cli/cmd/ao/eval_scenario_ab_agentic.go | 195 ---- cli/cmd/ao/eval_scenario_ab_agentic_test.go | 116 -- cli/cmd/ao/eval_scenario_ab_sandbox.go | 196 ---- cli/cmd/ao/eval_scenario_ab_sandbox_test.go | 243 ---- cli/cmd/ao/eval_scenario_ab_test.go | 396 ------- cli/cmd/ao/eval_scenario_moat.go | 73 -- cli/cmd/ao/eval_scenario_moat_test.go | 63 - cli/cmd/ao/eval_session_outcome.go | 23 - cli/cmd/ao/eval_suite.go | 286 ----- cli/cmd/ao/eval_task.go | 479 -------- cli/cmd/ao/eval_test.go | 1028 ----------------- cli/cmd/ao/eval_test_compat_test.go | 6 + cli/cmd/ao/retrieval_bench.go | 12 - cli/cmd/ao/scenario.go | 22 - cli/cmd/ao/scenario_add.go | 92 -- cli/cmd/ao/scenario_evaluate.go | 440 ------- cli/cmd/ao/scenario_evaluate_test.go | 437 ------- cli/cmd/ao/scenario_init.go | 55 - cli/cmd/ao/scenario_list.go | 82 -- cli/cmd/ao/scenario_test.go | 381 ------ cli/cmd/ao/scenario_validate.go | 103 -- .../adapters/eval/scenario_ab_runtime.go | 10 +- cli/internal/commands/eval/module.go | 8 +- cli/internal/eval/outcomes_service.go | 7 +- cli/internal/eval/scenario_service.go | 5 +- cli/internal/eval/task_service.go | 28 +- 48 files changed, 103 insertions(+), 7572 deletions(-) delete mode 100644 cli/cmd/ao/eval.go delete mode 100644 cli/cmd/ao/eval_chaos.go delete mode 100644 cli/cmd/ao/eval_cleanup.go delete mode 100644 cli/cmd/ao/eval_cleanup_test.go create mode 100644 cli/cmd/ao/eval_codex_exec_bridge.go create mode 100644 cli/cmd/ao/eval_composition.go create mode 100644 cli/cmd/ao/eval_composition_test.go delete mode 100644 cli/cmd/ao/eval_fold_surface_test.go delete mode 100644 cli/cmd/ao/eval_outcomes.go delete mode 100644 cli/cmd/ao/eval_outcomes_burn_test.go delete mode 100644 cli/cmd/ao/eval_outcomes_burn_wire_test.go delete mode 100644 cli/cmd/ao/eval_outcomes_codex.go delete mode 100644 cli/cmd/ao/eval_outcomes_codex_test.go delete mode 100644 cli/cmd/ao/eval_outcomes_gate2_warn_test.go delete mode 100644 cli/cmd/ao/eval_outcomes_hash_parity_test.go delete mode 100644 cli/cmd/ao/eval_outcomes_ingest.go delete mode 100644 cli/cmd/ao/eval_outcomes_ingest_test.go delete mode 100644 cli/cmd/ao/eval_outcomes_manifest.go delete mode 100644 cli/cmd/ao/eval_outcomes_manifest_test.go delete mode 100644 cli/cmd/ao/eval_outcomes_test.go delete mode 100644 cli/cmd/ao/eval_scenario_ab.go delete mode 100644 cli/cmd/ao/eval_scenario_ab_agentic.go delete mode 100644 cli/cmd/ao/eval_scenario_ab_agentic_test.go delete mode 100644 cli/cmd/ao/eval_scenario_ab_sandbox.go delete mode 100644 cli/cmd/ao/eval_scenario_ab_sandbox_test.go delete mode 100644 cli/cmd/ao/eval_scenario_ab_test.go delete mode 100644 cli/cmd/ao/eval_scenario_moat.go delete mode 100644 cli/cmd/ao/eval_scenario_moat_test.go delete mode 100644 cli/cmd/ao/eval_session_outcome.go delete mode 100644 cli/cmd/ao/eval_suite.go delete mode 100644 cli/cmd/ao/eval_task.go delete mode 100644 cli/cmd/ao/eval_test.go create mode 100644 cli/cmd/ao/eval_test_compat_test.go delete mode 100644 cli/cmd/ao/scenario.go delete mode 100644 cli/cmd/ao/scenario_add.go delete mode 100644 cli/cmd/ao/scenario_evaluate.go delete mode 100644 cli/cmd/ao/scenario_evaluate_test.go delete mode 100644 cli/cmd/ao/scenario_init.go delete mode 100644 cli/cmd/ao/scenario_list.go delete mode 100644 cli/cmd/ao/scenario_test.go delete mode 100644 cli/cmd/ao/scenario_validate.go diff --git a/cli/cmd/ao/cobra_commands_test.go b/cli/cmd/ao/cobra_commands_test.go index ec56a7fdd..5271458af 100644 --- a/cli/cmd/ao/cobra_commands_test.go +++ b/cli/cmd/ao/cobra_commands_test.go @@ -72,7 +72,6 @@ func executeCommand(args ...string) (string, error) { origFindingsPullForce := findingsPullForce origFindingsRetireBy := findingsRetireBy origForgeTier1LegacyLocalLLM := forgeTier1LegacyLocalLLM - origScenarioListStatus := scenarioListStatus // Struct-type flag vars (fields persist across Execute calls): origContextPacketFlags := contextPacketFlags origContextExplainFlags := contextExplainFlags @@ -142,7 +141,6 @@ func executeCommand(args ...string) (string, error) { findingsPullForce = origFindingsPullForce findingsRetireBy = origFindingsRetireBy forgeTier1LegacyLocalLLM = origForgeTier1LegacyLocalLLM - scenarioListStatus = origScenarioListStatus contextPacketFlags = origContextPacketFlags contextExplainFlags = origContextExplainFlags contextPacketStatusFlags = origContextPacketStatusFlags @@ -209,24 +207,6 @@ func executeCommand(args ...string) (string, error) { findingsPullForce = false findingsRetireBy = "" forgeTier1LegacyLocalLLM = false - scenarioListStatus = "" - evalRunOutput = "" - evalRunID = "" - evalRunRuntime = "" - evalRunBaseline = "" - evalCompareOutput = "" - evalCompareMaxAgg = 0 - evalCompareMaxDim = 0 - evalScorecardOutput = "" - evalScorecardKind = "rpi" - evalScorecardMaxCat = 0 - evalBaselineOutput = "" - evalBaselineBy = "" - evalBaselineReason = "" - evalCoverageRoot = "evals/agentops-core" - evalCoverageDomains = []string{"cli", "hook", "skill", "rpi", "runtime", "retrieval", "scenario", "mixed", "security"} - evalCoverageDims = []string{"correctness", "process_adherence", "artifact_quality", "runtime_compatibility", "efficiency", "safety", "learning_closure"} - evalCoverageRuntimes = []string{"static", "shell", "mock"} maturityApply = false maturityScan = false maturityCurate = false diff --git a/cli/cmd/ao/eval.go b/cli/cmd/ao/eval.go deleted file mode 100644 index 17dd0b996..000000000 --- a/cli/cmd/ao/eval.go +++ /dev/null @@ -1,469 +0,0 @@ -// practices: [llm-eval-harness, dora-metrics] -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/spf13/cobra" - - aoeval "github.com/boshu2/agentops/cli/internal/eval" -) - -var ( - evalRunOutput string - evalRunID string - evalRunRuntime string - evalRunBaseline string - evalRunBaselineMode string - evalRunContextMode string - evalRunContextOffDir string - evalRunContextOnDir string - evalRunDeltaOut string - evalCompareOutput string - evalCompareMaxAgg float64 - evalCompareMaxDim float64 - evalScorecardOutput string - evalScorecardKind string - evalScorecardMaxCat float64 - evalBaselineOutput string - evalBaselineBy string - evalBaselineReason string - evalBaselineAuditRoot string - evalBaselineAuditDir string - evalCoverageRoot string - evalCoverageDomains []string - evalCoverageEvidence []string - evalCoverageDims []string - evalCoverageRuntimes []string - evalConfigured bool -) - -var evalCmd = &cobra.Command{ - Use: "eval", - Short: "Run deterministic local evaluation suites", - Long: `Run deterministic AgentOps evaluation suites and compare run records. - -The eval surface intentionally supports only offline deterministic runs in this -release. Live Claude and Codex adapters are evaluated by a later runtime tier.`, -} - -var evalRunCmd = &cobra.Command{ - Use: "run ", - Short: "Run a deterministic eval suite", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - runtimeName, err := parseEvalRuntime(evalRunRuntime) - if err != nil { - return err - } - mode := aoeval.ABBaselineMode(evalRunBaselineMode) - if !aoeval.IsValidBaselineMode(string(mode)) { - return fmt.Errorf("invalid --baseline-mode %q (allowed: %s)", evalRunBaselineMode, strings.Join(aoeval.AllBaselineModes(), ", ")) - } - contextMode := aoeval.ContextMode(evalRunContextMode) - if !aoeval.IsValidContextMode(string(contextMode)) { - return fmt.Errorf("invalid --context-mode %q (allowed: %s)", evalRunContextMode, strings.Join(aoeval.AllContextModes(), ", ")) - } - baseOpts := aoeval.RunOptions{ - SuitePath: args[0], - RunID: evalRunID, - Runtime: runtimeName, - OutputPath: evalRunOutput, - BaselinePath: evalRunBaseline, - } - if contextMode == aoeval.ContextModeAB { - if mode != aoeval.BaselineModeSkillOn { - return fmt.Errorf("--context-mode=ab cannot be combined with --baseline-mode=%s", mode) - } - contextOpts := resolveEvalContextABOptions(args[0], evalRunContextOffDir, evalRunContextOnDir) - scorecard, offRun, onRun, err := aoeval.RunContextAB(baseOpts, contextOpts) - if err != nil { - return err - } - if err := aoeval.WriteContextDeltaScorecard(scorecard, evalRunDeltaOut); err != nil { - return err - } - if GetOutput() == "json" { - return writeEvalJSON(cmd, scorecard) - } - fmt.Fprintf(cmd.OutOrStdout(), - "Eval context-AB %s: context-off=%.4f (%s) context-on=%.4f (%s) delta=%+.4f cases=%d\n", - scorecard.SuiteID, - offRun.AggregateScore, offRun.Status, - onRun.AggregateScore, onRun.Status, - scorecard.AggregateDelta, - len(scorecard.PerCase), - ) - if evalRunDeltaOut != "" { - fmt.Fprintf(cmd.OutOrStdout(), "Context delta scorecard: %s\n", evalRunDeltaOut) - } - return nil - } - if mode == aoeval.BaselineModeBoth { - scorecard, onRun, offRun, err := aoeval.RunBaselineAB(baseOpts) - if err != nil { - return err - } - if err := aoeval.WriteDeltaScorecard(scorecard, evalRunDeltaOut); err != nil { - return err - } - if GetOutput() == "json" { - return writeEvalJSON(cmd, scorecard) - } - fmt.Fprintf(cmd.OutOrStdout(), - "Eval baseline-AB %s: skill-on=%.4f (%s) skill-off=%.4f (%s) delta=%+.4f cases=%d\n", - scorecard.SuiteID, - onRun.AggregateScore, onRun.Status, - offRun.AggregateScore, offRun.Status, - scorecard.AggregateDelta, - len(scorecard.PerCase), - ) - if evalRunDeltaOut != "" { - fmt.Fprintf(cmd.OutOrStdout(), "Delta scorecard: %s\n", evalRunDeltaOut) - } - return nil - } - opts := baseOpts - opts.OverrideDisableHooks = (mode == aoeval.BaselineModeSkillOff) - run, err := aoeval.RunSuite(opts) - if err != nil { - return err - } - if GetOutput() == "json" { - return writeEvalJSON(cmd, run) - } - fmt.Fprintf(cmd.OutOrStdout(), "Eval %s: %s (aggregate %.4f, cases %d)\n", run.RunID, run.Status, run.AggregateScore, len(run.CaseResults)) - if evalRunOutput != "" { - fmt.Fprintf(cmd.OutOrStdout(), "Run record: %s\n", evalRunOutput) - } - return nil - }, -} - -var evalCompareCmd = &cobra.Command{ - Use: "compare ", - Short: "Compare an eval run against a baseline", - Args: cobra.ExactArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { - candidate, err := aoeval.LoadRun(args[0]) - if err != nil { - return err - } - baseline, err := aoeval.LoadRun(args[1]) - if err != nil { - return err - } - compared, err := aoeval.CompareRuns(candidate, baseline, aoeval.CompareOptions{ - MaxAggregateRegression: evalCompareMaxAgg, - MaxDimensionRegression: evalCompareMaxDim, - OutputPath: evalCompareOutput, - }) - if err != nil { - return err - } - if GetOutput() == "json" { - return writeEvalJSON(cmd, compared) - } - delta := 0.0 - if compared.BaselineComparison != nil { - delta = compared.BaselineComparison.AggregateDelta - } - fmt.Fprintf(cmd.OutOrStdout(), "Eval compare %s vs %s: %s (aggregate delta %.4f)\n", compared.RunID, baseline.RunID, compared.Verdict, delta) - if evalCompareOutput != "" { - fmt.Fprintf(cmd.OutOrStdout(), "Comparison record: %s\n", evalCompareOutput) - } - return nil - }, -} - -var evalBaselineCmd = &cobra.Command{ - Use: "baseline ", - Short: "Promote an eval run record as a baseline", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - run, err := aoeval.LoadRun(args[0]) - if err != nil { - return err - } - cwd, err := os.Getwd() - if err != nil { - return err - } - promoted, err := aoeval.PromoteBaseline(run, aoeval.BaselineOptions{ - OutputPath: evalBaselineOutput, - PromotedBy: evalBaselineBy, - Rationale: evalBaselineReason, - WorkDir: cwd, - }) - if err != nil { - return err - } - if GetOutput() == "json" { - return writeEvalJSON(cmd, promoted) - } - path := "" - if promoted.Baseline != nil { - path = promoted.Baseline.BaselinePath - } - fmt.Fprintf(cmd.OutOrStdout(), "Eval baseline promoted: %s\n", path) - return nil - }, -} - -var evalBaselineAuditCmd = &cobra.Command{ - Use: "baseline-audit [suite.json ...]", - Short: "Audit eval suite baseline policy against promoted baselines", - Args: cobra.ArbitraryArgs, - RunE: func(cmd *cobra.Command, args []string) error { - roots := []string{} - if len(args) == 0 { - roots = append(roots, evalBaselineAuditRoot) - } - report, err := aoeval.AuditBaselinePolicy(aoeval.BaselineAuditOptions{ - SuitePaths: args, - Roots: roots, - BaselineDir: evalBaselineAuditDir, - }) - if err != nil { - return err - } - if GetOutput() == "json" { - return writeEvalJSON(cmd, report) - } - fmt.Fprintf(cmd.OutOrStdout(), "Eval baseline audit: %d suites, %d baselines, %d policy mismatches\n", report.SuiteCount, report.BaselineCount, report.PolicyMismatchCount) - if len(report.MissingCompareBaselines) > 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Missing compare baselines: %d\n", len(report.MissingCompareBaselines)) - } - if len(report.UnexpectedBaselinesForNone) > 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Unexpected baselines for none policy: %d\n", len(report.UnexpectedBaselinesForNone)) - } - if len(report.OrphanBaselines) > 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Orphan baselines: %d\n", len(report.OrphanBaselines)) - } - if len(report.StaleSuiteHashes) > 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Stale suite hashes: %d\n", len(report.StaleSuiteHashes)) - } - return nil - }, -} - -var evalScorecardCmd = &cobra.Command{ - Use: "scorecard [baseline-run.json]", - Short: "Build an eval scorecard from run records", - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { - kind, err := parseScorecardKind(evalScorecardKind) - if err != nil { - return err - } - candidate, err := aoeval.LoadRun(args[0]) - if err != nil { - return err - } - var baseline *aoeval.RunRecord - if len(args) == 2 { - baseline, err = aoeval.LoadRun(args[1]) - if err != nil { - return err - } - } - scorecard, err := aoeval.BuildScorecard(candidate, baseline, aoeval.ScorecardOptions{ - Kind: kind, - MaxCategoryRegression: evalScorecardMaxCat, - }) - if err != nil { - return err - } - if evalScorecardOutput != "" { - if err := aoeval.WriteScorecard(evalScorecardOutput, scorecard); err != nil { - return err - } - } - if GetOutput() == "json" { - return writeEvalJSON(cmd, scorecard) - } - fmt.Fprintf(cmd.OutOrStdout(), "Eval scorecard %s: %s (%s, categories %d)\n", scorecard.CandidateRunID, scorecard.Verdict, scorecard.Kind, len(scorecard.Categories)) - if evalScorecardOutput != "" { - fmt.Fprintf(cmd.OutOrStdout(), "Scorecard: %s\n", evalScorecardOutput) - } - return nil - }, -} - -var evalCoverageCmd = &cobra.Command{ - Use: "coverage [suite.json ...]", - Short: "Summarize eval suite coverage", - Args: cobra.ArbitraryArgs, - RunE: func(cmd *cobra.Command, args []string) error { - roots := []string{} - if len(args) == 0 { - roots = append(roots, evalCoverageRoot) - } - report, err := aoeval.BuildCoverageReport(aoeval.CoverageOptions{ - SuitePaths: args, - Roots: roots, - RequiredDomains: evalCoverageDomains, - RequiredEvidenceKinds: evalCoverageEvidence, - RequiredDimensions: evalCoverageDims, - RequiredRuntimes: evalCoverageRuntimes, - }) - if err != nil { - return err - } - if GetOutput() == "json" { - return writeEvalJSON(cmd, report) - } - fmt.Fprintf(cmd.OutOrStdout(), "Eval coverage: %d suites, %d cases, %d critical cases\n", report.SuiteCount, report.CaseCount, report.CriticalCaseCount) - if len(report.MissingRequiredDomains) > 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Missing required domains: %s\n", strings.Join(report.MissingRequiredDomains, ", ")) - } else if len(report.RequiredDomains) > 0 { - fmt.Fprintln(cmd.OutOrStdout(), "Required domains covered") - } - if len(report.MissingRequiredEvidenceKinds) > 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Missing required evidence kinds: %s\n", strings.Join(report.MissingRequiredEvidenceKinds, ", ")) - } else if len(report.RequiredEvidenceKinds) > 0 { - fmt.Fprintln(cmd.OutOrStdout(), "Required evidence kinds covered") - } - if len(report.MissingRequiredDimensions) > 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Missing required dimensions: %s\n", strings.Join(report.MissingRequiredDimensions, ", ")) - } else if len(report.RequiredDimensions) > 0 { - fmt.Fprintln(cmd.OutOrStdout(), "Required dimensions covered") - } - if len(report.MissingRequiredRuntimes) > 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Missing required runtimes: %s\n", strings.Join(report.MissingRequiredRuntimes, ", ")) - } else if len(report.RequiredRuntimes) > 0 { - fmt.Fprintln(cmd.OutOrStdout(), "Required runtimes covered") - } - return nil - }, -} - -func init() { - registerEvalCommand() -} - -func registerEvalCommand() { - if !evalConfigured { - configureEvalCommand() - evalConfigured = true - } - if evalCmd.Parent() == nil { - rootCmd.AddCommand(evalCmd) - } -} - -func configureEvalCommand() { - evalCmd.GroupID = "workflow" - - evalRunCmd.Flags().StringVar(&evalRunOutput, "out", "", "write eval run record to path") - evalRunCmd.Flags().StringVar(&evalRunID, "run-id", "", "stable run id to use in the run record") - evalRunCmd.Flags().StringVar(&evalRunRuntime, "runtime", "", "runtime override (static, mock, shell, claude, codex)") - evalRunCmd.Flags().StringVar(&evalRunBaseline, "baseline", "", "compare the run against a baseline run record") - evalRunCmd.Flags().StringVar(&evalRunBaselineMode, "baseline-mode", string(aoeval.BaselineModeSkillOn), "skill-on | skill-off | both — runs the suite once with skills loaded, once with hooks suppressed, or both for a delta scorecard") - evalRunCmd.Flags().StringVar(&evalRunContextMode, "context-mode", string(aoeval.ContextModeNone), "none | ab — run context-off/context-on legs over isolated AO_AGENTS_DIR roots") - evalRunCmd.Flags().StringVar(&evalRunContextOffDir, "context-off-agents-dir", "", "AO_AGENTS_DIR root for the context-off leg (defaults to suite fixtures)") - evalRunCmd.Flags().StringVar(&evalRunContextOnDir, "context-on-agents-dir", "", "AO_AGENTS_DIR root for the context-on leg (defaults to suite fixtures)") - evalRunCmd.Flags().StringVar(&evalRunDeltaOut, "delta-out", "", "write delta scorecard JSON to path (with --baseline-mode=both or --context-mode=ab)") - _ = evalRunCmd.RegisterFlagCompletionFunc("runtime", staticCompletionFunc("static", "mock", "shell", "claude", "codex")) - _ = evalRunCmd.RegisterFlagCompletionFunc("baseline-mode", staticCompletionFunc(aoeval.AllBaselineModes()...)) - _ = evalRunCmd.RegisterFlagCompletionFunc("context-mode", staticCompletionFunc(aoeval.AllContextModes()...)) - - evalCompareCmd.Flags().StringVar(&evalCompareOutput, "out", "", "write compared eval run record to path") - evalCompareCmd.Flags().Float64Var(&evalCompareMaxAgg, "max-aggregate-regression", 0, "allowed aggregate regression before verdict becomes regression") - evalCompareCmd.Flags().Float64Var(&evalCompareMaxDim, "max-dimension-regression", 0, "allowed per-dimension regression before verdict becomes regression") - - evalBaselineCmd.Flags().StringVar(&evalBaselineOutput, "out", "", "write promoted baseline run record to path") - evalBaselineCmd.Flags().StringVar(&evalBaselineBy, "promoted-by", "", "identity promoting the baseline") - evalBaselineCmd.Flags().StringVar(&evalBaselineReason, "rationale", "", "rationale for promoting the baseline") - - evalBaselineAuditCmd.Flags().StringVar(&evalBaselineAuditRoot, "root", "evals/agentops-core", "suite root to scan when no suite paths are provided") - evalBaselineAuditCmd.Flags().StringVar(&evalBaselineAuditDir, "baseline-dir", ".agents/evals/baselines", "promoted baseline directory") - - evalCoverageCmd.Flags().StringVar(&evalCoverageRoot, "root", "evals/agentops-core", "suite root to scan when no suite paths are provided") - evalCoverageCmd.Flags().StringArrayVar(&evalCoverageDomains, "require-domain", aoeval.DefaultCoverageDomains, "required product domain for missing-domain reporting") - evalCoverageCmd.Flags().StringArrayVar(&evalCoverageEvidence, "require-evidence-kind", nil, "required evidence kind for missing-evidence-kind reporting") - evalCoverageCmd.Flags().StringArrayVar(&evalCoverageDims, "require-dimension", aoeval.DefaultCoverageDimensions, "required score dimension for missing-dimension reporting") - evalCoverageCmd.Flags().StringArrayVar(&evalCoverageRuntimes, "require-runtime", aoeval.DefaultCoverageRuntimes, "required deterministic runtime for missing-runtime reporting") - - evalScorecardCmd.Flags().StringVar(&evalScorecardOutput, "out", "", "write scorecard JSON to path") - evalScorecardCmd.Flags().StringVar(&evalScorecardKind, "kind", string(aoeval.ScorecardKindRPI), "scorecard kind (rpi, skill-change)") - evalScorecardCmd.Flags().Float64Var(&evalScorecardMaxCat, "max-category-regression", 0, "allowed per-category regression before verdict becomes regression") - _ = evalScorecardCmd.RegisterFlagCompletionFunc("kind", staticCompletionFunc(string(aoeval.ScorecardKindRPI), string(aoeval.ScorecardKindSkillChange))) - - evalCmd.AddCommand(evalRunCmd, evalCompareCmd, evalBaselineCmd, evalBaselineAuditCmd, evalScorecardCmd, evalCoverageCmd) - - registerEvalTaskCmd() - registerEvalCleanupCmd() - registerEvalSuiteCmd() - evalCmd.AddCommand(evalTaskCmd, evalCleanupCmd, evalSuiteCmd) -} - -func resolveEvalContextABOptions(suitePath, offDir, onDir string) aoeval.ContextABOptions { - if offDir == "" { - offDir = defaultEvalContextAgentsDir(suitePath, "context-off") - } - if onDir == "" { - onDir = defaultEvalContextAgentsDir(suitePath, "context-on") - } - return aoeval.ContextABOptions{ - ContextOffAgentsDir: absoluteEvalContextAgentsDir(offDir), - ContextOnAgentsDir: absoluteEvalContextAgentsDir(onDir), - ContextOffLabel: "context-off", - ContextOnLabel: "context-on", - } -} - -func defaultEvalContextAgentsDir(suitePath, leg string) string { - base := filepath.Base(suitePath) - name := strings.TrimSuffix(base, filepath.Ext(base)) - return filepath.Join(filepath.Dir(suitePath), "fixtures", name, leg, "agents") -} - -func absoluteEvalContextAgentsDir(path string) string { - if path == "" { - return "" - } - abs, err := filepath.Abs(path) - if err != nil { - return path - } - return abs -} - -func parseEvalRuntime(value string) (aoeval.Runtime, error) { - value = strings.TrimSpace(value) - if value == "" { - return "", nil - } - switch aoeval.Runtime(value) { - case aoeval.RuntimeStatic, aoeval.RuntimeMock, aoeval.RuntimeShell: - return aoeval.Runtime(value), nil - case aoeval.RuntimeClaude, aoeval.RuntimeCodex: - return aoeval.Runtime(value), nil - default: - return "", fmt.Errorf("unknown runtime %q (use static, mock, shell, claude, or codex)", value) - } -} - -func parseScorecardKind(value string) (aoeval.ScorecardKind, error) { - value = strings.TrimSpace(value) - if value == "" { - return aoeval.ScorecardKindRPI, nil - } - switch aoeval.ScorecardKind(value) { - case aoeval.ScorecardKindRPI, aoeval.ScorecardKindSkillChange: - return aoeval.ScorecardKind(value), nil - default: - return "", fmt.Errorf("unsupported scorecard kind %q (use rpi or skill-change)", value) - } -} - -func writeEvalJSON(cmd *cobra.Command, value any) error { - enc := json.NewEncoder(cmd.OutOrStdout()) - enc.SetIndent("", " ") - return enc.Encode(value) -} diff --git a/cli/cmd/ao/eval_chaos.go b/cli/cmd/ao/eval_chaos.go deleted file mode 100644 index cc001b815..000000000 --- a/cli/cmd/ao/eval_chaos.go +++ /dev/null @@ -1,21 +0,0 @@ -// practices: [ai-assisted-dev, pragmatic-programmer] -package main - -import "github.com/spf13/cobra" - -// evalChaosCmd is the canonical `ao eval chaos` alias for the read-only tick -// membrane smoke test, folded under `ao eval` -// (age-focus-membrane-bookkeeper-m1wg.16). The hidden top-level `ao chaos-test` -// is preserved for back-compat (see tick.go). -var evalChaosCmd = &cobra.Command{ - Use: "chaos", - Short: "Run a read-only smoke test of the tick membrane", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, _ []string) error { - return tickSmoke(newTickRuntime(cmd)) - }, -} - -func init() { - evalCmd.AddCommand(evalChaosCmd) -} diff --git a/cli/cmd/ao/eval_cleanup.go b/cli/cmd/ao/eval_cleanup.go deleted file mode 100644 index 87c190801..000000000 --- a/cli/cmd/ao/eval_cleanup.go +++ /dev/null @@ -1,240 +0,0 @@ -// practices: [llm-eval-harness, dora-metrics] -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/spf13/cobra" - - evalsub "github.com/boshu2/agentops/cli/internal/evalsubstrate" -) - -var ( - evalCleanupDelete bool - evalCleanupTmpFiles bool - evalCleanupAge int64 - evalCleanupDryRun bool -) - -// CleanupReport summarizes a cleanup pass for JSON output. -type CleanupReport struct { - TransitionsAborted int `json:"transitions_to_aborted"` - TransitionsFailed int `json:"transitions_to_failed"` - RunsDeleted int `json:"runs_deleted"` - TmpFilesSwept int `json:"tmp_files_swept"` - Touched []string `json:"touched"` -} - -var evalCleanupCmd = &cobra.Command{ - Use: "cleanup", - Short: "Recover stale Runs (state transitions, --delete, --tmp-files)", - Long: `Per SCHEMA.md §4 cleanup state-transition rule (rc2): - - Stale pending (no running transition within 60s) - -> aborted (retraction_reason=never_started) - Stale running (no heartbeat within 5min OR Inspect process not alive) - -> failed (retraction_reason=orphaned_process) - -After transitions: - --delete Remove Run dirs whose status is failed OR aborted (NEVER retracted). - --tmp-files Sweep orphan manifest.json.tmp left from rename-step crashes. - --dry-run Print what would be done; no mutations. - -The cleanup procedure honors the §4 atomic-write contract on every state -transition (temp + fsync + rename + fsync-parent-dir). Retracted Runs are -never auto-removed — retraction is an audit trail.`, - RunE: runEvalCleanup, -} - -func runEvalCleanup(cmd *cobra.Command, args []string) error { - root := evalsRoot() - runsRoot := filepath.Join(root, "runs") - report := CleanupReport{Touched: []string{}} - - if err := cleanupStaleTransitions(root, runsRoot, &report); err != nil { - return err - } - - if evalCleanupDelete { - if err := cleanupDeleteRuns(root, runsRoot, &report); err != nil { - return err - } - } - - if evalCleanupTmpFiles { - if err := cleanupSweepTmpFiles(root, &report); err != nil { - return err - } - } - - return writeCleanupReport(cmd, &report) -} - -// cleanupStaleTransitions walks every run dir, ages its manifest, and applies -// the §4 stale → aborted/failed transitions when the cutoff has been crossed. -func cleanupStaleTransitions(root, runsRoot string, report *CleanupReport) error { - entries, err := os.ReadDir(runsRoot) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("eval cleanup: read runs/: %w", err) - } - stalePendingCutoff := 60 * time.Second - staleRunningCutoff := 5 * time.Minute - now := time.Now().UTC() - for _, e := range entries { - if !e.IsDir() { - continue - } - runID := e.Name() - m, err := evalsub.LoadManifest(evalsub.ManifestPath(root, runID)) - if err != nil { - report.Touched = append(report.Touched, runID+":unreadable") - continue - } - ageSinceStart := now.Sub(time.Unix(m.StartedAtUnixMs/1000, 0).UTC()) - if err := applyStaleTransition(root, runID, m.Status, ageSinceStart, stalePendingCutoff, staleRunningCutoff, report); err != nil { - return err - } - } - return nil -} - -// applyStaleTransition routes a single run through the §4 cleanup state -// machine. Returns nil when no transition is required. -func applyStaleTransition(root, runID string, status evalsub.RunStatus, age, pendingCutoff, runningCutoff time.Duration, report *CleanupReport) error { - switch status { - case evalsub.StatusPending: - if age < pendingCutoff { - return nil - } - if err := transitionStale(root, runID, evalsub.StatusAborted, "never_started"); err != nil { - return err - } - report.TransitionsAborted++ - report.Touched = append(report.Touched, runID+":pending->aborted") - case evalsub.StatusRunning: - if age < runningCutoff { - return nil - } - if err := transitionStale(root, runID, evalsub.StatusFailed, "orphaned_process"); err != nil { - return err - } - report.TransitionsFailed++ - report.Touched = append(report.Touched, runID+":running->failed") - } - return nil -} - -// cleanupDeleteRuns sweeps run directories whose status is failed or aborted. -// Retracted runs are never auto-removed (audit trail). -func cleanupDeleteRuns(root, runsRoot string, report *CleanupReport) error { - entries, err := os.ReadDir(runsRoot) - if err != nil { - return nil - } - for _, e := range entries { - if !e.IsDir() { - continue - } - runID := e.Name() - m, err := evalsub.LoadManifest(evalsub.ManifestPath(root, runID)) - if err != nil { - continue - } - if m.Status != evalsub.StatusFailed && m.Status != evalsub.StatusAborted { - continue - } - target := filepath.Join(runsRoot, runID) - if evalCleanupDryRun { - report.Touched = append(report.Touched, runID+":would-delete") - continue - } - if err := os.RemoveAll(target); err != nil { - return fmt.Errorf("eval cleanup: remove %q: %w", target, err) - } - report.RunsDeleted++ - report.Touched = append(report.Touched, runID+":deleted") - } - return nil -} - -// cleanupSweepTmpFiles sweeps orphaned manifest.json.tmp files older than -// --tmp-age. Dry-run mode records a placeholder touch only. -func cleanupSweepTmpFiles(root string, report *CleanupReport) error { - if evalCleanupDryRun { - report.Touched = append(report.Touched, "tmp-files: dry-run preview not implemented (sweep is a write op)") - return nil - } - swept, err := evalsub.SweepTempFiles(root, evalCleanupAge) - if err != nil { - return fmt.Errorf("eval cleanup: sweep tmp: %w", err) - } - report.TmpFilesSwept = len(swept) - for _, s := range swept { - report.Touched = append(report.Touched, "tmp:"+s) - } - return nil -} - -func writeCleanupReport(cmd *cobra.Command, report *CleanupReport) error { - if GetOutput() == "json" { - enc := json.NewEncoder(cmd.OutOrStdout()) - enc.SetIndent("", " ") - return enc.Encode(report) - } - fmt.Fprintf(cmd.OutOrStdout(), "Eval cleanup:\n transitions->aborted: %d\n transitions->failed: %d\n runs deleted: %d\n tmp files swept: %d\n", - report.TransitionsAborted, report.TransitionsFailed, report.RunsDeleted, report.TmpFilesSwept) - if len(report.Touched) > 0 && GetVerbose() { - fmt.Fprintln(cmd.OutOrStdout(), "Touched:") - for _, t := range report.Touched { - fmt.Fprintf(cmd.OutOrStdout(), " %s\n", t) - } - } - return nil -} - -// transitionStale reads + writes via the §4 atomic contract by routing -// through RunWriter — but RunWriter requires a fresh runID + dir. For -// cleanup we mutate an existing manifest in place, so we re-marshal + -// WriteAtomic directly with status + retraction_reason set. -func transitionStale(root, runID string, next evalsub.RunStatus, reason string) error { - path := evalsub.ManifestPath(root, runID) - m, err := evalsub.LoadManifest(path) - if err != nil { - return fmt.Errorf("transitionStale: load: %w", err) - } - if !legalStaleTransition(m.Status, next) { - return nil // nothing to do - } - m.Status = next - m.RetractionReason = reason - now := time.Now().UTC() - m.FinishedAt = now.Format(time.RFC3339) - m.FinishedAtUnixMs = now.UnixNano() / int64(time.Millisecond) - data, err := json.MarshalIndent(m, "", " ") - if err != nil { - return fmt.Errorf("transitionStale: marshal: %w", err) - } - data = append(data, '\n') - return evalsub.WriteAtomic(path, data) -} - -func legalStaleTransition(cur, next evalsub.RunStatus) bool { - switch cur { - case evalsub.StatusPending: - return next == evalsub.StatusAborted - case evalsub.StatusRunning: - return next == evalsub.StatusFailed - } - return false -} - -func registerEvalCleanupCmd() { - evalCleanupCmd.Flags().BoolVar(&evalCleanupDelete, "delete", false, "Remove Run directories whose status is failed or aborted (never retracted)") - evalCleanupCmd.Flags().BoolVar(&evalCleanupTmpFiles, "tmp-files", false, "Sweep orphan *.tmp files older than --tmp-age") - evalCleanupCmd.Flags().Int64Var(&evalCleanupAge, "tmp-age", 60, "Minimum tmp-file age in seconds before sweep (0 = sweep all)") - evalCleanupCmd.Flags().BoolVar(&evalCleanupDryRun, "dry-run", false, "Preview without mutations") -} diff --git a/cli/cmd/ao/eval_cleanup_test.go b/cli/cmd/ao/eval_cleanup_test.go deleted file mode 100644 index 3a25af9fb..000000000 --- a/cli/cmd/ao/eval_cleanup_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// practices: [llm-eval-harness, dora-metrics] -package main - -import ( - "bytes" - "encoding/json" - "os" - "path/filepath" - "testing" - "time" - - evalsub "github.com/boshu2/agentops/cli/internal/evalsubstrate" - "github.com/spf13/cobra" -) - -func TestRunEvalCleanupTransitionsStaleRuns(t *testing.T) { - root := t.TempDir() - configureEvalCleanupTest(t, root, "json", false) - oldStartedAt := time.Now().UTC().Add(-10 * time.Minute) - freshStartedAt := time.Now().UTC() - writeEvalCleanupManifest(t, root, "pending-old", evalsub.StatusPending, oldStartedAt) - writeEvalCleanupManifest(t, root, "running-old", evalsub.StatusRunning, oldStartedAt) - writeEvalCleanupManifest(t, root, "pending-fresh", evalsub.StatusPending, freshStartedAt) - - var out bytes.Buffer - cmd := &cobra.Command{} - cmd.SetOut(&out) - if err := runEvalCleanup(cmd, nil); err != nil { - t.Fatalf("runEvalCleanup: %v", err) - } - - var report CleanupReport - if err := json.Unmarshal(out.Bytes(), &report); err != nil { - t.Fatalf("decode report: %v\n%s", err, out.String()) - } - if report.TransitionsAborted != 1 || report.TransitionsFailed != 1 { - t.Fatalf("report = %#v, want one aborted and one failed transition", report) - } - assertEvalCleanupStatus(t, root, "pending-old", evalsub.StatusAborted, "never_started") - assertEvalCleanupStatus(t, root, "running-old", evalsub.StatusFailed, "orphaned_process") - assertEvalCleanupStatus(t, root, "pending-fresh", evalsub.StatusPending, "") -} - -func TestRunEvalCleanupDeletePreservesRetractedRuns(t *testing.T) { - root := t.TempDir() - configureEvalCleanupTest(t, root, "json", false) - oldStartedAt := time.Now().UTC().Add(-10 * time.Minute) - writeEvalCleanupManifest(t, root, "failed-run", evalsub.StatusFailed, oldStartedAt) - writeEvalCleanupManifest(t, root, "aborted-run", evalsub.StatusAborted, oldStartedAt) - writeEvalCleanupManifest(t, root, "retracted-run", evalsub.StatusRetracted, oldStartedAt) - evalCleanupDelete = true - - var out bytes.Buffer - cmd := &cobra.Command{} - cmd.SetOut(&out) - if err := runEvalCleanup(cmd, nil); err != nil { - t.Fatalf("runEvalCleanup: %v", err) - } - - var report CleanupReport - if err := json.Unmarshal(out.Bytes(), &report); err != nil { - t.Fatalf("decode report: %v\n%s", err, out.String()) - } - if report.RunsDeleted != 2 { - t.Fatalf("runs deleted = %d, want 2 (report=%#v)", report.RunsDeleted, report) - } - assertEvalCleanupRunMissing(t, root, "failed-run") - assertEvalCleanupRunMissing(t, root, "aborted-run") - assertEvalCleanupStatus(t, root, "retracted-run", evalsub.StatusRetracted, "") -} - -func configureEvalCleanupTest(t *testing.T, root, outputValue string, verboseValue bool) { - t.Helper() - t.Setenv("AGENTOPS_EVALS_ROOT", root) - oldDelete := evalCleanupDelete - oldTmpFiles := evalCleanupTmpFiles - oldAge := evalCleanupAge - oldDryRun := evalCleanupDryRun - oldOutput := output - oldVerbose := verbose - evalCleanupDelete = false - evalCleanupTmpFiles = false - evalCleanupAge = 60 - evalCleanupDryRun = false - output = outputValue - verbose = verboseValue - t.Cleanup(func() { - evalCleanupDelete = oldDelete - evalCleanupTmpFiles = oldTmpFiles - evalCleanupAge = oldAge - evalCleanupDryRun = oldDryRun - output = oldOutput - verbose = oldVerbose - }) -} - -func writeEvalCleanupManifest(t *testing.T, root, runID string, status evalsub.RunStatus, startedAt time.Time) { - t.Helper() - runDir := evalsub.RunDir(root, runID) - if err := os.MkdirAll(runDir, 0o755); err != nil { - t.Fatalf("mkdir run dir: %v", err) - } - m := evalsub.Manifest{ - SchemaVersion: evalsub.SchemaVersion, - ID: runID, - StartedAt: startedAt.Format(time.RFC3339), - StartedAtUnixMs: startedAt.UnixNano() / int64(time.Millisecond), - Kind: "task", - Status: status, - Seeds: []int{}, - CapturedBy: "eval cleanup test", - } - data, err := json.MarshalIndent(m, "", " ") - if err != nil { - t.Fatalf("marshal manifest: %v", err) - } - data = append(data, '\n') - if err := os.WriteFile(filepath.Join(runDir, "manifest.json"), data, 0o644); err != nil { - t.Fatalf("write manifest: %v", err) - } -} - -func assertEvalCleanupStatus(t *testing.T, root, runID string, want evalsub.RunStatus, wantReason string) { - t.Helper() - m, err := evalsub.LoadManifest(evalsub.ManifestPath(root, runID)) - if err != nil { - t.Fatalf("load %s: %v", runID, err) - } - if m.Status != want { - t.Fatalf("%s status = %s, want %s", runID, m.Status, want) - } - if m.RetractionReason != wantReason { - t.Fatalf("%s retraction reason = %q, want %q", runID, m.RetractionReason, wantReason) - } -} - -func assertEvalCleanupRunMissing(t *testing.T, root, runID string) { - t.Helper() - if _, err := os.Stat(evalsub.RunDir(root, runID)); !os.IsNotExist(err) { - t.Fatalf("run dir %s exists or stat failed with %v", runID, err) - } -} diff --git a/cli/cmd/ao/eval_codex_exec_bridge.go b/cli/cmd/ao/eval_codex_exec_bridge.go new file mode 100644 index 000000000..86992c81e --- /dev/null +++ b/cli/cmd/ao/eval_codex_exec_bridge.go @@ -0,0 +1,13 @@ +package main + +import ( + "context" + + evaladapter "github.com/boshu2/agentops/cli/internal/adapters/eval" +) + +const judgeOutputSchema = evaladapter.JudgeOutputSchema + +func runCodexExec(ctx context.Context, prompt, outputSchemaPath string) (string, int, error) { + return evaladapter.RunCodexExec(ctx, prompt, outputSchemaPath) +} diff --git a/cli/cmd/ao/eval_composition.go b/cli/cmd/ao/eval_composition.go new file mode 100644 index 000000000..fed465f74 --- /dev/null +++ b/cli/cmd/ao/eval_composition.go @@ -0,0 +1,32 @@ +package main + +import ( + "github.com/spf13/cobra" + + evaladapter "github.com/boshu2/agentops/cli/internal/adapters/eval" + evalcommand "github.com/boshu2/agentops/cli/internal/commands/eval" + aoeval "github.com/boshu2/agentops/cli/internal/eval" +) + +func newEvalCommand() *cobra.Command { + runtime := evaladapter.Runtime{} + return evalcommand.NewModule(evalcommand.UseCases{ + Core: aoeval.CoreService{Runtime: runtime}, + Cleanup: aoeval.CleanupService{Runtime: runtime}, + Task: aoeval.TaskService{Runtime: runtime}, + Suite: aoeval.SuiteService{Runtime: runtime}, + Outcomes: aoeval.OutcomesService{Runtime: runtime}, + Scenario: aoeval.ScenarioService{Runtime: runtime}, + ScenarioAB: aoeval.ScenarioABService{Runtime: runtime}, + Aliases: evalAliasAdapter{}, + Bench: evalBenchAdapter{}, + }, evalcommand.HostOptions{ + OutputMode: func(*cobra.Command) string { return GetOutput() }, + Verbose: func(*cobra.Command) bool { return GetVerbose() }, + ProjectRoot: measureProjectRoot, + GoalsPath: resolveGoalsFile, + DryRun: func(*cobra.Command) bool { return GetDryRun() }, + }).Command() +} + +func init() { rootCmd.AddCommand(newEvalCommand()) } diff --git a/cli/cmd/ao/eval_composition_test.go b/cli/cmd/ao/eval_composition_test.go new file mode 100644 index 000000000..bea12172a --- /dev/null +++ b/cli/cmd/ao/eval_composition_test.go @@ -0,0 +1,14 @@ +package main + +import "testing" + +func TestEvalCompositionOwnsCompleteCommandTree(t *testing.T) { + command := newEvalCommand() + want := []string{"baseline", "baseline-audit", "bench", "chaos", "cleanup", "compare", "coverage", "outcomes", "run", "scenario", "scenario-ab", "scenario-moat", "scorecard", "session-outcome", "suite", "task"} + for _, name := range want { + child, _, err := command.Find([]string{name}) + if err != nil || child == command || child.Name() != name { + t.Fatalf("missing eval child %q: child=%v err=%v", name, child, err) + } + } +} diff --git a/cli/cmd/ao/eval_fold_surface_test.go b/cli/cmd/ao/eval_fold_surface_test.go deleted file mode 100644 index 4eba28a61..000000000 --- a/cli/cmd/ao/eval_fold_surface_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// practices: [ai-assisted-dev, llm-eval-harness] -package main - -import "testing" - -// TestEvalFold_SubcommandsRegistered asserts the eval-family top-level commands -// folded under `ao eval` (age-focus-membrane-bookkeeper-m1wg.16) resolve at -// their canonical `ao eval …` paths, that the `scenario-ab` ADR-0004 revival -// path is preserved as a distinct sibling, and that `session-outcome` does not -// collide with the pre-existing `outcomes` rubric command. -func TestEvalFold_SubcommandsRegistered(t *testing.T) { - cases := []struct { - path []string - leaf string - }{ - {[]string{"eval", "bench"}, "bench"}, - {[]string{"eval", "chaos"}, "chaos"}, - {[]string{"eval", "session-outcome"}, "session-outcome"}, - {[]string{"eval", "scenario"}, "scenario"}, - // Preserved siblings — must not be shadowed by the fold. - {[]string{"eval", "scenario-ab"}, "scenario-ab"}, - {[]string{"eval", "outcomes"}, "outcomes"}, - // Reparented children of scenario auto-follow. - {[]string{"eval", "scenario", "add"}, "add"}, - {[]string{"eval", "scenario", "validate"}, "validate"}, - } - for _, tc := range cases { - cmd, rest, err := rootCmd.Find(tc.path) - if err != nil { - t.Errorf("%v: not registered: %v", tc.path, err) - continue - } - if len(rest) != 0 { - t.Errorf("%v: unresolved trailing args %v (path did not fully resolve)", tc.path, rest) - } - if cmd.Name() != tc.leaf { - t.Errorf("%v: found %q, want %q", tc.path, cmd.Name(), tc.leaf) - } - } -} - -// TestEvalFold_BenchAndSessionOutcomeFlagsBound verifies the eval aliases carry -// the same flag set as their back-compat top-level twins (they bind the same -// package-global vars via the shared bind helpers). -func TestEvalFold_BenchAndSessionOutcomeFlagsBound(t *testing.T) { - bench, _, err := rootCmd.Find([]string{"eval", "bench"}) - if err != nil { - t.Fatalf("eval bench not registered: %v", err) - } - for _, f := range []string{"live", "corpus", "json", "k"} { - if bench.Flags().Lookup(f) == nil { - t.Errorf("eval bench missing --%s flag", f) - } - } - - so, _, err := rootCmd.Find([]string{"eval", "session-outcome"}) - if err != nil { - t.Fatalf("eval session-outcome not registered: %v", err) - } - for _, f := range []string{"session", "output"} { - if so.Flags().Lookup(f) == nil { - t.Errorf("eval session-outcome missing --%s flag", f) - } - } -} - -// TestEvalFold_OldSpellingsStillResolveHidden asserts the pre-fold top-level -// spellings remain reachable (hidden) for back-compat, while `scenario` is fully -// reparented off the root. -func TestEvalFold_OldSpellingsStillResolveHidden(t *testing.T) { - for _, name := range []string{"retrieval-bench", "chaos-test", "session-outcome"} { - cmd, rest, err := rootCmd.Find([]string{name}) - if err != nil || len(rest) != 0 || cmd.Name() != name { - t.Errorf("back-compat top-level %q should still resolve (found %q, rest %v, err %v)", name, cmd.Name(), rest, err) - continue - } - if !cmd.Hidden { - t.Errorf("back-compat top-level %q should be Hidden", name) - } - } - - // scenario is reparented under eval — it must NOT remain a top-level command. - for _, c := range rootCmd.Commands() { - if c.Name() == "scenario" { - t.Errorf("scenario should be reparented under eval, still found at top level") - } - } -} diff --git a/cli/cmd/ao/eval_outcomes.go b/cli/cmd/ao/eval_outcomes.go deleted file mode 100644 index 303b85249..000000000 --- a/cli/cmd/ao/eval_outcomes.go +++ /dev/null @@ -1,84 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - - "github.com/boshu2/agentops/cli/internal/evalsubstrate" - "github.com/spf13/cobra" -) - -// compileOutcomesRubric projects a locked eval Task + its grading criteria into a -// holdout-safe Outcomes rubric, then runs two deny-by-default re-scans before -// returning it: layer 3 checks the payload for the run's holdout answer *values* -// (ContainsAny), and layer 4 checks it for any holdout-leak *key name* -// (HasForbiddenKey). It REFUSES to return a rubric that would carry either across -// the cloud boundary — Managed Agents are not ZDR, so a leak is permanent. -// ProjectRubric already strips ground truth by construction; these re-scans are -// the backstops that keep that true under future drift. -func compileOutcomesRubric(task evalsubstrate.Task, criteria []evalsubstrate.Criterion, judgeContentHash string, holdoutValues []string) (evalsubstrate.Rubric, error) { - r := evalsubstrate.ProjectRubric(task, criteria, judgeContentHash) - if hit, found := r.ContainsAny(holdoutValues); found { - return evalsubstrate.Rubric{}, fmt.Errorf("outcomes compile: holdout value %q would leak into the rubric payload; refusing (Managed Agents are not ZDR)", hit) - } - // Layer 4 (key-name backstop, ag-uux93): re-parse the emitted payload and - // refuse any object key matching a holdout-leak pattern. The allowlisted - // Rubric struct never carries one today; this pins that invariant so a - // future field that smuggles ground truth across the (non-ZDR) cloud - // boundary fails closed instead of leaking silently. - if key, found := r.HasForbiddenKey(); found { - return evalsubstrate.Rubric{}, fmt.Errorf("outcomes compile: emitted payload carries holdout-leak key %q; refusing (Managed Agents are not ZDR)", key) - } - return r, nil -} - -// outcomesCompileInput is the JSON shape accepted by `ao eval outcomes compile`. -// holdout_values feeds the re-scan guard and is NEVER copied into the output. -type outcomesCompileInput struct { - Task evalsubstrate.Task `json:"task"` - Criteria []evalsubstrate.Criterion `json:"criteria"` - JudgeContentHash string `json:"judge_content_hash"` - HoldoutValues []string `json:"holdout_values,omitempty"` -} - -var evalOutcomesCmd = &cobra.Command{ - Use: "outcomes", - Short: "Project the locked eval substrate into Outcomes grading payloads (holdout-safe)", - Long: "Outcomes is a derived projection of the locked eval substrate (SCHEMA.md), " + - "never an alternate authority. Subcommands compile holdout-safe rubric payloads " + - "and ingest returned scores into the one verdict format.", -} - -var evalOutcomesCompileCmd = &cobra.Command{ - Use: "compile ", - Short: "Compile a holdout-safe Outcomes rubric payload from a locked Task + criteria", - Args: cobra.ExactArgs(1), - RunE: runEvalOutcomesCompile, -} - -func runEvalOutcomesCompile(cmd *cobra.Command, args []string) error { - raw, err := os.ReadFile(args[0]) - if err != nil { - return fmt.Errorf("read %s: %w", args[0], err) - } - var in outcomesCompileInput - if err := json.Unmarshal(raw, &in); err != nil { - return fmt.Errorf("parse %s: %w", args[0], err) - } - rubric, err := compileOutcomesRubric(in.Task, in.Criteria, in.JudgeContentHash, in.HoldoutValues) - if err != nil { - return err - } - out, err := json.MarshalIndent(rubric, "", " ") - if err != nil { - return fmt.Errorf("encode rubric: %w", err) - } - fmt.Fprintln(cmd.OutOrStdout(), string(out)) - return nil -} - -func init() { - evalOutcomesCmd.AddCommand(evalOutcomesCompileCmd) - evalCmd.AddCommand(evalOutcomesCmd) -} diff --git a/cli/cmd/ao/eval_outcomes_burn_test.go b/cli/cmd/ao/eval_outcomes_burn_test.go deleted file mode 100644 index 96a96fbdf..000000000 --- a/cli/cmd/ao/eval_outcomes_burn_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package main - -import ( - "testing" - - "github.com/boshu2/agentops/cli/internal/evalsubstrate" -) - -// TestEvalOutcomesIngest_HoldoutRegistersBurn proves the burn write-side -// (ag-hdqu0.5): ingesting a grade produced against the holdout split registers -// exactly one burn in the global ledger, so a cloud/Codex Outcomes run cannot -// silently bypass holdout quota the way a local holdout run cannot. A dev-split -// grade registers NO burn. The registered burn is what gate #3 (the read-side, -// #607) later refuses against once the budget is spent. -func TestEvalOutcomesIngest_HoldoutRegistersBurn(t *testing.T) { - led := evalsubstrate.HoldoutBurnLedger{Budget: 5} - - holdout := outcomesScore{ - SourceTaskID: "task-x", - Split: "holdout", - SuiteRef: "suite-x", - GroundTruthVersion: "gt-1", - RunID: "run-cloud-1", - Aggregate: 0.9, - Threshold: 0.8, - } - after, err := registerOutcomesBurn(led, holdout) - if err != nil { - t.Fatalf("first holdout burn under budget must not refuse: %v", err) - } - if got := after.Spent("suite-x", "gt-1"); got != 1 { - t.Fatalf("holdout grade must register exactly +1 burn, Spent=%d want 1", got) - } - rec := after.Records[len(after.Records)-1] - if rec.SuiteRef != "suite-x" || rec.GTVersion != "gt-1" || rec.RunID != "run-cloud-1" { - t.Errorf("burn record identity wrong: %+v", rec) - } - - // A second distinct holdout run burns again (+1 each, no silent bypass) — - // still under the budget of 5, so it is allowed. - holdout2 := holdout - holdout2.RunID = "run-cloud-2" - after2, err := registerOutcomesBurn(after, holdout2) - if err != nil { - t.Fatalf("second holdout burn under budget must not refuse: %v", err) - } - if got := after2.Spent("suite-x", "gt-1"); got != 2 { - t.Errorf("each holdout ingest burns once: Spent=%d want 2", got) - } -} - -// TestEvalOutcomesIngest_DevSplitNoBurn: a dev-split grade registers no burn — -// the dev split is reusable, so ingesting against it must never consume holdout -// quota. -func TestEvalOutcomesIngest_DevSplitNoBurn(t *testing.T) { - led := evalsubstrate.HoldoutBurnLedger{Budget: 5} - dev := outcomesScore{ - SourceTaskID: "task-x", - Split: "dev", - SuiteRef: "suite-x", - GroundTruthVersion: "gt-1", - RunID: "run-dev-1", - } - after, err := registerOutcomesBurn(led, dev) - if err != nil { - t.Fatalf("dev-split ingest must never refuse: %v", err) - } - if got := after.Spent("suite-x", "gt-1"); got != 0 { - t.Errorf("dev-split grade must register NO burn, Spent=%d want 0", got) - } - if len(after.Records) != 0 { - t.Errorf("dev-split ingest must not append records, got %d", len(after.Records)) - } -} - -// TestRegisterOutcomesBurn_EmptySplitNoBurn: an unset split is treated as -// non-holdout (no burn) — fail safe toward not over-consuming quota, since gate -// #3 only fires on an explicit holdout split anyway. -func TestRegisterOutcomesBurn_EmptySplitNoBurn(t *testing.T) { - led := evalsubstrate.HoldoutBurnLedger{Budget: 5} - after, err := registerOutcomesBurn(led, outcomesScore{SourceTaskID: "t", SuiteRef: "s", GroundTruthVersion: "g"}) - if err != nil { - t.Fatalf("empty-split ingest must never refuse: %v", err) - } - if len(after.Records) != 0 { - t.Errorf("empty split must register no burn, got %d records", len(after.Records)) - } -} - -// TestRegisterOutcomesBurn_RefusesWhenQuotaExhausted is the gate #3 write-side -// guard (ag-62g68): once the (suite_ref, gt_version) holdout quota is spent, a -// further holdout ingest is REFUSED — symmetric with the read-side gate #3 -// (gates.go), which refuses a local run on the same condition. Without this, a -// cloud/Codex Outcomes run could silently re-burn an exhausted holdout split, -// the exact statistical leak the burn ledger exists to prevent (Managed Agents -// are not ZDR). The ledger is left unmutated on refusal (no partial append). -func TestRegisterOutcomesBurn_RefusesWhenQuotaExhausted(t *testing.T) { - // Budget 1, one burn already spent for (suite-x, gt-1). - led := evalsubstrate.HoldoutBurnLedger{ - Budget: 1, - Records: []evalsubstrate.BurnRecord{{SuiteRef: "suite-x", GTVersion: "gt-1", RunID: "run-prior"}}, - } - second := outcomesScore{ - SourceTaskID: "task-x", - Split: "holdout", - SuiteRef: "suite-x", - GroundTruthVersion: "gt-1", - RunID: "run-cloud-2", - Aggregate: 0.9, - Threshold: 0.8, - } - after, err := registerOutcomesBurn(led, second) - if err == nil { - t.Fatal("gate #3 write-side must refuse a holdout burn once the quota is exhausted") - } - if got := after.Spent("suite-x", "gt-1"); got != 1 { - t.Errorf("refused burn must not mutate the ledger, Spent=%d want 1", got) - } -} - -// TestRegisterOutcomesBurn_NoCeilingWhenBudgetUnset: a non-positive Budget means -// no enforceable ceiling is configured (Day-4 input absent → gate #3 is a -// no-op), so holdout burns are recorded without refusal — exactly the read-side -// gate #3 posture. -func TestRegisterOutcomesBurn_NoCeilingWhenBudgetUnset(t *testing.T) { - led := evalsubstrate.HoldoutBurnLedger{ - Records: []evalsubstrate.BurnRecord{{SuiteRef: "suite-x", GTVersion: "gt-1", RunID: "r0"}}, - } - s := outcomesScore{Split: "holdout", SuiteRef: "suite-x", GroundTruthVersion: "gt-1", RunID: "r1"} - after, err := registerOutcomesBurn(led, s) - if err != nil { - t.Fatalf("unset budget must not enforce a ceiling: %v", err) - } - if got := after.Spent("suite-x", "gt-1"); got != 2 { - t.Errorf("unset-budget holdout burn must append, Spent=%d want 2", got) - } -} diff --git a/cli/cmd/ao/eval_outcomes_burn_wire_test.go b/cli/cmd/ao/eval_outcomes_burn_wire_test.go deleted file mode 100644 index 636ad8ee9..000000000 --- a/cli/cmd/ao/eval_outcomes_burn_wire_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package main - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/boshu2/agentops/cli/internal/evalsubstrate" - "github.com/spf13/cobra" -) - -// writeBurnLedgerFile seeds a HoldoutBurnLedger JSON file. -func writeBurnLedgerFile(t *testing.T, path string, led evalsubstrate.HoldoutBurnLedger) { - t.Helper() - blob, err := json.Marshal(led) - if err != nil { - t.Fatalf("marshal ledger: %v", err) - } - if err := os.WriteFile(path, blob, 0o644); err != nil { - t.Fatalf("write ledger %s: %v", path, err) - } -} - -func readBurnLedgerFile(t *testing.T, path string) evalsubstrate.HoldoutBurnLedger { - t.Helper() - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read ledger %s: %v", path, err) - } - var led evalsubstrate.HoldoutBurnLedger - if err := json.Unmarshal(raw, &led); err != nil { - t.Fatalf("parse ledger %s: %v", path, err) - } - return led -} - -func writeHoldoutScore(t *testing.T, path, suite, gt, runID string) { - t.Helper() - s := outcomesScore{ - SourceTaskID: "task-x", - JudgeContentHash: "sha256:abc", - Aggregate: 0.9, - Threshold: 0.8, - Split: "holdout", - SuiteRef: suite, - GroundTruthVersion: gt, - RunID: runID, - } - blob, err := json.Marshal(s) - if err != nil { - t.Fatalf("marshal score: %v", err) - } - if err := os.WriteFile(path, blob, 0o644); err != nil { - t.Fatalf("write score %s: %v", path, err) - } -} - -// ingestCmdForTest returns a cobra command whose stdout is discarded and -// restores the process-global ingest flag vars after the test. -func ingestCmdForTest(t *testing.T, burnLedger string) *cobra.Command { - t.Helper() - origLedger, origHash := evalOutcomesIngestBurnLedger, evalOutcomesIngestExpectHash - t.Cleanup(func() { - evalOutcomesIngestBurnLedger = origLedger - evalOutcomesIngestExpectHash = origHash - }) - evalOutcomesIngestBurnLedger = burnLedger - evalOutcomesIngestExpectHash = "" - cmd := &cobra.Command{} - cmd.SetOut(&cobraDiscard{}) - return cmd -} - -// TestApplyOutcomesBurn_RefusesAtCommandLevel_WhenQuotaExhausted is the gate #3 -// RUNTIME proof (ag-vbwx): with a --burn-ledger whose (suite,gt) quota is already -// spent, `ao eval outcomes ingest` of a holdout score REFUSES at the command -// level (not merely in the unit-tested primitive), and leaves the ledger -// unmutated. This closes the council 2026-05-30 caveat that registerOutcomesBurn -// was wired into no command path. -func TestApplyOutcomesBurn_RefusesAtCommandLevel_WhenQuotaExhausted(t *testing.T) { - dir := t.TempDir() - ledgerPath := filepath.Join(dir, "ledger.json") - writeBurnLedgerFile(t, ledgerPath, evalsubstrate.HoldoutBurnLedger{ - Budget: 1, - Records: []evalsubstrate.BurnRecord{{SuiteRef: "suite-x", GTVersion: "gt-1", RunID: "r0"}}, - }) - scorePath := filepath.Join(dir, "score.json") - writeHoldoutScore(t, scorePath, "suite-x", "gt-1", "run-cloud-2") - - cmd := ingestCmdForTest(t, ledgerPath) - if err := runEvalOutcomesIngest(cmd, []string{scorePath}); err == nil { - t.Fatal("ingest must refuse at the command level when the holdout quota is exhausted (gate #3 runtime)") - } - after := readBurnLedgerFile(t, ledgerPath) - if got := after.Spent("suite-x", "gt-1"); got != 1 { - t.Errorf("refused ingest must not mutate the ledger, Spent=%d want 1", got) - } -} - -// TestApplyOutcomesBurn_PersistsAcrossInvocations proves the burn is durable: two -// distinct holdout ingests under a budget of 2 both succeed and each appends one -// record to the on-disk ledger; the third (quota now exhausted) refuses. This is -// the cross-invocation persistence the council required. -func TestApplyOutcomesBurn_PersistsAcrossInvocations(t *testing.T) { - dir := t.TempDir() - ledgerPath := filepath.Join(dir, "ledger.json") - writeBurnLedgerFile(t, ledgerPath, evalsubstrate.HoldoutBurnLedger{Budget: 2}) - scorePath := filepath.Join(dir, "score.json") - - cmd := ingestCmdForTest(t, ledgerPath) - - writeHoldoutScore(t, scorePath, "suite-x", "gt-1", "run-1") - if err := runEvalOutcomesIngest(cmd, []string{scorePath}); err != nil { - t.Fatalf("first holdout ingest under budget must succeed: %v", err) - } - if got := readBurnLedgerFile(t, ledgerPath).Spent("suite-x", "gt-1"); got != 1 { - t.Fatalf("first ingest must persist +1 burn, Spent=%d want 1", got) - } - - writeHoldoutScore(t, scorePath, "suite-x", "gt-1", "run-2") - if err := runEvalOutcomesIngest(cmd, []string{scorePath}); err != nil { - t.Fatalf("second holdout ingest under budget must succeed: %v", err) - } - if got := readBurnLedgerFile(t, ledgerPath).Spent("suite-x", "gt-1"); got != 2 { - t.Fatalf("second ingest must persist a 2nd burn, Spent=%d want 2", got) - } - - writeHoldoutScore(t, scorePath, "suite-x", "gt-1", "run-3") - if err := runEvalOutcomesIngest(cmd, []string{scorePath}); err == nil { - t.Fatal("third holdout ingest must refuse — budget 2 is exhausted") - } - if got := readBurnLedgerFile(t, ledgerPath).Spent("suite-x", "gt-1"); got != 2 { - t.Errorf("refused third ingest must not append, Spent=%d want 2", got) - } -} - -// TestApplyOutcomesBurn_EmptyPathNoOp: with no --burn-ledger configured, holdout -// ingest is unaffected (enforcement off; dev/legacy flows preserved). -func TestApplyOutcomesBurn_EmptyPathNoOp(t *testing.T) { - dir := t.TempDir() - scorePath := filepath.Join(dir, "score.json") - writeHoldoutScore(t, scorePath, "suite-x", "gt-1", "run-1") - - cmd := ingestCmdForTest(t, "") // empty burn-ledger path - if err := runEvalOutcomesIngest(cmd, []string{scorePath}); err != nil { - t.Fatalf("no --burn-ledger configured must not enforce gate #3: %v", err) - } -} - -// TestApplyOutcomesBurn_DevSplitNoBurn: a dev-split score never burns, even when -// the configured ledger is already at quota — only holdout grades consume quota. -func TestApplyOutcomesBurn_DevSplitNoBurn(t *testing.T) { - dir := t.TempDir() - ledgerPath := filepath.Join(dir, "ledger.json") - writeBurnLedgerFile(t, ledgerPath, evalsubstrate.HoldoutBurnLedger{ - Budget: 1, - Records: []evalsubstrate.BurnRecord{{SuiteRef: "suite-x", GTVersion: "gt-1", RunID: "r0"}}, - }) - scorePath := filepath.Join(dir, "score.json") - dev := outcomesScore{SourceTaskID: "t", Split: "dev", SuiteRef: "suite-x", GroundTruthVersion: "gt-1", RunID: "d1", Aggregate: 0.9, Threshold: 0.8} - blob, _ := json.Marshal(dev) - if err := os.WriteFile(scorePath, blob, 0o644); err != nil { - t.Fatalf("write dev score: %v", err) - } - - cmd := ingestCmdForTest(t, ledgerPath) - if err := runEvalOutcomesIngest(cmd, []string{scorePath}); err != nil { - t.Fatalf("dev-split ingest must never burn or refuse: %v", err) - } - if got := readBurnLedgerFile(t, ledgerPath).Spent("suite-x", "gt-1"); got != 1 { - t.Errorf("dev-split ingest must not append a burn, Spent=%d want 1", got) - } -} diff --git a/cli/cmd/ao/eval_outcomes_codex.go b/cli/cmd/ao/eval_outcomes_codex.go deleted file mode 100644 index eeedd838b..000000000 --- a/cli/cmd/ao/eval_outcomes_codex.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import "github.com/boshu2/agentops/cli/internal/evalsubstrate" - -// gradeLocally is the Codex/NTM offline grading path (ag-hdqu0.7). Codex has no -// Managed Agents loop and no Workflow tool, so it calls the SAME -// `ao eval outcomes compile` to get a holdout-safe Rubric, grades the candidate -// locally — Inspect AI over the dev split, or the bushido llama.cpp Qwen3.6 grader -// over tailnet — and writes through the SAME ingestOutcomesScore. This function is -// the output adapter for that local grade: it folds per-criterion scores into the -// weight-normalized outcomesScore that ingest consumes, so the Codex path yields a -// verdict byte-identical in shape to the cloud path with ZERO outbound cloud calls. -// -// perCriterion maps Criterion.ID -> [0,1] score; criteria the grader did not score -// count as 0. The aggregate is the weight-normalized mean over the rubric's own -// criteria (so a rubric and its grade can never disagree on which criteria exist). -func gradeLocally(r evalsubstrate.Rubric, perCriterion map[string]float64, threshold float64) outcomesScore { - scores := make(map[string]float64, len(r.Criteria)) - var weighted, weightSum float64 - for _, c := range r.Criteria { - s := perCriterion[c.ID] - scores[c.ID] = s - weighted += s * c.Weight - weightSum += c.Weight - } - aggregate := 0.0 - if weightSum > 0 { - aggregate = weighted / weightSum - } - return outcomesScore{ - SourceTaskID: r.SourceTaskID, - JudgeContentHash: r.JudgeContentHash, - Aggregate: aggregate, - Threshold: threshold, - CriterionScores: scores, - } -} diff --git a/cli/cmd/ao/eval_outcomes_codex_test.go b/cli/cmd/ao/eval_outcomes_codex_test.go deleted file mode 100644 index 25c17ea03..000000000 --- a/cli/cmd/ao/eval_outcomes_codex_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package main - -import ( - "testing" - - "github.com/boshu2/agentops/cli/internal/evalsubstrate" -) - -// TestEvalOutcomesCodexPath_LocalGradeProducesVerdict proves the Codex/NTM -// offline grading path (ag-hdqu0.7): the SAME compile (ProjectRubric) and ingest -// (ingestOutcomesScore) serve a locally-graded run with ZERO cloud dependency, -// producing a verdict identical in shape to the cloud Outcomes path. The local -// grader (gradeLocally) stands in for the Inspect AI dev-split / bushido Qwen3.6 -// graders — all offline, all funnelling through the one verdict format. -func TestEvalOutcomesCodexPath_LocalGradeProducesVerdict(t *testing.T) { - rubric := evalsubstrate.ProjectRubric( - evalsubstrate.Task{ID: "task-capital-cities", Description: "answer accurately"}, - []evalsubstrate.Criterion{ - {ID: "accuracy", Description: "names the correct capital", Weight: 0.7}, - {ID: "concision", Description: "one short sentence", Weight: 0.3}, - }, - "sha256:judge1", - ) - - // Local grader output (Inspect AI dev-split / Qwen3.6 stand-in) — offline. - score := gradeLocally(rubric, map[string]float64{"accuracy": 1.0, "concision": 1.0}, 0.8) - if score.SourceTaskID != rubric.SourceTaskID { - t.Errorf("score.SourceTaskID = %q, want %q", score.SourceTaskID, rubric.SourceTaskID) - } - if score.JudgeContentHash != rubric.JudgeContentHash { - t.Errorf("local grade must carry the rubric judge hash, got %q", score.JudgeContentHash) - } - if score.Aggregate != 1.0 { - t.Errorf("weighted aggregate of all-1.0 criteria = %v, want 1.0", score.Aggregate) - } - - // Same ingest path as the cloud Outcomes run → one verdict format. - v := ingestOutcomesScore(score) - if v.Verdict != "PASS" { - t.Errorf("Verdict = %q, want PASS", v.Verdict) - } - if v.SatisfactionScore == nil || *v.SatisfactionScore != 1.0 { - t.Errorf("SatisfactionScore = %v, want 1.0", v.SatisfactionScore) - } - if v.SatisfactionBreakdown["accuracy"] != 1.0 { - t.Errorf("breakdown[accuracy] = %v, want 1.0", v.SatisfactionBreakdown["accuracy"]) - } -} - -// TestGradeLocally_WeightedAggregate: the local grader computes a weight-normalized -// aggregate over exactly the rubric's criteria (unknown criteria score 0). -func TestGradeLocally_WeightedAggregate(t *testing.T) { - rubric := evalsubstrate.ProjectRubric( - evalsubstrate.Task{ID: "t"}, - []evalsubstrate.Criterion{{ID: "a", Weight: 3}, {ID: "b", Weight: 1}}, - "sha256:j", - ) - score := gradeLocally(rubric, map[string]float64{"a": 1.0, "b": 0.0}, 0.5) - // (1.0*3 + 0.0*1) / (3+1) = 0.75 - if score.Aggregate != 0.75 { - t.Errorf("weighted aggregate = %v, want 0.75", score.Aggregate) - } - if ingestOutcomesScore(score).Verdict != "PASS" { // 0.75 >= 0.5 - t.Error("0.75 vs threshold 0.5 should be PASS") - } -} diff --git a/cli/cmd/ao/eval_outcomes_gate2_warn_test.go b/cli/cmd/ao/eval_outcomes_gate2_warn_test.go deleted file mode 100644 index 15fd49d9f..000000000 --- a/cli/cmd/ao/eval_outcomes_gate2_warn_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package main - -import ( - "bytes" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/spf13/cobra" -) - -// TestUncheckedJudgeHashWarning is the gate #2 advisory unit (council -// 2026-05-30 caveat #2): a score that carries a judge_content_hash but no -// configured --expect-judge-hash warns; a configured expected hash (parity -// enforced by requireJudgeHashParity) or a hashless score does not. -func TestUncheckedJudgeHashWarning(t *testing.T) { - cases := []struct { - name string - scoreHash string - expected string - wantWarn bool - }{ - {"hash present, no expect → warn", "sha256:abc", "", true}, - {"hash present, expect set → no warn (parity enforced)", "sha256:abc", "sha256:abc", false}, - {"no hash, no expect → no warn", "", "", false}, - {"no hash, expect set → no warn", "", "sha256:abc", false}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - got := uncheckedJudgeHashWarning(c.scoreHash, c.expected) - if c.wantWarn && got == "" { - t.Errorf("uncheckedJudgeHashWarning(%q,%q) = \"\", want a warning", c.scoreHash, c.expected) - } - if !c.wantWarn && got != "" { - t.Errorf("uncheckedJudgeHashWarning(%q,%q) = %q, want \"\"", c.scoreHash, c.expected, got) - } - }) - } -} - -// TestRunEvalOutcomesIngest_WarnsWhenParityUnchecked exercises gate #2's advisory -// end-to-end through the command: a score carrying a judge_content_hash ingested -// WITHOUT --expect-judge-hash still succeeds (the warning is non-fatal) but emits -// a visible gate #2 warning to stderr, so the unenforced-parity gap is not -// silent. With --expect-judge-hash matching, no warning is emitted. -func TestRunEvalOutcomesIngest_WarnsWhenParityUnchecked(t *testing.T) { - dir := t.TempDir() - scorePath := filepath.Join(dir, "score.json") - if err := os.WriteFile(scorePath, []byte(`{"source_task_id":"t","judge_content_hash":"sha256:aaa","aggregate":0.9,"threshold":0.8}`), 0o644); err != nil { - t.Fatalf("write fixture: %v", err) - } - - origHash, origLedger := evalOutcomesIngestExpectHash, evalOutcomesIngestBurnLedger - t.Cleanup(func() { - evalOutcomesIngestExpectHash = origHash - evalOutcomesIngestBurnLedger = origLedger - }) - evalOutcomesIngestBurnLedger = "" - - // No --expect-judge-hash → warns on stderr, still ingests. - evalOutcomesIngestExpectHash = "" - var stderr, stdout bytes.Buffer - cmd := &cobra.Command{} - cmd.SetOut(&stdout) - cmd.SetErr(&stderr) - if err := runEvalOutcomesIngest(cmd, []string{scorePath}); err != nil { - t.Fatalf("ingest without --expect-judge-hash must still succeed (warning is non-fatal): %v", err) - } - if !strings.Contains(stderr.String(), "gate #2") || !strings.Contains(stderr.String(), "sha256:aaa") { - t.Errorf("expected a gate #2 unchecked-parity warning naming the hash on stderr, got: %q", stderr.String()) - } - if stdout.Len() == 0 { - t.Error("a verdict must still be emitted to stdout despite the warning") - } - - // With matching --expect-judge-hash → parity enforced, no warning. - evalOutcomesIngestExpectHash = "sha256:aaa" - var stderr2 bytes.Buffer - cmd2 := &cobra.Command{} - cmd2.SetOut(&cobraDiscard{}) - cmd2.SetErr(&stderr2) - if err := runEvalOutcomesIngest(cmd2, []string{scorePath}); err != nil { - t.Fatalf("matching --expect-judge-hash must ingest cleanly: %v", err) - } - if stderr2.Len() != 0 { - t.Errorf("no gate #2 warning expected when parity is configured, got: %q", stderr2.String()) - } -} diff --git a/cli/cmd/ao/eval_outcomes_hash_parity_test.go b/cli/cmd/ao/eval_outcomes_hash_parity_test.go deleted file mode 100644 index 10b5fb331..000000000 --- a/cli/cmd/ao/eval_outcomes_hash_parity_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "testing" - - "github.com/spf13/cobra" -) - -// TestRequireJudgeHashParity is the gate #2 (rubric-drift parity) unit: a score -// graded against a different judge_content_hash than the active rubric is -// refused; a matching hash passes; an empty expected hash is a no-op (parity not -// configured), so legacy/dev flows are unaffected. -func TestRequireJudgeHashParity(t *testing.T) { - cases := []struct { - name string - scoreHash string - expected string - wantErr bool - }{ - {"match", "sha256:abc", "sha256:abc", false}, - {"mismatch refuses", "sha256:abc", "sha256:def", true}, - {"empty expected is no-op", "sha256:abc", "", false}, - {"empty score vs configured expected refuses", "", "sha256:def", true}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - err := requireJudgeHashParity(c.scoreHash, c.expected) - if c.wantErr && err == nil { - t.Errorf("requireJudgeHashParity(%q,%q) = nil, want error", c.scoreHash, c.expected) - } - if !c.wantErr && err != nil { - t.Errorf("requireJudgeHashParity(%q,%q) = %v, want nil", c.scoreHash, c.expected, err) - } - }) - } -} - -// TestRunEvalOutcomesIngest_RefusesOnHashMismatch exercises gate #2 end-to-end -// through the command: with --expect-judge-hash set to a value that differs from -// the score's judge_content_hash, ingest refuses and emits NO verdict (a stale -// rubric must not feed the Knowledge Flywheel). With a matching hash, it ingests. -func TestRunEvalOutcomesIngest_RefusesOnHashMismatch(t *testing.T) { - dir := t.TempDir() - scorePath := filepath.Join(dir, "score.json") - if err := os.WriteFile(scorePath, []byte(`{"source_task_id":"t","judge_content_hash":"sha256:aaa","aggregate":0.9,"threshold":0.8}`), 0o644); err != nil { - t.Fatalf("write fixture: %v", err) - } - - // Restore the package flag var after the test (it is process-global). - orig := evalOutcomesIngestExpectHash - t.Cleanup(func() { evalOutcomesIngestExpectHash = orig }) - - cmd := &cobra.Command{} - cmd.SetOut(&cobraDiscard{}) - - // Mismatch → refuse. - evalOutcomesIngestExpectHash = "sha256:bbb" - if err := runEvalOutcomesIngest(cmd, []string{scorePath}); err == nil { - t.Fatal("ingest must refuse when judge_content_hash mismatches --expect-judge-hash (gate #2)") - } - - // Match → ingest succeeds. - evalOutcomesIngestExpectHash = "sha256:aaa" - if err := runEvalOutcomesIngest(cmd, []string{scorePath}); err != nil { - t.Fatalf("matching judge_content_hash must ingest cleanly: %v", err) - } - - // Unset → no parity check (legacy/dev path). - evalOutcomesIngestExpectHash = "" - if err := runEvalOutcomesIngest(cmd, []string{scorePath}); err != nil { - t.Fatalf("unset --expect-judge-hash must not enforce parity: %v", err) - } -} - -// cobraDiscard is an io.Writer that drops the verdict JSON the command prints. -type cobraDiscard struct{} - -func (cobraDiscard) Write(p []byte) (int, error) { return len(p), nil } diff --git a/cli/cmd/ao/eval_outcomes_ingest.go b/cli/cmd/ao/eval_outcomes_ingest.go deleted file mode 100644 index 7639fef7e..000000000 --- a/cli/cmd/ao/eval_outcomes_ingest.go +++ /dev/null @@ -1,265 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "time" - - "github.com/boshu2/agentops/cli/internal/evalsubstrate" - "github.com/spf13/cobra" -) - -// outcomesScore is the score payload returned by an Outcomes grader (cloud -// Managed Agent, async job, or the local Codex/NTM path). It carries only -// aggregate + per-criterion scores — never holdout answers. -type outcomesScore struct { - SourceTaskID string `json:"source_task_id"` - JudgeContentHash string `json:"judge_content_hash"` - Aggregate float64 `json:"aggregate"` - Threshold float64 `json:"threshold"` - CriterionScores map[string]float64 `json:"criterion_scores"` - - // Burn-ledger carriers (ag-hdqu0.5). Split is the eval split the candidate - // was graded against; only "holdout" consumes quota. SuiteRef + - // GroundTruthVersion key the burn; RunID identifies the consuming run. - // Absent on dev-split or pre-.5 payloads (→ no burn). - Split string `json:"split,omitempty"` - SuiteRef string `json:"suite_ref,omitempty"` - GroundTruthVersion string `json:"ground_truth_version,omitempty"` - RunID string `json:"run_id,omitempty"` -} - -// outcomesVerdict is the one verdict record (skills/council/schemas/verdict.json -// shape). ao eval outcomes ingest emits this so an Outcomes score feeds the -// Knowledge Flywheel exactly as a local `ao eval run` verdict does — no second -// verdict format, no alternate bar. -type outcomesVerdict struct { - Verdict string `json:"verdict"` - Confidence string `json:"confidence"` - KeyInsight string `json:"key_insight"` - Recommendation string `json:"recommendation"` - SchemaVersion int `json:"schema_version"` - SatisfactionScore *float64 `json:"satisfaction_score"` - SatisfactionBreakdown map[string]float64 `json:"satisfaction_breakdown"` - Findings []map[string]any `json:"findings"` -} - -// ingestOutcomesScore maps an Outcomes score onto the council verdict record. -// PASS when aggregate meets the rubric threshold, FAIL when it falls below 70% -// of it, WARN in between. The aggregate becomes satisfaction_score and the -// per-criterion scores become satisfaction_breakdown. -func ingestOutcomesScore(s outcomesScore) outcomesVerdict { - threshold := s.Threshold - if threshold <= 0 { - threshold = 1.0 - } - verdict := "WARN" - switch { - case s.Aggregate >= threshold: - verdict = "PASS" - case s.Aggregate < threshold*0.7: - verdict = "FAIL" - } - agg := s.Aggregate - return outcomesVerdict{ - Verdict: verdict, - Confidence: "HIGH", - KeyInsight: fmt.Sprintf("Outcomes aggregate %.4f vs threshold %.4f for task %q", - s.Aggregate, threshold, s.SourceTaskID), - Recommendation: fmt.Sprintf("Outcomes grade ingested as %s; feeds the corpus via the eval-verdict pipeline.", verdict), - SchemaVersion: 4, - SatisfactionScore: &agg, - SatisfactionBreakdown: s.CriterionScores, - Findings: []map[string]any{}, - } -} - -// registerOutcomesBurn is the write-side of the holdout burn ledger (ag-hdqu0.5), -// the counterpart to gate #3's read-side (#607). When an Outcomes grade was -// produced against the holdout split, ingest appends exactly one BurnRecord to -// the global ledger — so a cloud/async/Codex Outcomes run cannot silently bypass -// holdout quota the way a local holdout run cannot. Dev-split (and absent-split) -// grades register no burn, since the dev split is reusable. -// -// Gate #3 write-side enforcement (ag-62g68): if the (suite_ref, gt_version) -// holdout quota is already spent (Spent >= Budget, with a positive Budget), the -// burn is REFUSED and the ledger is returned unmutated — symmetric with the -// read-side gate #3 in gates.go, which refuses a local run on the same -// condition. Without this, a cloud/Codex Outcomes run could re-burn an exhausted -// holdout split, the exact statistical leak the ledger exists to prevent -// (Managed Agents are not ZDR). A non-positive Budget means no ceiling is -// configured (Day-4 input absent → no-op), so burns are recorded without refusal. -// -// Returns the updated ledger for the caller to persist; the global-Dolt store is -// a separate adapter concern, exactly as gate #3 reads an injected ledger. -func registerOutcomesBurn(led evalsubstrate.HoldoutBurnLedger, s outcomesScore) (evalsubstrate.HoldoutBurnLedger, error) { - if s.Split != "holdout" { - return led, nil - } - if led.Budget > 0 { - if spent := led.Spent(s.SuiteRef, s.GroundTruthVersion); spent >= led.Budget { - return led, fmt.Errorf("outcomes ingest: holdout burn refused — quota for (suite=%q, gt_version=%q) is exhausted (%d/%d spent); the holdout split is statistically spent and may not be re-observed (gate #3, no escape)", - s.SuiteRef, s.GroundTruthVersion, spent, led.Budget) - } - } - led.Records = append(led.Records, evalsubstrate.BurnRecord{ - SuiteRef: s.SuiteRef, - GTVersion: s.GroundTruthVersion, - RunID: s.RunID, - }) - return led, nil -} - -// requireJudgeHashParity is the gate #2 (rubric-drift parity) guard for ingest. -// An Outcomes score carries the judge_content_hash of the rubric it was graded -// against (SCHEMA rc2 drift key). If the caller supplies the active Suite's -// current judge hash and it differs from the score's, the rubric drifted between -// projection and grading — the score is stale and MUST be refused, exactly as a -// stale local judge self-invalidates. An empty expected hash means the caller -// did not configure a parity check (no-op), so legacy/dev flows are unaffected. -func requireJudgeHashParity(scoreHash, expected string) error { - if expected == "" || scoreHash == expected { - return nil - } - return fmt.Errorf("outcomes ingest: judge_content_hash mismatch — score was graded against %q but the active rubric is %q; the rubric drifted and this score is stale (gate #2 parity, refused)", - scoreHash, expected) -} - -// uncheckedJudgeHashWarning returns a gate #2 advisory (council 2026-05-30 -// caveat #2) for the case where a score carries a judge_content_hash but no -// --expect-judge-hash was configured: parity is then silently unenforced, so a -// stale drifted-rubric score would ingest with no signal. The warning makes the -// gap VISIBLE without requiring the operator to remember the flag. It returns "" -// when parity is configured (requireJudgeHashParity handles the refuse) or the -// score carries no hash to check. -func uncheckedJudgeHashWarning(scoreHash, expected string) string { - if expected != "" || scoreHash == "" { - return "" - } - return fmt.Sprintf("WARN outcomes ingest: score carries judge_content_hash %q but --expect-judge-hash was not provided; rubric-drift parity (gate #2) was NOT checked — pass --expect-judge-hash to enforce it", - scoreHash) -} - -// loadBurnLedger reads a HoldoutBurnLedger JSON file. A missing file is an empty -// ledger (Budget 0 → no enforceable ceiling, gate #3 no-op), so a first ingest -// against a fresh path does not error — the operator seeds Budget by writing the -// ledger file once. -func loadBurnLedger(path string) (evalsubstrate.HoldoutBurnLedger, error) { - var led evalsubstrate.HoldoutBurnLedger - raw, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - return led, nil - } - if err != nil { - return led, fmt.Errorf("read burn ledger %s: %w", path, err) - } - if err := json.Unmarshal(raw, &led); err != nil { - return led, fmt.Errorf("parse burn ledger %s: %w", path, err) - } - return led, nil -} - -// saveBurnLedger atomically persists the ledger (temp file + rename) so a crash -// mid-write cannot corrupt the global holdout-burn accounting. -func saveBurnLedger(path string, led evalsubstrate.HoldoutBurnLedger) error { - blob, err := json.MarshalIndent(led, "", " ") - if err != nil { - return fmt.Errorf("encode burn ledger: %w", err) - } - tmp := path + ".tmp" - if err := os.WriteFile(tmp, blob, 0o644); err != nil { - return fmt.Errorf("write burn ledger temp: %w", err) - } - if err := os.Rename(tmp, path); err != nil { - return fmt.Errorf("commit burn ledger: %w", err) - } - return nil -} - -// applyOutcomesBurn is the gate #3 RUNTIME enforcement (ag-vbwx) — the wiring -// that makes registerOutcomesBurn a live guard rather than an unexercised -// primitive (council post-mortem 2026-05-30 WARN). When a --burn-ledger path is -// configured and the score is a holdout grade, it loads the persisted ledger, -// registers the burn (which REFUSES on exhausted (suite,gt) quota), and persists -// the updated ledger across invocations so a cloud/Codex Outcomes run cannot -// re-observe a spent holdout split. An empty path is a no-op (enforcement not -// configured), preserving dev/legacy flows; dev-split scores register no burn, -// so the ledger is only rewritten when an actual burn was appended. -func applyOutcomesBurn(path string, s outcomesScore) error { - if path == "" || s.Split != "holdout" { - return nil - } - led, err := loadBurnLedger(path) - if err != nil { - return err - } - updated, err := registerOutcomesBurn(led, s) - if err != nil { - return err // gate #3 refuse — surfaced to the operator, no verdict emitted - } - return saveBurnLedger(path, updated) -} - -var ( - evalOutcomesIngestExpectHash string - evalOutcomesIngestBurnLedger string - evalOutcomesIngestManifestOut string - evalOutcomesIngestRunID string -) - -var evalOutcomesIngestCmd = &cobra.Command{ - Use: "ingest ", - Short: "Ingest an Outcomes score payload into the one council verdict record", - Args: cobra.ExactArgs(1), - RunE: runEvalOutcomesIngest, -} - -func runEvalOutcomesIngest(cmd *cobra.Command, args []string) error { - raw, err := os.ReadFile(args[0]) - if err != nil { - return fmt.Errorf("read %s: %w", args[0], err) - } - var s outcomesScore - if err := json.Unmarshal(raw, &s); err != nil { - return fmt.Errorf("parse %s: %w", args[0], err) - } - if w := uncheckedJudgeHashWarning(s.JudgeContentHash, evalOutcomesIngestExpectHash); w != "" { - fmt.Fprintln(cmd.ErrOrStderr(), w) - } - if err := requireJudgeHashParity(s.JudgeContentHash, evalOutcomesIngestExpectHash); err != nil { - return err - } - if err := applyOutcomesBurn(evalOutcomesIngestBurnLedger, s); err != nil { - return err - } - v := ingestOutcomesScore(s) - out, err := json.MarshalIndent(v, "", " ") - if err != nil { - return fmt.Errorf("encode verdict: %w", err) - } - fmt.Fprintln(cmd.OutOrStdout(), string(out)) - if evalOutcomesIngestManifestOut != "" { - runID := resolveOutcomesRunID(evalOutcomesIngestRunID, s) - rec := buildOutcomesRunManifest(s, v.Verdict, runID, time.Now().UTC()) - path, err := writeOutcomesRunManifest(evalOutcomesIngestManifestOut, runID, rec) - if err != nil { - return err - } - fmt.Fprintf(cmd.ErrOrStderr(), "wrote eval-run manifest: %s\n", path) - } - return nil -} - -func init() { - evalOutcomesIngestCmd.Flags().StringVar(&evalOutcomesIngestExpectHash, "expect-judge-hash", "", - "refuse the ingest if the score's judge_content_hash does not match this value (gate #2 rubric-drift parity)") - evalOutcomesIngestCmd.Flags().StringVar(&evalOutcomesIngestBurnLedger, "burn-ledger", "", - "path to a JSON HoldoutBurnLedger; when set, a holdout-split score registers a burn and is REFUSED if the (suite,gt) quota is exhausted (gate #3 runtime enforcement), persisted across invocations") - evalOutcomesIngestCmd.Flags().StringVar(&evalOutcomesIngestManifestOut, "manifest-out", "", - "also write an eval-run.v1 manifest to //manifest.json so the verdict pipeline feeds the Knowledge Flywheel (closes the Outcomes→Flywheel loop)") - evalOutcomesIngestCmd.Flags().StringVar(&evalOutcomesIngestRunID, "run-id", "", - "run id for the --manifest-out manifest; defaults to the score's run_id, then source_task_id (sanitized to the eval-run.v1 pattern)") - evalOutcomesCmd.AddCommand(evalOutcomesIngestCmd) -} diff --git a/cli/cmd/ao/eval_outcomes_ingest_test.go b/cli/cmd/ao/eval_outcomes_ingest_test.go deleted file mode 100644 index 9e61fcb63..000000000 --- a/cli/cmd/ao/eval_outcomes_ingest_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import "testing" - -// TestIngestOutcomesScore_ProducesVerdictRecord: an Outcomes score at/above the -// rubric threshold maps to a PASS verdict record in the one council verdict shape -// (skills/council/schemas/verdict.json) — closing the Outcomes → Knowledge -// Flywheel loop without forking the verdict format. -func TestIngestOutcomesScore_ProducesVerdictRecord(t *testing.T) { - s := outcomesScore{ - SourceTaskID: "task-capital-cities", - JudgeContentHash: "sha256:abc123", - Aggregate: 0.92, - Threshold: 0.8, - CriterionScores: map[string]float64{"accuracy": 0.95, "concision": 0.85}, - } - v := ingestOutcomesScore(s) - - if v.Verdict != "PASS" { - t.Errorf("Verdict = %q, want PASS", v.Verdict) - } - if v.SatisfactionScore == nil || *v.SatisfactionScore != 0.92 { - t.Errorf("SatisfactionScore = %v, want 0.92", v.SatisfactionScore) - } - if v.SatisfactionBreakdown["accuracy"] != 0.95 { - t.Errorf("breakdown[accuracy] = %v, want 0.95", v.SatisfactionBreakdown["accuracy"]) - } - if v.SchemaVersion != 4 { - t.Errorf("SchemaVersion = %d, want 4", v.SchemaVersion) - } - if v.Findings == nil { - t.Error("Findings must be a non-nil (possibly empty) slice for verdict.json validity") - } -} - -// TestIngestOutcomesScore_VerdictBands: aggregate below threshold downgrades to -// WARN, and far below (< 70% of threshold) to FAIL. -func TestIngestOutcomesScore_VerdictBands(t *testing.T) { - cases := []struct { - agg float64 - want string - }{ - {0.90, "PASS"}, // >= 0.8 - {0.70, "WARN"}, // < 0.8 but >= 0.56 - {0.40, "FAIL"}, // < 0.56 - } - for _, c := range cases { - got := ingestOutcomesScore(outcomesScore{Aggregate: c.agg, Threshold: 0.8}).Verdict - if got != c.want { - t.Errorf("aggregate %.2f: Verdict = %q, want %q", c.agg, got, c.want) - } - } -} diff --git a/cli/cmd/ao/eval_outcomes_manifest.go b/cli/cmd/ao/eval_outcomes_manifest.go deleted file mode 100644 index 783526c08..000000000 --- a/cli/cmd/ao/eval_outcomes_manifest.go +++ /dev/null @@ -1,227 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - "time" - - "github.com/boshu2/agentops/cli/internal/eval" -) - -// This file (ag-fy6e) gives `ao eval outcomes ingest` a second, optional output: -// an eval-run.v1 RUN MANIFEST alongside the council verdict record. With -// --manifest-out set, ingest ALSO writes //manifest.json -// conforming to schemas/eval-run.v1.schema.json, so the existing verdict -// pipeline (docs/contracts/eval-verdict-pipeline.md) picks it up and feeds the -// Knowledge Flywheel — closing the Outcomes→Flywheel loop end-to-end. The writer -// lives in this NEW file (diversifies off eval_outcomes_ingest.go); ingest.go -// only wires the thin flag. -// -// SCHEMA-vs-bead reconciliation (source-of-truth precedence, CLAUDE.md): the -// bead text said "status=complete", but that value belongs to the *other* -// SCHEMA.md run-manifest (the rc3 struct-verdict form). eval-run.v1's `status` -// enum is pass/fail/error/skipped/inconclusive — it has NO "complete" member, so -// this writer follows the schema and maps the PASS/WARN/FAIL band onto the -// allowed enum (see outcomesBandToStatus). An out-of-process Outcomes grade has -// no candidate git sha / live runtime / sandboxed environment, so git/runtime/ -// environment carry honest, schema-valid stubs and a `notes` entry records the -// provenance. - -// outcomesManifestDimensions is the set of dimension keys eval-run.v1's -// dimension_scores object permits (additionalProperties:false). A criterion -// score whose key is outside this set cannot be placed in the manifest, so -// outcomesDimensionScores filters to exactly these. Note context_comprehension -// is a valid eval.Dimension but is NOT a dimension_scores property in -// eval-run.v1, so it is intentionally excluded here. -var outcomesManifestDimensions = map[eval.Dimension]bool{ - eval.DimensionCorrectness: true, - eval.DimensionProcessAdherence: true, - eval.DimensionArtifactQuality: true, - eval.DimensionRuntimeCompatibility: true, - eval.DimensionEfficiency: true, - eval.DimensionSafety: true, - eval.DimensionLearningClosure: true, -} - -var outcomesRunIDInvalid = regexp.MustCompile(`[^a-zA-Z0-9._:-]`) - -// clampOutcomesScore bounds a score into the schema's [0,1] range so a payload -// carrying an out-of-band aggregate (or a malformed criterion score) still -// produces a schema-valid manifest rather than a validation failure downstream. -func clampOutcomesScore(v float64) float64 { - switch { - case v < 0: - return 0 - case v > 1: - return 1 - default: - return v - } -} - -// outcomesBandToVerdict maps the council PASS/WARN/FAIL band onto the -// eval-run.v1 verdict enum: PASS→pass, FAIL→fail, anything else (WARN)→advisory. -// improvement/regression are baseline-delta verdicts; an Outcomes grade carries -// no baseline, so this writer never emits them. -func outcomesBandToVerdict(band string) eval.Verdict { - switch band { - case "PASS": - return eval.VerdictPass - case "FAIL": - return eval.VerdictFail - default: - return eval.VerdictAdvisory - } -} - -// outcomesBandToStatus maps the band onto the eval-run.v1 status enum. The -// schema's status enum lacks "complete", so the run lifecycle status mirrors the -// band: PASS→pass, FAIL→fail, WARN→inconclusive. -func outcomesBandToStatus(band string) eval.Status { - switch band { - case "PASS": - return eval.StatusPass - case "FAIL": - return eval.StatusFail - default: - return eval.StatusInconclusive - } -} - -// outcomesVisibility maps the score's eval split onto the suite visibility: -// a holdout grade is a private_holdout suite, everything else a public_canary. -func outcomesVisibility(split string) eval.Visibility { - if split == "holdout" { - return eval.VisibilityPrivateHoldout - } - return eval.VisibilityPublicCanary -} - -// outcomesDimensionScores projects an Outcomes score's per-criterion scores onto -// the eval-run.v1 dimension_scores object, keeping only the schema-permitted -// dimension keys (additionalProperties:false). If no criterion maps to a known -// dimension, it synthesizes correctness=aggregate so the object satisfies the -// schema's minProperties:1 — an Outcomes grade always carries an aggregate. -func outcomesDimensionScores(crit map[string]float64, aggregate float64) map[eval.Dimension]float64 { - out := map[eval.Dimension]float64{} - for k, v := range crit { - d := eval.Dimension(k) - if outcomesManifestDimensions[d] { - out[d] = clampOutcomesScore(v) - } - } - if len(out) == 0 { - out[eval.DimensionCorrectness] = clampOutcomesScore(aggregate) - } - return out -} - -// resolveOutcomesRunID picks the run id in precedence order — explicit -// --run-id, then the score's own run_id, then its source_task_id — sanitizes it -// to the schema pattern (^[a-zA-Z0-9][a-zA-Z0-9._:-]*$), and falls back to a -// constant when nothing usable remains. -func resolveOutcomesRunID(explicit string, s outcomesScore) string { - for _, cand := range []string{explicit, s.RunID, s.SourceTaskID} { - if id := sanitizeOutcomesRunID(cand); id != "" { - return id - } - } - return "outcomes-run" -} - -// sanitizeOutcomesRunID coerces an arbitrary string into the eval-run.v1 run_id -// pattern: invalid characters become '-', then leading non-alphanumerics are -// trimmed (the first char must be [a-zA-Z0-9]). Returns "" when nothing valid -// remains, so the caller can fall through to the next candidate. -func sanitizeOutcomesRunID(raw string) string { - if raw == "" { - return "" - } - cleaned := outcomesRunIDInvalid.ReplaceAllString(raw, "-") - cleaned = strings.TrimLeft(cleaned, "-._:") - return cleaned -} - -// buildOutcomesRunManifest assembles an eval-run.v1 RunRecord from an Outcomes -// score and its council band. startedAt is injected (clock seam) so callers and -// tests are deterministic. git/runtime/environment are honest stubs because the -// grade was produced out-of-process; a notes entry records that provenance. -func buildOutcomesRunManifest(s outcomesScore, band, runID string, startedAt time.Time) eval.RunRecord { - completed := startedAt - dims := outcomesDimensionScores(s.CriterionScores, s.Aggregate) - caseID := s.SourceTaskID - if caseID == "" { - caseID = "aggregate" - } - suiteID := s.SuiteRef - if suiteID == "" { - suiteID = "outcomes" - } - return eval.RunRecord{ - SchemaVersion: 1, - RunID: runID, - Suite: eval.SuiteRef{ - ID: suiteID, - Path: "out-of-process", - Visibility: outcomesVisibility(s.Split), - Tier: eval.TierLive, - }, - StartedAt: startedAt, - CompletedAt: &completed, - Status: outcomesBandToStatus(band), - Verdict: outcomesBandToVerdict(band), - Git: eval.GitRecord{ - // No candidate sha is known for an out-of-process grade; a zeroed - // hex sha satisfies the schema pattern while signalling "unknown". - CandidateRef: "out-of-process-outcomes-grade", - CandidateSHA: "0000000", - Dirty: false, - }, - Runtime: eval.RuntimeRecord{ - Name: eval.RuntimeManual, - Live: false, - }, - Environment: eval.EnvironmentRecord{ - ScrubbedEnvPrefixes: []string{}, - IsolatedHome: false, - IsolatedCodexHome: false, - NetworkAccess: eval.NetworkUnknown, - }, - CaseResults: []eval.CaseResult{ - { - ID: caseID, - Status: outcomesBandToStatus(band), - Score: clampOutcomesScore(s.Aggregate), - DimensionScores: dims, - }, - }, - AggregateScore: clampOutcomesScore(s.Aggregate), - DimensionScores: dims, - Notes: []string{ - fmt.Sprintf("Out-of-process Outcomes grade ingested by `ao eval outcomes ingest` (band=%s); git/runtime/environment are stubs — the grade was produced outside this harness.", band), - }, - } -} - -// writeOutcomesRunManifest writes rec to //manifest.json (creating -// the directory) and returns the manifest path. It is the file-system adapter -// for buildOutcomesRunManifest. -func writeOutcomesRunManifest(dir, runID string, rec eval.RunRecord) (string, error) { - runDir := filepath.Join(dir, runID) - if err := os.MkdirAll(runDir, 0o750); err != nil { - return "", fmt.Errorf("create manifest dir %s: %w", runDir, err) - } - blob, err := json.MarshalIndent(rec, "", " ") - if err != nil { - return "", fmt.Errorf("encode eval-run manifest: %w", err) - } - path := filepath.Join(runDir, "manifest.json") - if err := os.WriteFile(path, append(blob, '\n'), 0o600); err != nil { - return "", fmt.Errorf("write eval-run manifest %s: %w", path, err) - } - return path, nil -} diff --git a/cli/cmd/ao/eval_outcomes_manifest_test.go b/cli/cmd/ao/eval_outcomes_manifest_test.go deleted file mode 100644 index 16d55af46..000000000 --- a/cli/cmd/ao/eval_outcomes_manifest_test.go +++ /dev/null @@ -1,254 +0,0 @@ -package main - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - "time" - - "github.com/boshu2/agentops/cli/internal/eval" -) - -func fixedManifestTime() time.Time { - return time.Date(2026, 6, 6, 12, 0, 0, 0, time.UTC) -} - -// findEvalRunSchema walks up from the working directory to locate -// schemas/eval-run.v1.schema.json so the schema-linked assertions work whether -// the test runs from cli/cmd/ao or the repo root. Returns "" when not found. -func findEvalRunSchema(t *testing.T) string { - t.Helper() - dir, err := os.Getwd() - if err != nil { - return "" - } - for i := 0; i < 8; i++ { - cand := filepath.Join(dir, "schemas", "eval-run.v1.schema.json") - if _, err := os.Stat(cand); err == nil { - return cand - } - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent - } - return "" -} - -func TestBuildOutcomesRunManifest_BandMapping(t *testing.T) { - cases := []struct { - name string - band string - wantStatus eval.Status - wantVerd eval.Verdict - }{ - {"pass", "PASS", eval.StatusPass, eval.VerdictPass}, - {"fail", "FAIL", eval.StatusFail, eval.VerdictFail}, - {"warn", "WARN", eval.StatusInconclusive, eval.VerdictAdvisory}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - s := outcomesScore{SourceTaskID: "task-1", Aggregate: 0.82} - rec := buildOutcomesRunManifest(s, tc.band, "run-1", fixedManifestTime()) - if rec.Status != tc.wantStatus { - t.Errorf("status = %q, want %q", rec.Status, tc.wantStatus) - } - if rec.Verdict != tc.wantVerd { - t.Errorf("verdict = %q, want %q", rec.Verdict, tc.wantVerd) - } - }) - } -} - -func TestBuildOutcomesRunManifest_RequiredFieldsAndStubs(t *testing.T) { - s := outcomesScore{ - SourceTaskID: "task-xyz", - Aggregate: 0.9, - Threshold: 0.8, - Split: "holdout", - SuiteRef: "suite-a", - CriterionScores: map[string]float64{"correctness": 0.95, "safety": 0.88}, - } - rec := buildOutcomesRunManifest(s, "PASS", "run-xyz", fixedManifestTime()) - - if rec.SchemaVersion != 1 { - t.Errorf("schema_version = %d, want 1", rec.SchemaVersion) - } - if rec.RunID != "run-xyz" { - t.Errorf("run_id = %q, want run-xyz", rec.RunID) - } - if rec.Suite.ID != "suite-a" { - t.Errorf("suite.id = %q, want suite-a", rec.Suite.ID) - } - if rec.Suite.Visibility != eval.VisibilityPrivateHoldout { - t.Errorf("holdout split must map to private_holdout, got %q", rec.Suite.Visibility) - } - if rec.Suite.Path == "" || rec.Suite.Tier != eval.TierLive { - t.Errorf("suite path/tier stubs wrong: path=%q tier=%q", rec.Suite.Path, rec.Suite.Tier) - } - if rec.AggregateScore != 0.9 { - t.Errorf("aggregate_score = %v, want 0.9", rec.AggregateScore) - } - if len(rec.CaseResults) != 1 { - t.Fatalf("case_results len = %d, want 1", len(rec.CaseResults)) - } - if rec.CaseResults[0].ID != "task-xyz" { - t.Errorf("case id = %q, want task-xyz", rec.CaseResults[0].ID) - } - if len(rec.DimensionScores) != 2 { - t.Errorf("dimension_scores = %v, want 2 known dims", rec.DimensionScores) - } - // Honest stubs that must still be schema-valid. - if rec.Git.CandidateSHA != "0000000" { - t.Errorf("candidate_sha stub = %q, want 0000000", rec.Git.CandidateSHA) - } - if rec.Runtime.Name != eval.RuntimeManual { - t.Errorf("runtime.name = %q, want manual", rec.Runtime.Name) - } - if rec.Environment.NetworkAccess != eval.NetworkUnknown { - t.Errorf("network_access = %q, want unknown", rec.Environment.NetworkAccess) - } - if rec.Environment.ScrubbedEnvPrefixes == nil { - t.Error("scrubbed_env_prefixes must be a non-nil slice (marshals to [] not null)") - } - if len(rec.Notes) == 0 { - t.Error("expected a provenance note recording the out-of-process grade") - } -} - -func TestOutcomesDimensionScores_FiltersAndFallback(t *testing.T) { - // Unknown keys are dropped; known keys (incl. clamping) are kept. - got := outcomesDimensionScores(map[string]float64{ - "correctness": 1.4, // clamps to 1 - "made_up_axis": 0.5, // not a schema dimension → dropped - "context_comprehension": 0.7, // valid Dimension but NOT an eval-run.v1 prop → dropped - }, 0.6) - if len(got) != 1 { - t.Fatalf("expected only correctness kept, got %v", got) - } - if got[eval.DimensionCorrectness] != 1.0 { - t.Errorf("correctness clamp = %v, want 1.0", got[eval.DimensionCorrectness]) - } - - // No known criterion → fallback synthesizes correctness from aggregate. - fb := outcomesDimensionScores(map[string]float64{"nope": 0.3}, 0.42) - if len(fb) != 1 || fb[eval.DimensionCorrectness] != 0.42 { - t.Errorf("fallback = %v, want {correctness:0.42}", fb) - } -} - -func TestResolveOutcomesRunID(t *testing.T) { - cases := []struct { - name string - explicit string - score outcomesScore - want string - }{ - {"explicit wins", "my-run", outcomesScore{RunID: "other", SourceTaskID: "task"}, "my-run"}, - {"falls back to run_id", "", outcomesScore{RunID: "from-payload", SourceTaskID: "task"}, "from-payload"}, - {"falls back to source_task_id", "", outcomesScore{SourceTaskID: "task-7"}, "task-7"}, - {"sanitizes invalid chars", "weird id/with spaces", outcomesScore{}, "weird-id-with-spaces"}, - {"trims leading invalid", " -lead", outcomesScore{}, "lead"}, - {"all-empty fallback", "", outcomesScore{}, "outcomes-run"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := resolveOutcomesRunID(tc.explicit, tc.score); got != tc.want { - t.Errorf("resolveOutcomesRunID(%q, %+v) = %q, want %q", tc.explicit, tc.score, got, tc.want) - } - }) - } -} - -func TestWriteOutcomesRunManifest_RoundTrip(t *testing.T) { - dir := t.TempDir() - s := outcomesScore{SourceTaskID: "task-rt", Aggregate: 0.77, Threshold: 0.7, - CriterionScores: map[string]float64{"efficiency": 0.8}} - rec := buildOutcomesRunManifest(s, "PASS", "run-rt", fixedManifestTime()) - - path, err := writeOutcomesRunManifest(dir, "run-rt", rec) - if err != nil { - t.Fatalf("write: %v", err) - } - want := filepath.Join(dir, "run-rt", "manifest.json") - if path != want { - t.Errorf("path = %q, want %q", path, want) - } - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read back: %v", err) - } - var back eval.RunRecord - if err := json.Unmarshal(raw, &back); err != nil { - t.Fatalf("unmarshal round-trip: %v", err) - } - if back.RunID != "run-rt" || back.AggregateScore != 0.77 || back.Verdict != eval.VerdictPass { - t.Errorf("round-trip mismatch: %+v", back) - } -} - -// TestOutcomesManifest_MatchesSchemaContract ties the manifest to the actual -// schema file: every top-level required key must be present and non-null in the -// marshaled manifest, and status/verdict must be members of the schema's enums. -// This is the Go-struct-roundtrip validation the bead permits in lieu of a -// vendored jsonschema validator. -func TestOutcomesManifest_MatchesSchemaContract(t *testing.T) { - schemaPath := findEvalRunSchema(t) - if schemaPath == "" { - t.Skip("eval-run.v1 schema not found from cwd; skipping schema-contract check") - } - raw, err := os.ReadFile(schemaPath) - if err != nil { - t.Fatalf("read schema: %v", err) - } - var schema struct { - Required []string `json:"required"` - Defs map[string]struct { - Enum []string `json:"enum"` - } `json:"$defs"` - } - if err := json.Unmarshal(raw, &schema); err != nil { - t.Fatalf("parse schema: %v", err) - } - - s := outcomesScore{SourceTaskID: "task-c", Aggregate: 0.5, Threshold: 0.8, - CriterionScores: map[string]float64{"correctness": 0.5}} - rec := buildOutcomesRunManifest(s, "WARN", "run-c", fixedManifestTime()) - blob, err := json.Marshal(rec) - if err != nil { - t.Fatalf("marshal manifest: %v", err) - } - var asMap map[string]json.RawMessage - if err := json.Unmarshal(blob, &asMap); err != nil { - t.Fatalf("unmarshal manifest to map: %v", err) - } - - for _, key := range schema.Required { - v, ok := asMap[key] - if !ok { - t.Errorf("required key %q missing from manifest", key) - continue - } - if string(v) == "null" { - t.Errorf("required key %q is null", key) - } - } - - assertEnumMember(t, schema.Defs["status"].Enum, string(rec.Status), "status") - assertEnumMember(t, schema.Defs["verdict"].Enum, string(rec.Verdict), "verdict") -} - -func assertEnumMember(t *testing.T, enum []string, got, label string) { - t.Helper() - if len(enum) == 0 { - t.Fatalf("%s enum not found in schema", label) - } - for _, e := range enum { - if e == got { - return - } - } - t.Errorf("%s = %q is not a member of schema enum %v", label, got, enum) -} diff --git a/cli/cmd/ao/eval_outcomes_test.go b/cli/cmd/ao/eval_outcomes_test.go deleted file mode 100644 index 003c0dd51..000000000 --- a/cli/cmd/ao/eval_outcomes_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/boshu2/agentops/cli/internal/evalsubstrate" -) - -func sampleTaskAndCriteria() (evalsubstrate.Task, []evalsubstrate.Criterion) { - task := evalsubstrate.Task{ - SchemaVersion: evalsubstrate.SchemaVersion, - ID: "task-capital-cities", - Domain: "geography", - Description: "Answer the capital-city question accurately.", - } - criteria := []evalsubstrate.Criterion{ - {ID: "accuracy", Description: "Names the correct capital city.", Weight: 0.7}, - {ID: "concision", Description: "Answers in one short sentence.", Weight: 0.3}, - } - return task, criteria -} - -// TestCompileOutcomesRubric_StripsHoldoutTarget: the compiled rubric carries the -// criteria but none of the holdout answer values — the holdout-isolation -// invariant at the cloud boundary (Managed Agents are not ZDR). -func TestCompileOutcomesRubric_StripsHoldoutTarget(t *testing.T) { - task, criteria := sampleTaskAndCriteria() - holdout := []string{"Ouagadougou", "Antananarivo"} - - r, err := compileOutcomesRubric(task, criteria, "sha256:abc123", holdout) - if err != nil { - t.Fatalf("compile: unexpected error: %v", err) - } - if r.SourceTaskID != task.ID { - t.Errorf("SourceTaskID = %q, want %q", r.SourceTaskID, task.ID) - } - if r.JudgeContentHash != "sha256:abc123" { - t.Errorf("JudgeContentHash = %q, want sha256:abc123", r.JudgeContentHash) - } - if len(r.Criteria) != 2 { - t.Fatalf("len(Criteria) = %d, want 2", len(r.Criteria)) - } - if hit, found := r.ContainsAny(holdout); found { - t.Errorf("compiled rubric leaked holdout value %q", hit) - } -} - -// TestCompileOutcomesRubric_RefusesLeak: deny-by-default — if a holdout value -// would appear in the payload (e.g. a criterion description accidentally embeds -// the answer), compile MUST refuse rather than emit a leaking rubric. -func TestCompileOutcomesRubric_RefusesLeak(t *testing.T) { - task, _ := sampleTaskAndCriteria() - leaky := []evalsubstrate.Criterion{ - {ID: "accuracy", Description: "The answer is Ouagadougou.", Weight: 1.0}, - } - _, err := compileOutcomesRubric(task, leaky, "sha256:abc123", []string{"Ouagadougou"}) - if err == nil { - t.Fatal("expected compile to refuse a leaking rubric, got nil error") - } - if !strings.Contains(err.Error(), "Ouagadougou") { - t.Errorf("error should name the leaked value, got: %v", err) - } -} diff --git a/cli/cmd/ao/eval_scenario_ab.go b/cli/cmd/ao/eval_scenario_ab.go deleted file mode 100644 index 0e7f42a91..000000000 --- a/cli/cmd/ao/eval_scenario_ab.go +++ /dev/null @@ -1,320 +0,0 @@ -// practices: [llm-eval-harness] -package main - -import ( - "context" - "encoding/json" - "fmt" - "os" - "os/exec" - "regexp" - "strconv" - "strings" - "time" - - "github.com/spf13/cobra" - - aoeval "github.com/boshu2/agentops/cli/internal/eval" - "github.com/boshu2/agentops/cli/internal/scenario" -) - -var ( - evalScenarioABScenario string - evalScenarioABOutput string - evalScenarioABBudget int - evalScenarioABTimeout time.Duration - evalScenarioABControlOnly bool -) - -// scenarioABRunnerFactory / scenarioABJudgeFactory are the injectable seam: -// production builds codex-exec-backed implementations; tests override these -// package vars with deterministic fakes (no live models, no hangs). -var ( - scenarioABRunnerFactory = func() aoeval.ScenarioRunner { return codexScenarioRunner{} } - scenarioABJudgeFactory = func() aoeval.ScenarioJudge { return codexScenarioJudge{} } -) - -// selectScenarioJudge picks the DETERMINISTIC answer-key judge when the scenario -// carries an answer_key (fact-recall OOD grading — no LLM, zero judge noise/cost), -// else the injected (codex) judge. age-k8u. -func selectScenarioJudge(sc scenario.Scenario) aoeval.ScenarioJudge { - if strings.TrimSpace(sc.AnswerKey) != "" { - return aoeval.AnswerKeyJudge{} - } - return scenarioABJudgeFactory() -} - -var evalScenarioABCmd = &cobra.Command{ - Use: "scenario-ab", - Short: "Run a knowledge-reuse holdout scenario with vs. without the gold pull (the discriminating A/B)", - Long: `Run one holdout scenario (scenario.v1) twice — a control arm WITHOUT the gold -pull and a treatment arm WITH it — grade each arm with a cross-family judge, -and emit a ScenarioDeltaScorecard. - -The gate is DETERMINISTIC over the (stochastic) judge output: it FAILS LOUDLY -(non-zero exit) when the treatment did not beat the control (delta <= 0 — the -ADR-0002 spray returning), when the treatment misses the scenario's -satisfaction threshold, or when the summed arm token cost exceeds the budget.`, - Args: cobra.NoArgs, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, _ []string) error { - if strings.TrimSpace(evalScenarioABScenario) == "" { - return fmt.Errorf("--scenario is required") - } - sc, err := scenario.Load(evalScenarioABScenario) - if err != nil { - return err - } - card, err := aoeval.RunScenarioAB(cmd.Context(), aoeval.ScenarioABOptions{ - Scenario: *sc, - ScenarioPath: evalScenarioABScenario, - Runner: selectScenarioRunner(*sc), - Judge: selectScenarioJudge(*sc), - Timeout: evalScenarioABTimeout, - TokenBudget: evalScenarioABBudget, - ControlOnly: evalScenarioABControlOnly, - }) - if err != nil { - return err - } - if err := aoeval.WriteScenarioDeltaScorecard(card, evalScenarioABOutput); err != nil { - return err - } - if card.CeilingViolation { - fmt.Fprintf(cmd.OutOrStdout(), - "scenario-ab %s [%s]: CEILING VIOLATION (without=%.4f >= threshold=%.4f) — invalid scenario, no delta emitted\n", - card.ScenarioID, card.VerdictClass, card.Without.Score, card.SatisfactionThreshold) - } else if card.ControlOnly { - fmt.Fprintf(cmd.OutOrStdout(), - "scenario-ab %s [%s]: control-only headroom=PASS (without=%.4f < threshold=%.4f) tokens=%d gate=%s\n", - card.ScenarioID, card.VerdictClass, card.Without.Score, card.SatisfactionThreshold, - card.Without.TokenCost, gateLabel(card.Gate.Pass)) - } else { - fmt.Fprintf(cmd.OutOrStdout(), - "scenario-ab %s [%s]: delta=%.4f (with=%.4f without=%.4f) tokens=%d gate=%s %s\n", - card.ScenarioID, card.VerdictClass, card.AggregateDelta, card.With.Score, card.Without.Score, - card.With.TokenCost+card.Without.TokenCost, gateLabel(card.Gate.Pass), moatLabel(card.MoatEligible)) - } - if !card.Gate.Pass { - for _, r := range card.Gate.Reasons { - fmt.Fprintf(cmd.ErrOrStderr(), " FAIL: %s\n", r) - } - return fmt.Errorf("scenario-ab gate failed for %s", card.ScenarioID) - } - return nil - }, -} - -func gateLabel(pass bool) string { - if pass { - return "PASS" - } - return "FAIL" -} - -// moatLabel surfaces whether a PASSING scorecard is admissible MOAT evidence. -// A fact-recall (sentinel) scorecard can pass its plumbing gate yet is NEVER -// moat evidence (age-6ys) — this label keeps that distinction loud at the CLI so -// a plumbing PASS is not misread as "the corpus improves agent work" verdict. -func moatLabel(eligible bool) string { - if eligible { - return "moat-eligible" - } - return "NOT-moat-evidence(plumbing)" -} - -func init() { - evalScenarioABCmd.Flags().StringVar(&evalScenarioABScenario, "scenario", "", "Path to the scenario.v1 JSON file (required)") - evalScenarioABCmd.Flags().StringVar(&evalScenarioABOutput, "output", "", "Write the ScenarioDeltaScorecard JSON to this path") - evalScenarioABCmd.Flags().IntVar(&evalScenarioABBudget, "token-budget", 0, "Fail the gate if summed arm token cost exceeds this (0 = default 200000)") - evalScenarioABCmd.Flags().DurationVar(&evalScenarioABTimeout, "timeout", 0, "Per-arm timeout (0 = default 5m)") - evalScenarioABCmd.Flags().BoolVar(&evalScenarioABControlOnly, "control-only", false, "Run only the without-gold control arm and fail on ceiling/no-headroom") - evalCmd.AddCommand(evalScenarioABCmd) -} - -// --- codex-exec-backed defaults (Law 0: codex exec only; no Claude print-mode) - -// These are the live path; they are exercised end-to-end in KF-C, not in this -// slice's tests (which inject fakes via the factory seam above). - -type codexScenarioRunner struct{} - -func (codexScenarioRunner) RunArm(ctx context.Context, sc scenario.Scenario, withGold bool) (aoeval.ArmOutcome, error) { - var prompt strings.Builder - if withGold { - // The treatment arm consumes the KF-A decision-point gold pull. - if pointers := goldPointers(ctx, sc.Goal); pointers != "" { - prompt.WriteString("Relevant prior knowledge (gold corpus pointers):\n") - prompt.WriteString(pointers) - prompt.WriteString("\n\n") - } - } - prompt.WriteString(sc.Narrative) - prompt.WriteString("\n\nGoal: ") - prompt.WriteString(sc.Goal) - prompt.WriteString("\n\nProduce your best attempt. Output only the result, no preamble.") - out, tokens, err := runCodexExec(ctx, prompt.String(), "") - if err != nil { - return aoeval.ArmOutcome{}, err - } - return aoeval.ArmOutcome{Output: out, TokenCost: tokens}, nil -} - -// taskSuccessJudgePrompt builds a per-arm TASK-SUCCESS grading prompt: it asks the -// judge whether the OUTPUT accomplishes the scenario's goal/expected outcome, scored -// 0..1 ABSOLUTELY and on its own merits. This replaces grading against the scenario's -// A/B-CONTRAST acceptance_vectors (treatment-engages / control-hangs / delta-attributable), -// which are properties of the COMPARISON, not of one arm — grading a single arm against -// them is incoherent (an arm was penalized for correctly NOT "hanging"; the KF-4 −0.34 -// noise). An absolute per-arm task-success score is coherent AND preserves the age-707 -// ceiling pre-screen, which needs the control's absolute score. (age-oe2) -func taskSuccessJudgePrompt(sc scenario.Scenario, output string) string { - var p strings.Builder - p.WriteString("You are a strict, fair judge. Grade ONLY whether the OUTPUT below accomplishes the task — score 0.0 to 1.0 (1.0 = the goal is fully achieved and the expected outcome met; 0.0 = not at all). Judge this OUTPUT on its own merits; do NOT compare it to any other answer.\n\n") - p.WriteString("Goal: " + sc.Goal + "\n") - if strings.TrimSpace(sc.ExpectedOutcome) != "" { - p.WriteString("Expected outcome: " + sc.ExpectedOutcome + "\n") - } - p.WriteString("\nOUTPUT:\n" + output + "\n\n") - p.WriteString(`Return ONLY strict JSON, no prose: {"vectors":[{"dimension":"task-success","pass":true,"score":0.0}],"aggregate_score":0.0}. Set pass=true iff score>=0.5; aggregate_score in [0,1] is your task-success grade.`) - return p.String() -} - -type codexScenarioJudge struct{} - -func (codexScenarioJudge) Judge(ctx context.Context, sc scenario.Scenario, arm aoeval.ScenarioArm, outcome aoeval.ArmOutcome) (aoeval.JudgeVerdict, error) { - prompt := taskSuccessJudgePrompt(sc, outcome.Output) - // Constrain the model's final message to the verdict shape so the parser - // reads a clean object, not codex's surrounding reasoning chrome. - schemaPath, cleanup, err := writeJudgeSchema() - if err != nil { - return aoeval.JudgeVerdict{}, err - } - defer cleanup() - out, _, err := runCodexExec(ctx, prompt, schemaPath) - if err != nil { - return aoeval.JudgeVerdict{}, err - } - verdict, err := parseJudgeJSON(out) - if err != nil { - return aoeval.JudgeVerdict{}, fmt.Errorf("judge (%s) returned unparseable output: %w", arm, err) - } - return verdict, nil -} - -// judgeOutputSchema constrains codex's final message to the JudgeVerdict shape -// (codex exec --output-schema), so the judge returns a clean JSON object. -// codex's strict structured-output mode requires "additionalProperties": false -// on every object AND every property listed in "required" (a missing -// additionalProperties yields a 400 invalid_json_schema). -const judgeOutputSchema = `{"type":"object","additionalProperties":false,"properties":{"vectors":{"type":"array","items":{"type":"object","additionalProperties":false,"properties":{"dimension":{"type":"string"},"pass":{"type":"boolean"},"score":{"type":"number"}},"required":["dimension","pass","score"]}},"aggregate_score":{"type":"number"}},"required":["vectors","aggregate_score"]}` - -func writeJudgeSchema() (string, func(), error) { - f, err := os.CreateTemp("", "scenario-ab-judge-schema-*.json") - if err != nil { - return "", func() {}, fmt.Errorf("judge schema temp: %w", err) - } - name := f.Name() - cleanup := func() { _ = os.Remove(name) } - if _, err := f.WriteString(judgeOutputSchema); err != nil { - _ = f.Close() - cleanup() - return "", func() {}, fmt.Errorf("write judge schema: %w", err) - } - if err := f.Close(); err != nil { - cleanup() - return "", func() {}, fmt.Errorf("close judge schema: %w", err) - } - return name, cleanup, nil -} - -// goldPointers shells the KF-A decision-point pull. Best-effort: a missing gold -// wiki yields no pointers (the control/treatment difference simply narrows). -func goldPointers(ctx context.Context, query string) string { - if _, err := os.Stat(".ao/wiki"); err != nil { - return "" - } - cmd := exec.CommandContext(ctx, "ao", "lookup", "--query", query, "--gold", "--pointers", "--limit", "3") - out, err := cmd.Output() - if err != nil { - return "" - } - return strings.TrimSpace(string(out)) -} - -var codexTokensRe = regexp.MustCompile(`(?i)tokens used[^\d]*([\d,]+)`) - -// runCodexExec runs a single non-interactive codex turn and returns the agent's -// FINAL message (captured via --output-last-message, so codex's reasoning/chrome -// is excluded) plus a best-effort token count (parsed from the full stdout's -// "tokens used" line, else estimated). When outputSchemaPath is set, codex is -// constrained to emit a final message matching that JSON Schema. Law 0: codex -// exec only; no Claude print-mode. -func runCodexExec(ctx context.Context, prompt, outputSchemaPath string) (string, int, error) { - msgFile, err := os.CreateTemp("", "scenario-ab-codex-msg-*.txt") - if err != nil { - return "", 0, fmt.Errorf("codex last-message temp: %w", err) - } - msgPath := msgFile.Name() - _ = msgFile.Close() - defer func() { _ = os.Remove(msgPath) }() - - // Inner codex runs WITHOUT its own sandbox — the OUTER OS sandbox (sandboxedCodexCmd) - // is the real confinement: it removes the corpus from the readable filesystem so the - // control arm cannot grep the answer off disk. The bypass flag is documented "solely - // for environments that are externally sandboxed" (age-9a9). - args := []string{"exec", "--dangerously-bypass-approvals-and-sandbox", "--skip-git-repo-check", "--output-last-message", msgPath} - if outputSchemaPath != "" { - args = append(args, "--output-schema", outputSchemaPath) - } - args = append(args, prompt) - - cwd, err := os.Getwd() - if err != nil { - return "", 0, fmt.Errorf("resolve cwd for arm isolation: %w", err) - } - denyPaths := corpusDenyPaths(cwd) // repo corpus (root resolved from any subdir) + global ~/.agents - cmd, err := sandboxedCodexCmd(ctx, denyPaths, args) - if err != nil { - return "", 0, err // fail closed: never run an unisolated arm - } - // Closed stdin: codex takes the prompt positionally and would otherwise block - // reading additional input from stdin in this non-TTY context - // (f-2026-06-15-codex-exec-stdin-stall). An empty reader gives immediate EOF. - cmd.Stdin = strings.NewReader("") - combined, err := cmd.CombinedOutput() - if err != nil { - return "", 0, fmt.Errorf("codex exec: %w", err) - } - tokens := parseCodexTokens(string(combined)) - // Prefer the clean final message; fall back to combined output if codex did - // not write one. - if msg, readErr := os.ReadFile(msgPath); readErr == nil && len(strings.TrimSpace(string(msg))) > 0 { - return strings.TrimSpace(string(msg)), tokens, nil - } - return string(combined), tokens, nil -} - -// parseJudgeJSON extracts the first JSON object from the judge's output (codex -// may wrap it in prose) and decodes it into a JudgeVerdict. -func parseJudgeJSON(text string) (aoeval.JudgeVerdict, error) { - start := strings.Index(text, "{") - end := strings.LastIndex(text, "}") - if start < 0 || end <= start { - return aoeval.JudgeVerdict{}, fmt.Errorf("no JSON object found") - } - var verdict aoeval.JudgeVerdict - if err := json.Unmarshal([]byte(text[start:end+1]), &verdict); err != nil { - return aoeval.JudgeVerdict{}, err - } - return verdict, nil -} - -func parseCodexTokens(text string) int { - if m := codexTokensRe.FindStringSubmatch(text); m != nil { - if n, err := strconv.Atoi(strings.ReplaceAll(m[1], ",", "")); err == nil { - return n - } - } - // Fallback estimate: ~4 chars/token. - return len(text) / 4 -} diff --git a/cli/cmd/ao/eval_scenario_ab_agentic.go b/cli/cmd/ao/eval_scenario_ab_agentic.go deleted file mode 100644 index 007bf013d..000000000 --- a/cli/cmd/ao/eval_scenario_ab_agentic.go +++ /dev/null @@ -1,195 +0,0 @@ -// practices: [llm-eval-harness] -package main - -import ( - "context" - "encoding/json" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - aoeval "github.com/boshu2/agentops/cli/internal/eval" - "github.com/boshu2/agentops/cli/internal/scenario" -) - -const ( - agenticMaxTurns = 8 - agenticStepSchemaJSON = `{"type":"object","additionalProperties":false,"properties":{"commands":{"type":"array","items":{"type":"string"}},"done":{"type":"boolean"},"summary":{"type":"string"}},"required":["commands","done","summary"]}` -) - -// agenticStep is one structured turn from the worker model. -type agenticStep struct { - Commands []string `json:"commands"` - Done bool `json:"done"` - Summary string `json:"summary"` -} - -// workspaceCommandRunner executes shell commands inside the isolated workspace. -type workspaceCommandRunner func(ctx context.Context, workDir, command string) (stdout string, exitCode int, err error) - -// agenticRunnerHooks is the injectable seam for tests (no live codex). -var agenticRunnerHooks = struct { - runCodex func(ctx context.Context, prompt, schemaPath string) (string, int, error) - runCmd workspaceCommandRunner -}{ - runCodex: runCodexExec, - runCmd: defaultWorkspaceCommandRunner, -} - -type agenticScenarioRunner struct{} - -func (agenticScenarioRunner) RunArm(ctx context.Context, sc scenario.Scenario, withGold bool) (aoeval.ArmOutcome, error) { - workDir, err := os.MkdirTemp("", "scenario-ab-agentic-*") - if err != nil { - return aoeval.ArmOutcome{}, fmt.Errorf("agentic workspace: %w", err) - } - defer func() { _ = os.RemoveAll(workDir) }() - - schemaPath, cleanup, err := writeAgenticStepSchema() - if err != nil { - return aoeval.ArmOutcome{}, err - } - defer cleanup() - - var history strings.Builder - totalTokens := 0 - for turn := 0; turn < agenticMaxTurns; turn++ { - prompt := buildAgenticPrompt(ctx, sc, withGold, workDir, history.String()) - out, tokens, err := agenticRunnerHooks.runCodex(ctx, prompt, schemaPath) - totalTokens += tokens - if err != nil { - return aoeval.ArmOutcome{}, fmt.Errorf("agentic turn %d: %w", turn+1, err) - } - step, err := parseAgenticStep(out) - if err != nil { - return aoeval.ArmOutcome{}, fmt.Errorf("agentic turn %d parse: %w", turn+1, err) - } - for _, command := range step.Commands { - command = strings.TrimSpace(command) - if command == "" { - continue - } - stdout, exitCode, runErr := agenticRunnerHooks.runCmd(ctx, workDir, command) - if runErr != nil { - return aoeval.ArmOutcome{}, fmt.Errorf("agentic command %q: %w", command, runErr) - } - fmt.Fprintf(&history, "$ %s\nexit=%d\n%s\n\n", command, exitCode, strings.TrimSpace(stdout)) - } - if step.Done { - summary := strings.TrimSpace(step.Summary) - if summary == "" { - summary = strings.TrimSpace(history.String()) - } - artifactSummary, _ := summarizeWorkspace(workDir) - if artifactSummary != "" { - summary = summary + "\n\nWorkspace artifacts:\n" + artifactSummary - } - return aoeval.ArmOutcome{Output: summary, TokenCost: totalTokens}, nil - } - } - return aoeval.ArmOutcome{}, fmt.Errorf("agentic runner exceeded %d turns without done=true", agenticMaxTurns) -} - -func buildAgenticPrompt(ctx context.Context, sc scenario.Scenario, withGold bool, workDir, history string) string { - var p strings.Builder - p.WriteString("You are an agentic worker in an isolated workspace. Execute the task by running shell commands in the workspace directory.\n") - p.WriteString("Return ONLY strict JSON matching the schema: commands (shell lines to run now), done (true when finished), summary (final result for grading when done=true).\n\n") - if withGold { - if pointers := goldPointers(ctx, sc.Goal); pointers != "" { - p.WriteString("Relevant prior knowledge (gold corpus pointers):\n") - p.WriteString(pointers) - p.WriteString("\n\n") - } - } - p.WriteString(sc.Narrative) - p.WriteString("\n\nGoal: ") - p.WriteString(sc.Goal) - if strings.TrimSpace(sc.ExpectedOutcome) != "" { - p.WriteString("\nExpected outcome: ") - p.WriteString(sc.ExpectedOutcome) - } - p.WriteString("\n\nWorkspace: ") - p.WriteString(workDir) - p.WriteString("\nRun commands relative to this directory. Do not read outside the workspace.\n") - if history != "" { - p.WriteString("\nPrior command history:\n") - p.WriteString(history) - } - return p.String() -} - -func writeAgenticStepSchema() (string, func(), error) { - f, err := os.CreateTemp("", "scenario-ab-agentic-schema-*.json") - if err != nil { - return "", func() {}, fmt.Errorf("agentic schema temp: %w", err) - } - name := f.Name() - cleanup := func() { _ = os.Remove(name) } - if _, err := f.WriteString(agenticStepSchemaJSON); err != nil { - _ = f.Close() - cleanup() - return "", func() {}, fmt.Errorf("write agentic schema: %w", err) - } - if err := f.Close(); err != nil { - cleanup() - return "", func() {}, fmt.Errorf("close agentic schema: %w", err) - } - return name, cleanup, nil -} - -func parseAgenticStep(text string) (agenticStep, error) { - start := strings.Index(text, "{") - end := strings.LastIndex(text, "}") - if start < 0 || end <= start { - return agenticStep{}, fmt.Errorf("no JSON object found") - } - var step agenticStep - if err := json.Unmarshal([]byte(text[start:end+1]), &step); err != nil { - return agenticStep{}, err - } - return step, nil -} - -func defaultWorkspaceCommandRunner(ctx context.Context, workDir, command string) (string, int, error) { - cmd := exec.CommandContext(ctx, "bash", "-lc", command) - cmd.Dir = workDir - out, err := cmd.CombinedOutput() - exitCode := 0 - if err != nil { - if ee, ok := err.(*exec.ExitError); ok { - exitCode = ee.ExitCode() - } else { - return "", 0, err - } - } - return string(out), exitCode, nil -} - -func summarizeWorkspace(workDir string) (string, error) { - var lines []string - _ = filepath.WalkDir(workDir, func(path string, d os.DirEntry, err error) error { - if err != nil || d.IsDir() { - return nil - } - rel, relErr := filepath.Rel(workDir, path) - if relErr != nil || strings.HasPrefix(rel, ".") { - return nil - } - info, statErr := os.Stat(path) - if statErr != nil { - return nil - } - lines = append(lines, fmt.Sprintf("- %s (%d bytes)", rel, info.Size())) - return nil - }) - return strings.Join(lines, "\n"), nil -} - -func selectScenarioRunner(sc scenario.Scenario) aoeval.ScenarioRunner { - if sc.EffectiveRunnerMode() == scenario.RunnerModeAgentic { - return agenticScenarioRunner{} - } - return scenarioABRunnerFactory() -} diff --git a/cli/cmd/ao/eval_scenario_ab_agentic_test.go b/cli/cmd/ao/eval_scenario_ab_agentic_test.go deleted file mode 100644 index d5358c03a..000000000 --- a/cli/cmd/ao/eval_scenario_ab_agentic_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - - aoeval "github.com/boshu2/agentops/cli/internal/eval" - "github.com/boshu2/agentops/cli/internal/scenario" -) - -func withAgenticHooks(t *testing.T, steps []agenticStep) { - t.Helper() - orig := agenticRunnerHooks - call := 0 - agenticRunnerHooks.runCodex = func(_ context.Context, _ string, _ string) (string, int, error) { - idx := call - if len(steps) == 0 { - return "", 0, context.Canceled - } - if idx >= len(steps) { - idx = len(steps) - 1 - } - step := steps[idx] - call++ - data, err := json.Marshal(step) - if err != nil { - return "", 0, err - } - return string(data), 10, nil - } - agenticRunnerHooks.runCmd = func(_ context.Context, workDir, command string) (string, int, error) { - if strings.Contains(command, "touch") { - name := strings.TrimSpace(strings.TrimPrefix(command, "touch")) - if err := os.WriteFile(filepath.Join(workDir, name), []byte("ok\n"), 0o644); err != nil { - return "", 1, err - } - return "", 0, nil - } - return "", 0, nil - } - t.Cleanup(func() { - agenticRunnerHooks = orig - }) -} - -func TestAgenticScenarioRunnerProducesWorkspaceArtifact(t *testing.T) { - withAgenticHooks(t, []agenticStep{ - {Commands: []string{"touch result.txt"}, Done: true, Summary: "created result.txt in workspace"}, - }) - sc := scenario.Scenario{ - ID: "s-agentic-test", - Goal: "create a file named result.txt", - Narrative: "Ship a file in the workspace.", - } - out, err := agenticScenarioRunner{}.RunArm(context.Background(), sc, false) - if err != nil { - t.Fatalf("RunArm: %v", err) - } - if !strings.Contains(out.Output, "result.txt") { - t.Errorf("output = %q, want result.txt mention", out.Output) - } -} - -func TestSelectScenarioRunnerAgentic(t *testing.T) { - sc := scenario.Scenario{RunnerMode: "agentic"} - if _, ok := selectScenarioRunner(sc).(agenticScenarioRunner); !ok { - t.Fatalf("selectScenarioRunner(agentic) = %T, want agenticScenarioRunner", selectScenarioRunner(sc)) - } -} - -func TestSelectScenarioRunnerOneShotDefault(t *testing.T) { - sc := scenario.Scenario{} - if _, ok := selectScenarioRunner(sc).(codexScenarioRunner); !ok { - t.Fatalf("selectScenarioRunner(default) = %T, want codexScenarioRunner", selectScenarioRunner(sc)) - } -} - -func TestScenarioEffectiveRunnerMode(t *testing.T) { - if got := (scenario.Scenario{}).EffectiveRunnerMode(); got != scenario.RunnerModeOneShot { - t.Errorf("default = %q", got) - } - if got := (scenario.Scenario{RunnerMode: "agentic"}).EffectiveRunnerMode(); got != scenario.RunnerModeAgentic { - t.Errorf("agentic = %q", got) - } -} - -func TestAgenticRunnerUsedInScenarioABHarness(t *testing.T) { - withAgenticHooks(t, []agenticStep{ - {Commands: []string{"touch gate.txt"}, Done: true, Summary: "gate artifact ready"}, - }) - withFakes(t, - agenticScenarioRunner{}, - fakeCmdJudge{ - with: aoeval.JudgeVerdict{AggregateScore: 0.9}, - without: aoeval.JudgeVerdict{AggregateScore: 0.3}, - }, - ) - dir := t.TempDir() - path := filepath.Join(dir, "agentic.json") - body := `{"id":"s-agentic-001","version":1,"date":"2026-06-18","goal":"create gate.txt","narrative":"execution required","expected_outcome":"gate.txt exists","acceptance_vectors":[{"dimension":"artifact","threshold":0.8}],"satisfaction_threshold":0.8,"runner_mode":"agentic","status":"active"}` - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - out := filepath.Join(t.TempDir(), "card.json") - stdout, err := runScenarioABCmd(t, path, out, 200000) - if err != nil { - t.Fatalf("scenario-ab with agentic runner: %v\n%s", err, stdout) - } - if !strings.Contains(stdout, "gate=PASS") { - t.Errorf("stdout = %q", stdout) - } -} diff --git a/cli/cmd/ao/eval_scenario_ab_sandbox.go b/cli/cmd/ao/eval_scenario_ab_sandbox.go deleted file mode 100644 index bc5c64c6a..000000000 --- a/cli/cmd/ao/eval_scenario_ab_sandbox.go +++ /dev/null @@ -1,196 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - - "github.com/boshu2/agentops/cli/internal/config" - "github.com/boshu2/agentops/cli/internal/wiki" -) - -// corpusMarkers are the dir names that mark a corpus root (and the per-root dirs -// an arm must not read). -var corpusMarkers = []string{".agents", ".ao"} - -// corpusRoot walks up from startDir to the nearest ancestor containing a corpus -// marker (.agents or .ao), so the deny set is correct no matter which subdir the -// command was invoked from (refuter: os.Getwd() under cli/ left /.agents -// readable). Falls back to startDir. -func corpusRoot(startDir string) string { - dir := startDir - for { - for _, m := range corpusMarkers { - if fi, err := os.Stat(filepath.Join(dir, m)); err == nil && fi.IsDir() { - return dir - } - } - parent := filepath.Dir(dir) - if parent == dir { - return startDir - } - dir = parent - } -} - -// corpusDenyPaths returns the absolute corpus directories a scenario-ab arm must -// NOT read: the repo corpus (/.agents, /.ao — resolved from any subdir) -// AND the global corpus (~/.agents), each plus its symlink-resolved canonical path -// (Seatbelt matches the real path). A leak in ANY of these voids the A/B. -func corpusDenyPaths(startDir string) []string { - root := corpusRoot(startDir) - cand := []string{filepath.Join(root, ".agents"), filepath.Join(root, ".ao")} - if home, err := os.UserHomeDir(); err == nil && home != "" { - cand = append(cand, filepath.Join(home, ".agents")) - } - // Deny the ENV-resolved corpus dir too. AO_AGENTS_DIR / AO_HOME override where - // `ao` reads the corpus (wiki.AgentsDirIn — the authoritative resolver); a - // NON-DEFAULT override points OUTSIDE /.agents, leaving it readable by the - // control arm (age-58r). Resolving the deny path through the same resolver `ao` - // uses (not a literal ".agents" join) closes that leak under any env config. - cand = append(cand, envCorpusDenyPaths(root)...) - // Deny the CONFIG-resolved global AND local corpus roots too — they may be - // configured OUTSIDE ~/.agents / /.agents, and `ao lookup` reads exactly - // these (refuter r2; age-58r for the local LearningsDir/PatternsDir). Sourcing - // the deny set from config (the authoritative resolver) — not a hand-enumerated - // list — is what closes the whack-a-mole of "another corpus path". - if cfg, err := config.Load(nil); err == nil && cfg != nil { - cand = append(cand, configCorpusDenyPaths(cfg.Paths.GlobalLearningsDir, cfg.Paths.GlobalPatternsDir)...) - cand = append(cand, localCorpusDenyPaths(root, cfg.Paths.LearningsDir, cfg.Paths.PatternsDir)...) - } - seen := map[string]bool{} - var out []string - for _, p := range cand { - for _, q := range []string{p, resolveSymlink(p)} { - if q != "" && !seen[q] { - seen[q] = true - out = append(out, q) - } - } - } - return out -} - -// configCorpusDenyPaths returns the global corpus directories `ao lookup` resolves -// from config — the global learnings dir, the DERIVED global findings dir -// (Dir(learnings)/findings, exactly as lookup computes it), and the global patterns -// dir. Denying precisely these (sourced from config, not hand-enumerated) closes the -// gap where a global corpus configured outside ~/.agents stayed readable (refuter r2). -// Pure (config values passed in) so it is unit-testable without loading from disk. -func configCorpusDenyPaths(globalLearningsDir, globalPatternsDir string) []string { - var out []string - if d := strings.TrimSpace(globalLearningsDir); d != "" { - out = append(out, d, filepath.Join(filepath.Dir(d), SectionFindings)) - } - if d := strings.TrimSpace(globalPatternsDir); d != "" { - out = append(out, d) - } - return out -} - -// envCorpusDenyPaths returns the corpus dir an AO_AGENTS_DIR / AO_HOME env override -// redirects `ao` to read from (resolved via wiki.AgentsDirIn — the same resolver the -// real code uses). It returns the override only when it resolves to something OTHER -// than the default /.agents (which is already denied): a non-default override is -// the age-58r leak path. With no env override set the override equals the default and -// nothing is added. -func envCorpusDenyPaths(root string) []string { - resolved := wiki.AgentsDirIn(root) - if resolved == "" || resolved == filepath.Join(root, ".agents") { - return nil - } - return []string{resolved} -} - -// localCorpusDenyPaths returns the config-resolved LOCAL (per-repo) corpus dirs — -// cfg.Paths.LearningsDir / PatternsDir — that fall OUTSIDE the already-denied -// /.agents subtree. These default to ".agents/learnings" / ".agents/patterns" -// (relative, under the denied .agents root), but a config can point them at an -// ABSOLUTE path elsewhere; that absolute local corpus stayed readable by the control -// arm (age-58r). A relative dir resolves under root/.agents and is dropped (already -// covered); an absolute dir not under /.agents is denied. Pure (config values -// passed in) so it is unit-testable without loading from disk. -func localCorpusDenyPaths(root, learningsDir, patternsDir string) []string { - var out []string - for _, d := range []string{learningsDir, patternsDir} { - d = strings.TrimSpace(d) - if d == "" || !filepath.IsAbs(d) { - // Relative dirs resolve under /.agents (already denied). - continue - } - if within(d, filepath.Join(root, ".agents")) { - continue - } - out = append(out, d) - } - return out -} - -// within reports whether path p is the same as, or nested under, dir. -func within(p, dir string) bool { - rel, err := filepath.Rel(dir, p) - if err != nil { - return false - } - return rel == "." || (!strings.HasPrefix(rel, ".."+string(filepath.Separator)) && rel != "..") -} - -func resolveSymlink(p string) string { - if r, err := filepath.EvalSymlinks(p); err == nil { - return r - } - return "" -} - -// sandboxExecDenyProfile builds a macOS Seatbelt profile that allows everything -// EXCEPT reads of the given corpus dirs. A denylist (not allowlist) keeps codex -// functional (auth, binary, system libs, temp writes) while the corpus is -// unreadable (age-9a9). All paths must be absolute. -func sandboxExecDenyProfile(denyPaths []string) string { - var b strings.Builder - b.WriteString("(version 1)(allow default)(deny file-read*") - for _, p := range denyPaths { - fmt.Fprintf(&b, " (subpath %q)", p) - } - b.WriteString(")") - return b.String() -} - -// sandboxedCodexArgs returns the argv for sandbox-exec wrapping `codex ` -// with the given profile. Pure (no exec) so the wrapping is unit-testable. -func sandboxedCodexArgs(profile string, codexArgs []string) []string { - return append([]string{"-p", profile, "codex"}, codexArgs...) -} - -// macOSSandboxExec is the Seatbelt binary. -const macOSSandboxExec = "/usr/bin/sandbox-exec" - -// sandboxedCodexCmd builds the *exec.Cmd that runs codex CONFINED so it cannot read -// any of denyPaths. The inner codex runs --dangerously-bypass-approvals-and-sandbox -// (documented for externally-sandboxed environments — the OUTER sandbox-exec is the -// real confinement, and macOS Seatbelt is inherited by child processes). -// -// FAIL-CLOSED: returns an error — never a bare, unconfined codex command — when the -// platform isolation is unavailable or no corpus path was resolved. A control arm -// that silently reads the corpus is precisely the failure this guards. -func sandboxedCodexCmd(ctx context.Context, denyPaths []string, codexArgs []string) (*exec.Cmd, error) { - if len(denyPaths) == 0 { - return nil, fmt.Errorf("arm isolation resolved no corpus paths to deny; refusing to run a potentially-unisolated control arm") - } - switch runtime.GOOS { - case "darwin": - if _, err := os.Stat(macOSSandboxExec); err != nil { - return nil, fmt.Errorf("arm isolation requires %s (macOS Seatbelt); refusing to run an unisolated control arm: %w", macOSSandboxExec, err) - } - profile := sandboxExecDenyProfile(denyPaths) - return exec.CommandContext(ctx, macOSSandboxExec, sandboxedCodexArgs(profile, codexArgs)...), nil - default: - // Linux/WSL bwrap path is age-9a9 S2. Until it lands, fail closed rather - // than run an unisolated (corpus-readable) control arm. - return nil, fmt.Errorf("scenario-ab arm isolation is not yet implemented for GOOS=%q (age-9a9 S2: bubblewrap); refusing to run an unisolated control arm", runtime.GOOS) - } -} diff --git a/cli/cmd/ao/eval_scenario_ab_sandbox_test.go b/cli/cmd/ao/eval_scenario_ab_sandbox_test.go deleted file mode 100644 index aaa3e0f5b..000000000 --- a/cli/cmd/ao/eval_scenario_ab_sandbox_test.go +++ /dev/null @@ -1,243 +0,0 @@ -package main - -import ( - "context" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "testing" -) - -// TestCorpusRoot_WalksUpFromSubdir: the corpus root is found by walking UP to the -// nearest .agents/.ao ancestor, so isolation is correct no matter which subdir the -// command ran from (refuter: os.Getwd() under cli/ left /.agents readable). -func TestCorpusRoot_WalksUpFromSubdir(t *testing.T) { - root := t.TempDir() - if err := os.MkdirAll(filepath.Join(root, ".agents"), 0o755); err != nil { - t.Fatal(err) - } - sub := filepath.Join(root, "cli", "cmd", "ao") - if err := os.MkdirAll(sub, 0o755); err != nil { - t.Fatal(err) - } - if got := corpusRoot(sub); got != root { - t.Errorf("corpusRoot(%q) = %q, want %q (must walk up to the .agents root)", sub, got, root) - } -} - -// TestCorpusDenyPaths_RepoAndGlobal: the deny set covers the repo corpus (both -// .agents and .ao under the resolved root) AND the global ~/.agents. -func TestCorpusDenyPaths_RepoAndGlobal(t *testing.T) { - root := t.TempDir() - if err := os.MkdirAll(filepath.Join(root, ".agents"), 0o755); err != nil { - t.Fatal(err) - } - joined := strings.Join(corpusDenyPaths(root), "|") - for _, want := range []string{filepath.Join(root, ".agents"), filepath.Join(root, ".ao")} { - if !strings.Contains(joined, want) { - t.Errorf("deny set must include repo %q; got %s", want, joined) - } - } - if home, err := os.UserHomeDir(); err == nil && home != "" { - if !strings.Contains(joined, filepath.Join(home, ".agents")) { - t.Errorf("deny set must include the global ~/.agents; got %s", joined) - } - } -} - -// TestConfigCorpusDenyPaths_CustomGlobalRoots: a global corpus configured OUTSIDE -// ~/.agents (refuter r2) is denied, including the derived global findings dir that -// `ao lookup` computes as Dir(globalLearnings)/findings. -func TestConfigCorpusDenyPaths_CustomGlobalRoots(t *testing.T) { - got := configCorpusDenyPaths("/data/corpus/learnings", "/opt/patterns") - joined := strings.Join(got, "|") - for _, want := range []string{"/data/corpus/learnings", filepath.Join("/data/corpus", SectionFindings), "/opt/patterns"} { - if !strings.Contains(joined, want) { - t.Errorf("must deny configured global corpus root %q; got %v", want, got) - } - } - if len(configCorpusDenyPaths("", "")) != 0 { - t.Error("empty config global dirs must add no deny paths") - } -} - -// TestEnvCorpusDenyPaths_NonDefaultOverride: a NON-DEFAULT AO_AGENTS_DIR (or AO_HOME) -// redirects where `ao` reads the corpus OUTSIDE /.agents, so the resolved -// override must enter the deny set; with no override (or the default), nothing is -// added (it equals the already-denied /.agents). age-58r. -func TestEnvCorpusDenyPaths_NonDefaultOverride(t *testing.T) { - root := t.TempDir() - custom := filepath.Join(t.TempDir(), "elsewhere-corpus") - - // AO_AGENTS_DIR override → denied verbatim (the path `ao` actually reads). - t.Setenv("AO_HOME", "") - t.Setenv("AO_AGENTS_DIR", custom) - got := envCorpusDenyPaths(root) - if len(got) != 1 || got[0] != custom { - t.Errorf("AO_AGENTS_DIR override must be denied; got %v want [%q]", got, custom) - } - - // AO_HOME override (no AO_AGENTS_DIR) → also denied. - t.Setenv("AO_AGENTS_DIR", "") - t.Setenv("AO_HOME", custom) - if got := envCorpusDenyPaths(root); len(got) != 1 || got[0] != custom { - t.Errorf("AO_HOME override must be denied; got %v want [%q]", got, custom) - } - - // No override → resolves to the default /.agents (already denied) → nothing added. - t.Setenv("AO_AGENTS_DIR", "") - t.Setenv("AO_HOME", "") - if got := envCorpusDenyPaths(root); len(got) != 0 { - t.Errorf("no env override must add no deny path; got %v", got) - } -} - -// TestEnvCorpusDenyPaths_FlowsIntoCorpusDenyPaths: end-to-end, a non-default -// AO_AGENTS_DIR appears in the full deny set corpusDenyPaths builds (not just the -// helper). The control arm cannot read the override corpus. age-58r. -func TestEnvCorpusDenyPaths_FlowsIntoCorpusDenyPaths(t *testing.T) { - root := t.TempDir() - if err := os.MkdirAll(filepath.Join(root, ".agents"), 0o755); err != nil { - t.Fatal(err) - } - custom := filepath.Join(t.TempDir(), "elsewhere-corpus") - t.Setenv("AO_HOME", "") - t.Setenv("AO_AGENTS_DIR", custom) - joined := strings.Join(corpusDenyPaths(root), "|") - if !strings.Contains(joined, custom) { - t.Errorf("full deny set must include the AO_AGENTS_DIR override %q; got %s", custom, joined) - } -} - -// TestLocalCorpusDenyPaths_AbsoluteOutsideRoot: a config-local LearningsDir/PatternsDir -// pointed at an ABSOLUTE path outside /.agents is denied; a relative dir (the -// default ".agents/learnings", resolving under the already-denied .agents) and an -// absolute dir nested inside /.agents are NOT re-added. age-58r. -func TestLocalCorpusDenyPaths_AbsoluteOutsideRoot(t *testing.T) { - root := t.TempDir() - - // Absolute, outside /.agents → denied. - got := localCorpusDenyPaths(root, "/custom/learnings", "/opt/patterns") - joined := strings.Join(got, "|") - for _, want := range []string{"/custom/learnings", "/opt/patterns"} { - if !strings.Contains(joined, want) { - t.Errorf("must deny absolute local corpus root %q; got %v", want, got) - } - } - - // Relative dirs resolve under /.agents (already denied) → not re-added. - if out := localCorpusDenyPaths(root, ".agents/learnings", ".agents/patterns"); len(out) != 0 { - t.Errorf("relative local dirs must add no deny path (covered by .agents); got %v", out) - } - - // Absolute but nested inside /.agents → already covered, not re-added. - inside := filepath.Join(root, ".agents", "learnings") - if out := localCorpusDenyPaths(root, inside, ""); len(out) != 0 { - t.Errorf("absolute dir inside /.agents must not be re-added; got %v", out) - } - - // Empty → nothing. - if out := localCorpusDenyPaths(root, "", ""); len(out) != 0 { - t.Errorf("empty local dirs must add no deny path; got %v", out) - } -} - -// TestSandboxExecDenyProfile_DeniesAllPaths: every deny path appears in a -// (deny file-read* (subpath ...)) clause, and non-corpus reads remain allowed. -func TestSandboxExecDenyProfile_DeniesAllPaths(t *testing.T) { - p := sandboxExecDenyProfile([]string{"/r/.agents", "/r/.ao", "/home/u/.agents"}) - if !strings.Contains(p, "(allow default)") || !strings.Contains(p, "deny file-read*") { - t.Errorf("profile must allow-default then deny corpus reads; got %s", p) - } - for _, d := range []string{"/r/.agents", "/r/.ao", "/home/u/.agents"} { - if !strings.Contains(p, `(subpath "`+d+`")`) { - t.Errorf("profile must deny subpath %q; got %s", d, p) - } - } -} - -// TestSandboxedCodexArgs_WrapsCodex: codex runs under sandbox-exec with the profile. -func TestSandboxedCodexArgs_WrapsCodex(t *testing.T) { - got := sandboxedCodexArgs("PROFILE", []string{"exec", "--foo", "the prompt"}) - want := []string{"-p", "PROFILE", "codex", "exec", "--foo", "the prompt"} - if len(got) != len(want) { - t.Fatalf("argv = %v (len %d), want len %d", got, len(got), len(want)) - } - for i := range want { - if got[i] != want[i] { - t.Errorf("argv[%d] = %q, want %q", i, got[i], want[i]) - } - } -} - -// TestSandboxedCodexCmd_FailClosed: a sandboxed command on macOS; an ERROR (never a -// bare codex) when isolation is unavailable — empty deny set, or a non-darwin GOOS. -func TestSandboxedCodexCmd_FailClosed(t *testing.T) { - // empty deny set → fail closed on every platform - if cmd, err := sandboxedCodexCmd(context.Background(), nil, []string{"exec"}); err == nil || cmd != nil { - t.Errorf("empty deny set must fail closed; got cmd=%v err=%v", cmd, err) - } - - cmd, err := sandboxedCodexCmd(context.Background(), []string{"/r/.agents", "/r/.ao"}, []string{"exec", "--output-last-message", "/tmp/x"}) - switch runtime.GOOS { - case "darwin": - if err != nil || cmd == nil { - t.Fatalf("darwin: expected a sandboxed command; got cmd=%v err=%v", cmd, err) - } - if !strings.Contains(cmd.Path, "sandbox-exec") { - t.Errorf("darwin: must wrap sandbox-exec; got path %q", cmd.Path) - } - joined := strings.Join(cmd.Args, " ") - if !strings.Contains(joined, "deny file-read*") || !strings.Contains(joined, "codex") { - t.Errorf("darwin: args must carry the deny profile + codex; got %q", joined) - } - default: - if err == nil || cmd != nil { - t.Errorf("non-darwin must fail closed (no unisolated arm); got cmd=%v err=%v", cmd, err) - } - } -} - -// TestSandboxExec_Integration_DeniesCorpusRead (age-wp1): on darwin, sandbox-exec -// with the scenario-ab deny profile must block reads under .agents while allowing -// reads outside the denied corpus paths. String-only profile tests are insufficient. -func TestSandboxExec_Integration_DeniesCorpusRead(t *testing.T) { - if runtime.GOOS != "darwin" { - t.Skip("darwin-only: macOS Seatbelt sandbox-exec integration") - } - if _, err := os.Stat(macOSSandboxExec); err != nil { - t.Skipf("sandbox-exec unavailable: %v", err) - } - - root := t.TempDir() - agents := filepath.Join(root, ".agents") - if err := os.MkdirAll(agents, 0o755); err != nil { - t.Fatal(err) - } - secret := filepath.Join(agents, "sentinel.txt") - if err := os.WriteFile(secret, []byte("CORPUS-SECRET-MUST-NOT-READ"), 0o644); err != nil { - t.Fatal(err) - } - allowed := filepath.Join(root, "allowed.txt") - if err := os.WriteFile(allowed, []byte("allowed-ok"), 0o644); err != nil { - t.Fatal(err) - } - - profile := sandboxExecDenyProfile(corpusDenyPaths(root)) - - denyCmd := exec.Command(macOSSandboxExec, "-p", profile, "/bin/cat", secret) - if err := denyCmd.Run(); err == nil { - t.Fatal("expected sandbox-exec to deny read of corpus file under .agents") - } - - allowCmd := exec.Command(macOSSandboxExec, "-p", profile, "/bin/cat", allowed) - out, err := allowCmd.CombinedOutput() - if err != nil { - t.Fatalf("expected read outside deny set to succeed: %v output=%q", err, out) - } - if !strings.Contains(string(out), "allowed-ok") { - t.Errorf("allowed read output = %q, want allowed-ok", out) - } -} diff --git a/cli/cmd/ao/eval_scenario_ab_test.go b/cli/cmd/ao/eval_scenario_ab_test.go deleted file mode 100644 index 0399d7d68..000000000 --- a/cli/cmd/ao/eval_scenario_ab_test.go +++ /dev/null @@ -1,396 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" - - aoeval "github.com/boshu2/agentops/cli/internal/eval" - "github.com/boshu2/agentops/cli/internal/scenario" -) - -type fakeCmdRunner struct { - with, without aoeval.ArmOutcome -} - -func (f fakeCmdRunner) RunArm(_ context.Context, _ scenario.Scenario, withGold bool) (aoeval.ArmOutcome, error) { - if withGold { - return f.with, nil - } - return f.without, nil -} - -type fakeCmdJudge struct { - with, without aoeval.JudgeVerdict -} - -func (f fakeCmdJudge) Judge(_ context.Context, _ scenario.Scenario, arm aoeval.ScenarioArm, _ aoeval.ArmOutcome) (aoeval.JudgeVerdict, error) { - if arm == aoeval.ArmWithGold { - return f.with, nil - } - return f.without, nil -} - -func writeCmdFixture(t *testing.T, threshold float64) string { - t.Helper() - dir := t.TempDir() - res, err := scenario.Create(scenario.CreateOptions{ - Goal: "design a reward mechanism that gates on a verified outcome", - Threshold: threshold, - Status: "active", - Source: "human", - Dir: dir, - Now: func() time.Time { return time.Date(2026, 6, 16, 0, 0, 0, 0, time.UTC) }, - }) - if err != nil { - t.Fatalf("create fixture: %v", err) - } - return res.Path -} - -// withFakes overrides the injectable factory seam for the duration of a test. -func withFakes(t *testing.T, runner aoeval.ScenarioRunner, judge aoeval.ScenarioJudge) { - t.Helper() - origR, origJ := scenarioABRunnerFactory, scenarioABJudgeFactory - scenarioABRunnerFactory = func() aoeval.ScenarioRunner { return runner } - scenarioABJudgeFactory = func() aoeval.ScenarioJudge { return judge } - t.Cleanup(func() { - scenarioABRunnerFactory = origR - scenarioABJudgeFactory = origJ - evalScenarioABScenario = "" - evalScenarioABOutput = "" - evalScenarioABBudget = 0 - evalScenarioABControlOnly = false - }) -} - -func runScenarioABCmd(t *testing.T, scenarioPath, outPath string, budget int) (string, error) { - t.Helper() - cmd := evalScenarioABCmd - var buf bytes.Buffer - cmd.SetOut(&buf) - cmd.SetErr(&buf) - t.Cleanup(func() { cmd.SetOut(nil); cmd.SetErr(nil) }) // age-ztf8: shared command; don't leak the writer - cmd.SetContext(context.Background()) - evalScenarioABScenario = scenarioPath - evalScenarioABOutput = outPath - evalScenarioABBudget = budget - err := cmd.RunE(cmd, nil) - return buf.String(), err -} - -func runScenarioABControlOnlyCmd(t *testing.T, scenarioPath, outPath string, budget int) (string, error) { - t.Helper() - cmd := evalScenarioABCmd - var buf bytes.Buffer - cmd.SetOut(&buf) - cmd.SetErr(&buf) - t.Cleanup(func() { cmd.SetOut(nil); cmd.SetErr(nil) }) // age-ztf8: shared command; don't leak the writer - cmd.SetContext(context.Background()) - evalScenarioABScenario = scenarioPath - evalScenarioABOutput = outPath - evalScenarioABBudget = budget - evalScenarioABControlOnly = true - err := cmd.RunE(cmd, nil) - return buf.String(), err -} - -func TestEvalScenarioABCmd_GatePassWritesScorecard(t *testing.T) { - withFakes(t, - fakeCmdRunner{ - with: aoeval.ArmOutcome{Output: "treatment", TokenCost: 100}, - without: aoeval.ArmOutcome{Output: "control", TokenCost: 100}, - }, - fakeCmdJudge{ - with: aoeval.JudgeVerdict{AggregateScore: 0.9}, - without: aoeval.JudgeVerdict{AggregateScore: 0.4}, - }, - ) - out := filepath.Join(t.TempDir(), "card.json") - stdout, err := runScenarioABCmd(t, writeCmdFixture(t, 0.8), out, 200000) - if err != nil { - t.Fatalf("expected pass, got error: %v\n%s", err, stdout) - } - if !strings.Contains(stdout, "gate=PASS") { - t.Errorf("stdout missing gate=PASS: %q", stdout) - } - if !strings.Contains(stdout, "delta=0.5000") { - t.Errorf("stdout missing expected delta: %q", stdout) - } - // Scorecard must be persisted and decode back into the typed struct. - data, readErr := os.ReadFile(out) - if readErr != nil { - t.Fatalf("read scorecard: %v", readErr) - } - var card aoeval.ScenarioDeltaScorecard - if jsonErr := json.Unmarshal(data, &card); jsonErr != nil { - t.Fatalf("decode scorecard: %v", jsonErr) - } - if !card.Gate.Pass || card.AggregateDelta != 0.5 { - t.Errorf("scorecard = pass:%v delta:%.4f, want pass:true delta:0.5", card.Gate.Pass, card.AggregateDelta) - } -} - -func TestEvalScenarioABCmd_GateFailReturnsError(t *testing.T) { - withFakes(t, - fakeCmdRunner{ - with: aoeval.ArmOutcome{TokenCost: 100}, - without: aoeval.ArmOutcome{TokenCost: 100}, - }, - // Equal scores -> delta 0 -> fail-loud. Both BELOW the 0.8 threshold so this - // exercises the delta<=0 gate path, not the ceiling pre-screen (control above - // threshold is a ceiling violation — a different code path, covered separately). - fakeCmdJudge{ - with: aoeval.JudgeVerdict{AggregateScore: 0.5}, - without: aoeval.JudgeVerdict{AggregateScore: 0.5}, - }, - ) - stdout, err := runScenarioABCmd(t, writeCmdFixture(t, 0.8), "", 200000) - if err == nil { - t.Fatalf("expected gate-fail error, got nil\n%s", stdout) - } - if !strings.Contains(stdout, "gate=FAIL") { - t.Errorf("stdout missing gate=FAIL: %q", stdout) - } -} - -// TestEvalScenarioABCmd_CeilingViolation: when the control arm clears the -// threshold (no headroom), the CLI must surface CEILING VIOLATION (not a -// misleading delta) and exit non-zero (age-707 validity certificate). -func TestEvalScenarioABCmd_CeilingViolation(t *testing.T) { - withFakes(t, - fakeCmdRunner{ - with: aoeval.ArmOutcome{TokenCost: 100}, - without: aoeval.ArmOutcome{TokenCost: 100}, - }, - // control already clears the 0.8 threshold -> ceiling violation. - fakeCmdJudge{ - with: aoeval.JudgeVerdict{AggregateScore: 0.6}, - without: aoeval.JudgeVerdict{AggregateScore: 0.9}, - }, - ) - stdout, err := runScenarioABCmd(t, writeCmdFixture(t, 0.8), "", 200000) - if err == nil { - t.Fatalf("expected non-zero exit on ceiling violation, got nil\n%s", stdout) - } - if !strings.Contains(stdout, "CEILING VIOLATION") { - t.Errorf("stdout missing CEILING VIOLATION: %q", stdout) - } -} - -func TestEvalScenarioABCmd_ControlOnlyHeadroomPass(t *testing.T) { - withFakes(t, - fakeCmdRunner{ - with: aoeval.ArmOutcome{Output: "treatment should not run", TokenCost: 10000}, - without: aoeval.ArmOutcome{Output: "control misses", TokenCost: 100}, - }, - fakeCmdJudge{ - with: aoeval.JudgeVerdict{AggregateScore: 1.0}, - without: aoeval.JudgeVerdict{AggregateScore: 0.3}, - }, - ) - out := filepath.Join(t.TempDir(), "card.json") - stdout, err := runScenarioABControlOnlyCmd(t, writeCmdFixture(t, 0.8), out, 200000) - if err != nil { - t.Fatalf("control-only headroom should pass, got error: %v\n%s", err, stdout) - } - if !strings.Contains(stdout, "control-only headroom=PASS") { - t.Errorf("stdout missing control-only PASS: %q", stdout) - } - data, readErr := os.ReadFile(out) - if readErr != nil { - t.Fatalf("read scorecard: %v", readErr) - } - var card aoeval.ScenarioDeltaScorecard - if jsonErr := json.Unmarshal(data, &card); jsonErr != nil { - t.Fatalf("decode scorecard: %v", jsonErr) - } - if !card.ControlOnly || !card.Gate.Pass || card.CeilingViolation { - t.Fatalf("scorecard = control_only:%v pass:%v ceiling:%v, want true/true/false", card.ControlOnly, card.Gate.Pass, card.CeilingViolation) - } -} - -func TestEvalScenarioABCmd_ControlOnlyCeilingViolationFails(t *testing.T) { - withFakes(t, - fakeCmdRunner{ - with: aoeval.ArmOutcome{Output: "treatment should not run", TokenCost: 10000}, - without: aoeval.ArmOutcome{Output: "control solves it", TokenCost: 100}, - }, - fakeCmdJudge{ - with: aoeval.JudgeVerdict{AggregateScore: 1.0}, - without: aoeval.JudgeVerdict{AggregateScore: 0.9}, - }, - ) - stdout, err := runScenarioABControlOnlyCmd(t, writeCmdFixture(t, 0.8), "", 200000) - if err == nil { - t.Fatalf("expected control-only ceiling to fail\n%s", stdout) - } - if !strings.Contains(stdout, "CEILING VIOLATION") { - t.Errorf("stdout missing CEILING VIOLATION: %q", stdout) - } -} - -// TestEvalScenarioABCmd_AnswerKeyDeterministic: a scenario carrying answer_key is -// graded by the deterministic AnswerKeyJudge (no LLM). With-gold output contains the -// key (pass), without-gold does not (fail) → a clean positive delta that passes the -// gate. This is the OOD fact-recall path (age-k8u). -func TestEvalScenarioABCmd_AnswerKeyDeterministic(t *testing.T) { - withFakes(t, - fakeCmdRunner{ - with: aoeval.ArmOutcome{Output: "the private value is WIDGET-42", TokenCost: 10}, - without: aoeval.ArmOutcome{Output: "I do not have that information", TokenCost: 10}, - }, - fakeCmdJudge{}, // ignored — a scenario answer_key selects the deterministic judge - ) - dir := t.TempDir() - res, err := scenario.Create(scenario.CreateOptions{ - Goal: "recall a private fleet-specific value the base model cannot know", - Threshold: 0.8, AnswerKey: "WIDGET-42", - Status: "active", Source: "human", Dir: dir, - Now: func() time.Time { return time.Date(2026, 6, 16, 0, 0, 0, 0, time.UTC) }, - }) - if err != nil { - t.Fatalf("create fixture: %v", err) - } - stdout, err := runScenarioABCmd(t, res.Path, filepath.Join(dir, "card.json"), 200000) - if err != nil { - t.Fatalf("valid OOD answer-key A/B should pass the gate; got err %v\n%s", err, stdout) - } - if !strings.Contains(stdout, "gate=PASS") { - t.Errorf("with-gold (key present) over without-gold (absent) should PASS; got %q", stdout) - } - // age-6ys: a fact-recall scorecard can PASS its plumbing gate but the CLI must - // loudly mark it NOT moat evidence so a plumbing PASS is never the moat verdict. - if !strings.Contains(stdout, "[fact-recall]") { - t.Errorf("stdout missing [fact-recall] verdict class: %q", stdout) - } - if !strings.Contains(stdout, "NOT-moat-evidence(plumbing)") { - t.Errorf("a passing fact-recall card must print NOT-moat-evidence(plumbing): %q", stdout) - } -} - -// TestMoatLabel locks the age-6ys CLI signal: an applied-OOD (moat-eligible) card -// reads moat-eligible; a fact-recall/plumbing card reads NOT-moat-evidence so it -// cannot be misread as the moat verdict. -func TestMoatLabel(t *testing.T) { - if got := moatLabel(true); got != "moat-eligible" { - t.Errorf("moatLabel(true) = %q, want moat-eligible", got) - } - if got := moatLabel(false); got != "NOT-moat-evidence(plumbing)" { - t.Errorf("moatLabel(false) = %q, want NOT-moat-evidence(plumbing)", got) - } -} - -// TestTaskSuccessJudgePrompt_GradesPerArmSuccess locks the age-oe2 fix: the LLM -// judge grades each arm's ABSOLUTE task-success against the goal/expected-outcome, -// NOT the scenario's A/B-contrast acceptance_vectors (which describe the comparison -// and are incoherent per-arm — the KF-4 −0.34 noise). It also keeps the grade -// absolute (per-arm) so the age-707 ceiling pre-screen still has a control score. -func TestTaskSuccessJudgePrompt_GradesPerArmSuccess(t *testing.T) { - sc := scenario.Scenario{ - Goal: "dispatch codex exec in a non-TTY context", - ExpectedOutcome: "the process engages and completes", - AcceptanceVectors: []scenario.AcceptanceVector{ - {Dimension: "control-hangs", Threshold: 0.8}, - {Dimension: "delta-attributable", Threshold: 0.8}, - }, - } - p := taskSuccessJudgePrompt(sc, "some arm output") - if !strings.Contains(p, sc.Goal) || !strings.Contains(p, sc.ExpectedOutcome) { - t.Error("prompt must include the goal and expected outcome") - } - if !strings.Contains(strings.ToLower(p), "task") || !strings.Contains(p, "do NOT compare") { - t.Error("prompt must instruct absolute, per-arm, on-its-own-merits task-success grading") - } - if strings.Contains(p, "control-hangs") || strings.Contains(p, "delta-attributable") { - t.Error("prompt must NOT enumerate the A/B-contrast acceptance vectors (the per-arm incoherence)") - } -} - -func TestEvalScenarioABCmd_MissingScenarioFlag(t *testing.T) { - withFakes(t, fakeCmdRunner{}, fakeCmdJudge{}) - if _, err := runScenarioABCmd(t, "", "", 0); err == nil { - t.Error("expected error when --scenario is empty") - } -} - -func TestParseJudgeJSON(t *testing.T) { - cases := []struct { - name string - in string - wantAgg float64 - wantErr bool - }{ - {"bare json", `{"vectors":[],"aggregate_score":0.75}`, 0.75, false}, - {"json wrapped in prose", "Here is my verdict:\n{\"aggregate_score\":0.9}\nDone.", 0.9, false}, - {"no json", "no object here", 0, true}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - v, err := parseJudgeJSON(tc.in) - if tc.wantErr { - if err == nil { - t.Error("expected error") - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if v.AggregateScore != tc.wantAgg { - t.Errorf("aggregate = %.4f, want %.4f", v.AggregateScore, tc.wantAgg) - } - }) - } -} - -func TestParseCodexTokens(t *testing.T) { - cases := []struct { - in string - want int - }{ - {"... \ntokens used\n147,396\n", 147396}, - {"tokens used: 1024", 1024}, - {"no token line here, just 12 chars-ish", len("no token line here, just 12 chars-ish") / 4}, - } - for _, tc := range cases { - if got := parseCodexTokens(tc.in); got != tc.want { - t.Errorf("parseCodexTokens(%q) = %d, want %d", tc.in, got, tc.want) - } - } -} - -// TestJudgeOutputSchema_StrictMode guards the exact codex strict-output -// requirement that every object set "additionalProperties": false and list all -// its properties in "required" — a missing additionalProperties yields a 400 -// invalid_json_schema (this cost a live run to discover, KF-C). -func TestJudgeOutputSchema_StrictMode(t *testing.T) { - var root map[string]any - if err := json.Unmarshal([]byte(judgeOutputSchema), &root); err != nil { - t.Fatalf("judgeOutputSchema is not valid JSON: %v", err) - } - assertStrictObject := func(t *testing.T, obj map[string]any, where string) { - t.Helper() - if ap, ok := obj["additionalProperties"]; !ok || ap != false { - t.Errorf("%s: additionalProperties must be present and false (codex strict mode), got %v", where, ap) - } - props, _ := obj["properties"].(map[string]any) - req, _ := obj["required"].([]any) - if len(props) != len(req) { - t.Errorf("%s: every property must be in required (strict mode): %d props, %d required", where, len(props), len(req)) - } - } - assertStrictObject(t, root, "root") - props, _ := root["properties"].(map[string]any) - vectors, _ := props["vectors"].(map[string]any) - if items, ok := vectors["items"].(map[string]any); ok { - assertStrictObject(t, items, "vectors.items") - } else { - t.Error("vectors.items object missing") - } -} diff --git a/cli/cmd/ao/eval_scenario_moat.go b/cli/cmd/ao/eval_scenario_moat.go deleted file mode 100644 index 9a3b73c71..000000000 --- a/cli/cmd/ao/eval_scenario_moat.go +++ /dev/null @@ -1,73 +0,0 @@ -// practices: [llm-eval-harness] -package main - -import ( - "errors" - "fmt" - - "github.com/spf13/cobra" - - aoeval "github.com/boshu2/agentops/cli/internal/eval" -) - -var ( - evalScenarioMoatScorecards []string - evalScenarioMoatOutput string -) - -var evalScenarioMoatCmd = &cobra.Command{ - Use: "scenario-moat", - Short: "Aggregate moat-eligible scenario A/B scorecards into a publication verdict", - Long: `Render a moat positive/null/inconclusive verdict over one or more -ScenarioDeltaScorecard JSON artifacts from ao eval scenario-ab. - -The claim surface fail-closes on any scorecard with moat_eligible=false — a -fact-recall/plumbing scorecard can pass its own gate but must NEVER be aggregated -into a moat verdict (age-6ys/age-sb0). See docs/evals/applied-ood-claim-rule.md.`, - Args: cobra.NoArgs, - SilenceUsage: true, - RunE: func(cmd *cobra.Command, _ []string) error { - if len(evalScenarioMoatScorecards) == 0 { - return fmt.Errorf("at least one --scorecard path is required") - } - cards := make([]aoeval.ScenarioDeltaScorecard, 0, len(evalScenarioMoatScorecards)) - for _, path := range evalScenarioMoatScorecards { - card, err := aoeval.LoadScenarioDeltaScorecard(path) - if err != nil { - return err - } - cards = append(cards, card) - } - result, err := aoeval.AggregateMoatClaim(cards) - if err != nil { - var ineligible aoeval.ErrMoatIneligibleScorecard - if errors.As(err, &ineligible) { - fmt.Fprintf(cmd.ErrOrStderr(), "REJECTED: %s\n", err) - } - return err - } - if err := aoeval.WriteMoatClaimResult(evalScenarioMoatOutput, result); err != nil { - return err - } - if GetOutput() == "json" { - return writeEvalJSON(cmd, result) - } - fmt.Fprintf(cmd.OutOrStdout(), - "scenario-moat: verdict=%s scenarios=%d mean_delta=%.4f\n %s\n", - result.Verdict, result.ScenarioCount, result.MeanAggregateDelta, result.Reason, - ) - if evalScenarioMoatOutput != "" { - fmt.Fprintf(cmd.OutOrStdout(), "Moat claim result: %s\n", evalScenarioMoatOutput) - } - if result.Verdict == aoeval.MoatVerdictInconclusive { - return fmt.Errorf("moat claim inconclusive — cannot publish positive or honest null") - } - return nil - }, -} - -func init() { - evalScenarioMoatCmd.Flags().StringArrayVar(&evalScenarioMoatScorecards, "scorecard", nil, "Path to a ScenarioDeltaScorecard JSON (repeatable)") - evalScenarioMoatCmd.Flags().StringVar(&evalScenarioMoatOutput, "output", "", "Write the MoatClaimResult JSON to this path") - evalCmd.AddCommand(evalScenarioMoatCmd) -} diff --git a/cli/cmd/ao/eval_scenario_moat_test.go b/cli/cmd/ao/eval_scenario_moat_test.go deleted file mode 100644 index 851b3b388..000000000 --- a/cli/cmd/ao/eval_scenario_moat_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package main - -import ( - "bytes" - "path/filepath" - "runtime" - "strings" - "testing" -) - -func withMoatCmdReset(t *testing.T) { - t.Helper() - t.Cleanup(func() { - evalScenarioMoatScorecards = nil - evalScenarioMoatOutput = "" - }) -} - -func runScenarioMoatCmd(t *testing.T, scorecards []string, outPath string) (string, error) { - t.Helper() - cmd := evalScenarioMoatCmd - var buf bytes.Buffer - cmd.SetOut(&buf) - cmd.SetErr(&buf) - t.Cleanup(func() { cmd.SetOut(nil); cmd.SetErr(nil) }) // age-ztf8: shared command; don't leak the writer - evalScenarioMoatScorecards = scorecards - evalScenarioMoatOutput = outPath - err := cmd.RunE(cmd, nil) - return buf.String(), err -} - -func TestEvalScenarioMoatRejectsPlumbingScorecard(t *testing.T) { - withMoatCmdReset(t) - root := repoRootForEvalCmd(t) - plumbing := filepath.Join(root, "evals/scenarios/fixtures/scenario-ab-fact-recall-plumbing.scorecard.json") - _, err := runScenarioMoatCmd(t, []string{plumbing}, "") - if err == nil { - t.Fatal("expected error when aggregating moat_eligible=false scorecard") - } - if !strings.Contains(err.Error(), "moat_eligible=false") { - t.Fatalf("error = %q, want moat_eligible=false rejection", err.Error()) - } -} - -func TestEvalScenarioMoatPositiveFromFixture(t *testing.T) { - withMoatCmdReset(t) - root := repoRootForEvalCmd(t) - valid := filepath.Join(root, "evals/scenarios/fixtures/scenario-ab-valid-redacted.scorecard.json") - out := filepath.Join(t.TempDir(), "moat-claim.json") - _, err := runScenarioMoatCmd(t, []string{valid}, out) - if err != nil { - t.Fatalf("runScenarioMoatCmd: %v", err) - } -} - -func repoRootForEvalCmd(t *testing.T) string { - t.Helper() - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - return filepath.Join(filepath.Dir(file), "..", "..", "..") -} diff --git a/cli/cmd/ao/eval_session_outcome.go b/cli/cmd/ao/eval_session_outcome.go deleted file mode 100644 index afdedf7ad..000000000 --- a/cli/cmd/ao/eval_session_outcome.go +++ /dev/null @@ -1,23 +0,0 @@ -// practices: [ai-assisted-dev, pragmatic-programmer] -package main - -import "github.com/spf13/cobra" - -// evalSessionOutcomeCmd is the canonical `ao eval session-outcome` alias for the -// transcript reward-signal analyzer, folded under `ao eval` -// (age-focus-membrane-bookkeeper-m1wg.16). The hidden top-level -// `ao session-outcome` is preserved for back-compat (see session_outcome.go). -// Use "session-outcome" (not "outcomes") to avoid colliding with the existing -// `ao eval outcomes` rubric command. -var evalSessionOutcomeCmd = &cobra.Command{ - Use: "session-outcome [transcript-path]", - Short: "Analyze session transcript to derive reward signal", - Long: sessionOutcomeCmd.Long, - Args: cobra.MaximumNArgs(1), - RunE: runSessionOutcome, -} - -func init() { - bindSessionOutcomeFlags(evalSessionOutcomeCmd) - evalCmd.AddCommand(evalSessionOutcomeCmd) -} diff --git a/cli/cmd/ao/eval_suite.go b/cli/cmd/ao/eval_suite.go deleted file mode 100644 index d335172dc..000000000 --- a/cli/cmd/ao/eval_suite.go +++ /dev/null @@ -1,286 +0,0 @@ -// practices: [llm-eval-harness, dora-metrics] -package main - -import ( - "encoding/json" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/spf13/cobra" - - evalsub "github.com/boshu2/agentops/cli/internal/evalsubstrate" -) - -var ( - evalSuiteVerdictBootstrapInputs string - evalSuiteVerdictMDE float64 - evalSuiteVerdictB int - evalSuiteVerdictNRequired int -) - -var evalSuiteCmd = &cobra.Command{ - Use: "suite", - Short: "Suite-level operations (verdict, n-required)", - Long: `Suite-level operations against the §6.5 statistical contract. - -Subcommands: - - ao eval suite verdict --arms a,b --inputs - Compute the §6.5 paired cluster-bootstrap verdict for two Run arms. - Reads canonical-JSON inputs (one row per (sample_id, seed) pair with - both arm scores), derives bootstrap_seed deterministically per §6.5, - runs B=10000 resamples (override via --B), maps to the §6.5 5-verdict - enum, and emits all manifest fields the run writer needs. - - ao eval suite n-required --baseline-rate

--mde --alpha - Compute power-derived n_required (Day-3 graduates gate #6 to use this - instead of Task.stats.min_n_samples). - -The actual bootstrap math + RNG (numpy.random.PCG64) lives in -~/.agents/evals/_stats/. The Go CLI shells out so both Go and shell-script -callers go through the same code path — bit-exact reproducibility is a -property of the Python module, not the CLI surface.`, -} - -var evalSuiteVerdictCmd = &cobra.Command{ - Use: "verdict --arms a,b --inputs ", - Short: "Compute the §6.5 paired cluster-bootstrap verdict", - Args: cobra.ExactArgs(1), - RunE: runEvalSuiteVerdict, -} - -func runEvalSuiteVerdict(cmd *cobra.Command, args []string) error { - suiteID := args[0] - if evalSuiteVerdictBootstrapInputs == "" { - return fmt.Errorf("eval suite verdict: --inputs is required") - } - suite, err := loadSuite(suiteID) - if err != nil { - // Suite ID may not be on disk yet (ad-hoc verdict); skip lookup if so. - suite = nil - } - - armsCSV := strings.Join(extractArmIDs(suite, evalSuiteVerdictArmsRaw), ",") - if strings.TrimSpace(armsCSV) == "" { - return fmt.Errorf("eval suite verdict: --arms required when suite has no varied_axis on disk") - } - - rule, err := decisionRuleJSON(suite) - if err != nil { - return fmt.Errorf("eval suite verdict: %w", err) - } - nReq := evalSuiteVerdictNRequired - if nReq <= 0 { - nReq = derivedNRequired(suite) - } - - pyArgs := []string{ - "-m", "_stats.cli", "verdict", - "--suite-id", suiteID, - "--arms", armsCSV, - "--inputs", evalSuiteVerdictBootstrapInputs, - "--decision-rule", rule, - "--n-required", fmt.Sprintf("%d", nReq), - "--B", fmt.Sprintf("%d", evalSuiteVerdictB), - } - if evalSuiteVerdictMDE > 0 { - pyArgs = append(pyArgs, "--mde", fmt.Sprintf("%g", evalSuiteVerdictMDE)) - } - - out, err := runStatsCLI(pyArgs) - if err != nil { - return err - } - if GetOutput() == "json" { - _, _ = cmd.OutOrStdout().Write(out) - _, _ = cmd.OutOrStdout().Write([]byte("\n")) - return nil - } - - var result map[string]interface{} - if err := json.Unmarshal(out, &result); err != nil { - return fmt.Errorf("eval suite verdict: parse stats output: %w", err) - } - fmt.Fprintf(cmd.OutOrStdout(), - "Suite verdict: %v\n delta_point: %v\n ci: [%v, %v]\n n_clusters: %v\n n_required: %v\n bootstrap_seed: %v\n bootstrap_inputs_hash: %v\n", - result["verdict"], result["delta_point"], result["ci_low"], result["ci_high"], - result["n_clusters"], result["n_required"], result["bootstrap_seed"], - result["bootstrap_inputs_hash"]) - return nil -} - -var evalSuiteNRequiredCmd = &cobra.Command{ - Use: "n-required", - Short: "Compute power-derived n_required (gate #6 input on Day 3+)", - RunE: runEvalSuiteNRequired, -} - -var ( - evalSuiteNRReqBaselineRate float64 - evalSuiteNRReqMDE float64 - evalSuiteNRReqAlpha float64 - evalSuiteNRReqPower float64 - evalSuiteNRReqPaired bool -) - -func runEvalSuiteNRequired(cmd *cobra.Command, args []string) error { - pyArgs := []string{ - "-m", "_stats.cli", "n-required", - "--baseline-rate", fmt.Sprintf("%g", evalSuiteNRReqBaselineRate), - "--mde", fmt.Sprintf("%g", evalSuiteNRReqMDE), - "--alpha", fmt.Sprintf("%g", evalSuiteNRReqAlpha), - "--power", fmt.Sprintf("%g", evalSuiteNRReqPower), - "--paired", fmt.Sprintf("%v", evalSuiteNRReqPaired), - } - out, err := runStatsCLI(pyArgs) - if err != nil { - return err - } - if GetOutput() == "json" { - _, _ = cmd.OutOrStdout().Write(out) - _, _ = cmd.OutOrStdout().Write([]byte("\n")) - return nil - } - var got struct { - NRequired int `json:"n_required"` - } - if err := json.Unmarshal(out, &got); err != nil { - return fmt.Errorf("eval suite n-required: parse: %w", err) - } - fmt.Fprintf(cmd.OutOrStdout(), "n_required: %d (baseline_rate=%g MDE=%g alpha=%g power=%g paired=%v)\n", - got.NRequired, evalSuiteNRReqBaselineRate, evalSuiteNRReqMDE, - evalSuiteNRReqAlpha, evalSuiteNRReqPower, evalSuiteNRReqPaired) - return nil -} - -var evalSuiteVerdictArmsRaw string - -func registerEvalSuiteCmd() { - evalSuiteVerdictCmd.Flags().StringVar(&evalSuiteVerdictArmsRaw, "arms", "", "Comma-separated arm ids (default: from suite varied_axis)") - evalSuiteVerdictCmd.Flags().StringVar(&evalSuiteVerdictBootstrapInputs, "inputs", "", "Path to canonical bootstrap-inputs JSON (REQUIRED)") - evalSuiteVerdictCmd.Flags().Float64Var(&evalSuiteVerdictMDE, "mde", 0, "Minimum detectable effect (used for inconclusive_high_variance)") - evalSuiteVerdictCmd.Flags().IntVar(&evalSuiteVerdictB, "B", 10000, "Bootstrap resamples") - evalSuiteVerdictCmd.Flags().IntVar(&evalSuiteVerdictNRequired, "n-required", 0, "Override n_required (default: derived from suite power block)") - - evalSuiteNRequiredCmd.Flags().Float64Var(&evalSuiteNRReqBaselineRate, "baseline-rate", 0.5, "Baseline rate (binomial worst-case fallback)") - evalSuiteNRequiredCmd.Flags().Float64Var(&evalSuiteNRReqMDE, "mde", 0.05, "Minimum detectable effect") - evalSuiteNRequiredCmd.Flags().Float64Var(&evalSuiteNRReqAlpha, "alpha", 0.05, "Type-I error rate") - evalSuiteNRequiredCmd.Flags().Float64Var(&evalSuiteNRReqPower, "power", 0.80, "Statistical power (1-beta)") - evalSuiteNRequiredCmd.Flags().BoolVar(&evalSuiteNRReqPaired, "paired", true, "Paired comparison") - - evalSuiteCmd.AddCommand(evalSuiteVerdictCmd, evalSuiteNRequiredCmd) -} - -func extractArmIDs(suite *evalsub.Suite, override string) []string { - if override != "" { - parts := strings.Split(override, ",") - out := make([]string, 0, len(parts)) - for _, p := range parts { - if t := strings.TrimSpace(p); t != "" { - out = append(out, t) - } - } - return out - } - if suite != nil && len(suite.VariedAxis.Values) > 0 { - return suite.VariedAxis.Values - } - return nil -} - -func decisionRuleJSON(suite *evalsub.Suite) (string, error) { - rule := map[string]interface{}{"kind": "ci_excludes_zero", "confidence": 0.95} - if suite != nil { - if suite.Stats.DecisionRule.Kind != "" { - rule["kind"] = suite.Stats.DecisionRule.Kind - } - if suite.Stats.DecisionRule.Confidence > 0 { - rule["confidence"] = suite.Stats.DecisionRule.Confidence - } - if suite.Stats.DecisionRule.MinDelta > 0 { - rule["min_delta"] = suite.Stats.DecisionRule.MinDelta - } - } - // rule carries float64 values (e.g. DecisionRule.Confidence / .MinDelta), - // and json.Marshal DOES fail on non-finite floats (NaN/+Inf/-Inf). Do not - // swallow it: propagate so a bad suite surfaces instead of silently - // emitting an empty decision rule. - bs, err := json.Marshal(rule) - if err != nil { - return "", fmt.Errorf("marshal decision rule: %w", err) - } - return string(bs), nil -} - -func derivedNRequired(suite *evalsub.Suite) int { - if suite == nil || suite.Stats.Power == nil { - return 0 - } - pyArgs := []string{ - "-m", "_stats.cli", "n-required", - "--baseline-rate", "0.5", - "--mde", fmt.Sprintf("%g", suite.Stats.Power.MinimumDetectableEffect), - "--alpha", fmt.Sprintf("%g", suite.Stats.Power.Alpha), - "--power", "0.80", - "--paired", "true", - } - out, err := runStatsCLI(pyArgs) - if err != nil { - return 0 - } - var got struct { - NRequired int `json:"n_required"` - } - if err := json.Unmarshal(out, &got); err != nil { - return 0 - } - return got.NRequired -} - -// runStatsCLI shells out to the substrate venv's `python -m _stats.cli ...`. -// Looks for AGENTOPS_EVALS_VENV (override) -> $AGENTOPS_EVALS_ROOT/.venv/bin/python -> -// ~/.agents/evals/.venv/bin/python. -func runStatsCLI(args []string) ([]byte, error) { - py := pythonBinary() - if py == "" { - return nil, fmt.Errorf("eval suite: substrate venv not found; provision via `python3 -m venv ~/.agents/evals/.venv && pip install numpy scipy`") - } - root := evalsRoot() - cmd := exec.Command(py, args...) - // PYTHONPATH so `_stats` is importable. - cmd.Env = append(os.Environ(), "PYTHONPATH="+root) - out, err := cmd.Output() - if err != nil { - stderr := "" - if ee, ok := err.(*exec.ExitError); ok { - stderr = string(ee.Stderr) - } - return nil, fmt.Errorf("eval suite: stats CLI failed: %w (stderr: %s)", err, stderr) - } - return out, nil -} - -func pythonBinary() string { - if v := strings.TrimSpace(os.Getenv("AGENTOPS_EVALS_VENV")); v != "" { - return v - } - candidates := []string{ - filepath.Join(evalsRoot(), ".venv", "bin", "python"), - filepath.Join(evalsRoot(), ".venv", "bin", "python3"), - } - if home, err := os.UserHomeDir(); err == nil { - candidates = append(candidates, - filepath.Join(home, ".agents", "evals", ".venv", "bin", "python"), - filepath.Join(home, ".agents", "evals", ".venv", "bin", "python3"), - ) - } - for _, c := range candidates { - if _, err := os.Stat(c); err == nil { - return c - } - } - return "" -} diff --git a/cli/cmd/ao/eval_task.go b/cli/cmd/ao/eval_task.go deleted file mode 100644 index cda606195..000000000 --- a/cli/cmd/ao/eval_task.go +++ /dev/null @@ -1,479 +0,0 @@ -// practices: [llm-eval-harness, dora-metrics] -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/spf13/cobra" - "gopkg.in/yaml.v3" - - evalsub "github.com/boshu2/agentops/cli/internal/evalsubstrate" -) - -// AGENTOPS_EVALS_ROOT lets callers (and tests) override the substrate -// root directory. Default ~/.agents/evals. -func evalsRoot() string { - if r := strings.TrimSpace(os.Getenv("AGENTOPS_EVALS_ROOT")); r != "" { - return r - } - home, err := os.UserHomeDir() - if err != nil { - return ".agents/evals" - } - return filepath.Join(home, ".agents", "evals") -} - -var ( - evalTaskRunSuiteRef string - evalTaskRunRigID string - evalTaskRunSeeds string - evalTaskRunHarnessRef string - evalTaskRunHarnessDir string - evalTaskRunModelSpecID string - evalTaskRunGTRef string - evalTaskRunSampleSplit string - evalTaskRunNSamples int - evalTaskRunInspectVersion string - evalTaskRunInspectCommand string - evalTaskRunCrossSpec bool - evalTaskRunAllowWeak bool - evalTaskRunQuickSession bool - evalTaskRunDryRun bool -) - -var evalTaskCmd = &cobra.Command{ - Use: "task", - Short: "Manage evaluation Tasks (add, list, show, run)", - Long: `Operate on the §3 Task primitive of the eval substrate. - -Tasks live under $AGENTOPS_EVALS_ROOT/tasks//task.yaml and define the -input/output contract a Run will be evaluated against. The command surface -exposes: - - ao eval task add Register a Task by copying its file - ao eval task list List registered Task ids - ao eval task show Print a Task summary - ao eval task run ... Open a new Run manifest under runs/ - -Run-time gates 1, 6, 7, 8, 9 (per SCHEMA.md §6) refuse the Run before any -Inspect launch when their preconditions fail. Refusal messages match §6 -format and are checked by ao eval doctor.`, -} - -var evalTaskAddCmd = &cobra.Command{ - Use: "add ", - Short: "Register a Task by copying its yaml + samples into the substrate", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - src := args[0] - raw, err := os.ReadFile(src) - if err != nil { - return fmt.Errorf("eval task add: read %q: %w", src, err) - } - var task evalsub.Task - if err := yaml.Unmarshal(raw, &task); err != nil { - return fmt.Errorf("eval task add: parse: %w", err) - } - if task.ID == "" { - return fmt.Errorf("eval task add: missing required field id") - } - if task.Stats.MinNSamples <= 0 { - return fmt.Errorf("eval task add: task %q has no stats.min_n_samples (gate #6 cannot enforce)", task.ID) - } - dest := filepath.Join(evalsRoot(), "tasks", task.ID, "task.yaml") - canon, err := evalsub.CanonicalizeYAML(raw) - if err != nil { - return fmt.Errorf("eval task add: canonicalize: %w", err) - } - if err := evalsub.WriteAtomic(dest, canon); err != nil { - return fmt.Errorf("eval task add: write: %w", err) - } - fmt.Fprintf(cmd.OutOrStdout(), "Task registered: %s\n path: %s\n min_n_samples: %d\n", task.ID, dest, task.Stats.MinNSamples) - return nil - }, -} - -var evalTaskListCmd = &cobra.Command{ - Use: "list", - Short: "List registered Task ids", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - root := filepath.Join(evalsRoot(), "tasks") - entries, err := os.ReadDir(root) - if err != nil { - if os.IsNotExist(err) { - fmt.Fprintln(cmd.OutOrStdout(), "No tasks registered") - return nil - } - return fmt.Errorf("eval task list: %w", err) - } - var ids []string - for _, e := range entries { - if e.IsDir() { - ids = append(ids, e.Name()) - } - } - sort.Strings(ids) - if GetOutput() == "json" { - enc := json.NewEncoder(cmd.OutOrStdout()) - enc.SetIndent("", " ") - return enc.Encode(map[string]interface{}{"tasks": ids, "root": root}) - } - if len(ids) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No tasks registered") - return nil - } - fmt.Fprintf(cmd.OutOrStdout(), "Tasks under %s:\n", root) - for _, id := range ids { - task, err := loadTask(id) - if err != nil { - fmt.Fprintf(cmd.OutOrStdout(), " %s\t\n", id, err) - continue - } - fmt.Fprintf(cmd.OutOrStdout(), " %s\tschema_version=%d min_n_samples=%d metric=%s\n", - task.ID, task.SchemaVersion, task.Stats.MinNSamples, task.Stats.Metric) - } - return nil - }, -} - -var evalTaskShowCmd = &cobra.Command{ - Use: "show ", - Short: "Print a registered Task summary", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - task, err := loadTask(args[0]) - if err != nil { - return err - } - if GetOutput() == "json" { - enc := json.NewEncoder(cmd.OutOrStdout()) - enc.SetIndent("", " ") - return enc.Encode(task) - } - fmt.Fprintf(cmd.OutOrStdout(), "Task: %s\n", task.ID) - fmt.Fprintf(cmd.OutOrStdout(), " schema_version: %d\n", task.SchemaVersion) - fmt.Fprintf(cmd.OutOrStdout(), " domain: %s\n", task.Domain) - fmt.Fprintf(cmd.OutOrStdout(), " description: %s\n", task.Description) - fmt.Fprintf(cmd.OutOrStdout(), " harness_ref: %s\n", task.HarnessRef) - fmt.Fprintf(cmd.OutOrStdout(), " stats.metric: %s\n", task.Stats.Metric) - fmt.Fprintf(cmd.OutOrStdout(), " stats.paired: %v\n", task.Stats.Paired) - fmt.Fprintf(cmd.OutOrStdout(), " stats.min_n_samples: %d\n", task.Stats.MinNSamples) - fmt.Fprintf(cmd.OutOrStdout(), " stats.decision_rule: kind=%s confidence=%.2f\n", - task.Stats.DecisionRule.Kind, task.Stats.DecisionRule.Confidence) - return nil - }, -} - -var evalTaskRunCmd = &cobra.Command{ - Use: "run ", - Short: "Open a new Run manifest for ; refuses on gate failure", - Long: `Opens a new Run under $AGENTOPS_EVALS_ROOT/runs//manifest.json -in pending state, runs §6 manifest-checkable gates 1/6/7/8/9, and on pass -transitions the manifest to running. - -This command opens and gates the Run manifest; it does not yet launch -Inspect itself. The atomic-write contract, manifest fields, and refusal -format are all fully exercised here. Use --dry-run to refuse-test without -creating the run directory.`, - Args: cobra.ExactArgs(1), - RunE: runEvalTaskRun, -} - -func runEvalTaskRun(cmd *cobra.Command, args []string) error { - gateInputs, suite, task, seeds, err := prepareTaskRunInputs(args[0]) - if err != nil { - return err - } - - refusals := evalsub.RunGates(gateInputs) - if !refusals.Empty() { - fmt.Fprintln(cmd.ErrOrStderr(), refusals.Format()) - return fmt.Errorf("eval task run: %d gate refusal(s)", len(refusals)) - } - - if evalTaskRunDryRun { - fmt.Fprintln(cmd.OutOrStdout(), "Dry run: gates passed; no Run manifest written.") - return nil - } - - rigID := evalTaskRunRigID - if rigID == "" { - rigID = "unknown-rig" - } - runID := evalsub.GenerateRunID(rigID) - manifest := buildTaskRunManifest(task, suite, seeds, rigID, gateInputs) - - w, err := evalsub.NewRunWriter(evalsRoot(), runID, manifest) - if err != nil { - return fmt.Errorf("eval task run: open run: %w", err) - } - if err := w.Transition(evalsub.StatusRunning, func(m *evalsub.Manifest) { - m.ValidityGatesPassed = []string{ - "held_constant_declared", - "min_n_samples", - "ground_truth_immutable", - "harness_lock_match", - "multi_comparison_correction", - } - }); err != nil { - return fmt.Errorf("eval task run: transition->running: %w", err) - } - - return writeTaskRunOutput(cmd, w) -} - -// prepareTaskRunInputs loads the task + suite, parses seeds, and assembles the -// gate-input bundle (harness snapshot + GT) for runEvalTaskRun. -func prepareTaskRunInputs(taskID string) (evalsub.GateInputs, *evalsub.Suite, *evalsub.Task, []int, error) { - task, err := loadTask(taskID) - if err != nil { - return evalsub.GateInputs{}, nil, nil, nil, err - } - if evalTaskRunSuiteRef == "" { - return evalsub.GateInputs{}, nil, nil, nil, fmt.Errorf("eval task run: --suite is required") - } - suite, err := loadSuite(evalTaskRunSuiteRef) - if err != nil { - return evalsub.GateInputs{}, nil, nil, nil, err - } - seeds, err := parseSeeds(evalTaskRunSeeds) - if err != nil { - return evalsub.GateInputs{}, nil, nil, nil, err - } - if len(seeds) < 3 { - return evalsub.GateInputs{}, nil, nil, nil, fmt.Errorf("eval task run: --seeds requires >=3 values, got %d (per §4 manifest required field)", len(seeds)) - } - - gateInputs := evalsub.GateInputs{ - Suite: suite, - Task: task, - AllowWeak: evalTaskRunAllowWeak, - GTRequested: evalTaskRunGTRef, - } - if err := attachHarnessSnapshot(&gateInputs); err != nil { - return evalsub.GateInputs{}, nil, nil, nil, err - } - if evalTaskRunGTRef != "" { - if gtRows, gtErr := loadGroundTruthFile(); gtErr == nil { - gateInputs.GroundTruth = gtRows - } - } - return gateInputs, suite, task, seeds, nil -} - -// attachHarnessSnapshot snapshots --harness-dir (when set) into the gate -// inputs, expanding ~ and recording the lock + content hash. -func attachHarnessSnapshot(gateInputs *evalsub.GateInputs) error { - if evalTaskRunHarnessDir == "" { - return nil - } - expanded := evalTaskRunHarnessDir - if strings.HasPrefix(expanded, "~") { - home, _ := os.UserHomeDir() - expanded = filepath.Join(home, strings.TrimPrefix(expanded, "~")) - } - harness, lock, err := evalsub.SnapshotHarness(expanded, evalTaskRunHarnessRef, "ao eval task run") - if err != nil { - return fmt.Errorf("eval task run: harness snapshot: %w", err) - } - gateInputs.Harness = harness - gateInputs.HarnessLock = lock - gateInputs.HarnessDir = expanded - return nil -} - -// buildTaskRunManifest assembles the §4 Run manifest from the gathered inputs, -// including model-spec/ground-truth hashes and multi-comparison fields when -// the suite arm count exceeds 2. -func buildTaskRunManifest(task *evalsub.Task, suite *evalsub.Suite, seeds []int, rigID string, gateInputs evalsub.GateInputs) evalsub.Manifest { - manifest := evalsub.Manifest{ - TaskRef: task.ID, - SuiteRef: suite.ID, - HarnessRef: evalTaskRunHarnessRef, - ModelSpecRef: evalTaskRunModelSpecID, - GroundTruthRef: evalTaskRunGTRef, - SampleSplit: pickSampleSplit(suite, evalTaskRunSampleSplit), - NSamples: pickNSamples(suite, evalTaskRunNSamples), - Seeds: seeds, - RigID: rigID, - InspectCommand: evalTaskRunInspectCommand, - InspectVersion: evalTaskRunInspectVersion, - QuickSession: evalTaskRunQuickSession, - } - if gateInputs.Harness != nil { - manifest.HarnessContentHash = gateInputs.Harness.ContentHash - } - if evalTaskRunModelSpecID != "" { - if spec, err := evalsub.LoadModelSpec(evalsRoot(), evalTaskRunModelSpecID); err == nil { - manifest.ModelSpecHash = spec.ContentHash - } - } - if evalTaskRunGTRef != "" { - manifest.GroundTruthHash = hashGroundTruthRef(gateInputs.GroundTruth, evalTaskRunGTRef) - } - if n := len(suite.VariedAxis.Values); n > 2 { - manifest.MultiComparisonMethod = suite.Stats.MultiComparisonMethod - manifest.ComparisonFamily = suite.Stats.ComparisonFamily - manifest.ReferenceArm = suite.Stats.ReferenceArm - manifest.FamilySizeK = evalsub.FamilySizeK(suite.Stats.ComparisonFamily, n) - } - return manifest -} - -func writeTaskRunOutput(cmd *cobra.Command, w *evalsub.RunWriter) error { - out := w.Manifest() - if GetOutput() == "json" { - enc := json.NewEncoder(cmd.OutOrStdout()) - enc.SetIndent("", " ") - return enc.Encode(out) - } - fmt.Fprintf(cmd.OutOrStdout(), "Run opened: %s\n status: %s\n manifest: %s\n rig_id: %s\n task_ref: %s\n suite_ref: %s\n", - out.ID, out.Status, w.Path(), out.RigID, out.TaskRef, out.SuiteRef) - return nil -} - -// loadTask reads the Task by id from the substrate. -func loadTask(id string) (*evalsub.Task, error) { - path := filepath.Join(evalsRoot(), "tasks", id, "task.yaml") - raw, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("loadTask %q: %w", id, err) - } - var task evalsub.Task - if err := yaml.Unmarshal(raw, &task); err != nil { - return nil, fmt.Errorf("loadTask %q: parse: %w", id, err) - } - return &task, nil -} - -// loadSuite resolves a Suite ref against $EVALS_ROOT/suites//suite.yaml, -// or treats it as a literal path when it ends in .yaml. -func loadSuite(ref string) (*evalsub.Suite, error) { - candidate := ref - if !strings.HasSuffix(ref, ".yaml") && !strings.HasSuffix(ref, ".yml") { - candidate = filepath.Join(evalsRoot(), "suites", ref, "suite.yaml") - } - raw, err := os.ReadFile(candidate) - if err != nil { - return nil, fmt.Errorf("loadSuite %q: %w", ref, err) - } - var s evalsub.Suite - if err := yaml.Unmarshal(raw, &s); err != nil { - return nil, fmt.Errorf("loadSuite %q: parse: %w", ref, err) - } - return &s, nil -} - -// hashGroundTruthRef computes a manifest-level ground_truth_hash by serializing -// every matching row id (id == requested OR supersedes-chain head pointing to -// it) in deterministic order, then hashing. Day-2 minimal hash; Day-4 graduates -// to Dolt-resolved supersession + supersedes-chain canonicalization. -func hashGroundTruthRef(rows []evalsub.GroundTruthRow, ref string) string { - if ref == "" || len(rows) == 0 { - return "" - } - var matches []evalsub.GroundTruthRow - for _, r := range rows { - if r.ID == ref || r.Supersedes == ref { - matches = append(matches, r) - } - } - if len(matches) == 0 { - return "" - } - // Stable serialization: sort by id, marshal each as canonical JSON. - var buf []byte - for _, m := range matches { - bs, _ := json.Marshal(m) //nolint:errcheck // m is an in-memory record struct; marshal cannot fail and CanonicalizeJSON below re-validates the bytes - canon, err := evalsub.CanonicalizeJSON(bs) - if err != nil { - canon = bs - } - buf = append(buf, canon...) - } - return evalsub.ContentHash(buf) -} - -// loadGroundTruthFile loads ground-truth rows from the canonical jsonl path -// when --ground-truth is set on the run command. Day-2 keeps this thin; Day-4 -// graduates to Dolt-resolved supersession chains. -func loadGroundTruthFile() ([]evalsub.GroundTruthRow, error) { - path := filepath.Join(evalsRoot(), "ground-truth", "ground-truth.jsonl") - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer func() { _ = f.Close() }() - var rows []evalsub.GroundTruthRow - dec := json.NewDecoder(f) - for dec.More() { - var row evalsub.GroundTruthRow - if err := dec.Decode(&row); err != nil { - return nil, err - } - rows = append(rows, row) - } - return rows, nil -} - -func parseSeeds(s string) ([]int, error) { - if s == "" { - return nil, nil - } - parts := strings.Split(s, ",") - out := make([]int, 0, len(parts)) - for _, p := range parts { - var n int - if _, err := fmt.Sscanf(strings.TrimSpace(p), "%d", &n); err != nil { - return nil, fmt.Errorf("parseSeeds: bad seed %q: %w", p, err) - } - out = append(out, n) - } - return out, nil -} - -func pickSampleSplit(s *evalsub.Suite, override string) string { - if override != "" { - return override - } - if s != nil && s.SampleSplit != "" { - return s.SampleSplit - } - return "dev" -} - -func pickNSamples(s *evalsub.Suite, override int) int { - if override > 0 { - return override - } - if s != nil && s.NSamples > 0 { - return s.NSamples - } - return 0 -} - -func registerEvalTaskCmd() { - evalTaskRunCmd.Flags().StringVar(&evalTaskRunSuiteRef, "suite", "", "Suite id or path to suite.yaml (required)") - evalTaskRunCmd.Flags().StringVar(&evalTaskRunRigID, "rig-id", "", "Rig identifier stamped into the Run manifest") - evalTaskRunCmd.Flags().StringVar(&evalTaskRunSeeds, "seeds", "", "Comma-separated seeds (>=3, per §4)") - evalTaskRunCmd.Flags().StringVar(&evalTaskRunHarnessRef, "harness", "", "Harness id (recorded into manifest)") - evalTaskRunCmd.Flags().StringVar(&evalTaskRunHarnessDir, "harness-dir", "", "Path to harness source dir for snapshot + gate #8") - evalTaskRunCmd.Flags().StringVar(&evalTaskRunModelSpecID, "model-spec", "", "ModelSpec id (already captured via ao eval models capture)") - evalTaskRunCmd.Flags().StringVar(&evalTaskRunGTRef, "ground-truth", "", "Ground-truth row id (head of supersession chain)") - evalTaskRunCmd.Flags().StringVar(&evalTaskRunSampleSplit, "sample-split", "", "Sample split (dev|holdout); default from suite") - evalTaskRunCmd.Flags().IntVar(&evalTaskRunNSamples, "n-samples", 0, "Override Suite.n_samples") - evalTaskRunCmd.Flags().StringVar(&evalTaskRunInspectVersion, "inspect-version", "0.3.216", "Inspect AI version stamped into manifest") - evalTaskRunCmd.Flags().StringVar(&evalTaskRunInspectCommand, "inspect-command", "", "Inspect command recorded into the Run manifest (not executed yet)") - evalTaskRunCmd.Flags().BoolVar(&evalTaskRunCrossSpec, "cross-spec", false, "Allow ModelSpec drift (gate #4)") - evalTaskRunCmd.Flags().BoolVar(&evalTaskRunAllowWeak, "allow-weak-labels", false, "Allow runs against confidence=weak ground-truth rows (gate #7)") - evalTaskRunCmd.Flags().BoolVar(&evalTaskRunQuickSession, "quick", false, "Mark Run as quick_session=true (excluded from --vs auto-baseline pool)") - evalTaskRunCmd.Flags().BoolVar(&evalTaskRunDryRun, "dry-run", false, "Run gates and exit without writing a Run manifest") - - evalTaskCmd.AddCommand(evalTaskAddCmd, evalTaskListCmd, evalTaskShowCmd, evalTaskRunCmd) -} diff --git a/cli/cmd/ao/eval_test.go b/cli/cmd/ao/eval_test.go deleted file mode 100644 index 51f75727c..000000000 --- a/cli/cmd/ao/eval_test.go +++ /dev/null @@ -1,1028 +0,0 @@ -// practices: [llm-eval-harness, dora-metrics] -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - "time" - - aoeval "github.com/boshu2/agentops/cli/internal/eval" - evalsub "github.com/boshu2/agentops/cli/internal/evalsubstrate" - "github.com/spf13/cobra" -) - -func TestEvalRunCommandJSON(t *testing.T) { - withEvalCommand(t) - dir := t.TempDir() - writeEvalCmdFile(t, filepath.Join(dir, "fixture.txt"), "needle\n") - suitePath := writeEvalCmdSuite(t, dir, `{ - "schema_version": 1, - "id": "cmd.pass", - "name": "Command pass", - "domain": "cli", - "visibility": "public_canary", - "tier": "deterministic", - "scoring": { - "aggregate_threshold": 1, - "dimensions": [ - {"name": "correctness", "weight": 1, "threshold": 1} - ] - }, - "baseline_policy": {"mode": "none"}, - "cases": [ - { - "id": "contains", - "title": "fixture contains needle", - "kind": "artifact_check", - "objective": "Score fixture deterministically.", - "expectations": [ - {"type": "artifact_contains", "target": "fixture.txt", "value": "needle"} - ] - } - ] -}`) - - out, err := executeCommand("eval", "run", suitePath, "--run-id", "cmd-pass", "--json") - if err != nil { - t.Fatalf("ao eval run failed: %v\noutput: %s", err, out) - } - var run aoeval.RunRecord - if err := json.Unmarshal([]byte(out), &run); err != nil { - t.Fatalf("eval run JSON parse failed: %v\noutput: %s", err, out) - } - if run.RunID != "cmd-pass" || run.Status != aoeval.StatusPass { - t.Fatalf("run_id/status = %q/%s, want cmd-pass/pass", run.RunID, run.Status) - } -} - -func TestEvalRunContextModeABCommandJSON(t *testing.T) { - if os.Getenv("GO_WANT_EVAL_CMD_CONTEXT_HELPER_PROCESS") == "1" { - root := os.Getenv("AO_AGENTS_DIR") - variant := os.Getenv("AO_CONTEXT_VARIANT") - data, err := os.ReadFile(filepath.Join(root, "variant.txt")) - if err != nil { - os.Exit(1) - } - ok := string(data) == variant+"\n" && os.Getenv("AGENTOPS_HOOKS_DISABLED") != "1" - if ok { - fmt.Fprintf(os.Stdout, `{"ok":true,"variant":%q}`, variant) - os.Exit(0) - } - fmt.Fprintf(os.Stdout, `{"ok":false,"variant":%q}`, variant) - os.Exit(1) - } - - withEvalCommand(t) - dir := t.TempDir() - writeEvalCmdFile(t, filepath.Join(dir, "fixtures", "context-ab", "context-off", "agents", "variant.txt"), string(aoeval.ContextVariantOff)+"\n") - writeEvalCmdFile(t, filepath.Join(dir, "fixtures", "context-ab", "context-on", "agents", "variant.txt"), string(aoeval.ContextVariantOn)+"\n") - exe, err := os.Executable() - if err != nil { - t.Fatalf("os.Executable: %v", err) - } - suitePath := writeEvalCmdSuiteNamed(t, dir, "context-ab.json", fmt.Sprintf(`{ - "schema_version": 1, - "id": "context.ab.cmd", - "name": "Context AB command", - "domain": "cli", - "visibility": "public_canary", - "tier": "deterministic", - "scoring": { - "aggregate_threshold": 1, - "dimensions": [ - {"name": "correctness", "weight": 1, "threshold": 1} - ] - }, - "baseline_policy": {"mode": "none"}, - "cases": [ - { - "id": "agents-dir", - "title": "agents dir is isolated per context leg", - "kind": "command", - "objective": "Verify context-mode runs with isolated AO_AGENTS_DIR roots.", - "runtime": "shell", - "inputs": { - "argv": [%q, "-test.run=TestEvalRunContextModeABCommandJSON"], - "env": {"GO_WANT_EVAL_CMD_CONTEXT_HELPER_PROCESS": "1"} - }, - "expectations": [ - {"type": "exit_code", "value": 0}, - {"type": "json_path", "target": "stdout.ok", "value": true} - ] - } - ] -}`, exe)) - outputPath := filepath.Join(dir, "run.json") - - out, err := executeCommand("eval", "run", suitePath, "--context-mode", "ab", "--run-id", "cmd-context", "--out", outputPath, "--json") - if err != nil { - t.Fatalf("ao eval run --context-mode=ab failed: %v\noutput: %s", err, out) - } - var scorecard aoeval.ContextDeltaScorecard - if err := json.Unmarshal([]byte(out), &scorecard); err != nil { - t.Fatalf("context scorecard JSON parse failed: %v\noutput: %s", err, out) - } - if scorecard.ContextOff.RunID != "cmd-context-context-off" || scorecard.ContextOn.RunID != "cmd-context-context-on" { - t.Fatalf("run ids = off:%q on:%q", scorecard.ContextOff.RunID, scorecard.ContextOn.RunID) - } - for _, path := range []string{ - filepath.Join(dir, "run-context-off.json"), - filepath.Join(dir, "run-context-on.json"), - } { - if _, err := os.Stat(path); err != nil { - t.Fatalf("expected context leg output %s: %v", path, err) - } - } -} - -func TestEvalRunContextModeRejectsSkillOffBaseline(t *testing.T) { - withEvalCommand(t) - dir := t.TempDir() - suitePath := writeEvalCmdSuite(t, dir, `{ - "schema_version": 1, - "id": "context.reject", - "name": "Context reject", - "domain": "cli", - "visibility": "public_canary", - "tier": "deterministic", - "scoring": { - "aggregate_threshold": 1, - "dimensions": [ - {"name": "correctness", "weight": 1, "threshold": 1} - ] - }, - "baseline_policy": {"mode": "none"}, - "cases": [ - { - "id": "static", - "title": "static", - "kind": "artifact_check", - "objective": "static", - "expectations": [ - {"type": "file_exists", "target": "suite.json"} - ] - } - ] -}`) - - out, err := executeCommand("eval", "run", suitePath, "--context-mode", "ab", "--baseline-mode", "skill-off") - if err == nil { - t.Fatalf("ao eval run accepted incompatible context/baseline modes; output: %s", out) - } - if !strings.Contains(out, "--context-mode=ab cannot be combined") { - t.Fatalf("unexpected error output: %s", out) - } -} - -func TestEvalRunCommandMissingSuiteFails(t *testing.T) { - withEvalCommand(t) - out, err := executeCommand("eval", "run", filepath.Join(t.TempDir(), "missing.json"), "--json") - if err == nil { - t.Fatalf("ao eval run missing suite succeeded; output: %s", out) - } -} - -func TestEvalCompareCommandJSON(t *testing.T) { - withEvalCommand(t) - dir := t.TempDir() - baselinePath := filepath.Join(dir, "baseline.json") - candidatePath := filepath.Join(dir, "candidate.json") - writeEvalRunRecord(t, baselinePath, "baseline-run", 0.80) - writeEvalRunRecord(t, candidatePath, "candidate-run", 0.95) - - out, err := executeCommand("eval", "compare", candidatePath, baselinePath, "--json") - if err != nil { - t.Fatalf("ao eval compare failed: %v\noutput: %s", err, out) - } - var compared aoeval.RunRecord - if err := json.Unmarshal([]byte(out), &compared); err != nil { - t.Fatalf("eval compare JSON parse failed: %v\noutput: %s", err, out) - } - if compared.Verdict != aoeval.VerdictImprovement { - t.Fatalf("verdict = %s, want improvement", compared.Verdict) - } - if compared.BaselineComparison == nil || compared.BaselineComparison.BaselineRunID != "baseline-run" { - t.Fatalf("baseline comparison = %+v", compared.BaselineComparison) - } -} - -func TestEvalCompareCommandJSONIncludesZeroAggregateDelta(t *testing.T) { - withEvalCommand(t) - dir := t.TempDir() - baselinePath := filepath.Join(dir, "baseline.json") - candidatePath := filepath.Join(dir, "candidate.json") - writeEvalRunRecord(t, baselinePath, "baseline-run", 1.0) - writeEvalRunRecord(t, candidatePath, "candidate-run", 1.0) - - out, err := executeCommand("eval", "compare", candidatePath, baselinePath, "--json") - if err != nil { - t.Fatalf("ao eval compare failed: %v\noutput: %s", err, out) - } - var raw map[string]any - if err := json.Unmarshal([]byte(out), &raw); err != nil { - t.Fatalf("eval compare JSON parse failed: %v\noutput: %s", err, out) - } - comparison, ok := raw["baseline_comparison"].(map[string]any) - if !ok { - t.Fatalf("baseline_comparison missing or invalid: %#v", raw["baseline_comparison"]) - } - if delta, ok := comparison["aggregate_delta"].(float64); !ok || delta != 0 { - t.Fatalf("aggregate_delta = %#v, want explicit 0", comparison["aggregate_delta"]) - } -} - -func TestEvalScorecardCommandJSON(t *testing.T) { - withEvalCommand(t) - dir := t.TempDir() - baselinePath := filepath.Join(dir, "baseline.json") - candidatePath := filepath.Join(dir, "candidate.json") - writeEvalScorecardRunRecord(t, candidatePath, "candidate-run", map[string]float64{ - "artifact-completeness": 1, - "phase-order": 1, - "objective-spine": 1, - "validation-separation": 1, - "scenario-satisfaction": 1, - "runtime-safety": 1, - }) - writeEvalScorecardRunRecord(t, baselinePath, "baseline-run", map[string]float64{ - "artifact-completeness": 0.8, - "phase-order": 1, - "objective-spine": 1, - "validation-separation": 1, - "scenario-satisfaction": 1, - "runtime-safety": 1, - }) - - out, err := executeCommand("eval", "scorecard", candidatePath, baselinePath, "--kind", "rpi", "--json") - if err != nil { - t.Fatalf("ao eval scorecard failed: %v\noutput: %s", err, out) - } - var scorecard aoeval.Scorecard - if err := json.Unmarshal([]byte(out), &scorecard); err != nil { - t.Fatalf("eval scorecard JSON parse failed: %v\noutput: %s", err, out) - } - if scorecard.Kind != aoeval.ScorecardKindRPI { - t.Fatalf("kind = %s, want rpi", scorecard.Kind) - } - if scorecard.CandidateRunID != "candidate-run" || scorecard.BaselineRunID != "baseline-run" { - t.Fatalf("run ids = %q/%q, want candidate-run/baseline-run", scorecard.CandidateRunID, scorecard.BaselineRunID) - } - if scorecard.Verdict != aoeval.VerdictImprovement { - t.Fatalf("verdict = %s, want improvement", scorecard.Verdict) - } - if len(scorecard.Categories) != 6 { - t.Fatalf("category count = %d, want 6", len(scorecard.Categories)) - } - category := evalScorecardCategory(t, scorecard, "artifact completeness") - if category.CandidateScore != 1 { - t.Fatalf("artifact completeness candidate_score = %v, want 1", category.CandidateScore) - } - if category.BaselineScore == nil || *category.BaselineScore != 0.8 { - t.Fatalf("artifact completeness baseline_score = %v, want 0.8", category.BaselineScore) - } - if category.Delta == nil || *category.Delta != 0.2 { - t.Fatalf("artifact completeness delta = %v, want 0.2", category.Delta) - } -} - -func TestEvalBaselineCommandJSON(t *testing.T) { - withEvalCommand(t) - dir := t.TempDir() - runPath := filepath.Join(dir, "run.json") - outputPath := filepath.Join(dir, "baseline.json") - writeEvalRunRecord(t, runPath, "candidate-run", 1.0) - - out, err := executeCommand( - "eval", "baseline", runPath, - "--out", outputPath, - "--promoted-by", "tester", - "--rationale", "stable canary", - "--json", - ) - if err != nil { - t.Fatalf("ao eval baseline failed: %v\noutput: %s", err, out) - } - var promoted aoeval.RunRecord - if err := json.Unmarshal([]byte(out), &promoted); err != nil { - t.Fatalf("eval baseline JSON parse failed: %v\noutput: %s", err, out) - } - if promoted.Baseline == nil || promoted.Baseline.Mode != aoeval.BaselineModePromote { - t.Fatalf("baseline metadata = %+v, want promote", promoted.Baseline) - } - if _, err := os.Stat(outputPath); err != nil { - t.Fatalf("baseline output not written: %v", err) - } -} - -func TestEvalCoverageCommandJSON(t *testing.T) { - withEvalCommand(t) - dir := t.TempDir() - writeEvalCmdFile(t, filepath.Join(dir, "fixture.txt"), "ok\n") - writeEvalCmdFile(t, filepath.Join(dir, "suite.json"), `{ - "schema_version": 1, - "id": "coverage.cli", - "name": "Coverage CLI", - "domain": "cli", - "visibility": "public_canary", - "tier": "deterministic", - "allowed_runtimes": ["static"], - "scoring": { - "aggregate_threshold": 1, - "dimensions": [ - {"name": "correctness", "weight": 1, "threshold": 1} - ] - }, - "baseline_policy": {"mode": "none"}, - "cases": [ - { - "id": "fixture", - "title": "fixture", - "kind": "artifact_check", - "objective": "fixture", - "expectations": [ - {"type": "file_exists", "target": "fixture.txt"} - ], - "critical": true - } - ] -}`) - - out, err := executeCommand("eval", "coverage", "--root", dir, "--json") - if err != nil { - t.Fatalf("ao eval coverage failed: %v\noutput: %s", err, out) - } - var report aoeval.CoverageReport - if err := json.Unmarshal([]byte(out), &report); err != nil { - t.Fatalf("eval coverage JSON parse failed: %v\noutput: %s", err, out) - } - if report.SuiteCount != 1 || report.CaseCount != 1 || report.CriticalCaseCount != 1 { - t.Fatalf("coverage counts = suites:%d cases:%d critical:%d, want 1/1/1", report.SuiteCount, report.CaseCount, report.CriticalCaseCount) - } - if report.Domains["cli"].SuiteCount != 1 { - t.Fatalf("cli domain coverage = %+v, want one suite", report.Domains["cli"]) - } - if report.EvidenceKinds[string(aoeval.EvidenceKindContractCanary)].CaseCount != 1 { - t.Fatalf("contract_canary evidence coverage = %+v, want one case", report.EvidenceKinds[string(aoeval.EvidenceKindContractCanary)]) - } - if len(report.MissingRequiredDomains) == 0 { - t.Fatalf("missing required domains empty; want gaps for non-cli domains") - } -} - -func TestEvalBaselineAuditCommandJSON(t *testing.T) { - withEvalCommand(t) - dir := t.TempDir() - writeEvalCmdFile(t, filepath.Join(dir, "fixture.txt"), "ok\n") - writeEvalCmdFile(t, filepath.Join(dir, "suite.json"), `{ - "schema_version": 1, - "id": "baseline.audit", - "name": "Baseline Audit", - "domain": "cli", - "visibility": "public_canary", - "tier": "deterministic", - "allowed_runtimes": ["static"], - "scoring": { - "aggregate_threshold": 1, - "dimensions": [ - {"name": "correctness", "weight": 1, "threshold": 1} - ] - }, - "baseline_policy": {"mode": "compare"}, - "cases": [ - { - "id": "fixture", - "title": "fixture", - "kind": "artifact_check", - "objective": "fixture", - "expectations": [ - {"type": "file_exists", "target": "fixture.txt"} - ], - "critical": true - } - ] -}`) - - out, err := executeCommand("eval", "baseline-audit", "--root", dir, "--baseline-dir", filepath.Join(dir, "baselines"), "--json") - if err != nil { - t.Fatalf("ao eval baseline-audit failed: %v\noutput: %s", err, out) - } - var report aoeval.BaselineAuditReport - if err := json.Unmarshal([]byte(out), &report); err != nil { - t.Fatalf("baseline audit JSON parse failed: %v\noutput: %s", err, out) - } - if report.PolicyMismatchCount != 1 || len(report.MissingCompareBaselines) != 1 { - t.Fatalf("baseline audit report = %+v, want one missing compare baseline", report) - } -} - -func TestEvalCleanupTransitionsDeleteTmpAndReport(t *testing.T) { - root := t.TempDir() - runsRoot := filepath.Join(root, "runs") - now := time.Now().UTC() - writeEvalSubManifest(t, root, "old-pending", evalsub.StatusPending, now.Add(-2*time.Minute)) - writeEvalSubManifest(t, root, "old-running", evalsub.StatusRunning, now.Add(-10*time.Minute)) - writeEvalSubManifest(t, root, "fresh-pending", evalsub.StatusPending, now.Add(-10*time.Second)) - writeEvalSubManifest(t, root, "failed-delete", evalsub.StatusFailed, now.Add(-20*time.Minute)) - writeEvalSubManifest(t, root, "retracted-keep", evalsub.StatusRetracted, now.Add(-20*time.Minute)) - - report := CleanupReport{Touched: []string{}} - if err := cleanupStaleTransitions(root, runsRoot, &report); err != nil { - t.Fatalf("cleanup stale transitions failed: %v", err) - } - if report.TransitionsAborted != 1 || report.TransitionsFailed != 1 { - t.Fatalf("transition counts = aborted:%d failed:%d, want 1/1", report.TransitionsAborted, report.TransitionsFailed) - } - assertEvalSubManifestStatus(t, root, "old-pending", evalsub.StatusAborted, "never_started") - assertEvalSubManifestStatus(t, root, "old-running", evalsub.StatusFailed, "orphaned_process") - assertEvalSubManifestStatus(t, root, "fresh-pending", evalsub.StatusPending, "") - - oldDryRun := evalCleanupDryRun - oldAge := evalCleanupAge - evalCleanupDryRun = false - evalCleanupAge = 0 - t.Cleanup(func() { - evalCleanupDryRun = oldDryRun - evalCleanupAge = oldAge - }) - if err := cleanupDeleteRuns(root, runsRoot, &report); err != nil { - t.Fatalf("cleanup delete runs failed: %v", err) - } - for _, runID := range []string{"old-pending", "old-running", "failed-delete"} { - if _, err := os.Stat(evalsub.RunDir(root, runID)); !os.IsNotExist(err) { - t.Fatalf("%s dir still exists after delete cleanup: %v", runID, err) - } - } - if _, err := os.Stat(evalsub.RunDir(root, "retracted-keep")); err != nil { - t.Fatalf("retracted run should be kept: %v", err) - } - - tmpPath := filepath.Join(root, "runs", "fresh-pending", "manifest.json.tmp") - writeEvalCmdFile(t, tmpPath, "{}\n") - if err := cleanupSweepTmpFiles(root, &report); err != nil { - t.Fatalf("cleanup tmp files failed: %v", err) - } - if report.TmpFilesSwept != 1 { - t.Fatalf("tmp files swept = %d, want 1", report.TmpFilesSwept) - } - if _, err := os.Stat(tmpPath); !os.IsNotExist(err) { - t.Fatalf("tmp file still exists after sweep: %v", err) - } - - oldOutput := output - oldVerbose := verbose - output = "json" - verbose = false - t.Cleanup(func() { - output = oldOutput - verbose = oldVerbose - }) - cmd := &cobra.Command{} - var jsonBuf bytes.Buffer - cmd.SetOut(&jsonBuf) - if err := writeCleanupReport(cmd, &report); err != nil { - t.Fatalf("write cleanup JSON report failed: %v", err) - } - var decoded CleanupReport - if err := json.Unmarshal(jsonBuf.Bytes(), &decoded); err != nil { - t.Fatalf("cleanup JSON did not parse: %v\n%s", err, jsonBuf.String()) - } - if decoded.RunsDeleted != 3 || decoded.TmpFilesSwept != 1 { - t.Fatalf("decoded cleanup report = %+v, want runs_deleted=3 tmp_files_swept=1", decoded) - } - - output = "table" - verbose = true - var textBuf bytes.Buffer - cmd.SetOut(&textBuf) - if err := writeCleanupReport(cmd, &report); err != nil { - t.Fatalf("write cleanup text report failed: %v", err) - } - text := textBuf.String() - if !strings.Contains(text, "Eval cleanup:") || !strings.Contains(text, "Touched:") { - t.Fatalf("cleanup text report missing summary or verbose touches:\n%s", text) - } -} - -func TestEvalTaskHelpersBuildManifestAndPrepareInputs(t *testing.T) { - root := t.TempDir() - restoreEvalTaskRunGlobals(t) - t.Setenv("AGENTOPS_EVALS_ROOT", root) - - task := evalsub.Task{ - SchemaVersion: 1, - ID: "task-a", - Domain: "cli", - Description: "task fixture", - HarnessRef: "harness-a", - Stats: evalsub.TaskStat{ - Metric: "accuracy", - Paired: true, - MinNSamples: 3, - DecisionRule: evalsub.DecisionRule{ - Kind: "bootstrap_ci", - Confidence: 0.95, - MinDelta: 0.05, - }, - }, - } - suite := evalsub.Suite{ - SchemaVersion: 1, - ID: "suite-a", - Kind: "comparison", - VariedAxis: evalsub.VariedAxis{ - Kind: "model", - Values: []string{"baseline", "candidate-a", "candidate-b"}, - }, - HeldConstant: evalsub.HeldConstant{ - Task: "task-a", - Harness: "harness-a", - GroundTruthVersion: "gt-v1", - Decoding: map[string]interface{}{"temperature": 0}, - }, - SampleSplit: "holdout", - NSamples: 7, - Stats: evalsub.SuiteStat{ - DecisionRule: evalsub.DecisionRule{ - Kind: "bootstrap_ci", - Confidence: 0.95, - }, - MultiComparisonMethod: "holm", - ComparisonFamily: "all_pairs", - ReferenceArm: "baseline", - Paired: true, - }, - } - writeEvalSubTask(t, root, task) - writeEvalSubSuite(t, root, suite) - writeEvalSubGroundTruth(t, root, []evalsub.GroundTruthRow{ - {ID: "gt-a", Value: "expected", Source: "test", Confidence: "strong", Split: "holdout"}, - {ID: "gt-a-v2", Value: "expected newer", Source: "test", Confidence: "strong", Split: "holdout", Supersedes: "gt-a"}, - }) - _, modelHash, err := evalsub.CaptureModelSpec(root, &evalsub.ModelSpec{ - ID: "model-a", - Provider: "local", - ModelName: "fixture-model", - ToolCallSupport: true, - SamplingDefaults: map[string]interface{}{ - "temperature": 0, - }, - }) - if err != nil { - t.Fatalf("capture model spec failed: %v", err) - } - - evalTaskRunSuiteRef = "suite-a" - evalTaskRunSeeds = "1, 2, 3" - evalTaskRunHarnessRef = "harness-a" - evalTaskRunModelSpecID = "model-a" - evalTaskRunGTRef = "gt-a" - evalTaskRunSampleSplit = "dev" - evalTaskRunNSamples = 5 - evalTaskRunInspectCommand = "inspect eval fixture" - evalTaskRunInspectVersion = "0.3.216" - evalTaskRunQuickSession = true - evalTaskRunAllowWeak = false - - seeds, err := parseSeeds(evalTaskRunSeeds) - if err != nil { - t.Fatalf("parse seeds failed: %v", err) - } - gateInputs := evalsub.GateInputs{ - Suite: &suite, - Task: &task, - Harness: &evalsub.Harness{ID: "harness-a", ContentHash: "sha256:harness"}, - GroundTruth: []evalsub.GroundTruthRow{{ID: "gt-a", Value: "expected", Source: "test", Confidence: "strong", Split: "holdout"}}, - GTRequested: "gt-a", - } - manifest := buildTaskRunManifest(&task, &suite, seeds, "rig-a", gateInputs) - if manifest.SampleSplit != "dev" || manifest.NSamples != 5 || !manifest.QuickSession { - t.Fatalf("manifest overrides = split:%q n:%d quick:%v, want dev/5/true", manifest.SampleSplit, manifest.NSamples, manifest.QuickSession) - } - if manifest.HarnessContentHash != "sha256:harness" || manifest.ModelSpecHash != modelHash { - t.Fatalf("manifest hashes = harness:%q model:%q, want harness/model %q/%q", manifest.HarnessContentHash, manifest.ModelSpecHash, "sha256:harness", modelHash) - } - if manifest.GroundTruthHash == "" { - t.Fatalf("manifest ground_truth_hash is empty") - } - if manifest.MultiComparisonMethod != "holm" || manifest.FamilySizeK != 3 { - t.Fatalf("manifest multi-comparison = method:%q family_k:%d, want holm/3", manifest.MultiComparisonMethod, manifest.FamilySizeK) - } - - preparedInputs, preparedSuite, preparedTask, preparedSeeds, err := prepareTaskRunInputs("task-a") - if err != nil { - t.Fatalf("prepare task run inputs failed: %v", err) - } - if preparedTask.ID != "task-a" || preparedSuite.ID != "suite-a" { - t.Fatalf("prepared refs = task:%q suite:%q, want task-a/suite-a", preparedTask.ID, preparedSuite.ID) - } - if len(preparedSeeds) != 3 || preparedSeeds[2] != 3 { - t.Fatalf("prepared seeds = %v, want [1 2 3]", preparedSeeds) - } - if len(preparedInputs.GroundTruth) != 2 { - t.Fatalf("prepared ground truth rows = %d, want 2", len(preparedInputs.GroundTruth)) - } - - if _, err := parseSeeds("1,bad"); err == nil { - t.Fatalf("parseSeeds accepted an invalid seed") - } - if split := pickSampleSplit(nil, ""); split != "dev" { - t.Fatalf("pickSampleSplit(nil, empty) = %q, want dev", split) - } - if n := pickNSamples(nil, 0); n != 0 { - t.Fatalf("pickNSamples(nil, 0) = %d, want 0", n) - } - if got := hashGroundTruthRef(nil, "gt-a"); got != "" { - t.Fatalf("hashGroundTruthRef(nil) = %q, want empty", got) - } -} - -func TestEvalTaskRunDryRunCommand(t *testing.T) { - withEvalCommand(t) - root := t.TempDir() - restoreEvalTaskRunGlobals(t) - t.Setenv("AGENTOPS_EVALS_ROOT", root) - writeEvalSubTask(t, root, evalsub.Task{ - SchemaVersion: 1, - ID: "task-dry-run", - Domain: "cli", - HarnessRef: "harness-dry-run", - Stats: evalsub.TaskStat{ - Metric: "accuracy", - MinNSamples: 3, - DecisionRule: evalsub.DecisionRule{ - Kind: "bootstrap_ci", - Confidence: 0.95, - }, - }, - }) - writeEvalSubSuite(t, root, evalsub.Suite{ - SchemaVersion: 1, - ID: "suite-dry-run", - Kind: "comparison", - VariedAxis: evalsub.VariedAxis{Kind: "model", Values: []string{"a", "b"}}, - HeldConstant: evalsub.HeldConstant{Task: "task-dry-run", Harness: "harness-dry-run", GroundTruthVersion: "gt-v1"}, - SampleSplit: "dev", - NSamples: 3, - Stats: evalsub.SuiteStat{DecisionRule: evalsub.DecisionRule{Kind: "bootstrap_ci", Confidence: 0.95}}, - }) - writeEvalSubGroundTruth(t, root, []evalsub.GroundTruthRow{ - {ID: "gt-dry-run", Value: "expected", Source: "test", Confidence: "strong", Split: "dev"}, - }) - - out, err := executeCommand( - "eval", "task", "run", "task-dry-run", - "--suite", "suite-dry-run", - "--seeds", "11,12,13", - "--harness", "harness-dry-run", - "--ground-truth", "gt-dry-run", - "--dry-run", - ) - if err != nil { - t.Fatalf("ao eval task run --dry-run failed: %v\noutput: %s", err, out) - } - if !strings.Contains(out, "Dry run: gates passed") { - t.Fatalf("dry-run output missing success message:\n%s", out) - } -} - -func writeEvalCmdSuite(t *testing.T, dir, body string) string { - t.Helper() - return writeEvalCmdSuiteNamed(t, dir, "suite.json", body) -} - -func writeEvalCmdSuiteNamed(t *testing.T, dir, name, body string) string { - t.Helper() - path := filepath.Join(dir, name) - writeEvalCmdFile(t, path, body) - return path -} - -func writeEvalCmdFile(t *testing.T, path, body string) { - t.Helper() - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) - } - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatalf("write %s: %v", path, err) - } -} - -func writeEvalRunRecord(t *testing.T, path, runID string, score float64) { - t.Helper() - run := aoeval.RunRecord{ - SchemaVersion: 1, - RunID: runID, - Suite: aoeval.SuiteRef{ - ID: "cmd.pass", - Path: "suite.json", - Visibility: aoeval.VisibilityPublicCanary, - Tier: aoeval.TierDeterministic, - }, - StartedAt: fixedEvalCmdTime(), - Status: aoeval.StatusPass, - Verdict: aoeval.VerdictPass, - Git: aoeval.GitRecord{ - CandidateRef: "test", - CandidateSHA: "0000000", - Dirty: false, - }, - Runtime: aoeval.RuntimeRecord{ - Name: aoeval.RuntimeStatic, - Live: false, - }, - Environment: aoeval.EnvironmentRecord{ - ScrubbedEnvPrefixes: []string{"AGENTOPS_RPI_RUNTIME"}, - IsolatedHome: false, - IsolatedCodexHome: false, - NetworkAccess: aoeval.NetworkDisabled, - }, - CaseResults: []aoeval.CaseResult{ - { - ID: "case", - Status: aoeval.StatusPass, - Score: score, - DimensionScores: map[aoeval.Dimension]float64{ - aoeval.DimensionCorrectness: score, - }, - }, - }, - AggregateScore: score, - DimensionScores: map[aoeval.Dimension]float64{ - aoeval.DimensionCorrectness: score, - }, - } - data, err := json.MarshalIndent(run, "", " ") - if err != nil { - t.Fatalf("marshal run: %v", err) - } - writeEvalCmdFile(t, path, string(data)) -} - -func writeEvalScorecardRunRecord(t *testing.T, path, runID string, scores map[string]float64) { - t.Helper() - slugs := []string{ - "artifact-completeness", - "phase-order", - "objective-spine", - "validation-separation", - "scenario-satisfaction", - "runtime-safety", - } - sum := 0.0 - results := make([]aoeval.CaseResult, 0, len(slugs)) - for _, slug := range slugs { - score := scores[slug] - sum += score - results = append(results, aoeval.CaseResult{ - ID: "scorecard." + slug + ".surface", - Status: aoeval.StatusPass, - Score: score, - DimensionScores: map[aoeval.Dimension]float64{ - aoeval.DimensionCorrectness: score, - }, - }) - } - aggregate := sum / float64(len(slugs)) - run := aoeval.RunRecord{ - SchemaVersion: 1, - RunID: runID, - Suite: aoeval.SuiteRef{ - ID: "agentops-core.rpi-scorecard", - Path: "suite.json", - Visibility: aoeval.VisibilityPublicCanary, - Tier: aoeval.TierDeterministic, - }, - StartedAt: fixedEvalCmdTime(), - Status: aoeval.StatusPass, - Verdict: aoeval.VerdictPass, - Git: aoeval.GitRecord{ - CandidateRef: "test", - CandidateSHA: "0000000", - Dirty: false, - }, - Runtime: aoeval.RuntimeRecord{ - Name: aoeval.RuntimeStatic, - Live: false, - }, - Environment: aoeval.EnvironmentRecord{ - ScrubbedEnvPrefixes: []string{"AGENTOPS_RPI_RUNTIME"}, - IsolatedHome: false, - IsolatedCodexHome: false, - NetworkAccess: aoeval.NetworkDisabled, - }, - CaseResults: results, - AggregateScore: aggregate, - DimensionScores: map[aoeval.Dimension]float64{ - aoeval.DimensionCorrectness: aggregate, - aoeval.DimensionProcessAdherence: aggregate, - aoeval.DimensionArtifactQuality: aggregate, - aoeval.DimensionRuntimeCompatibility: aggregate, - aoeval.DimensionSafety: aggregate, - aoeval.DimensionLearningClosure: aggregate, - }, - } - data, err := json.MarshalIndent(run, "", " ") - if err != nil { - t.Fatalf("marshal scorecard run: %v", err) - } - writeEvalCmdFile(t, path, string(data)) -} - -func evalScorecardCategory(t *testing.T, scorecard aoeval.Scorecard, name string) aoeval.ScorecardCategory { - t.Helper() - for _, category := range scorecard.Categories { - if category.Category == name { - return category - } - } - t.Fatalf("category %q not found in %+v", name, scorecard.Categories) - return aoeval.ScorecardCategory{} -} - -func fixedEvalCmdTime() time.Time { - return time.Date(2026, 4, 24, 12, 0, 0, 0, time.UTC) -} - -func withEvalCommand(t *testing.T) { - t.Helper() - resetEvalRunGlobals() - registerEvalCommand() -} - -func resetEvalRunGlobals() { - evalRunOutput = "" - evalRunID = "" - evalRunRuntime = "" - evalRunBaseline = "" - evalRunBaselineMode = string(aoeval.BaselineModeSkillOn) - evalRunContextMode = string(aoeval.ContextModeNone) - evalRunContextOffDir = "" - evalRunContextOnDir = "" - evalRunDeltaOut = "" -} - -func writeEvalSubManifest(t *testing.T, root, runID string, status evalsub.RunStatus, started time.Time) { - t.Helper() - manifest := evalsub.Manifest{ - SchemaVersion: evalsub.SchemaVersion, - ID: runID, - Kind: "task", - Status: status, - StartedAt: started.Format(time.RFC3339), - StartedAtUnixMs: started.UnixNano() / int64(time.Millisecond), - TaskRef: "task-a", - SuiteRef: "suite-a", - HarnessRef: "harness-a", - HarnessContentHash: "sha256:harness", - ModelSpecRef: "model-a", - ModelSpecHash: "sha256:model", - GroundTruthRef: "gt-a", - GroundTruthHash: "sha256:gt", - SampleSplit: "dev", - NSamples: 3, - Seeds: []int{1, 2, 3}, - InspectCommand: "inspect eval", - InspectVersion: "0.3.216", - ValidityGatesPassed: []string{"held_constant_declared"}, - RigID: "rig-a", - CapturedBy: "test", - } - data, err := json.MarshalIndent(manifest, "", " ") - if err != nil { - t.Fatalf("marshal eval substrate manifest: %v", err) - } - data = append(data, '\n') - if err := evalsub.WriteAtomic(evalsub.ManifestPath(root, runID), data); err != nil { - t.Fatalf("write eval substrate manifest: %v", err) - } -} - -func assertEvalSubManifestStatus(t *testing.T, root, runID string, status evalsub.RunStatus, reason string) { - t.Helper() - manifest, err := evalsub.LoadManifest(evalsub.ManifestPath(root, runID)) - if err != nil { - t.Fatalf("load %s manifest: %v", runID, err) - } - if manifest.Status != status || manifest.RetractionReason != reason { - t.Fatalf("%s status/reason = %s/%q, want %s/%q", runID, manifest.Status, manifest.RetractionReason, status, reason) - } -} - -func writeEvalSubTask(t *testing.T, root string, task evalsub.Task) { - t.Helper() - body, err := json.Marshal(task) - if err != nil { - t.Fatalf("marshal eval substrate task: %v", err) - } - if err := evalsub.WriteAtomic(filepath.Join(root, "tasks", task.ID, "task.yaml"), body); err != nil { - t.Fatalf("write eval substrate task: %v", err) - } -} - -func writeEvalSubSuite(t *testing.T, root string, suite evalsub.Suite) { - t.Helper() - body, err := json.Marshal(suite) - if err != nil { - t.Fatalf("marshal eval substrate suite: %v", err) - } - if err := evalsub.WriteAtomic(filepath.Join(root, "suites", suite.ID, "suite.yaml"), body); err != nil { - t.Fatalf("write eval substrate suite: %v", err) - } -} - -func writeEvalSubGroundTruth(t *testing.T, root string, rows []evalsub.GroundTruthRow) { - t.Helper() - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - for _, row := range rows { - if err := enc.Encode(row); err != nil { - t.Fatalf("encode eval substrate ground truth: %v", err) - } - } - if err := evalsub.WriteAtomic(filepath.Join(root, "ground-truth", "ground-truth.jsonl"), buf.Bytes()); err != nil { - t.Fatalf("write eval substrate ground truth: %v", err) - } -} - -func restoreEvalTaskRunGlobals(t *testing.T) { - t.Helper() - oldSuiteRef := evalTaskRunSuiteRef - oldRigID := evalTaskRunRigID - oldSeeds := evalTaskRunSeeds - oldHarnessRef := evalTaskRunHarnessRef - oldHarnessDir := evalTaskRunHarnessDir - oldModelSpecID := evalTaskRunModelSpecID - oldGTRef := evalTaskRunGTRef - oldSampleSplit := evalTaskRunSampleSplit - oldNSamples := evalTaskRunNSamples - oldInspectVersion := evalTaskRunInspectVersion - oldInspectCommand := evalTaskRunInspectCommand - oldCrossSpec := evalTaskRunCrossSpec - oldAllowWeak := evalTaskRunAllowWeak - oldQuickSession := evalTaskRunQuickSession - oldDryRun := evalTaskRunDryRun - evalTaskRunSuiteRef = "" - evalTaskRunRigID = "" - evalTaskRunSeeds = "" - evalTaskRunHarnessRef = "" - evalTaskRunHarnessDir = "" - evalTaskRunModelSpecID = "" - evalTaskRunGTRef = "" - evalTaskRunSampleSplit = "" - evalTaskRunNSamples = 0 - evalTaskRunInspectVersion = "0.3.216" - evalTaskRunInspectCommand = "" - evalTaskRunCrossSpec = false - evalTaskRunAllowWeak = false - evalTaskRunQuickSession = false - evalTaskRunDryRun = false - t.Cleanup(func() { - evalTaskRunSuiteRef = oldSuiteRef - evalTaskRunRigID = oldRigID - evalTaskRunSeeds = oldSeeds - evalTaskRunHarnessRef = oldHarnessRef - evalTaskRunHarnessDir = oldHarnessDir - evalTaskRunModelSpecID = oldModelSpecID - evalTaskRunGTRef = oldGTRef - evalTaskRunSampleSplit = oldSampleSplit - evalTaskRunNSamples = oldNSamples - evalTaskRunInspectVersion = oldInspectVersion - evalTaskRunInspectCommand = oldInspectCommand - evalTaskRunCrossSpec = oldCrossSpec - evalTaskRunAllowWeak = oldAllowWeak - evalTaskRunQuickSession = oldQuickSession - evalTaskRunDryRun = oldDryRun - }) -} - -func TestParseEvalRuntime(t *testing.T) { - tests := []struct { - input string - want aoeval.Runtime - wantErr bool - }{ - {"static", aoeval.RuntimeStatic, false}, - {"mock", aoeval.RuntimeMock, false}, - {"shell", aoeval.RuntimeShell, false}, - {"claude", aoeval.RuntimeClaude, false}, - {"codex", aoeval.RuntimeCodex, false}, - {"", "", false}, - {"unknown", "", true}, - } - for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { - got, err := parseEvalRuntime(tt.input) - if (err != nil) != tt.wantErr { - t.Fatalf("parseEvalRuntime(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) - } - if got != tt.want { - t.Fatalf("parseEvalRuntime(%q) = %q, want %q", tt.input, got, tt.want) - } - }) - } -} diff --git a/cli/cmd/ao/eval_test_compat_test.go b/cli/cmd/ao/eval_test_compat_test.go new file mode 100644 index 000000000..9efc5e086 --- /dev/null +++ b/cli/cmd/ao/eval_test_compat_test.go @@ -0,0 +1,6 @@ +package main + +// scenarioListStatus remains only as a reset-harness compatibility cell for +// older package-main tests. The production eval command owns closure-local +// state in internal/commands/eval. +var scenarioListStatus string diff --git a/cli/cmd/ao/retrieval_bench.go b/cli/cmd/ao/retrieval_bench.go index 7a40ba339..5e0177d91 100644 --- a/cli/cmd/ao/retrieval_bench.go +++ b/cli/cmd/ao/retrieval_bench.go @@ -713,20 +713,8 @@ func bindRetrievalBenchFlags(cmd *cobra.Command) { cmd.Flags().StringVar(&benchSearchCompareBackends, "search-compare-backends", "", "Comma-separated search backends to compare for --search-eval") } -// evalBenchCmd is the canonical `ao eval bench` alias for the retrieval-quality -// benchmark, folded under `ao eval` (age-focus-membrane-bookkeeper-m1wg.16). -var evalBenchCmd = &cobra.Command{ - Use: "bench", - Short: "Run retrieval quality benchmarks", - Long: retrievalBenchCmd.Long, - RunE: runRetrievalBench, -} - func init() { retrievalBenchCmd.GroupID = "knowledge" rootCmd.AddCommand(retrievalBenchCmd) bindRetrievalBenchFlags(retrievalBenchCmd) - - bindRetrievalBenchFlags(evalBenchCmd) - evalCmd.AddCommand(evalBenchCmd) } diff --git a/cli/cmd/ao/scenario.go b/cli/cmd/ao/scenario.go deleted file mode 100644 index 283282670..000000000 --- a/cli/cmd/ao/scenario.go +++ /dev/null @@ -1,22 +0,0 @@ -// practices: [property-based-testing, llm-eval-harness] -package main - -import "github.com/spf13/cobra" - -var scenarioCmd = &cobra.Command{ - Use: "scenario", - Short: "Manage holdout scenarios for behavioral validation", - Long: `Create, list, and validate holdout scenarios stored in .agents/holdout/. - -Scenarios are behavioral validation specs that implementing agents never see. -They are evaluated by council judges during validation (STEP 1.8) to assess -whether the implementation satisfies user intent.`, -} - -func init() { - // Folded under `ao eval` (age-focus-membrane-bookkeeper-m1wg.16). The - // add/init/list/validate children are attached to scenarioCmd elsewhere and - // auto-follow this reparent. `ao eval scenario-ab` (ADR-0004 revival path) is - // a distinct sibling and is unaffected. - evalCmd.AddCommand(scenarioCmd) -} diff --git a/cli/cmd/ao/scenario_add.go b/cli/cmd/ao/scenario_add.go deleted file mode 100644 index 4853278c1..000000000 --- a/cli/cmd/ao/scenario_add.go +++ /dev/null @@ -1,92 +0,0 @@ -// practices: [property-based-testing, llm-eval-harness] -package main - -import ( - "encoding/json" - "fmt" - "path/filepath" - "time" - - "github.com/boshu2/agentops/cli/internal/scenario" - "github.com/spf13/cobra" -) - -// scenarioFile aliases the scenario package's Scenario type so other files in -// this package (and tests) keep a stable local name. -type scenarioFile = scenario.Scenario - -// scenarioAcceptanceVector aliases the scenario package's AcceptanceVector. -type scenarioAcceptanceVector = scenario.AcceptanceVector - -// scenarioIDPattern aliases the scenario package's ID pattern (used by -// scenario validate). -var scenarioIDPattern = scenario.IDPattern - -// scenarioAddNow is the injectable clock for deterministic IDs in tests. -var scenarioAddNow = time.Now - -var scenarioAddFlags = struct { - Narrative string - ExpectedOutcome string - Threshold float64 - Status string - Source string -}{ - Threshold: 0.8, - Status: "draft", - Source: "human", -} - -var scenarioAddCmd = &cobra.Command{ - Use: "add ", - Short: "Author a holdout scenario from a goal description", - Long: `Author a schema-compliant holdout scenario in .agents/holdout/. - -The command infers narrative and expected-outcome text from the provided goal -unless explicit values are supplied. New scenarios default to draft so a human -or evaluator can review them before activation.`, - Args: cobra.ExactArgs(1), - RunE: runScenarioAdd, -} - -// runScenarioAdd authors an ad hoc holdout scenario by delegating to the shared -// scenario.Create path — the same path `ao goals scenarios --create` uses. -func runScenarioAdd(cmd *cobra.Command, args []string) error { - res, err := scenario.Create(scenario.CreateOptions{ - Goal: args[0], - Narrative: scenarioAddFlags.Narrative, - ExpectedOutcome: scenarioAddFlags.ExpectedOutcome, - Threshold: scenarioAddFlags.Threshold, - Status: scenarioAddFlags.Status, - Source: scenarioAddFlags.Source, - Dir: filepath.Join(".agents", "holdout"), - Now: scenarioAddNow, - }) - if err != nil { - return err - } - if GetOutput() == "json" { - enc := json.NewEncoder(cmd.OutOrStdout()) - enc.SetIndent("", " ") - return enc.Encode(res.Scenario) - } - fmt.Fprintf(cmd.OutOrStdout(), "Created scenario %s at %s\n", res.Scenario.ID, res.Path) - return nil -} - -// validScenarioStatus reports whether status is an allowed scenario status. -func validScenarioStatus(status string) bool { return scenario.ValidStatus(status) } - -// validScenarioSource reports whether source is an allowed scenario source. -func validScenarioSource(source string) bool { return scenario.ValidSource(source) } - -func init() { - scenarioAddCmd.Flags().StringVar(&scenarioAddFlags.Narrative, "narrative", "", "Narrative description (default: inferred from goal)") - scenarioAddCmd.Flags().StringVar(&scenarioAddFlags.ExpectedOutcome, "expected-outcome", "", "Expected observable outcome (default: inferred from goal)") - scenarioAddCmd.Flags().Float64Var(&scenarioAddFlags.Threshold, "threshold", 0.8, "Satisfaction threshold in [0,1]") - scenarioAddCmd.Flags().StringVar(&scenarioAddFlags.Status, "status", "draft", "Scenario status (active, draft, retired)") - scenarioAddCmd.Flags().StringVar(&scenarioAddFlags.Source, "source", "human", "Scenario source (human, agent, prod-telemetry)") - _ = scenarioAddCmd.RegisterFlagCompletionFunc("status", staticCompletionFunc("active", "draft", "retired")) - _ = scenarioAddCmd.RegisterFlagCompletionFunc("source", staticCompletionFunc("human", "agent", "prod-telemetry")) - scenarioCmd.AddCommand(scenarioAddCmd) -} diff --git a/cli/cmd/ao/scenario_evaluate.go b/cli/cmd/ao/scenario_evaluate.go deleted file mode 100644 index 17205279d..000000000 --- a/cli/cmd/ao/scenario_evaluate.go +++ /dev/null @@ -1,440 +0,0 @@ -// practices: [bdd-gherkin, llm-eval-harness] -package main - -import ( - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "time" - - "github.com/boshu2/agentops/cli/internal/goals" - "github.com/boshu2/agentops/cli/internal/goalsfitness" - "github.com/boshu2/agentops/cli/internal/scenario" - "github.com/boshu2/agentops/cli/internal/scenarioresults" - "github.com/spf13/cobra" -) - -// gateCheckPrefix marks an acceptance-vector check that references a GOALS.md -// Gates-table entry by ID (e.g. "gate:go-cli-builds") instead of carrying a -// literal shell command. The referenced gate's Check column is executed. -const gateCheckPrefix = "gate:" - -// scenarioShape classifies how a scenario can be evaluated. -type scenarioShape string - -const ( - // shapeGate means the scenario carries >=1 acceptance vector with a - // mechanical check command — it is evaluated by running those commands. - shapeGate scenarioShape = "gate" - // shapeJudgment means the scenario has no mechanical check — it cannot be - // evaluated by this command and is recorded as skip/attestation-needed. - shapeJudgment scenarioShape = "judgment" -) - -// scenarioEvaluateNow is the injectable clock for deterministic judged_at / -// generated_at timestamps in tests. -var scenarioEvaluateNow = time.Now - -// scenarioEvaluateFlags holds the evaluate subcommand's flag values. -var scenarioEvaluateFlags = struct { - DirectiveID string - All bool - JSON bool - Timeout time.Duration - RunID string -}{ - Timeout: 2 * time.Minute, - RunID: "ao-scenario-evaluate", -} - -// scenarioEvaluation is the per-scenario outcome reported by the command. It -// mirrors the persisted scenarioresults.ScenarioResult plus the shape and any -// non-persisted skip reason (missing/retired links write no result). -type scenarioEvaluation struct { - ScenarioID string `json:"scenario_id"` - DirectiveID string `json:"directive_id"` - Shape string `json:"shape,omitempty"` - Verdict string `json:"verdict,omitempty"` - Score float64 `json:"score"` - Threshold float64 `json:"threshold"` - Evidence []string `json:"evidence,omitempty"` - // Recorded is false when no result was written (missing spec file, retired - // scenario, or an unwritable directive ID) — the artifact stays untouched - // so the downstream aggregator keeps reporting "missing", never a verdict. - Recorded bool `json:"recorded"` - Note string `json:"note,omitempty"` -} - -// scenarioEvaluateReport is the full command output. -type scenarioEvaluateReport struct { - RunID string `json:"run_id"` - Iteration int `json:"iteration"` - Artifact string `json:"artifact"` - Written int `json:"written"` - Evaluations []scenarioEvaluation `json:"evaluations"` -} - -var scenarioEvaluateCmd = &cobra.Command{ - Use: "evaluate", - Short: "Evaluate directive-linked scenarios and record satisfaction results", - Long: `Evaluate the executable-spec scenarios linked to GOALS.md directives and -append their verdicts to ` + scenarioresults.ArtifactRelPath + ` — the producer -side of the 'ao goals measure --scenarios-only' satisfaction panel. - -A scenario is GATE-SHAPED when it declares acceptance_vectors carrying a -mechanical "check" command. Each check runs under a per-check timeout; a check -of the form "gate:" resolves through the GOALS.md Gates table and runs -that gate's Check command. The scenario's score is the fraction of checks -that passed, and its verdict is pass/fail against its own -satisfaction_threshold. A check that could not run (timeout, skip exit 77, -unresolvable gate reference) yields verdict "skip" — never a fabricated pass -or fail. - -A scenario with no mechanical check is JUDGMENT-SHAPED: it is recorded as -verdict "skip" with attestation-needed evidence, because this command cannot -honestly evaluate it. Missing or retired scenarios write no result at all, so -the downstream aggregator keeps reporting them as missing. - - ao eval scenario evaluate --all evaluate every directive's scenarios - ao eval scenario evaluate --directive d-foo evaluate one directive's scenarios - ao eval scenario evaluate --all --json machine-readable report`, - Args: cobra.NoArgs, - RunE: runScenarioEvaluate, -} - -// runScenarioEvaluate drives the evaluate flow: resolve directives, evaluate -// each linked scenario, and append the results via the production writer. -func runScenarioEvaluate(cmd *cobra.Command, _ []string) error { - if !scenarioEvaluateFlags.All && scenarioEvaluateFlags.DirectiveID == "" { - return fmt.Errorf("choose a scope: --all for every directive, or --directive ") - } - - projectRoot := measureProjectRoot() - goalsPath := resolveGoalsFile() - patcher, _, err := goals.LoadGoalsPatcher(goalsPath) - if err != nil { - return fmt.Errorf("loading directives: %w", err) - } - directives := goals.FilterDirectives(patcher.Directives(), 0, scenarioEvaluateFlags.DirectiveID) - if len(directives) == 0 { - return fmt.Errorf("no directive matches --directive %q", scenarioEvaluateFlags.DirectiveID) - } - - gates, err := loadGatesByID(goalsPath) - if err != nil { - return fmt.Errorf("loading gates table: %w", err) - } - - evaluations := evaluateDirectiveLinks(directives, gates) - report, err := appendScenarioResults(projectRoot, evaluations) - if err != nil { - return err - } - return emitScenarioEvaluateReport(cmd.OutOrStdout(), report) -} - -// evaluateDirectiveLinks evaluates every scenario linked from directives. -func evaluateDirectiveLinks(directives []goals.ParsedDirective, gates map[string]string) []scenarioEvaluation { - var evaluations []scenarioEvaluation - dirs := goals.DefaultScenarioDirs() - for _, d := range directives { - for _, sid := range d.Scenarios { - evaluations = append(evaluations, evaluateLinkedScenario(d, sid, dirs, gates)) - } - } - return evaluations -} - -// evaluateLinkedScenario evaluates one directive→scenario link. Missing, -// unreadable, or retired scenarios produce an unrecorded evaluation (no -// artifact write) so "could not evaluate" is never converted into a verdict. -func evaluateLinkedScenario(d goals.ParsedDirective, sid string, dirs []string, gates map[string]string) scenarioEvaluation { - ev := scenarioEvaluation{ScenarioID: sid, DirectiveID: d.StableID} - spec, err := loadScenarioSpec(dirs, sid) - switch { - case err != nil: - ev.Note = fmt.Sprintf("unreadable scenario spec: %v", err) - return ev - case spec == nil: - ev.Note = "no scenario file found on the search path" - return ev - case spec.Status == "retired": - ev.Note = "scenario is retired; not evaluated" - return ev - } - if ev.DirectiveID == "" { - ev.DirectiveID = spec.DirectiveID - } - if !scenarioresults.ValidDirectiveID(ev.DirectiveID) || !scenarioresults.ValidScenarioID(sid) { - ev.Note = "no valid stable directive/scenario ID; result would not validate" - return ev - } - - if spec.SatisfactionThreshold < 0 || spec.SatisfactionThreshold > 1 { - // An out-of-range threshold is a malformed spec. Silently normalizing it - // and then certifying a pass would fabricate satisfaction for a contract - // nobody wrote; record skip with the defect named instead. - ev.Shape = string(shapeJudgment) - ev.Threshold = normalizeScenarioThreshold(spec.SatisfactionThreshold) - ev.Verdict = scenarioresults.VerdictSkip - ev.Evidence = []string{fmt.Sprintf( - "invalid satisfaction_threshold %g (must be in (0,1]); spec must be fixed before this scenario can certify", - spec.SatisfactionThreshold)} - ev.Recorded = true - return ev - } - - ev.Threshold = normalizeScenarioThreshold(spec.SatisfactionThreshold) - checks := mechanicalChecks(spec) - if len(checks) == 0 { - ev.Shape = string(shapeJudgment) - ev.Verdict = scenarioresults.VerdictSkip - ev.Evidence = []string{"attestation-needed: no mechanical acceptance-vector check; requires judge/human attestation"} - ev.Recorded = true - return ev - } - - if len(checks) < len(spec.AcceptanceVectors) { - // Mixed shape: some vectors are mechanical, some need attestation. A pass - // scored over the mechanical subset alone would certify the scenario with - // incomplete evidence (fail-open). Run the mechanical checks for their - // evidence value, but the recorded verdict stays skip/attestation-needed. - ev.Shape = string(shapeJudgment) - _, score, evidence := runScenarioChecks(checks, gates, ev.Threshold) - ev.Score = score - unchecked := len(spec.AcceptanceVectors) - len(checks) - ev.Evidence = append([]string{fmt.Sprintf( - "attestation-needed: %d of %d acceptance vectors carry no mechanical check; mechanical subset ran for evidence only", - unchecked, len(spec.AcceptanceVectors))}, evidence...) - ev.Verdict = scenarioresults.VerdictSkip - ev.Recorded = true - return ev - } - - ev.Shape = string(shapeGate) - ev.Verdict, ev.Score, ev.Evidence = runScenarioChecks(checks, gates, ev.Threshold) - ev.Recorded = true - return ev -} - -// mechanicalChecks returns the scenario's acceptance vectors that carry a -// non-empty mechanical check command. -func mechanicalChecks(spec *scenario.Scenario) []scenario.AcceptanceVector { - var out []scenario.AcceptanceVector - for _, v := range spec.AcceptanceVectors { - if strings.TrimSpace(v.Check) != "" { - out = append(out, v) - } - } - return out -} - -// runScenarioChecks executes each mechanical check and folds the per-check -// outcomes into a scenario verdict, score, and evidence trail. -// -// Score is the fraction of checks that passed. The verdict is pass/fail by -// score vs threshold ONLY when every check produced real pass/fail evidence; -// if any check could not run (timeout, exit 77 skip, unresolvable gate -// reference) the scenario verdict is "skip" — incomplete evidence is never -// laundered into a pass or a fail. -func runScenarioChecks(checks []scenario.AcceptanceVector, gates map[string]string, threshold float64) (verdict string, score float64, evidence []string) { - passed, skipped := 0, 0 - for _, v := range checks { - outcome, detail := runOneCheck(v, gates) - switch outcome { - case resultPassOutcome: - passed++ - case resultSkipOutcome: - skipped++ - } - evidence = append(evidence, fmt.Sprintf("%s: %s — %s", v.Dimension, outcome, detail)) - } - score = float64(passed) / float64(len(checks)) - if skipped > 0 { - return scenarioresults.VerdictSkip, score, evidence - } - return verdictForScore(score, threshold), score, evidence -} - -// Per-check outcome labels (aligned with goals.Measurement.Result values). -const ( - resultPassOutcome = "pass" - resultFailOutcome = "fail" - resultSkipOutcome = "skip" -) - -// runOneCheck resolves and executes a single acceptance-vector check, returning -// its pass/fail/skip outcome and a human-readable evidence detail. -func runOneCheck(v scenario.AcceptanceVector, gates map[string]string) (outcome, detail string) { - command := strings.TrimSpace(v.Check) - if gateID, ok := strings.CutPrefix(command, gateCheckPrefix); ok { - resolved, found := gates[strings.TrimSpace(gateID)] - if !found { - return resultSkipOutcome, fmt.Sprintf("unresolvable gate reference %q (not in the GOALS.md Gates table)", command) - } - command = resolved - } - - m := goals.MeasureOne(goals.Goal{ID: "scenario-check", Check: command}, scenarioEvaluateFlags.Timeout) - detail = fmt.Sprintf("`%s` -> %s", command, m.Result) - if m.Result != resultPassOutcome && m.Output != "" { - detail += ": " + m.Output - } - return m.Result, detail -} - -// verdictForScore maps a completed-evidence score onto pass/fail against the -// scenario's own satisfaction threshold (equality passes). It is only called -// when every check produced pass/fail evidence, and it deliberately matches -// the goalsfitness countSatisfied comparison (score >= threshold) so a -// persisted verdict never disagrees with the aggregator's satisfied count. -func verdictForScore(score, threshold float64) string { - if score >= threshold { - return scenarioresults.VerdictPass - } - return scenarioresults.VerdictFail -} - -// normalizeScenarioThreshold clamps an absent or out-of-range spec threshold to -// the directive-layer default so a persisted result always validates and a -// zero threshold can never make an unevaluated scenario count as satisfied. -func normalizeScenarioThreshold(t float64) float64 { - if t <= 0 || t > 1 { - return goalsfitness.DefaultScenarioThreshold - } - return t -} - -// appendScenarioResults persists the recorded evaluations through the -// production scenarioresults.Writer and assembles the command report. When -// nothing was recorded the artifact is left untouched (no empty-run bump). -func appendScenarioResults(projectRoot string, evaluations []scenarioEvaluation) (*scenarioEvaluateReport, error) { - report := &scenarioEvaluateReport{ - RunID: scenarioEvaluateFlags.RunID, - Artifact: scenarioresults.ArtifactRelPath, - Evaluations: evaluations, - } - if report.Evaluations == nil { - report.Evaluations = []scenarioEvaluation{} - } - - results := persistableResults(evaluations) - report.Written = len(results) - if len(results) == 0 { - return report, nil - } - - report.Iteration = nextIteration(projectRoot) - writer := scenarioresults.Writer{Now: scenarioEvaluateNow} - if _, err := writer.Append(projectRoot, report.RunID, report.Iteration, results); err != nil { - return nil, fmt.Errorf("writing scenario results: %w", err) - } - return report, nil -} - -// persistableResults converts recorded evaluations into contract results. -func persistableResults(evaluations []scenarioEvaluation) []scenarioresults.ScenarioResult { - judgedAt := scenarioEvaluateNow().UTC().Format(time.RFC3339) - var results []scenarioresults.ScenarioResult - for _, ev := range evaluations { - if !ev.Recorded { - continue - } - results = append(results, scenarioresults.ScenarioResult{ - ScenarioID: ev.ScenarioID, - DirectiveID: ev.DirectiveID, - Score: ev.Score, - Threshold: ev.Threshold, - Verdict: ev.Verdict, - JudgedAt: judgedAt, - Evidence: ev.Evidence, - }) - } - return results -} - -// nextIteration returns the prior artifact's iteration + 1 (1 for a fresh -// artifact). A malformed artifact yields 1 here; the Writer itself still -// refuses to clobber a malformed artifact, so no data is lost. -func nextIteration(projectRoot string) int { - loaded, err := scenarioresults.Load(projectRoot, false) - if err == nil && loaded.Artifact != nil { - return loaded.Artifact.Iteration + 1 - } - return 1 -} - -// loadScenarioSpec reads the full scenario JSON for id from the ordered search -// dirs. A nil spec with nil error means "not found". -func loadScenarioSpec(dirs []string, id string) (*scenario.Scenario, error) { - for _, dir := range dirs { - path := filepath.Join(dir, id+".json") - data, err := os.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - continue - } - return nil, fmt.Errorf("%s: %w", path, err) - } - var spec scenario.Scenario - if err := json.Unmarshal(data, &spec); err != nil { - return nil, fmt.Errorf("%s: invalid JSON: %w", path, err) - } - return &spec, nil - } - return nil, nil -} - -// loadGatesByID parses the GOALS.md Gates table into an ID→Check map for -// resolving "gate:" acceptance-vector checks. -func loadGatesByID(goalsPath string) (map[string]string, error) { - data, err := os.ReadFile(goalsPath) - if err != nil { - return nil, err - } - gf, err := goals.ParseMarkdownGoals(data) - if err != nil { - return nil, err - } - gates := make(map[string]string, len(gf.Goals)) - for _, g := range gf.Goals { - gates[g.ID] = g.Check - } - return gates, nil -} - -// emitScenarioEvaluateReport renders the report as JSON or a human summary. -func emitScenarioEvaluateReport(w io.Writer, report *scenarioEvaluateReport) error { - if scenarioEvaluateFlags.JSON || GetOutput() == "json" { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(report) - } - for _, ev := range report.Evaluations { - if !ev.Recorded { - fmt.Fprintf(w, "%-20s %-10s not recorded — %s\n", ev.ScenarioID, "-", ev.Note) - continue - } - fmt.Fprintf(w, "%-20s %-10s shape=%s score=%.2f threshold=%.2f\n", - ev.ScenarioID, ev.Verdict, ev.Shape, ev.Score, ev.Threshold) - } - if report.Written == 0 { - fmt.Fprintln(w, "No results recorded; artifact untouched.") - return nil - } - fmt.Fprintf(w, "Wrote %d result(s) to %s (run %s, iteration %d)\n", - report.Written, report.Artifact, report.RunID, report.Iteration) - return nil -} - -func init() { - scenarioEvaluateCmd.Flags().StringVar(&scenarioEvaluateFlags.DirectiveID, "directive", "", "Evaluate only the directive with this stable Directive ID") - scenarioEvaluateCmd.Flags().BoolVar(&scenarioEvaluateFlags.All, "all", false, "Evaluate every directive's linked scenarios") - scenarioEvaluateCmd.Flags().BoolVar(&scenarioEvaluateFlags.JSON, "json", false, "Emit the machine-readable evaluation report") - scenarioEvaluateCmd.Flags().DurationVar(&scenarioEvaluateFlags.Timeout, "timeout", 2*time.Minute, "Per-check execution timeout") - scenarioEvaluateCmd.Flags().StringVar(&scenarioEvaluateFlags.RunID, "run-id", "ao-scenario-evaluate", "run_id recorded in the results artifact") - scenarioCmd.AddCommand(scenarioEvaluateCmd) -} diff --git a/cli/cmd/ao/scenario_evaluate_test.go b/cli/cmd/ao/scenario_evaluate_test.go deleted file mode 100644 index 8b2cd7541..000000000 --- a/cli/cmd/ao/scenario_evaluate_test.go +++ /dev/null @@ -1,437 +0,0 @@ -// practices: [tdd, bdd-gherkin] -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/boshu2/agentops/cli/internal/scenarioresults" -) - -// evaluateGoalsMD is a GOALS.md fixture with a Gates table (one passing stub -// gate) and three directives whose linked scenarios cover the gate-shaped -// pass, judgment-shaped, and gate-shaped fail lanes. -const evaluateGoalsMD = "# Goals\n\n" + - "Mission.\n\n" + - "## Gates\n\n" + - "| ID | Check | Weight | Description |\n" + - "|----|-------|--------|-------------|\n" + - "| stub-pass | `exit 0` | 5 | Always green stub gate |\n\n" + - "## Directives\n\n" + - "### 1. Gate-shaped passing directive\n\n" + - "**Directive ID:** d-gate-pass\n" + - "**Steer:** maintain\n" + - "**Scenarios:** s-2026-05-01-001\n\n" + - "### 2. Judgment-shaped directive\n\n" + - "**Directive ID:** d-judgment\n" + - "**Steer:** maintain\n" + - "**Scenarios:** s-2026-05-01-002\n\n" + - "### 3. Gate-shaped failing directive\n\n" + - "**Directive ID:** d-gate-fail\n" + - "**Steer:** maintain\n" + - "**Scenarios:** s-2026-05-01-003\n" - -// writeScenarioSpec writes one scenario JSON into spec/scenarios/ under root. -func writeScenarioSpec(t *testing.T, root, id, directiveID string, threshold float64, vectorsJSON string) { - t.Helper() - specDir := filepath.Join(root, "spec", "scenarios") - if err := os.MkdirAll(specDir, 0o755); err != nil { - t.Fatalf("mkdir spec/scenarios: %v", err) - } - doc := fmt.Sprintf(`{ - "id": %q, - "directive_id": %q, - "version": 1, - "date": "2026-05-01", - "goal": "fixture goal", - "narrative": "fixture narrative", - "expected_outcome": "fixture outcome", - "satisfaction_threshold": %g, - %s - "source": "human", - "status": "active" -}`, id, directiveID, threshold, vectorsJSON) - if err := os.WriteFile(filepath.Join(specDir, id+".json"), []byte(doc), 0o644); err != nil { - t.Fatalf("write scenario %s: %v", id, err) - } -} - -// setupScenarioEvaluateProject builds a temp project with GOALS.md, chdirs into -// it, and scopes the goalsFile/output globals the command reads. -func setupScenarioEvaluateProject(t *testing.T, goalsMD string) string { - t.Helper() - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "GOALS.md"), []byte(goalsMD), 0o644); err != nil { - t.Fatalf("write GOALS.md: %v", err) - } - t.Chdir(dir) - oldGoalsFile := goalsFile - oldOutput := output - t.Cleanup(func() { - goalsFile = oldGoalsFile - output = oldOutput - }) - goalsFile = "GOALS.md" - return dir -} - -// decodeEvaluateReport parses the --json report out of command output. -func decodeEvaluateReport(t *testing.T, out string) scenarioEvaluateReport { - t.Helper() - var report scenarioEvaluateReport - dec := json.NewDecoder(strings.NewReader(out)) - if err := dec.Decode(&report); err != nil { - t.Fatalf("decode evaluate report: %v\noutput: %s", err, out) - } - return report -} - -// consumerReports runs the REAL consumer (`ao goals measure --scenarios-only` -// path) against root and returns its per-directive reports keyed by ID. -func consumerReports(t *testing.T, root string) map[string]directiveScenarioReport { - t.Helper() - var buf bytes.Buffer - if err := runScenariosOnly("GOALS.md", root, true, &buf); err != nil { - t.Fatalf("runScenariosOnly: %v", err) - } - var payload measureScenarioJSON - if err := json.Unmarshal(buf.Bytes(), &payload); err != nil { - t.Fatalf("decode consumer payload: %v\noutput: %s", err, buf.String()) - } - byID := make(map[string]directiveScenarioReport, len(payload.Directives)) - for _, d := range payload.Directives { - byID[d.DirectiveID] = d - } - return byID -} - -// TestScenarioEvaluate_MixedVectorsNeverCertifyPass pins the fail-open the -// cross-family pawl caught: a scenario mixing one passing mechanical vector -// with one attestation-only vector must record skip/attestation-needed, never -// a pass scored over the mechanical subset alone (incomplete evidence). -// TestScenarioEvaluate_OutOfRangeThresholdDoesNotCertify pins the pawl's -// round-3 catch: a malformed spec (threshold outside (0,1]) must never have -// its threshold silently normalized and then certify a pass. -func TestScenarioEvaluate_OutOfRangeThresholdDoesNotCertify(t *testing.T) { - root := setupScenarioEvaluateProject(t, evaluateGoalsMD) - writeScenarioSpec(t, root, "s-2026-05-01-001", "d-gate-pass", 1.2, - `"acceptance_vectors": [{"dimension": "mechanical", "threshold": 1.0, "check": "exit 0"}],`) - - out, err := executeCommand("eval", "scenario", "evaluate", "--all", "--json") - if err != nil { - t.Fatalf("scenario evaluate failed: %v\noutput: %s", err, out) - } - loaded, err := scenarioresults.Load(root, true) - if err != nil { - t.Fatalf("production loader rejected artifact: %v", err) - } - for _, r := range loaded.Artifact.Results { - if r.ScenarioID != "s-2026-05-01-001" { - continue - } - if r.Verdict != scenarioresults.VerdictSkip { - t.Fatalf("out-of-range threshold verdict = %q, want skip (never certify a malformed spec)", r.Verdict) - } - joined := strings.Join(r.Evidence, " | ") - if !strings.Contains(joined, "invalid satisfaction_threshold") { - t.Fatalf("evidence must name the invalid threshold, got: %s", joined) - } - return - } - t.Fatalf("result for s-2026-05-01-001 not written") -} - -func TestScenarioEvaluate_MixedVectorsNeverCertifyPass(t *testing.T) { - root := setupScenarioEvaluateProject(t, evaluateGoalsMD) - writeScenarioSpec(t, root, "s-2026-05-01-001", "d-gate-pass", 1.0, - `"acceptance_vectors": [ - {"dimension": "mechanical", "threshold": 1.0, "check": "exit 0"}, - {"dimension": "judgment-only", "threshold": 1.0} - ],`) - - out, err := executeCommand("eval", "scenario", "evaluate", "--all", "--json") - if err != nil { - t.Fatalf("scenario evaluate failed: %v\noutput: %s", err, out) - } - report := decodeEvaluateReport(t, out) - if report.Written != 1 { - t.Fatalf("expected 1 written result, got %d (report: %+v)", report.Written, report) - } - loaded, err := scenarioresults.Load(root, true) - if err != nil { - t.Fatalf("production loader rejected artifact: %v", err) - } - var got scenarioresults.ScenarioResult - found := false - for _, r := range loaded.Artifact.Results { - if r.ScenarioID == "s-2026-05-01-001" { - got, found = r, true - } - } - if !found { - t.Fatalf("mixed-vector scenario result not written") - } - if got.Verdict != scenarioresults.VerdictSkip { - t.Fatalf("mixed-vector scenario verdict = %q, want skip (never pass on partial evidence)", got.Verdict) - } - joined := strings.Join(got.Evidence, " | ") - if !strings.Contains(joined, "attestation-needed") || !strings.Contains(joined, "1 of 2") { - t.Fatalf("evidence must name the unevaluated vectors, got: %s", joined) - } - // The consumer must count it skipped, never satisfied. - reports := consumerReports(t, root) - d := reports["d-gate-pass"] - if d.EvaluatedCount != 0 || d.ScenarioVerdict != "unknown" { - t.Fatalf("consumer counted mixed scenario as evaluated (%d/%q); want 0/unknown", d.EvaluatedCount, d.ScenarioVerdict) - } -} - -func TestScenarioEvaluate_ProducerConsumerRoundTrip(t *testing.T) { - root := setupScenarioEvaluateProject(t, evaluateGoalsMD) - // Gate-shaped, passing: one gate-reference check + one direct command. - writeScenarioSpec(t, root, "s-2026-05-01-001", "d-gate-pass", 0.9, - `"acceptance_vectors": [ - {"dimension": "gate-ref", "threshold": 0.9, "check": "gate:stub-pass"}, - {"dimension": "direct", "threshold": 0.9, "check": "exit 0"} - ],`) - // Judgment-shaped: no mechanical check anywhere. - writeScenarioSpec(t, root, "s-2026-05-01-002", "d-judgment", 0.8, - `"acceptance_vectors": [{"dimension": "usability", "threshold": 0.8}],`) - // Gate-shaped, failing. - writeScenarioSpec(t, root, "s-2026-05-01-003", "d-gate-fail", 0.9, - `"acceptance_vectors": [{"dimension": "correctness", "threshold": 0.9, "check": "exit 1"}],`) - - out, err := executeCommand("eval", "scenario", "evaluate", "--all", "--json") - if err != nil { - t.Fatalf("scenario evaluate failed: %v\noutput: %s", err, out) - } - report := decodeEvaluateReport(t, out) - if report.Written != 3 { - t.Fatalf("expected 3 written results, got %d (report: %+v)", report.Written, report) - } - if report.Iteration != 1 { - t.Errorf("fresh artifact iteration = %d, want 1", report.Iteration) - } - if report.RunID != "ao-scenario-evaluate" { - t.Errorf("run_id = %q, want ao-scenario-evaluate", report.RunID) - } - - // The artifact must load through the PRODUCTION loader, strict mode. - loaded, err := scenarioresults.Load(root, true) - if err != nil { - t.Fatalf("production loader rejected artifact: %v", err) - } - if loaded.Status != scenarioresults.StatusOK { - t.Fatalf("artifact load status = %s, want ok", loaded.Status) - } - byScenario := map[string]scenarioresults.ScenarioResult{} - for _, r := range loaded.Artifact.Results { - byScenario[r.ScenarioID] = r - } - if got := byScenario["s-2026-05-01-001"]; got.Verdict != scenarioresults.VerdictPass || got.Score != 1.0 { - t.Errorf("s-001 = verdict %q score %v, want pass 1.0", got.Verdict, got.Score) - } - judged := byScenario["s-2026-05-01-002"] - if judged.Verdict != scenarioresults.VerdictSkip { - t.Errorf("judgment-shaped verdict = %q, want skip (never a fabricated pass)", judged.Verdict) - } - if len(judged.Evidence) == 0 || !strings.Contains(judged.Evidence[0], "attestation-needed") { - t.Errorf("judgment-shaped evidence missing attestation-needed marker: %v", judged.Evidence) - } - if got := byScenario["s-2026-05-01-003"]; got.Verdict != scenarioresults.VerdictFail || got.Score != 0.0 { - t.Errorf("s-003 = verdict %q score %v, want fail 0.0", got.Verdict, got.Score) - } - - // Full round trip: the real consumer/aggregator reads nonzero evaluated - // counts back from what the producer wrote. - reports := consumerReports(t, root) - pass := reports["d-gate-pass"] - if pass.EvaluatedCount != 1 || pass.ScenarioVerdict != "pass" || pass.ScenarioSatisfaction != 1.0 { - t.Errorf("d-gate-pass = evaluated %d verdict %q satisfaction %v, want 1/pass/1.0", - pass.EvaluatedCount, pass.ScenarioVerdict, pass.ScenarioSatisfaction) - } - fail := reports["d-gate-fail"] - if fail.EvaluatedCount != 1 || fail.ScenarioVerdict != "fail" { - t.Errorf("d-gate-fail = evaluated %d verdict %q, want 1/fail", fail.EvaluatedCount, fail.ScenarioVerdict) - } - judgment := reports["d-judgment"] - if judgment.EvaluatedCount != 0 || judgment.ScenarioVerdict != "unknown" { - t.Errorf("d-judgment = evaluated %d verdict %q, want 0/unknown (attestation lane stays honest)", - judgment.EvaluatedCount, judgment.ScenarioVerdict) - } -} - -func TestScenarioEvaluate_TimeoutYieldsSkipNeverFail(t *testing.T) { - root := setupScenarioEvaluateProject(t, evaluateGoalsMD) - writeScenarioSpec(t, root, "s-2026-05-01-001", "d-gate-pass", 0.9, - `"acceptance_vectors": [{"dimension": "slow", "threshold": 0.9, "check": "sleep 5"}],`) - - out, err := executeCommand("eval", "scenario", "evaluate", - "--directive", "d-gate-pass", "--timeout", "100ms", "--json") - if err != nil { - t.Fatalf("scenario evaluate failed: %v\noutput: %s", err, out) - } - report := decodeEvaluateReport(t, out) - if report.Written != 1 { - t.Fatalf("expected 1 written result, got %d", report.Written) - } - loaded, err := scenarioresults.Load(root, true) - if err != nil || loaded.Artifact == nil { - t.Fatalf("load artifact: %v", err) - } - got := loaded.Artifact.Results[0] - if got.Verdict != scenarioresults.VerdictSkip { - t.Fatalf("timed-out check verdict = %q, want skip — a check that never ran must not become pass or fail", got.Verdict) - } - - // Downstream, a skip stays non-evidence: evaluated 0, verdict unknown. - reports := consumerReports(t, root) - if d := reports["d-gate-pass"]; d.EvaluatedCount != 0 || d.ScenarioVerdict != "unknown" { - t.Errorf("consumer after timeout = evaluated %d verdict %q, want 0/unknown", d.EvaluatedCount, d.ScenarioVerdict) - } -} - -func TestScenarioEvaluate_UnresolvableGateRefYieldsSkip(t *testing.T) { - root := setupScenarioEvaluateProject(t, evaluateGoalsMD) - writeScenarioSpec(t, root, "s-2026-05-01-001", "d-gate-pass", 0.9, - `"acceptance_vectors": [{"dimension": "ghost", "threshold": 0.9, "check": "gate:does-not-exist"}],`) - - out, err := executeCommand("eval", "scenario", "evaluate", "--all", "--json") - if err != nil { - t.Fatalf("scenario evaluate failed: %v\noutput: %s", err, out) - } - loaded, err := scenarioresults.Load(root, true) - if err != nil || loaded.Artifact == nil { - t.Fatalf("load artifact: %v", err) - } - got := loaded.Artifact.Results[0] - if got.Verdict != scenarioresults.VerdictSkip { - t.Fatalf("unresolvable gate ref verdict = %q, want skip", got.Verdict) - } - if len(got.Evidence) == 0 || !strings.Contains(got.Evidence[0], "unresolvable gate reference") { - t.Errorf("expected unresolvable-gate evidence, got %v", got.Evidence) - } -} - -func TestScenarioEvaluate_RerunSupersedesLatestPerScenario(t *testing.T) { - root := setupScenarioEvaluateProject(t, evaluateGoalsMD) - - // A fake check command on PATH that flips from failing to passing between - // runs — the supersede case is a real state change, not a replay. - binDir := filepath.Join(root, "bin") - if err := os.MkdirAll(binDir, 0o755); err != nil { - t.Fatalf("mkdir bin: %v", err) - } - fake := filepath.Join(binDir, "fakecheck") - writeFake := func(exitCode int) { - script := fmt.Sprintf("#!/bin/sh\nexit %d\n", exitCode) - if err := os.WriteFile(fake, []byte(script), 0o755); err != nil { // #nosec G306 -- test fixture executable - t.Fatalf("write fakecheck: %v", err) - } - } - writeFake(1) - t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) - - writeScenarioSpec(t, root, "s-2026-05-01-001", "d-gate-pass", 0.9, - `"acceptance_vectors": [{"dimension": "flip", "threshold": 0.9, "check": "fakecheck"}],`) - - if _, err := executeCommand("eval", "scenario", "evaluate", "--directive", "d-gate-pass"); err != nil { - t.Fatalf("first evaluate: %v", err) - } - loaded, err := scenarioresults.Load(root, true) - if err != nil || loaded.Artifact == nil { - t.Fatalf("load after first run: %v", err) - } - if got := loaded.Artifact.Results[0].Verdict; got != scenarioresults.VerdictFail { - t.Fatalf("first run verdict = %q, want fail", got) - } - - writeFake(0) - out, err := executeCommand("eval", "scenario", "evaluate", "--directive", "d-gate-pass", "--json") - if err != nil { - t.Fatalf("second evaluate: %v", err) - } - report := decodeEvaluateReport(t, out) - if report.Iteration != 2 { - t.Errorf("second run iteration = %d, want 2", report.Iteration) - } - - loaded, err = scenarioresults.Load(root, true) - if err != nil || loaded.Artifact == nil { - t.Fatalf("load after second run: %v", err) - } - if n := len(loaded.Artifact.Results); n != 1 { - t.Fatalf("expected exactly 1 merged result for the scenario, got %d", n) - } - if got := loaded.Artifact.Results[0].Verdict; got != scenarioresults.VerdictPass { - t.Errorf("superseded verdict = %q, want pass (latest wins)", got) - } -} - -func TestScenarioEvaluate_MissingScenarioWritesNothing(t *testing.T) { - root := setupScenarioEvaluateProject(t, evaluateGoalsMD) - // No spec files at all: every link is missing. - out, err := executeCommand("eval", "scenario", "evaluate", "--all", "--json") - if err != nil { - t.Fatalf("scenario evaluate failed: %v\noutput: %s", err, out) - } - report := decodeEvaluateReport(t, out) - if report.Written != 0 { - t.Fatalf("expected 0 written results for missing specs, got %d", report.Written) - } - for _, ev := range report.Evaluations { - if ev.Recorded { - t.Errorf("missing scenario %s reported as recorded", ev.ScenarioID) - } - } - if _, err := os.Stat(filepath.Join(root, ".agents", "rpi", "scenario-results.json")); !os.IsNotExist(err) { - t.Fatalf("artifact must not be created when nothing was evaluated (stat err: %v)", err) - } - // Downstream stays fully unknown: zero evidence was never turned into a verdict. - reports := consumerReports(t, root) - if d := reports["d-gate-pass"]; d.ScenarioVerdict != "unknown" { - t.Errorf("consumer verdict for unwritten scenario = %q, want unknown", d.ScenarioVerdict) - } -} - -func TestScenarioEvaluate_PartialScoreJudgedAgainstScenarioThreshold(t *testing.T) { - root := setupScenarioEvaluateProject(t, evaluateGoalsMD) - // 2 checks, 1 passes -> score 0.5. Threshold 0.5 -> pass (equality passes), - // matching the aggregator's countSatisfied comparison exactly. - writeScenarioSpec(t, root, "s-2026-05-01-001", "d-gate-pass", 0.5, - `"acceptance_vectors": [ - {"dimension": "green", "threshold": 0.5, "check": "exit 0"}, - {"dimension": "red", "threshold": 0.5, "check": "exit 1"} - ],`) - - out, err := executeCommand("eval", "scenario", "evaluate", "--directive", "d-gate-pass", "--json") - if err != nil { - t.Fatalf("scenario evaluate failed: %v\noutput: %s", err, out) - } - loaded, err := scenarioresults.Load(root, true) - if err != nil || loaded.Artifact == nil { - t.Fatalf("load artifact: %v", err) - } - got := loaded.Artifact.Results[0] - if got.Verdict != scenarioresults.VerdictPass || got.Score != 0.5 { - t.Fatalf("partial result = verdict %q score %v, want pass 0.5", got.Verdict, got.Score) - } - // The consumer agrees: evaluated 1, satisfied at its own threshold. - reports := consumerReports(t, root) - if d := reports["d-gate-pass"]; d.EvaluatedCount != 1 || d.ScenarioVerdict != "pass" { - t.Errorf("consumer = evaluated %d verdict %q, want 1/pass", d.EvaluatedCount, d.ScenarioVerdict) - } -} - -func TestScenarioEvaluate_RequiresScopeFlag(t *testing.T) { - setupScenarioEvaluateProject(t, evaluateGoalsMD) - _, err := executeCommand("eval", "scenario", "evaluate") - if err == nil || !strings.Contains(err.Error(), "--all") { - t.Fatalf("expected scope-flag error mentioning --all, got %v", err) - } -} diff --git a/cli/cmd/ao/scenario_init.go b/cli/cmd/ao/scenario_init.go deleted file mode 100644 index 57b6ce400..000000000 --- a/cli/cmd/ao/scenario_init.go +++ /dev/null @@ -1,55 +0,0 @@ -// practices: [property-based-testing, llm-eval-harness] -package main - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/spf13/cobra" -) - -var scenarioInitCmd = &cobra.Command{ - Use: "init", - Short: "Initialize .agents/holdout/ directory for scenario storage", - RunE: func(cmd *cobra.Command, args []string) error { - holdoutDir := filepath.Join(".agents", "holdout") - if err := os.MkdirAll(holdoutDir, 0755); err != nil { - return fmt.Errorf("creating holdout directory: %w", err) - } - - readmePath := filepath.Join(holdoutDir, "README.md") - if _, err := os.Stat(readmePath); os.IsNotExist(err) { - readme := `# Holdout Scenarios - -This directory contains behavioral validation scenarios that implementing -agents cannot see. Access is blocked by the holdout-isolation-gate hook. - -Evaluator agents (with AGENTOPS_HOLDOUT_EVALUATOR=1) can read these files -during validation STEP 1.8. - -## Schema - -Scenarios follow schemas/scenario.v1.schema.json. - -## Usage - -` + "```" + `bash -ao scenario init # Create this directory -ao scenario list # List active scenarios -ao scenario validate # Validate schema compliance -` + "```" + ` -` - if err := os.WriteFile(readmePath, []byte(readme), 0644); err != nil { - return fmt.Errorf("writing README: %w", err) - } - } - - fmt.Fprintf(cmd.OutOrStdout(), "Initialized holdout directory at %s\n", holdoutDir) - return nil - }, -} - -func init() { - scenarioCmd.AddCommand(scenarioInitCmd) -} diff --git a/cli/cmd/ao/scenario_list.go b/cli/cmd/ao/scenario_list.go deleted file mode 100644 index 8b1e6558a..000000000 --- a/cli/cmd/ao/scenario_list.go +++ /dev/null @@ -1,82 +0,0 @@ -// practices: [property-based-testing, llm-eval-harness] -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/spf13/cobra" -) - -var scenarioListStatus string - -var scenarioListCmd = &cobra.Command{ - Use: "list", - Short: "List holdout scenarios", - RunE: func(cmd *cobra.Command, args []string) error { - holdoutDir := filepath.Join(".agents", "holdout") - entries, err := os.ReadDir(holdoutDir) - if err != nil { - if os.IsNotExist(err) { - fmt.Fprintln(cmd.OutOrStdout(), "No holdout directory found. Run 'ao scenario init' first.") - return nil - } - return fmt.Errorf("reading holdout directory: %w", err) - } - - type scenarioSummary struct { - ID string `json:"id"` - Goal string `json:"goal"` - Status string `json:"status"` - Date string `json:"date"` - } - - var scenarios []scenarioSummary - for _, entry := range entries { - if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { - continue - } - data, err := os.ReadFile(filepath.Join(holdoutDir, entry.Name())) - if err != nil { - continue - } - var s struct { - ID string `json:"id"` - Goal string `json:"goal"` - Status string `json:"status"` - Date string `json:"date"` - } - if err := json.Unmarshal(data, &s); err != nil { - continue - } - if scenarioListStatus != "" && s.Status != scenarioListStatus { - continue - } - scenarios = append(scenarios, scenarioSummary{ - ID: s.ID, - Goal: s.Goal, - Status: s.Status, - Date: s.Date, - }) - } - - if len(scenarios) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No scenarios found.") - return nil - } - - out, err := json.MarshalIndent(scenarios, "", " ") - if err != nil { - return fmt.Errorf("marshal scenarios: %w", err) - } - fmt.Fprintln(cmd.OutOrStdout(), string(out)) - return nil - }, -} - -func init() { - scenarioListCmd.Flags().StringVar(&scenarioListStatus, "status", "", "Filter by status (active, draft, retired)") - scenarioCmd.AddCommand(scenarioListCmd) -} diff --git a/cli/cmd/ao/scenario_test.go b/cli/cmd/ao/scenario_test.go deleted file mode 100644 index ff6858945..000000000 --- a/cli/cmd/ao/scenario_test.go +++ /dev/null @@ -1,381 +0,0 @@ -// practices: [property-based-testing, llm-eval-harness] -package main - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -func TestScenarioInit_CreatesDirectory(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - out, err := executeCommand("eval", "scenario", "init") - if err != nil { - t.Fatalf("scenario init failed: %v", err) - } - - holdoutDir := filepath.Join(".agents", "holdout") - if _, err := os.Stat(holdoutDir); os.IsNotExist(err) { - t.Fatal("holdout directory not created") - } - if _, err := os.Stat(filepath.Join(holdoutDir, "README.md")); os.IsNotExist(err) { - t.Fatal("README.md not created") - } - if !strings.Contains(out, "Initialized holdout directory") { - t.Errorf("expected init confirmation, got: %s", out) - } -} - -func TestScenarioInit_Idempotent(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - // Run twice - executeCommand("eval", "scenario", "init") - _, err := executeCommand("eval", "scenario", "init") - if err != nil { - t.Fatalf("second init should not error: %v", err) - } -} - -func TestScenarioList_EmptyDir(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - os.MkdirAll(filepath.Join(".agents", "holdout"), 0755) - - out, err := executeCommand("eval", "scenario", "list") - if err != nil { - t.Fatalf("list should not error on empty dir: %v", err) - } - if !strings.Contains(out, "No scenarios found") { - t.Fatalf("expected 'No scenarios found', got: %s", out) - } -} - -func TestScenarioList_NoDirectory(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - out, err := executeCommand("eval", "scenario", "list") - if err != nil { - t.Fatalf("list should not error when dir missing: %v", err) - } - if !strings.Contains(out, "No holdout directory found") { - t.Fatalf("expected missing-dir message, got: %s", out) - } -} - -func TestScenarioList_WithScenarios(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - holdoutDir := filepath.Join(".agents", "holdout") - os.MkdirAll(holdoutDir, 0755) - - scenario := map[string]interface{}{ - "id": "s-2026-04-05-001", "version": 1, "date": "2026-04-05", - "goal": "test goal", "narrative": "test narrative", - "expected_outcome": "test outcome", "satisfaction_threshold": 0.8, - "status": "active", - } - data, _ := json.Marshal(scenario) - os.WriteFile(filepath.Join(holdoutDir, "s-2026-04-05-001.json"), data, 0644) - - out, err := executeCommand("eval", "scenario", "list") - if err != nil { - t.Fatalf("list failed: %v", err) - } - if !strings.Contains(out, "test goal") { - t.Fatalf("expected scenario in output, got: %s", out) - } -} - -func TestScenarioList_FilterByStatus(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - holdoutDir := filepath.Join(".agents", "holdout") - os.MkdirAll(holdoutDir, 0755) - - active := map[string]interface{}{ - "id": "s-2026-04-05-001", "version": 1, "date": "2026-04-05", - "goal": "active goal", "narrative": "n", "expected_outcome": "o", - "satisfaction_threshold": 0.8, "status": "active", - } - draft := map[string]interface{}{ - "id": "s-2026-04-05-002", "version": 1, "date": "2026-04-05", - "goal": "draft goal", "narrative": "n", "expected_outcome": "o", - "satisfaction_threshold": 0.5, "status": "draft", - } - d1, _ := json.Marshal(active) - d2, _ := json.Marshal(draft) - os.WriteFile(filepath.Join(holdoutDir, "s1.json"), d1, 0644) - os.WriteFile(filepath.Join(holdoutDir, "s2.json"), d2, 0644) - - out, err := executeCommand("eval", "scenario", "list", "--status", "draft") - if err != nil { - t.Fatalf("list failed: %v", err) - } - if !strings.Contains(out, "draft goal") { - t.Fatalf("expected draft scenario, got: %s", out) - } - if strings.Contains(out, "active goal") { - t.Fatalf("should not contain active scenario when filtering by draft") - } -} - -func TestScenarioAdd_CreatesSchemaCompliantScenario(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - withScenarioClock(t, time.Date(2026, 4, 24, 10, 30, 0, 0, time.UTC)) - - out, err := executeCommand( - "eval", "scenario", "add", "CLI can author holdout scenarios", - "--narrative", "Evaluator authors a scenario through the CLI.", - "--expected-outcome", "A schema-compliant scenario file is written.", - "--status", "active", - "--source", "agent", - "--threshold", "0.9", - "--json", - ) - if err != nil { - t.Fatalf("scenario add failed: %v\noutput: %s", err, out) - } - - var scenario scenarioFile - if err := json.Unmarshal([]byte(out), &scenario); err != nil { - t.Fatalf("parse scenario add JSON: %v\noutput: %s", err, out) - } - if scenario.ID != "s-2026-04-24-001" { - t.Fatalf("id = %q, want s-2026-04-24-001", scenario.ID) - } - if scenario.Date != "2026-04-24" || scenario.Status != "active" || scenario.Source != "agent" { - t.Fatalf("unexpected metadata: %+v", scenario) - } - if scenario.SatisfactionThreshold != 0.9 { - t.Fatalf("threshold = %.2f, want 0.90", scenario.SatisfactionThreshold) - } - - path := filepath.Join(".agents", "holdout", "s-2026-04-24-001.json") - if _, err := os.Stat(path); err != nil { - t.Fatalf("scenario file not written: %v", err) - } - validateOut, err := executeCommand("eval", "scenario", "validate") - if err != nil { - t.Fatalf("written scenario should validate: %v\noutput: %s", err, validateOut) - } -} - -func TestScenarioAdd_IncrementsSameDayID(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - withScenarioClock(t, time.Date(2026, 4, 24, 10, 30, 0, 0, time.UTC)) - - holdoutDir := filepath.Join(".agents", "holdout") - if err := os.MkdirAll(holdoutDir, 0755); err != nil { - t.Fatalf("mkdir holdout: %v", err) - } - existing := map[string]interface{}{ - "id": "s-2026-04-24-003", "version": 1, "date": "2026-04-24", - "goal": "existing", "narrative": "existing", "expected_outcome": "existing", - "satisfaction_threshold": 0.8, "status": "draft", - } - data, _ := json.Marshal(existing) - if err := os.WriteFile(filepath.Join(holdoutDir, "legacy.json"), data, 0644); err != nil { - t.Fatalf("write existing scenario: %v", err) - } - - out, err := executeCommand("eval", "scenario", "add", "next same-day scenario", "--json") - if err != nil { - t.Fatalf("scenario add failed: %v\noutput: %s", err, out) - } - var scenario scenarioFile - if err := json.Unmarshal([]byte(out), &scenario); err != nil { - t.Fatalf("parse scenario add JSON: %v\noutput: %s", err, out) - } - if scenario.ID != "s-2026-04-24-004" { - t.Fatalf("id = %q, want s-2026-04-24-004", scenario.ID) - } -} - -func TestScenarioAdd_RejectsInvalidFlags(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - if out, err := executeCommand("eval", "scenario", "add", "bad threshold", "--threshold", "1.2"); err == nil { - t.Fatalf("scenario add should reject invalid threshold; output: %s", out) - } - if out, err := executeCommand("eval", "scenario", "add", "bad status", "--status", "blocked"); err == nil { - t.Fatalf("scenario add should reject invalid status; output: %s", out) - } -} - -func TestScenarioValidate_ValidSchema(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - holdoutDir := filepath.Join(".agents", "holdout") - os.MkdirAll(holdoutDir, 0755) - - scenario := map[string]interface{}{ - "id": "s-2026-04-05-001", "version": 1, "date": "2026-04-05", - "goal": "test", "narrative": "test", "expected_outcome": "test", - "satisfaction_threshold": 0.7, "status": "active", "source": "human", - } - data, _ := json.Marshal(scenario) - os.WriteFile(filepath.Join(holdoutDir, "test.json"), data, 0644) - - out, err := executeCommand("eval", "scenario", "validate") - if err != nil { - t.Fatalf("validate should pass: %v", err) - } - if !strings.Contains(out, "all pass") { - t.Fatalf("expected 'all pass', got: %s", out) - } -} - -func TestScenarioValidate_AcceptsAutoID(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - holdoutDir := filepath.Join(".agents", "holdout") - os.MkdirAll(holdoutDir, 0755) - - scenario := map[string]interface{}{ - "id": "auto-agentops-core-cli", "version": 1, "date": "2026-04-24", - "goal": "test", "narrative": "test", "expected_outcome": "test", - "satisfaction_threshold": 0.7, "status": "draft", "source": "agent", - } - data, _ := json.Marshal(scenario) - os.WriteFile(filepath.Join(holdoutDir, "auto.json"), data, 0644) - - out, err := executeCommand("eval", "scenario", "validate") - if err != nil { - t.Fatalf("validate should accept auto-* IDs: %v\noutput: %s", err, out) - } - if !strings.Contains(out, "all pass") { - t.Fatalf("expected 'all pass', got: %s", out) - } -} - -func TestScenarioValidate_InvalidSchema(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - holdoutDir := filepath.Join(".agents", "holdout") - os.MkdirAll(holdoutDir, 0755) - - // Missing required fields and bad id pattern - scenario := map[string]interface{}{"id": "bad-id"} - data, _ := json.Marshal(scenario) - os.WriteFile(filepath.Join(holdoutDir, "bad.json"), data, 0644) - - _, err := executeCommand("eval", "scenario", "validate") - if err == nil { - t.Fatal("validate should fail for invalid schema") - } -} - -func TestScenarioValidate_MalformedJSON(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - holdoutDir := filepath.Join(".agents", "holdout") - os.MkdirAll(holdoutDir, 0755) - - os.WriteFile(filepath.Join(holdoutDir, "bad.json"), []byte("{invalid"), 0644) - - _, err := executeCommand("eval", "scenario", "validate") - if err == nil { - t.Fatal("validate should fail for malformed JSON") - } -} - -func TestScenarioValidate_NoDirectory(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - out, err := executeCommand("eval", "scenario", "validate") - if err != nil { - t.Fatalf("validate should not error when dir missing: %v", err) - } - if !strings.Contains(out, "No holdout directory found") { - t.Fatalf("expected missing-dir message, got: %s", out) - } -} - -func TestScenarioValidate_EmptyDir(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - os.MkdirAll(filepath.Join(".agents", "holdout"), 0755) - - out, err := executeCommand("eval", "scenario", "validate") - if err != nil { - t.Fatalf("validate should not error on empty dir: %v", err) - } - if !strings.Contains(out, "No scenario files found") { - t.Fatalf("expected empty message, got: %s", out) - } -} - -func withScenarioClock(t *testing.T, now time.Time) { - t.Helper() - orig := scenarioAddNow - scenarioAddNow = func() time.Time { return now } - t.Cleanup(func() { - scenarioAddNow = orig - }) -} - -// TestScenarioSchema_ExecutableSpecFields locks the scenario.v1 schema contract -// after F1.1: directive_id + given/when/then are accepted, additionalProperties -// stays false (so unknown keys are still rejected), and the satisfaction -// threshold default is reconciled with the CLI default (0.8). -func TestScenarioSchema_ExecutableSpecFields(t *testing.T) { - data, err := os.ReadFile(filepath.Join("..", "..", "..", "schemas", "scenario.v1.schema.json")) - if err != nil { - t.Fatalf("reading scenario schema: %v", err) - } - var schema struct { - AdditionalProperties bool `json:"additionalProperties"` - Properties map[string]struct { - Type string `json:"type"` - Pattern string `json:"pattern"` - Default json.RawMessage `json:"default"` - } `json:"properties"` - } - if err := json.Unmarshal(data, &schema); err != nil { - t.Fatalf("parsing scenario schema: %v", err) - } - - if schema.AdditionalProperties { - t.Error("scenario schema must keep additionalProperties:false") - } - - did, ok := schema.Properties["directive_id"] - if !ok { - t.Fatal("scenario schema missing directive_id property") - } - if did.Pattern != "^d-[a-z0-9][a-z0-9-]*$" { - t.Errorf("directive_id pattern = %q, want ^d-[a-z0-9][a-z0-9-]*$", did.Pattern) - } - - for _, key := range []string{"given", "when", "then"} { - p, ok := schema.Properties[key] - if !ok || p.Type != "array" { - t.Errorf("scenario schema %q property missing or not an array (type=%q ok=%v)", key, p.Type, ok) - } - } - - st, ok := schema.Properties["satisfaction_threshold"] - if !ok || string(st.Default) != "0.8" { - t.Errorf("satisfaction_threshold default = %q, want 0.8", string(st.Default)) - } -} diff --git a/cli/cmd/ao/scenario_validate.go b/cli/cmd/ao/scenario_validate.go deleted file mode 100644 index 48294beb2..000000000 --- a/cli/cmd/ao/scenario_validate.go +++ /dev/null @@ -1,103 +0,0 @@ -// practices: [property-based-testing, llm-eval-harness] -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/spf13/cobra" -) - -var scenarioValidateCmd = &cobra.Command{ - Use: "validate", - Short: "Validate holdout scenarios against schema", - RunE: func(cmd *cobra.Command, args []string) error { - holdoutDir := filepath.Join(".agents", "holdout") - entries, err := os.ReadDir(holdoutDir) - if err != nil { - if os.IsNotExist(err) { - fmt.Fprintln(cmd.OutOrStdout(), "No holdout directory found. Run 'ao scenario init' first.") - return nil - } - return fmt.Errorf("reading holdout directory: %w", err) - } - - var validationErrors []string - var validated int - - for _, entry := range entries { - if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { - continue - } - filePath := filepath.Join(holdoutDir, entry.Name()) - data, err := os.ReadFile(filePath) - if err != nil { - validationErrors = append(validationErrors, fmt.Sprintf("%s: read error: %v", entry.Name(), err)) - continue - } - - var s map[string]interface{} - if err := json.Unmarshal(data, &s); err != nil { - validationErrors = append(validationErrors, fmt.Sprintf("%s: invalid JSON: %v", entry.Name(), err)) - continue - } - - // Check required fields - required := []string{"id", "version", "date", "goal", "narrative", "expected_outcome", "satisfaction_threshold", "status"} - for _, field := range required { - if _, ok := s[field]; !ok { - validationErrors = append(validationErrors, fmt.Sprintf("%s: missing required field '%s'", entry.Name(), field)) - } - } - - // Validate id pattern - if id, ok := s["id"].(string); ok { - if !scenarioIDPattern.MatchString(id) { - validationErrors = append(validationErrors, fmt.Sprintf("%s: id '%s' does not match pattern s-YYYY-MM-DD-NNN or auto-*", entry.Name(), id)) - } - } - - // Validate status - if status, ok := s["status"].(string); ok { - if !validScenarioStatus(status) { - validationErrors = append(validationErrors, fmt.Sprintf("%s: invalid status '%s' (must be active, draft, or retired)", entry.Name(), status)) - } - } - - // Validate source if present - if source, ok := s["source"].(string); ok { - if !validScenarioSource(source) { - validationErrors = append(validationErrors, fmt.Sprintf("%s: invalid source '%s' (must be human, agent, or prod-telemetry)", entry.Name(), source)) - } - } - - // Validate satisfaction_threshold range - if threshold, ok := s["satisfaction_threshold"].(float64); ok { - if threshold < 0 || threshold > 1 { - validationErrors = append(validationErrors, fmt.Sprintf("%s: satisfaction_threshold %.2f out of range [0, 1]", entry.Name(), threshold)) - } - } - - validated++ - } - - if len(validationErrors) > 0 { - fmt.Fprintf(cmd.ErrOrStderr(), "Validation errors:\n%s\n", strings.Join(validationErrors, "\n")) - return fmt.Errorf("%d validation error(s) found", len(validationErrors)) - } - - if validated == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No scenario files found to validate.") - } else { - fmt.Fprintf(cmd.OutOrStdout(), "Validated %d scenario(s): all pass\n", validated) - } - return nil - }, -} - -func init() { - scenarioCmd.AddCommand(scenarioValidateCmd) -} diff --git a/cli/internal/adapters/eval/scenario_ab_runtime.go b/cli/internal/adapters/eval/scenario_ab_runtime.go index baef02b68..953d5c0d2 100644 --- a/cli/internal/adapters/eval/scenario_ab_runtime.go +++ b/cli/internal/adapters/eval/scenario_ab_runtime.go @@ -82,7 +82,7 @@ func taskSuccessJudgePrompt(spec scenario.Scenario, output string) string { return prompt.String() } -const judgeOutputSchema = `{"type":"object","additionalProperties":false,"properties":{"vectors":{"type":"array","items":{"type":"object","additionalProperties":false,"properties":{"dimension":{"type":"string"},"pass":{"type":"boolean"},"score":{"type":"number"}},"required":["dimension","pass","score"]}},"aggregate_score":{"type":"number"}},"required":["vectors","aggregate_score"]}` +const JudgeOutputSchema = `{"type":"object","additionalProperties":false,"properties":{"vectors":{"type":"array","items":{"type":"object","additionalProperties":false,"properties":{"dimension":{"type":"string"},"pass":{"type":"boolean"},"score":{"type":"number"}},"required":["dimension","pass","score"]}},"aggregate_score":{"type":"number"}},"required":["vectors","aggregate_score"]}` func writeJudgeSchema() (string, func(), error) { file, err := os.CreateTemp("", "scenario-ab-judge-schema-*.json") @@ -91,7 +91,7 @@ func writeJudgeSchema() (string, func(), error) { } name := file.Name() cleanup := func() { _ = os.Remove(name) } - if _, err := file.WriteString(judgeOutputSchema); err != nil { + if _, err := file.WriteString(JudgeOutputSchema); err != nil { _ = file.Close() cleanup() return "", func() {}, fmt.Errorf("write judge schema: %w", err) @@ -145,6 +145,12 @@ func runCodexExec(ctx context.Context, prompt, outputSchemaPath string) (string, } return string(combined), tokens, nil } + +// RunCodexExec exposes the sandboxed Codex execution adapter to other legacy +// composition shims while their command families are migrated. +func RunCodexExec(ctx context.Context, prompt, outputSchemaPath string) (string, int, error) { + return runCodexExec(ctx, prompt, outputSchemaPath) +} func codexExecArgs(messagePath, outputSchemaPath, prompt string) []string { args := []string{"exec", "--dangerously-bypass-approvals-and-sandbox", "--skip-git-repo-check", "--output-last-message", messagePath} if outputSchemaPath != "" { diff --git a/cli/internal/commands/eval/module.go b/cli/internal/commands/eval/module.go index dea12e0df..cf7bfd48b 100644 --- a/cli/internal/commands/eval/module.go +++ b/cli/internal/commands/eval/module.go @@ -559,9 +559,9 @@ func (module Module) outcomesIngestCommand() *cobra.Command { var options aoeval.OutcomesIngestRequest command := &cobra.Command{Use: "ingest ", Short: "Ingest an Outcomes score payload into the one council verdict record", Args: cobra.ExactArgs(1)} command.Flags().StringVar(&options.ExpectedJudgeHash, "expect-judge-hash", "", "refuse the ingest if the score's judge_content_hash does not match this value (gate #2 rubric-drift parity)") - command.Flags().StringVar(&options.BurnLedgerPath, "burn-ledger", "", "path to a JSON HoldoutBurnLedger; enforce and persist holdout burn quota") - command.Flags().StringVar(&options.ManifestDir, "manifest-out", "", "also write an eval-run.v1 manifest to

//manifest.json") - command.Flags().StringVar(&options.RunID, "run-id", "", "run id for the --manifest-out manifest") + command.Flags().StringVar(&options.BurnLedgerPath, "burn-ledger", "", "path to a JSON HoldoutBurnLedger; when set, a holdout-split score registers a burn and is REFUSED if the (suite,gt) quota is exhausted (gate #3 runtime enforcement), persisted across invocations") + command.Flags().StringVar(&options.ManifestDir, "manifest-out", "", "also write an eval-run.v1 manifest to //manifest.json so the verdict pipeline feeds the Knowledge Flywheel (closes the Outcomes→Flywheel loop)") + command.Flags().StringVar(&options.RunID, "run-id", "", "run id for the --manifest-out manifest; defaults to the score's run_id, then source_task_id (sanitized to the eval-run.v1 pattern)") command.RunE = func(command *cobra.Command, args []string) error { options.ScorePath = args[0] result, err := module.useCases.Outcomes.Ingest(command.Context(), options) @@ -850,7 +850,7 @@ func (module Module) benchCommand() *cobra.Command { command.Flags().BoolVar(&options.Global, "global", false, "Include ~/.agents/learnings/ (cross-rig aggregated store, requires --live)") command.Flags().StringVar(&options.SearchEval, "search-eval", "", "Path to an ao-search eval manifest with queries and ground_truth paths") command.Flags().StringVar(&options.SearchRoot, "search-root", "", "Repo root to search for --search-eval (defaults to current directory)") - command.Flags().StringVar(&options.SearchBackend, "search-backend", "local-lexical", "Search backend for --search-eval") + command.Flags().StringVar(&options.SearchBackend, "search-backend", "local-lexical", "Search backend for --search-eval (local-lexical, ao-auto, agentic-rg, wiki-link-expand, rerank-llamacpp)") command.Flags().StringVar(&options.CompareBackends, "search-compare-backends", "", "Comma-separated search backends to compare for --search-eval") command.RunE = func(command *cobra.Command, _ []string) error { options.KChanged = command.Flags().Changed("k") diff --git a/cli/internal/eval/outcomes_service.go b/cli/internal/eval/outcomes_service.go index 1cae29536..87002e604 100644 --- a/cli/internal/eval/outcomes_service.go +++ b/cli/internal/eval/outcomes_service.go @@ -51,7 +51,7 @@ func (score *OutcomesScore) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(data, &value); err != nil { return err } - *score = OutcomesScore{SourceTaskID: value.SourceTaskID, JudgeContentHash: value.JudgeContentHash, Aggregate: value.Aggregate, Threshold: value.Threshold, CriterionScores: value.CriterionScores, Split: value.Split, SuiteRef: value.SuiteRef, GroundTruthVersion: value.GroundTruthVersion, RunID: value.RunID} + *score = OutcomesScore(value) return nil } @@ -189,9 +189,10 @@ func buildOutcomeManifest(score OutcomesScore, band, runID string, started time. dimensions[DimensionCorrectness] = clampOutcomeScore(score.Aggregate) } status, verdict := StatusInconclusive, VerdictAdvisory - if band == "PASS" { + switch band { + case "PASS": status, verdict = StatusPass, VerdictPass - } else if band == "FAIL" { + case "FAIL": status, verdict = StatusFail, VerdictFail } caseID := score.SourceTaskID diff --git a/cli/internal/eval/scenario_service.go b/cli/internal/eval/scenario_service.go index 9bb716f74..a31bf2d74 100644 --- a/cli/internal/eval/scenario_service.go +++ b/cli/internal/eval/scenario_service.go @@ -300,9 +300,10 @@ func (service ScenarioService) runChecks(checks []scenario.AcceptanceVector, gat command = resolved } outcome, detail := service.Runtime.Measure(command, timeout) - if outcome == "pass" { + switch outcome { + case "pass": passed++ - } else if outcome == "skip" { + case "skip": skipped++ } text := fmt.Sprintf("`%s` -> %s", command, outcome) diff --git a/cli/internal/eval/task_service.go b/cli/internal/eval/task_service.go index fd496de40..dcbcefe7a 100644 --- a/cli/internal/eval/task_service.go +++ b/cli/internal/eval/task_service.go @@ -159,7 +159,10 @@ func (service TaskService) Run(_ context.Context, request TaskRunRequest) (TaskR if rigID == "" { rigID = "unknown-rig" } - manifest := service.taskManifest(request, task, suite, seeds, rigID, gateInputs, groundTruth) + manifest, err := service.taskManifest(request, task, suite, seeds, rigID, gateInputs, groundTruth) + if err != nil { + return TaskRunResult{}, fmt.Errorf("eval task run: build manifest: %w", err) + } runID := service.Runtime.GenerateRunID(rigID) writer, err := service.Runtime.OpenRun(service.Runtime.Root(), runID, manifest) if err != nil { @@ -228,7 +231,7 @@ func (service TaskService) loadGroundTruth() ([]evalsubstrate.GroundTruthRow, er return rows, nil } -func (service TaskService) taskManifest(request TaskRunRequest, task *evalsubstrate.Task, suite *evalsubstrate.Suite, seeds []int, rigID string, gates evalsubstrate.GateInputs, rows []evalsubstrate.GroundTruthRow) evalsubstrate.Manifest { +func (service TaskService) taskManifest(request TaskRunRequest, task *evalsubstrate.Task, suite *evalsubstrate.Suite, seeds []int, rigID string, gates evalsubstrate.GateInputs, rows []evalsubstrate.GroundTruthRow) (evalsubstrate.Manifest, error) { manifest := evalsubstrate.Manifest{TaskRef: task.ID, SuiteRef: suite.ID, HarnessRef: request.HarnessRef, ModelSpecRef: request.ModelSpecID, GroundTruthRef: request.GroundTruthRef, SampleSplit: pickTaskSplit(suite, request.SampleSplit), NSamples: pickTaskSamples(suite, request.NSamples), Seeds: seeds, RigID: rigID, InspectCommand: request.InspectCommand, InspectVersion: request.InspectVersion, QuickSession: request.QuickSession} if gates.Harness != nil { manifest.HarnessContentHash = gates.Harness.ContentHash @@ -238,11 +241,15 @@ func (service TaskService) taskManifest(request TaskRunRequest, task *evalsubstr manifest.ModelSpecHash = spec.ContentHash } } - manifest.GroundTruthHash = taskGroundTruthHash(rows, request.GroundTruthRef) + groundTruthHash, err := taskGroundTruthHash(rows, request.GroundTruthRef) + if err != nil { + return evalsubstrate.Manifest{}, err + } + manifest.GroundTruthHash = groundTruthHash if count := len(suite.VariedAxis.Values); count > 2 { manifest.MultiComparisonMethod, manifest.ComparisonFamily, manifest.ReferenceArm, manifest.FamilySizeK = suite.Stats.MultiComparisonMethod, suite.Stats.ComparisonFamily, suite.Stats.ReferenceArm, evalsubstrate.FamilySizeK(suite.Stats.ComparisonFamily, count) } - return manifest + return manifest, nil } func parseTaskSeeds(value string) ([]int, error) { @@ -278,14 +285,17 @@ func pickTaskSamples(suite *evalsubstrate.Suite, override int) int { } return 0 } -func taskGroundTruthHash(rows []evalsubstrate.GroundTruthRow, ref string) string { +func taskGroundTruthHash(rows []evalsubstrate.GroundTruthRow, ref string) (string, error) { if ref == "" || len(rows) == 0 { - return "" + return "", nil } var data []byte for _, row := range rows { if row.ID == ref || row.Supersedes == ref { - encoded, _ := json.Marshal(row) + encoded, err := json.Marshal(row) + if err != nil { + return "", fmt.Errorf("marshal ground truth row %q: %w", row.ID, err) + } canonical, err := evalsubstrate.CanonicalizeJSON(encoded) if err != nil { canonical = encoded @@ -294,7 +304,7 @@ func taskGroundTruthHash(rows []evalsubstrate.GroundTruthRow, ref string) string } } if len(data) == 0 { - return "" + return "", nil } - return evalsubstrate.ContentHash(data) + return evalsubstrate.ContentHash(data), nil } From 0a27b8f41b1ea01befce2d08bd6bfad381fb6fb2 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:26:46 -0400 Subject: [PATCH 099/108] chore(eval): seal accepted migration lineage --- cli/testdata/compatibility-baseline/families/eval/lineage.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/testdata/compatibility-baseline/families/eval/lineage.json b/cli/testdata/compatibility-baseline/families/eval/lineage.json index 5096f7561..97a8f3d92 100644 --- a/cli/testdata/compatibility-baseline/families/eval/lineage.json +++ b/cli/testdata/compatibility-baseline/families/eval/lineage.json @@ -6,5 +6,6 @@ "fixture_sha256": "906273234e3fc94a9fec4e4f7762ce1ec7ea58485ccfeb080aa338931bbf0a1d", "ownership_sha256": "e8eb03162ee4fded8e6aeb2c582ef265103b35cb5836c162ddae1a2803178565", "scope_sha256": "42d515c799e9f1a50bcc5471d3df4837cd2c1b8a1b7c6d000dc02877ab8ce86e", - "migration_state": "migrating" + "migration_state": "migrated", + "accepted_sha": "39e7a6885a01ee459d702a386775fdffd20f411b" } From 81cebb04275a20d55f2640609d0811d69b9684e8 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:27:14 -0400 Subject: [PATCH 100/108] test(eval): remove legacy reset compatibility cell --- cli/cmd/ao/eval_test_compat_test.go | 6 ------ cli/cmd/ao/testutil_test.go | 3 --- 2 files changed, 9 deletions(-) delete mode 100644 cli/cmd/ao/eval_test_compat_test.go diff --git a/cli/cmd/ao/eval_test_compat_test.go b/cli/cmd/ao/eval_test_compat_test.go deleted file mode 100644 index 9efc5e086..000000000 --- a/cli/cmd/ao/eval_test_compat_test.go +++ /dev/null @@ -1,6 +0,0 @@ -package main - -// scenarioListStatus remains only as a reset-harness compatibility cell for -// older package-main tests. The production eval command owns closure-local -// state in internal/commands/eval. -var scenarioListStatus string diff --git a/cli/cmd/ao/testutil_test.go b/cli/cmd/ao/testutil_test.go index 32ece1c6b..0464276cb 100644 --- a/cli/cmd/ao/testutil_test.go +++ b/cli/cmd/ao/testutil_test.go @@ -291,7 +291,6 @@ func resetCommandState(t *testing.T) { origFindingsPullAll := findingsPullAll origFindingsPullForce := findingsPullForce origFindingsRetireBy := findingsRetireBy - origScenarioListStatus := scenarioListStatus origContextPacketFlags := contextPacketFlags origContextExplainFlags := contextExplainFlags origContextPacketStatusFlags := contextPacketStatusFlags @@ -361,7 +360,6 @@ func resetCommandState(t *testing.T) { findingsPullAll = origFindingsPullAll findingsPullForce = origFindingsPullForce findingsRetireBy = origFindingsRetireBy - scenarioListStatus = origScenarioListStatus contextPacketFlags = origContextPacketFlags contextExplainFlags = origContextExplainFlags contextPacketStatusFlags = origContextPacketStatusFlags @@ -435,7 +433,6 @@ func resetCommandState(t *testing.T) { findingsPullAll = false findingsPullForce = false findingsRetireBy = "" - scenarioListStatus = "" maturityApply = false maturityScan = false maturityCurate = false From 8f779fea9e7802e7d2410281ff0ab112e48eb6ea Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:33:01 -0400 Subject: [PATCH 101/108] test(eval): cover migrated command surface --- cli/cmd/ao/eval_composition_test.go | 42 ++++++++++++++++--- cli/docs/COMMANDS.md | 18 ++++---- .../adapters/eval/scenario_ab_sandbox_test.go | 1 + 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/cli/cmd/ao/eval_composition_test.go b/cli/cmd/ao/eval_composition_test.go index bea12172a..ce8c433f8 100644 --- a/cli/cmd/ao/eval_composition_test.go +++ b/cli/cmd/ao/eval_composition_test.go @@ -1,14 +1,44 @@ package main -import "testing" +import ( + "strings" + "testing" +) func TestEvalCompositionOwnsCompleteCommandTree(t *testing.T) { command := newEvalCommand() - want := []string{"baseline", "baseline-audit", "bench", "chaos", "cleanup", "compare", "coverage", "outcomes", "run", "scenario", "scenario-ab", "scenario-moat", "scorecard", "session-outcome", "suite", "task"} - for _, name := range want { - child, _, err := command.Find([]string{name}) - if err != nil || child == command || child.Name() != name { - t.Fatalf("missing eval child %q: child=%v err=%v", name, child, err) + want := []string{ + "ao eval baseline", + "ao eval baseline-audit", + "ao eval bench", + "ao eval chaos", + "ao eval cleanup", + "ao eval compare", + "ao eval coverage", + "ao eval outcomes compile", + "ao eval outcomes ingest", + "ao eval run", + "ao eval scenario add", + "ao eval scenario evaluate", + "ao eval scenario init", + "ao eval scenario list", + "ao eval scenario validate", + "ao eval scenario-ab", + "ao eval scenario-moat", + "ao eval scorecard", + "ao eval session-outcome", + "ao eval suite n-required", + "ao eval suite verdict", + "ao eval task add", + "ao eval task list", + "ao eval task run", + "ao eval task show", + } + for _, path := range want { + args := strings.Fields(strings.TrimPrefix(path, "ao eval ")) + child, remaining, err := command.Find(args) + if err != nil || child == command || len(remaining) != 0 { + t.Fatalf("missing eval command %q: child=%v remaining=%v err=%v", path, child, remaining, err) } } } diff --git a/cli/docs/COMMANDS.md b/cli/docs/COMMANDS.md index 2c8953e84..977cb951a 100644 --- a/cli/docs/COMMANDS.md +++ b/cli/docs/COMMANDS.md @@ -414,7 +414,7 @@ ao eval baseline-audit [suite.json ...] [flags] #### `ao eval bench` -Measure Precision@K and MRR against a curated corpus of learning artifacts. +Run retrieval quality benchmarks ``` ao eval bench [flags] @@ -499,7 +499,7 @@ ao eval coverage [suite.json ...] [flags] #### `ao eval outcomes` -Outcomes is a derived projection of the locked eval substrate (SCHEMA.md), never an alternate authority. Subcommands compile holdout-safe rubric payloads and ingest returned scores into the one verdict format. +Outcomes is a derived projection of the locked eval substrate (SCHEMA.md), never an alternate authority. ``` ao eval outcomes [command] @@ -556,7 +556,7 @@ ao eval run [flags] #### `ao eval scenario` -Create, list, and validate holdout scenarios stored in .agents/holdout/. +Create, list, validate, and evaluate holdout scenarios stored in .agents/holdout/. ``` ao eval scenario [command] @@ -564,7 +564,7 @@ ao eval scenario [command] ##### `ao eval scenario add` -Author a schema-compliant holdout scenario in .agents/holdout/. +Author a holdout scenario from a goal description ``` ao eval scenario add [flags] @@ -583,7 +583,7 @@ ao eval scenario add [flags] ##### `ao eval scenario evaluate` -Evaluate the executable-spec scenarios linked to GOALS.md directives and +Evaluate directive-linked scenarios and record satisfaction results ``` ao eval scenario evaluate [flags] @@ -633,7 +633,7 @@ ao eval scenario validate [flags] #### `ao eval scenario-ab` -Run one holdout scenario (scenario.v1) twice — a control arm WITHOUT the gold +Run a knowledge-reuse holdout scenario with vs. without the gold pull (the discriminating A/B) ``` ao eval scenario-ab [flags] @@ -652,7 +652,7 @@ ao eval scenario-ab [flags] #### `ao eval scenario-moat` -Render a moat positive/null/inconclusive verdict over one or more +Aggregate moat-eligible scenario A/B scorecards into a publication verdict ``` ao eval scenario-moat [flags] @@ -685,7 +685,7 @@ ao eval scorecard [baseline-run.json] [flags] #### `ao eval session-outcome` -Parse a Claude Code session transcript and derive a composite reward signal. +Analyze session transcript to derive reward signal ``` ao eval session-outcome [transcript-path] [flags] @@ -771,7 +771,7 @@ ao eval task list [flags] ##### `ao eval task run` -Opens a new Run under $AGENTOPS_EVALS_ROOT/runs//manifest.json +Open a new Run manifest for ; refuses on gate failure ``` ao eval task run [flags] diff --git a/cli/internal/adapters/eval/scenario_ab_sandbox_test.go b/cli/internal/adapters/eval/scenario_ab_sandbox_test.go index d5accbf49..de7a3a626 100644 --- a/cli/internal/adapters/eval/scenario_ab_sandbox_test.go +++ b/cli/internal/adapters/eval/scenario_ab_sandbox_test.go @@ -30,6 +30,7 @@ func TestCorpusRoot_WalksUpFromSubdir(t *testing.T) { // TestCorpusDenyPaths_RepoAndGlobal: the deny set covers the repo corpus (both // .agents and .ao under the resolved root) AND the global ~/.agents. func TestCorpusDenyPaths_RepoAndGlobal(t *testing.T) { + t.Setenv("HOME", t.TempDir()) root := t.TempDir() if err := os.MkdirAll(filepath.Join(root, ".agents"), 0o755); err != nil { t.Fatal(err) From 121911c89c0dccf18a6e6229b6b92086fcd9d168 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:56:46 -0400 Subject: [PATCH 102/108] test(gate): freeze CLI compatibility before migration --- .../families/gate/case.json | 25 +++++++ .../families/gate/ownership.json | 68 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/gate/case.json create mode 100644 cli/testdata/compatibility-baseline/families/gate/ownership.json diff --git a/cli/testdata/compatibility-baseline/families/gate/case.json b/cli/testdata/compatibility-baseline/families/gate/case.json new file mode 100644 index 000000000..86163534d --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/gate/case.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "family": "gate", + "checks": [ + { + "id": "gate-black-box", + "command": "tmp=$(mktemp -d /tmp/ao-gate.XXXXXX); (cd cli && go build -o \"$tmp/ao\" ./cmd/ao); \"$tmp/ao\" gate --help >\"$tmp/help\"; grep -Fq 'ao gate [command]' \"$tmp/help\"; for child in approve bulk-approve check pending reject run; do grep -Eq \"^[[:space:]]+$child([[:space:]]|$)\" \"$tmp/help\"; done; \"$tmp/ao\" gate approve --help >\"$tmp/approve-help\"; grep -Fq -- '--note' \"$tmp/approve-help\"; \"$tmp/ao\" gate reject --help >\"$tmp/reject-help\"; grep -Fq -- '--reason' \"$tmp/reject-help\"; \"$tmp/ao\" gate bulk-approve --help >\"$tmp/bulk-help\"; grep -Fq -- '--older-than' \"$tmp/bulk-help\"; grep -Fq -- '--tier' \"$tmp/bulk-help\"; \"$tmp/ao\" gate check --help >\"$tmp/check-help\"; for flag in fail-fast fast full github-annotations json require-workflow-parity scope workflow-coverage workflow-path; do grep -Fq -- \"--$flag\" \"$tmp/check-help\"; done; test \"$(\"$tmp/ao\" gate pending --dry-run)\" = '[dry-run] Would list pending gate reviews'; test \"$(\"$tmp/ao\" gate approve cand-1 --note ok --dry-run)\" = '[dry-run] Would approve candidate cand-1 with note: ok'; test \"$(\"$tmp/ao\" gate reject cand-1 --reason no --dry-run)\" = '[dry-run] Would reject candidate cand-1 with reason: no'; set +e; \"$tmp/ao\" gate reject cand-1 >\"$tmp/out\" 2>\"$tmp/err\"; code=$?; set -e; test \"$code\" -eq 1; grep -Fq 'required flag(s) \"reason\" not set' \"$tmp/err\"; \"$tmp/ao\" gate run does-not-exist >\"$tmp/run\"; grep -Fq '\"Status\":\"UNKNOWN\"' \"$tmp/run\"" + }, + { + "id": "gate-go-behavior", + "command": "cd cli && go test ./cmd/ao -run 'Test(Gate|ProductionGateRunner|ApplyRangeScope)' -count=1" + } + ], + "dimensions": { + "path": {"status": "evidence", "evidence": ["gate-black-box", "gate-go-behavior"]}, + "aliases": {"status": "not_applicable", "reason": "The gate family declares no command aliases."}, + "accepted_args": {"status": "evidence", "evidence": ["gate-black-box", "gate-go-behavior"]}, + "rejected_args": {"status": "evidence", "evidence": ["gate-black-box", "gate-go-behavior"]}, + "stdout": {"status": "evidence", "evidence": ["gate-black-box", "gate-go-behavior"]}, + "stderr": {"status": "evidence", "evidence": ["gate-black-box", "gate-go-behavior"]}, + "exit_classes": {"status": "evidence", "evidence": ["gate-black-box", "gate-go-behavior"]}, + "tracker_selection": {"status": "not_applicable", "reason": "The gate family does not select a repository issue tracker."}, + "ordered_effects": {"status": "evidence", "evidence": ["gate-go-behavior"]} + } +} diff --git a/cli/testdata/compatibility-baseline/families/gate/ownership.json b/cli/testdata/compatibility-baseline/families/gate/ownership.json new file mode 100644 index 000000000..7e0cdf418 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/gate/ownership.json @@ -0,0 +1,68 @@ +{ + "schema_version": 1, + "family": "gate", + "profiles": { + "default": "present", + "flywheel": "present", + "legacy": "present", + "combined": "present" + }, + "legacy_symbols": [ + "gateNote", + "gateReason", + "gateOlderThan", + "gateTier", + "gateCmd", + "gatePendingCmd", + "gateApproveCmd", + "gateRejectCmd", + "gateBulkApproveCmd", + "outputGatePending", + "entryUrgency", + "outputGatePendingTable", + "printAutoPromoteWarning", + "gateRunCmd", + "gateRunOptions", + "runGateRun", + "gateRunRun", + "gateRunViaPort", + "productionGateRunner", + "newProductionGateRunner", + "exitCodeToVerdict", + "tailBytes", + "gateCheckFast", + "gateCheckFull", + "gateCheckJSON", + "gateCheckGitHub", + "gateCheckFailFast", + "gateCheckScope", + "gateCheckWorkflowCoverage", + "gateCheckRequireWorkflowParity", + "gateCheckWorkflowPath", + "gateCheckRegistry", + "gateCheckCmd", + "applyRangeScope", + "gateCheckMode", + "gateRepoRoot", + "runGateCheck" + ], + "live_owner": "cli/internal/commands/gate", + "allowed_paths": [ + "cli/cmd/ao/gate*.go", + "cli/cmd/ao/root.go", + "cli/cmd/ao/cobra_commands_test.go", + "cli/cmd/ao/completion_values_test.go", + "cli/cmd/ao/default_spine_test.go", + "cli/cmd/ao/json_validity_test.go", + "cli/cmd/ao/refinery.go", + "cli/cmd/ao/testutil_test.go", + "cli/internal/commands/gate/**", + "cli/internal/adapters/gate/**", + "cli/internal/gate/**", + "cli/internal/gates/**", + "cli/internal/pool/**", + "cli/internal/ports/**", + "cli/docs/COMMANDS.md", + "cli/testdata/compatibility-baseline/families/gate/**" + ] +} From 5bbb770eb32e799a6a897f3f6e2a46ad9da0b2be Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:57:03 -0400 Subject: [PATCH 103/108] chore(gate): bind migration lineage --- .../compatibility-baseline/families/gate/lineage.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cli/testdata/compatibility-baseline/families/gate/lineage.json diff --git a/cli/testdata/compatibility-baseline/families/gate/lineage.json b/cli/testdata/compatibility-baseline/families/gate/lineage.json new file mode 100644 index 000000000..4a2441004 --- /dev/null +++ b/cli/testdata/compatibility-baseline/families/gate/lineage.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "family": "gate", + "old_implementation_sha": "8f779fea9e7802e7d2410281ff0ab112e48eb6ea", + "freeze_sha": "121911c89c0dccf18a6e6229b6b92086fcd9d168", + "fixture_sha256": "67198187ec7bc117bc9190eca6238ab2c11a2853cfef3c48e89a89af46d65c7f", + "ownership_sha256": "0461b6c20f5477fde5b2fef03ecafaea9740c3f732d75092590877d4f0828a96", + "scope_sha256": "d286af8b81c202bea8c5a1a85d215d5da5aeedbaaac8f9ba1ebef9246b2cb2ad", + "migration_state": "migrating" +} From cdd86a0c56be9fc41496c559d214f855e1d16e63 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:57:26 -0400 Subject: [PATCH 104/108] refactor(gate): extract human review service --- cli/internal/adapters/gate/review_pool.go | 69 +++++++ .../adapters/gate/review_pool_test.go | 56 ++++++ cli/internal/gate/review_service.go | 175 ++++++++++++++++++ cli/internal/gate/review_service_test.go | 108 +++++++++++ 4 files changed, 408 insertions(+) create mode 100644 cli/internal/adapters/gate/review_pool.go create mode 100644 cli/internal/adapters/gate/review_pool_test.go create mode 100644 cli/internal/gate/review_service.go create mode 100644 cli/internal/gate/review_service_test.go diff --git a/cli/internal/adapters/gate/review_pool.go b/cli/internal/adapters/gate/review_pool.go new file mode 100644 index 000000000..5a4b368cc --- /dev/null +++ b/cli/internal/adapters/gate/review_pool.go @@ -0,0 +1,69 @@ +package gate + +import ( + "context" + + gateapp "github.com/boshu2/agentops/cli/internal/gate" + "github.com/boshu2/agentops/cli/internal/pool" +) + +type ReviewPool struct { + pool *pool.Pool + reviewer func() string +} + +func NewReviewPool(root string, reviewer func() string) *ReviewPool { + return &ReviewPool{pool: pool.NewPool(root), reviewer: reviewer} +} + +func (adapter *ReviewPool) ListPending(ctx context.Context) ([]gateapp.ReviewEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + entries, err := adapter.pool.ListPendingReview() + if err != nil { + return nil, err + } + result := make([]gateapp.ReviewEntry, 0, len(entries)) + for _, entry := range entries { + result = append(result, gateapp.ReviewEntry{ + ID: entry.Candidate.ID, + Tier: string(entry.Candidate.Tier), + Age: entry.Age, + AgeString: entry.AgeString, + Utility: entry.Candidate.Utility, + ApproachingAutoPromote: entry.ApproachingAutoPromote, + }) + } + return result, nil +} + +func (adapter *ReviewPool) Approve(ctx context.Context, request gateapp.ApproveRequest) error { + if err := ctx.Err(); err != nil { + return err + } + return adapter.pool.Approve(request.CandidateID, request.Note, request.Reviewer) +} + +func (adapter *ReviewPool) Reject(ctx context.Context, request gateapp.RejectRequest) error { + if err := ctx.Err(); err != nil { + return err + } + return adapter.pool.Reject(request.CandidateID, request.Reason, request.Reviewer) +} + +func (adapter *ReviewPool) BulkApprove(ctx context.Context, request gateapp.BulkApproveRequest) ([]string, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return adapter.pool.BulkApprove(request.OlderThan, request.Reviewer, request.DryRun) +} + +func (adapter *ReviewPool) Reviewer() string { + if adapter.reviewer == nil { + return "unknown" + } + return adapter.reviewer() +} + +var _ gateapp.ReviewPort = (*ReviewPool)(nil) diff --git a/cli/internal/adapters/gate/review_pool_test.go b/cli/internal/adapters/gate/review_pool_test.go new file mode 100644 index 000000000..7ee361b3b --- /dev/null +++ b/cli/internal/adapters/gate/review_pool_test.go @@ -0,0 +1,56 @@ +package gate + +import ( + "context" + "testing" + "time" + + gateapp "github.com/boshu2/agentops/cli/internal/gate" + "github.com/boshu2/agentops/cli/internal/pool" + "github.com/boshu2/agentops/cli/internal/types" +) + +func TestReviewPoolListsAndMapsPendingEntries(t *testing.T) { + root := t.TempDir() + p := pool.NewPool(root) + candidate := types.Candidate{ID: "cand-1", Tier: types.TierBronze, Content: "review me", Utility: 0.61} + if err := p.AddAt(candidate, types.Scoring{GateRequired: true}, time.Now().Add(-13*time.Hour)); err != nil { + t.Fatalf("AddAt: %v", err) + } + adapter := NewReviewPool(root, func() string { return "reviewer" }) + entries, err := adapter.ListPending(context.Background()) + if err != nil { + t.Fatalf("ListPending: %v", err) + } + if len(entries) != 1 || entries[0].ID != "cand-1" || entries[0].Tier != string(types.TierBronze) || entries[0].Utility != 0.61 { + t.Fatalf("entries = %+v", entries) + } +} + +func TestReviewPoolDelegatesApproveWithReviewer(t *testing.T) { + root := t.TempDir() + p := pool.NewPool(root) + if err := p.Add(types.Candidate{ID: "cand-1", Tier: types.TierBronze, Content: "review me"}, types.Scoring{GateRequired: true}); err != nil { + t.Fatalf("Add: %v", err) + } + adapter := NewReviewPool(root, func() string { return "system-reviewer" }) + if got := adapter.Reviewer(); got != "system-reviewer" { + t.Fatalf("Reviewer = %q", got) + } + if err := adapter.Approve(context.Background(), gateapp.ApproveRequest{CandidateID: "cand-1", Note: "good", Reviewer: adapter.Reviewer()}); err != nil { + t.Fatalf("Approve: %v", err) + } + entry, err := p.Get("cand-1") + if err != nil || entry.HumanReview == nil || entry.HumanReview.Reviewer != "system-reviewer" { + t.Fatalf("entry=%+v err=%v", entry, err) + } +} + +func TestReviewPoolHonorsCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + adapter := NewReviewPool(t.TempDir(), nil) + if _, err := adapter.ListPending(ctx); err == nil { + t.Fatal("expected cancellation error") + } +} diff --git a/cli/internal/gate/review_service.go b/cli/internal/gate/review_service.go new file mode 100644 index 000000000..22a1df1fb --- /dev/null +++ b/cli/internal/gate/review_service.go @@ -0,0 +1,175 @@ +package gate + +import ( + "context" + "fmt" + "time" +) + +const defaultBulkApproveThreshold = 24 * time.Hour + +type ReviewEntry struct { + ID string `json:"id" yaml:"id"` + Tier string `json:"tier" yaml:"tier"` + Age time.Duration `json:"-" yaml:"-"` + AgeString string `json:"age,omitempty" yaml:"age,omitempty"` + Utility float64 `json:"utility" yaml:"utility"` + ApproachingAutoPromote bool `json:"approaching_auto_promote,omitempty" yaml:"approaching_auto_promote,omitempty"` + Urgency string `json:"urgency" yaml:"urgency"` +} + +type ApproveRequest struct { + CandidateID string + Note string + Reviewer string +} + +type RejectRequest struct { + CandidateID string + Reason string + Reviewer string +} + +type BulkApproveRequest struct { + OlderThan time.Duration + Reviewer string + DryRun bool +} + +type ReviewPort interface { + ListPending(context.Context) ([]ReviewEntry, error) + Approve(context.Context, ApproveRequest) error + Reject(context.Context, RejectRequest) error + BulkApprove(context.Context, BulkApproveRequest) ([]string, error) + Reviewer() string +} + +type ReviewService struct { + Port ReviewPort +} + +type PendingRequest struct { + DryRun bool +} + +type PendingResult struct { + DryRun bool `json:"dry_run" yaml:"dry_run"` + Entries []ReviewEntry `json:"entries" yaml:"entries"` +} + +func (service ReviewService) Pending(ctx context.Context, request PendingRequest) (PendingResult, error) { + if request.DryRun { + return PendingResult{DryRun: true}, nil + } + entries, err := service.Port.ListPending(ctx) + if err != nil { + return PendingResult{}, fmt.Errorf("gate pending: %w", err) + } + for index := range entries { + entries[index].Urgency = reviewUrgency(entries[index]) + } + return PendingResult{Entries: entries}, nil +} + +func reviewUrgency(entry ReviewEntry) string { + switch { + case entry.ApproachingAutoPromote: + return "HIGH (approaching 24h)" + case entry.Age > 12*time.Hour: + return "MEDIUM" + default: + return "LOW" + } +} + +type ApproveInput struct { + CandidateID string + Note string + DryRun bool +} + +type ApproveResult struct { + CandidateID string `json:"candidate_id" yaml:"candidate_id"` + Note string `json:"note,omitempty" yaml:"note,omitempty"` + Reviewer string `json:"reviewer,omitempty" yaml:"reviewer,omitempty"` + DryRun bool `json:"dry_run" yaml:"dry_run"` +} + +func (service ReviewService) Approve(ctx context.Context, input ApproveInput) (ApproveResult, error) { + if input.CandidateID == "" { + return ApproveResult{}, fmt.Errorf("gate approve: candidate ID is required") + } + result := ApproveResult{CandidateID: input.CandidateID, Note: input.Note, DryRun: input.DryRun} + if input.DryRun { + return result, nil + } + result.Reviewer = service.Port.Reviewer() + if err := service.Port.Approve(ctx, ApproveRequest{CandidateID: input.CandidateID, Note: input.Note, Reviewer: result.Reviewer}); err != nil { + return ApproveResult{}, fmt.Errorf("approve candidate: %w", err) + } + return result, nil +} + +type RejectInput struct { + CandidateID string + Reason string + DryRun bool +} + +type RejectResult struct { + CandidateID string `json:"candidate_id" yaml:"candidate_id"` + Reason string `json:"reason" yaml:"reason"` + Reviewer string `json:"reviewer,omitempty" yaml:"reviewer,omitempty"` + DryRun bool `json:"dry_run" yaml:"dry_run"` +} + +func (service ReviewService) Reject(ctx context.Context, input RejectInput) (RejectResult, error) { + if input.CandidateID == "" { + return RejectResult{}, fmt.Errorf("gate reject: candidate ID is required") + } + if input.Reason == "" { + return RejectResult{}, fmt.Errorf("--reason is required for rejection") + } + result := RejectResult{CandidateID: input.CandidateID, Reason: input.Reason, DryRun: input.DryRun} + if input.DryRun { + return result, nil + } + result.Reviewer = service.Port.Reviewer() + if err := service.Port.Reject(ctx, RejectRequest{CandidateID: input.CandidateID, Reason: input.Reason, Reviewer: result.Reviewer}); err != nil { + return RejectResult{}, fmt.Errorf("reject candidate: %w", err) + } + return result, nil +} + +type BulkApproveInput struct { + OlderThan string + Tier string + DryRun bool +} + +type BulkApproveResult struct { + Approved []string `json:"approved" yaml:"approved"` + Threshold time.Duration `json:"-" yaml:"-"` + Tier string `json:"tier" yaml:"tier"` + DryRun bool `json:"dry_run" yaml:"dry_run"` +} + +func (service ReviewService) BulkApprove(ctx context.Context, input BulkApproveInput) (BulkApproveResult, error) { + threshold := defaultBulkApproveThreshold + if input.OlderThan != "" { + parsed, err := time.ParseDuration(input.OlderThan) + if err != nil { + return BulkApproveResult{}, fmt.Errorf("invalid duration %q: %w", input.OlderThan, err) + } + threshold = parsed + } + tier := input.Tier + if tier == "" { + tier = "silver" + } + approved, err := service.Port.BulkApprove(ctx, BulkApproveRequest{OlderThan: threshold, Reviewer: service.Port.Reviewer(), DryRun: input.DryRun}) + if err != nil { + return BulkApproveResult{}, fmt.Errorf("bulk approve: %w", err) + } + return BulkApproveResult{Approved: approved, Threshold: threshold, Tier: tier, DryRun: input.DryRun}, nil +} diff --git a/cli/internal/gate/review_service_test.go b/cli/internal/gate/review_service_test.go new file mode 100644 index 000000000..b387cd127 --- /dev/null +++ b/cli/internal/gate/review_service_test.go @@ -0,0 +1,108 @@ +package gate + +import ( + "context" + "errors" + "testing" + "time" +) + +type reviewPortSpy struct { + entries []ReviewEntry + listErr error + listCalls int + approveCalls []ApproveRequest + rejectCalls []RejectRequest + bulkCalls []BulkApproveRequest + reviewer string +} + +func (port *reviewPortSpy) ListPending(context.Context) ([]ReviewEntry, error) { + port.listCalls++ + return port.entries, port.listErr +} + +func (port *reviewPortSpy) Approve(_ context.Context, request ApproveRequest) error { + port.approveCalls = append(port.approveCalls, request) + return nil +} + +func (port *reviewPortSpy) Reject(_ context.Context, request RejectRequest) error { + port.rejectCalls = append(port.rejectCalls, request) + return nil +} + +func (port *reviewPortSpy) BulkApprove(_ context.Context, request BulkApproveRequest) ([]string, error) { + port.bulkCalls = append(port.bulkCalls, request) + return []string{"cand-1"}, nil +} + +func (port *reviewPortSpy) Reviewer() string { return port.reviewer } + +func TestReviewServicePendingComputesUrgency(t *testing.T) { + port := &reviewPortSpy{entries: []ReviewEntry{ + {ID: "high", ApproachingAutoPromote: true}, + {ID: "medium", Age: 13 * time.Hour}, + {ID: "low", Age: time.Hour}, + }} + result, err := (ReviewService{Port: port}).Pending(context.Background(), PendingRequest{}) + if err != nil { + t.Fatalf("Pending: %v", err) + } + want := []string{"HIGH (approaching 24h)", "MEDIUM", "LOW"} + for index := range want { + if result.Entries[index].Urgency != want[index] { + t.Errorf("entry %d urgency = %q, want %q", index, result.Entries[index].Urgency, want[index]) + } + } +} + +func TestReviewServicePendingDryRunDoesNotReadStore(t *testing.T) { + port := &reviewPortSpy{listErr: errors.New("must not be called")} + result, err := (ReviewService{Port: port}).Pending(context.Background(), PendingRequest{DryRun: true}) + if err != nil || !result.DryRun || port.listCalls != 0 { + t.Fatalf("result=%+v err=%v listCalls=%d", result, err, port.listCalls) + } +} + +func TestReviewServiceApproveDryRunDoesNotResolveReviewerOrMutate(t *testing.T) { + port := &reviewPortSpy{reviewer: "reviewer"} + result, err := (ReviewService{Port: port}).Approve(context.Background(), ApproveInput{CandidateID: "cand-1", Note: "ok", DryRun: true}) + if err != nil || !result.DryRun || len(port.approveCalls) != 0 { + t.Fatalf("result=%+v err=%v calls=%v", result, err, port.approveCalls) + } +} + +func TestReviewServiceRejectRequiresReason(t *testing.T) { + port := &reviewPortSpy{} + _, err := (ReviewService{Port: port}).Reject(context.Background(), RejectInput{CandidateID: "cand-1"}) + if err == nil || len(port.rejectCalls) != 0 { + t.Fatalf("err=%v calls=%v", err, port.rejectCalls) + } +} + +func TestReviewServiceBulkApproveParsesThresholdAndPreservesDryRun(t *testing.T) { + port := &reviewPortSpy{reviewer: "reviewer"} + result, err := (ReviewService{Port: port}).BulkApprove(context.Background(), BulkApproveInput{OlderThan: "12h", Tier: "gold", DryRun: true}) + if err != nil { + t.Fatalf("BulkApprove: %v", err) + } + if len(port.bulkCalls) != 1 { + t.Fatalf("bulk calls = %v", port.bulkCalls) + } + call := port.bulkCalls[0] + if call.OlderThan != 12*time.Hour || call.Reviewer != "reviewer" || !call.DryRun { + t.Errorf("call = %+v", call) + } + if result.Tier != "gold" || len(result.Approved) != 1 { + t.Errorf("result = %+v", result) + } +} + +func TestReviewServiceBulkApproveRejectsInvalidDurationBeforeCallingPort(t *testing.T) { + port := &reviewPortSpy{} + _, err := (ReviewService{Port: port}).BulkApprove(context.Background(), BulkApproveInput{OlderThan: "later"}) + if err == nil || len(port.bulkCalls) != 0 { + t.Fatalf("err=%v calls=%v", err, port.bulkCalls) + } +} From f10486ad591a25c57696bf10107e162a31f5351a Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:57:28 -0400 Subject: [PATCH 105/108] refactor(gate): extract named check runner --- cli/internal/adapters/gate/runner.go | 93 +++++++++++++++++++++++ cli/internal/adapters/gate/runner_test.go | 79 +++++++++++++++++++ cli/internal/gate/run_service.go | 31 ++++++++ cli/internal/gate/run_service_test.go | 56 ++++++++++++++ 4 files changed, 259 insertions(+) create mode 100644 cli/internal/adapters/gate/runner.go create mode 100644 cli/internal/adapters/gate/runner_test.go create mode 100644 cli/internal/gate/run_service.go create mode 100644 cli/internal/gate/run_service_test.go diff --git a/cli/internal/adapters/gate/runner.go b/cli/internal/adapters/gate/runner.go new file mode 100644 index 000000000..1e9578c66 --- /dev/null +++ b/cli/internal/adapters/gate/runner.go @@ -0,0 +1,93 @@ +package gate + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "path/filepath" + "strings" + + "github.com/boshu2/agentops/cli/internal/ports" + "github.com/boshu2/agentops/cli/internal/quality" +) + +const gateLogTailLimit = 4096 + +type Runner struct { + repoRoot string +} + +func NewRunner(repoRoot string) *Runner { + return &Runner{repoRoot: repoRoot} +} + +func (runner *Runner) Run(ctx context.Context, request ports.GateRunRequest) (ports.GateVerdict, error) { + if err := ctx.Err(); err != nil { + return ports.GateVerdict{}, err + } + if request.Name == "" { + return ports.GateVerdict{Status: ports.GateStatusUnknown, Reason: "empty GateName"}, nil + } + if runner.repoRoot == "" { + return ports.GateVerdict{}, fmt.Errorf("gate runner: repo root required") + } + scriptPath := filepath.Join(runner.repoRoot, "scripts", "check-"+string(request.Name)+".sh") + if !quality.FileExists(scriptPath) { + return ports.GateVerdict{ + Status: ports.GateStatusUnknown, + Reason: fmt.Sprintf("no script for gate %q at %s", request.Name, scriptPath), + }, nil + } + + command := exec.CommandContext(ctx, "bash", scriptPath) + command.Dir = runner.repoRoot + if len(request.Env) > 0 { + environment := make([]string, 0, len(request.Env)) + for key, value := range request.Env { + environment = append(environment, key+"="+value) + } + command.Env = append(command.Environ(), environment...) + } + var output bytes.Buffer + command.Stdout = &output + command.Stderr = &output + runErr := command.Run() + + exitCode := 0 + if exitErr, ok := runErr.(*exec.ExitError); ok { + exitCode = exitErr.ExitCode() + } else if runErr != nil { + return ports.GateVerdict{ + Status: ports.GateStatusUnknown, + Reason: fmt.Sprintf("subprocess error: %v", runErr), + LogTail: gateLogTail(output.Bytes()), + }, nil + } + + verdict := gateExitVerdict(exitCode) + verdict.LogTail = gateLogTail(output.Bytes()) + return verdict, nil +} + +func gateExitVerdict(code int) ports.GateVerdict { + switch code { + case 0: + return ports.GateVerdict{Status: ports.GateStatusPass, Reason: "exit 0"} + case 2: + return ports.GateVerdict{Status: ports.GateStatusWarn, Reason: "exit 2 (advisory)"} + case 75: + return ports.GateVerdict{Status: ports.GateStatusSkip, Reason: "exit 75 (structural skip)"} + default: + return ports.GateVerdict{Status: ports.GateStatusFail, Reason: fmt.Sprintf("exit %d", code)} + } +} + +func gateLogTail(output []byte) string { + if len(output) <= gateLogTailLimit { + return string(output) + } + return strings.TrimSpace(string(output[len(output)-gateLogTailLimit:])) +} + +var _ ports.GateRunnerPort = (*Runner)(nil) diff --git a/cli/internal/adapters/gate/runner_test.go b/cli/internal/adapters/gate/runner_test.go new file mode 100644 index 000000000..8adf5d654 --- /dev/null +++ b/cli/internal/adapters/gate/runner_test.go @@ -0,0 +1,79 @@ +package gate + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/boshu2/agentops/cli/internal/ports" +) + +func tempGateRepo(t *testing.T, name, script string) string { + t.Helper() + root := t.TempDir() + dir := filepath.Join(root, "scripts") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "check-"+name+".sh"), []byte("#!/usr/bin/env bash\n"+script+"\n"), 0o755); err != nil { + t.Fatal(err) + } + return root +} + +func TestRunnerMapsExitClasses(t *testing.T) { + tests := []struct { + name string + exit string + status ports.GateStatus + }{ + {name: "pass", exit: "0", status: ports.GateStatusPass}, + {name: "warn", exit: "2", status: ports.GateStatusWarn}, + {name: "skip", exit: "75", status: ports.GateStatusSkip}, + {name: "fail", exit: "1", status: ports.GateStatusFail}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runner := NewRunner(tempGateRepo(t, test.name, "echo marker; exit "+test.exit)) + verdict, err := runner.Run(context.Background(), ports.GateRunRequest{Name: ports.GateName(test.name)}) + if err != nil || verdict.Status != test.status || !strings.Contains(verdict.LogTail, "marker") { + t.Fatalf("verdict=%+v err=%v", verdict, err) + } + }) + } +} + +func TestRunnerMissingAndEmptyNamesAreUnknown(t *testing.T) { + runner := NewRunner(t.TempDir()) + for _, name := range []ports.GateName{"", "missing"} { + verdict, err := runner.Run(context.Background(), ports.GateRunRequest{Name: name}) + if err != nil || verdict.Status != ports.GateStatusUnknown { + t.Fatalf("name=%q verdict=%+v err=%v", name, verdict, err) + } + } +} + +func TestRunnerForwardsEnvironmentAndCapsLogTail(t *testing.T) { + runner := NewRunner(tempGateRepo(t, "env", `printf '%05000d' 0; echo "KEY=$KEY"`)) + verdict, err := runner.Run(context.Background(), ports.GateRunRequest{Name: "env", Env: map[string]string{"KEY": "value"}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(verdict.LogTail) > 4096 || !strings.Contains(verdict.LogTail, "KEY=value") { + t.Fatalf("tail len=%d suffix=%q", len(verdict.LogTail), verdict.LogTail[len(verdict.LogTail)-32:]) + } +} + +func TestRunnerHonorsCancellationAndRequiresRoot(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := NewRunner(t.TempDir()).Run(ctx, ports.GateRunRequest{Name: "x"}); !errors.Is(err, context.Canceled) { + t.Fatalf("cancel err=%v", err) + } + if _, err := NewRunner("").Run(context.Background(), ports.GateRunRequest{Name: "x"}); err == nil { + t.Fatal("expected empty-root error") + } +} diff --git a/cli/internal/gate/run_service.go b/cli/internal/gate/run_service.go new file mode 100644 index 000000000..c60df086c --- /dev/null +++ b/cli/internal/gate/run_service.go @@ -0,0 +1,31 @@ +package gate + +import ( + "context" + "fmt" + + "github.com/boshu2/agentops/cli/internal/ports" +) + +type RunRequest struct { + Name string + Env map[string]string +} + +type RunService struct { + Runner ports.GateRunnerPort +} + +func (service RunService) Execute(ctx context.Context, request RunRequest) (ports.GateVerdict, error) { + if request.Name == "" { + return ports.GateVerdict{}, fmt.Errorf("gate run: name required") + } + if service.Runner == nil { + return ports.GateVerdict{}, fmt.Errorf("gate run: runner required") + } + verdict, err := service.Runner.Run(ctx, ports.GateRunRequest{Name: ports.GateName(request.Name), Env: request.Env}) + if err != nil { + return ports.GateVerdict{}, fmt.Errorf("gate run: %w", err) + } + return verdict, nil +} diff --git a/cli/internal/gate/run_service_test.go b/cli/internal/gate/run_service_test.go new file mode 100644 index 000000000..147be6332 --- /dev/null +++ b/cli/internal/gate/run_service_test.go @@ -0,0 +1,56 @@ +package gate + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/boshu2/agentops/cli/internal/ports" +) + +type gateRunnerStub struct { + request ports.GateRunRequest + verdict ports.GateVerdict + err error +} + +func (runner *gateRunnerStub) Run(_ context.Context, request ports.GateRunRequest) (ports.GateVerdict, error) { + runner.request = request + return runner.verdict, runner.err +} + +func TestRunServiceReturnsTypedVerdict(t *testing.T) { + runner := &gateRunnerStub{verdict: ports.GateVerdict{Status: ports.GateStatusPass, Reason: "exit 0"}} + verdict, err := (RunService{Runner: runner}).Execute(context.Background(), RunRequest{Name: "compile-health"}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if runner.request.Name != "compile-health" || verdict.Status != ports.GateStatusPass { + t.Fatalf("request=%+v verdict=%+v", runner.request, verdict) + } +} + +func TestRunServiceRejectsEmptyName(t *testing.T) { + runner := &gateRunnerStub{} + _, err := (RunService{Runner: runner}).Execute(context.Background(), RunRequest{}) + if err == nil || runner.request.Name != "" { + t.Fatalf("err=%v request=%+v", err, runner.request) + } +} + +func TestRunServiceWrapsRunnerFailure(t *testing.T) { + runner := &gateRunnerStub{err: errors.New("subprocess died")} + _, err := (RunService{Runner: runner}).Execute(context.Background(), RunRequest{Name: "broken"}) + if err == nil || !strings.Contains(err.Error(), "gate run: subprocess died") { + t.Fatalf("err=%v", err) + } +} + +func TestRunServiceForwardsEnvironment(t *testing.T) { + runner := &gateRunnerStub{} + _, err := (RunService{Runner: runner}).Execute(context.Background(), RunRequest{Name: "env", Env: map[string]string{"KEY": "value"}}) + if err != nil || runner.request.Env["KEY"] != "value" { + t.Fatalf("err=%v request=%+v", err, runner.request) + } +} From 2177051f57c8dc822783d771cb262582ae288ec6 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:57:29 -0400 Subject: [PATCH 106/108] refactor(gate): extract registry check service --- cli/internal/adapters/gate/check_runtime.go | 47 ++++++++ .../adapters/gate/check_runtime_test.go | 66 +++++++++++ cli/internal/gate/check_service.go | 75 +++++++++++++ cli/internal/gate/check_service_test.go | 105 ++++++++++++++++++ 4 files changed, 293 insertions(+) create mode 100644 cli/internal/adapters/gate/check_runtime.go create mode 100644 cli/internal/adapters/gate/check_runtime_test.go create mode 100644 cli/internal/gate/check_service.go create mode 100644 cli/internal/gate/check_service_test.go diff --git a/cli/internal/adapters/gate/check_runtime.go b/cli/internal/adapters/gate/check_runtime.go new file mode 100644 index 000000000..f759cfc1f --- /dev/null +++ b/cli/internal/adapters/gate/check_runtime.go @@ -0,0 +1,47 @@ +package gate + +import ( + "fmt" + "os" + "os/exec" + "strings" + + gateapp "github.com/boshu2/agentops/cli/internal/gate" + "github.com/boshu2/agentops/cli/internal/gates" +) + +type CheckRuntime struct{} + +func (CheckRuntime) ApplyRangeScope(scope gates.Scope) error { + spec, ok := gates.ScopeRange(scope) + if !ok { + return nil + } + if err := gates.ValidateRangeSpec(spec); err != nil { + return err + } + if err := os.Setenv("AGENTOPS_GATE_RANGE", spec); err != nil { + return fmt.Errorf("set AGENTOPS_GATE_RANGE: %w", err) + } + return nil +} + +func (CheckRuntime) WorkflowCoverage(registry *gates.Registry, root, workflowPath string) (*gates.WorkflowCoverage, error) { + return gates.RegistryWorkflowCoverage(registry, root, workflowPath) +} + +func ResolveRepoRoot(start string) (string, error) { + if start == "" { + return "", fmt.Errorf("resolve gate repo root: start directory required") + } + command := exec.Command("git", "-C", start, "rev-parse", "--show-toplevel") + output, err := command.Output() + if err == nil { + if root := strings.TrimSpace(string(output)); root != "" { + return root, nil + } + } + return start, nil +} + +var _ gateapp.CheckRuntime = CheckRuntime{} diff --git a/cli/internal/adapters/gate/check_runtime_test.go b/cli/internal/adapters/gate/check_runtime_test.go new file mode 100644 index 000000000..98f1b0434 --- /dev/null +++ b/cli/internal/adapters/gate/check_runtime_test.go @@ -0,0 +1,66 @@ +package gate + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/boshu2/agentops/cli/internal/gates" +) + +func TestCheckRuntimeAppliesOnlyValidatedRangeScope(t *testing.T) { + t.Setenv("AGENTOPS_GATE_RANGE", "sentinel") + runtime := CheckRuntime{} + if err := runtime.ApplyRangeScope("head"); err != nil { + t.Fatalf("head: %v", err) + } + if got := os.Getenv("AGENTOPS_GATE_RANGE"); got != "sentinel" { + t.Fatalf("non-range changed env to %q", got) + } + if err := runtime.ApplyRangeScope("range:main..HEAD"); err != nil { + t.Fatalf("range: %v", err) + } + if got := os.Getenv("AGENTOPS_GATE_RANGE"); got != "main..HEAD" { + t.Fatalf("range env = %q", got) + } + if err := runtime.ApplyRangeScope("range:HEAD"); err == nil { + t.Fatal("expected malformed range error") + } +} + +func TestResolveRepoRootFindsLinkedDirectoryRootAndFallsBack(t *testing.T) { + root := t.TempDir() + if err := exec.Command("git", "-C", root, "init", "-q").Run(); err != nil { + t.Fatalf("git init: %v", err) + } + subdir := filepath.Join(root, "cli", "cmd") + if err := os.MkdirAll(subdir, 0o755); err != nil { + t.Fatal(err) + } + got, err := ResolveRepoRoot(subdir) + canonicalRoot, canonicalErr := filepath.EvalSymlinks(root) + if canonicalErr != nil { + t.Fatal(canonicalErr) + } + if err != nil || got != canonicalRoot { + t.Fatalf("got=%q err=%v want=%q", got, err, root) + } + nonRepo := t.TempDir() + got, err = ResolveRepoRoot(nonRepo) + if err != nil || got != nonRepo { + t.Fatalf("fallback got=%q err=%v want=%q", got, err, nonRepo) + } +} + +func TestCheckRuntimeWorkflowCoverageDelegates(t *testing.T) { + root := t.TempDir() + workflow := filepath.Join(root, "validate.yml") + if err := os.WriteFile(workflow, []byte("name: validate\njobs: {}\n"), 0o644); err != nil { + t.Fatal(err) + } + coverage, err := (CheckRuntime{}).WorkflowCoverage(gates.NewRegistry(), root, workflow) + if err != nil || coverage.WorkflowScriptCount != 0 { + t.Fatalf("coverage=%+v err=%v", coverage, err) + } +} diff --git a/cli/internal/gate/check_service.go b/cli/internal/gate/check_service.go new file mode 100644 index 000000000..1b623007d --- /dev/null +++ b/cli/internal/gate/check_service.go @@ -0,0 +1,75 @@ +package gate + +import ( + "context" + "fmt" + + "github.com/boshu2/agentops/cli/internal/gates" + "github.com/boshu2/agentops/cli/internal/ports" +) + +type CheckRuntime interface { + ApplyRangeScope(gates.Scope) error + WorkflowCoverage(*gates.Registry, string, string) (*gates.WorkflowCoverage, error) +} + +type CheckService struct { + Registry *gates.Registry + Runner ports.GateRunnerPort + Files gates.ChangedFilesPort + RepoRoot string + Runtime CheckRuntime +} + +type CheckRequest struct { + Full bool + Scope string + FailFast bool + WorkflowCoverage bool + RequireWorkflowParity bool + WorkflowPath string +} + +type CheckResult struct { + Report *gates.Report + ExitCode int + WorkflowParityMissing int +} + +func (service CheckService) Execute(ctx context.Context, request CheckRequest) (CheckResult, error) { + if service.Registry == nil { + return CheckResult{}, fmt.Errorf("gate check: registry required") + } + if service.Runtime == nil { + return CheckResult{}, fmt.Errorf("gate check: runtime required") + } + scope := gates.Scope(request.Scope) + if scope == "" { + scope = gates.ScopeHead + } + if err := service.Runtime.ApplyRangeScope(scope); err != nil { + return CheckResult{}, fmt.Errorf("gate check: %w", err) + } + mode := gates.Fast + if request.Full { + mode = gates.Full + } + orchestrator := gates.NewOrchestrator(service.Registry, service.Runner, service.Files, service.RepoRoot) + report, err := orchestrator.Run(ctx, gates.RunOptions{Mode: mode, Scope: scope, FailFast: request.FailFast}) + if err != nil { + return CheckResult{}, fmt.Errorf("gate check: %w", err) + } + result := CheckResult{Report: report, ExitCode: report.ExitCode()} + if request.WorkflowCoverage || request.RequireWorkflowParity { + coverage, coverageErr := service.Runtime.WorkflowCoverage(service.Registry, service.RepoRoot, request.WorkflowPath) + if coverageErr != nil { + return CheckResult{}, fmt.Errorf("gate check: workflow coverage: %w", coverageErr) + } + report.Coverage = coverage + if request.RequireWorkflowParity && coverage.MissingBlockingCount > 0 { + result.ExitCode = 1 + result.WorkflowParityMissing = coverage.MissingBlockingCount + } + } + return result, nil +} diff --git a/cli/internal/gate/check_service_test.go b/cli/internal/gate/check_service_test.go new file mode 100644 index 000000000..475a85fe4 --- /dev/null +++ b/cli/internal/gate/check_service_test.go @@ -0,0 +1,105 @@ +package gate + +import ( + "context" + "errors" + "testing" + + "github.com/boshu2/agentops/cli/internal/gates" + "github.com/boshu2/agentops/cli/internal/ports" +) + +type changedFilesStub struct { + files []string + err error +} + +func (stub changedFilesStub) Changed(context.Context, gates.Scope) ([]string, error) { + return stub.files, stub.err +} + +type checkRuntimeStub struct { + scope gates.Scope + applyErr error + coverage *gates.WorkflowCoverage + coverErr error +} + +func (runtime *checkRuntimeStub) ApplyRangeScope(scope gates.Scope) error { + runtime.scope = scope + return runtime.applyErr +} + +func (runtime *checkRuntimeStub) WorkflowCoverage(*gates.Registry, string, string) (*gates.WorkflowCoverage, error) { + return runtime.coverage, runtime.coverErr +} + +func registryWithVerdict(t *testing.T, status ports.GateStatus, blocking bool) *gates.Registry { + t.Helper() + registry := gates.NewRegistry() + if err := registry.Add(gates.Check{ + ID: "native", Tiers: gates.Fast | gates.Full, Blocking: blocking, + Run: func(context.Context, gates.RunContext) (ports.GateVerdict, error) { + return ports.GateVerdict{Status: status, Reason: "test"}, nil + }, + }); err != nil { + t.Fatal(err) + } + return registry +} + +func TestCheckServiceRunsFullAndReturnsBlockingExit(t *testing.T) { + runtime := &checkRuntimeStub{} + result, err := (CheckService{ + Registry: registryWithVerdict(t, ports.GateStatusFail, true), + Files: changedFilesStub{}, + Runtime: runtime, + }).Execute(context.Background(), CheckRequest{Full: true, Scope: "head"}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result.Report.Mode != gates.Full || result.ExitCode != 1 || runtime.scope != gates.ScopeHead { + t.Fatalf("result=%+v scope=%q", result, runtime.scope) + } +} + +func TestCheckServiceRunsFastWithChangedFileRouting(t *testing.T) { + result, err := (CheckService{ + Registry: registryWithVerdict(t, ports.GateStatusPass, true), + Files: changedFilesStub{files: []string{"cli/main.go"}}, + Runtime: &checkRuntimeStub{}, + }).Execute(context.Background(), CheckRequest{Scope: "head"}) + if err != nil || result.Report.Mode != gates.Fast || result.Report.ChangedCount != 1 || result.ExitCode != 0 { + t.Fatalf("result=%+v err=%v", result, err) + } +} + +func TestCheckServiceWorkflowParityCanFailResult(t *testing.T) { + runtime := &checkRuntimeStub{coverage: &gates.WorkflowCoverage{MissingBlockingCount: 2}} + result, err := (CheckService{ + Registry: registryWithVerdict(t, ports.GateStatusPass, true), + Files: changedFilesStub{}, + Runtime: runtime, + }).Execute(context.Background(), CheckRequest{Full: true, WorkflowCoverage: true, RequireWorkflowParity: true, WorkflowPath: "validate.yml"}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result.ExitCode != 1 || result.Report.Coverage != runtime.coverage || result.WorkflowParityMissing != 2 { + t.Fatalf("result=%+v", result) + } +} + +func TestCheckServiceReturnsRuntimeAndOrchestratorErrors(t *testing.T) { + t.Run("range environment", func(t *testing.T) { + _, err := (CheckService{Registry: gates.NewRegistry(), Runtime: &checkRuntimeStub{applyErr: errors.New("bad range")}}).Execute(context.Background(), CheckRequest{Scope: "range:HEAD"}) + if err == nil { + t.Fatal("expected error") + } + }) + t.Run("changed files", func(t *testing.T) { + _, err := (CheckService{Registry: gates.NewRegistry(), Files: changedFilesStub{err: errors.New("git failed")}, Runtime: &checkRuntimeStub{}}).Execute(context.Background(), CheckRequest{Scope: "head"}) + if err == nil { + t.Fatal("expected error") + } + }) +} From bb6f4b58cfe3efb9489d61512b3bb9af5cfd8d45 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 01:28:31 -0400 Subject: [PATCH 107/108] refactor(gate): cut over to internal command module --- cli/cmd/ao/completion_values_test.go | 2 +- cli/cmd/ao/gate.go | 326 -------- cli/cmd/ao/gate_check.go | 171 ---- cli/cmd/ao/gate_check_test.go | 309 -------- cli/cmd/ao/gate_cmd.go | 95 --- cli/cmd/ao/gate_cmd_test.go | 105 --- cli/cmd/ao/gate_composition.go | 112 +++ cli/cmd/ao/gate_composition_test.go | 29 + cli/cmd/ao/gate_runner_adapter.go | 116 --- cli/cmd/ao/gate_runner_adapter_test.go | 140 ---- cli/cmd/ao/gate_test.go | 728 ------------------ cli/cmd/ao/refinery.go | 15 +- cli/docs/COMMANDS.md | 12 +- cli/internal/adapters/gate/review_pool.go | 10 +- .../adapters/gate/review_pool_test.go | 12 +- cli/internal/commands/gate/module.go | 429 +++++++++++ cli/internal/commands/gate/module_test.go | 147 ++++ cli/internal/gate/review_service.go | 15 +- cli/internal/gate/review_service_test.go | 6 +- 19 files changed, 764 insertions(+), 2015 deletions(-) delete mode 100644 cli/cmd/ao/gate.go delete mode 100644 cli/cmd/ao/gate_check.go delete mode 100644 cli/cmd/ao/gate_check_test.go delete mode 100644 cli/cmd/ao/gate_cmd.go delete mode 100644 cli/cmd/ao/gate_cmd_test.go create mode 100644 cli/cmd/ao/gate_composition.go create mode 100644 cli/cmd/ao/gate_composition_test.go delete mode 100644 cli/cmd/ao/gate_runner_adapter.go delete mode 100644 cli/cmd/ao/gate_runner_adapter_test.go delete mode 100644 cli/cmd/ao/gate_test.go create mode 100644 cli/internal/commands/gate/module.go create mode 100644 cli/internal/commands/gate/module_test.go diff --git a/cli/cmd/ao/completion_values_test.go b/cli/cmd/ao/completion_values_test.go index 85d6dfd78..3a2c8255a 100644 --- a/cli/cmd/ao/completion_values_test.go +++ b/cli/cmd/ao/completion_values_test.go @@ -119,7 +119,7 @@ func TestFlagCompletions_Registered(t *testing.T) { sortedCompletionValues("health", "architecture", "quality", "meta")}, {"goals steer add --steer", goalsSteerAddCmd, "steer", sortedCompletionValues("increase", "decrease", "hold", "explore")}, - {"gate bulk-approve --tier", gateBulkApproveCmd, "tier", tierValues}, + {"gate bulk-approve --tier", mustFindCompletionCommand(t, "gate", "bulk-approve"), "tier", tierValues}, {"pool list --tier", poolListCmd, "tier", tierValues}, {"pool list --status", poolListCmd, "status", sortedCompletionValues("pending", "staged", "promoted", "rejected")}, diff --git a/cli/cmd/ao/gate.go b/cli/cmd/ao/gate.go deleted file mode 100644 index d5c593e4b..000000000 --- a/cli/cmd/ao/gate.go +++ /dev/null @@ -1,326 +0,0 @@ -// practices: [design-by-contract, continuous-delivery] -package main - -import ( - "encoding/json" - "fmt" - "os" - "text/tabwriter" - "time" - - "github.com/spf13/cobra" - "gopkg.in/yaml.v3" - - "github.com/boshu2/agentops/cli/internal/pool" -) - -var ( - gateNote string - gateReason string - gateOlderThan string - gateTier string -) - -var gateCmd = &cobra.Command{ - Use: "gate", - Short: "Human review gates", - Long: `Manage human review gates for bronze-tier candidates. - -Bronze-tier candidates (score 0.50-0.69) require human review -before promotion. The gate command provides the review interface. - -Examples: - ao gate pending - ao gate approve - ao gate reject --reason="Too vague"`, -} - -var gatePendingCmd = &cobra.Command{ - Use: "pending", - Short: "List candidates pending review", - Long: `List bronze-tier candidates awaiting human review. - -Shows age/urgency with oldest items first. -Highlights items approaching 24h auto-promote threshold. - -Examples: - ao gate pending - ao gate pending --json`, - RunE: func(cmd *cobra.Command, args []string) error { - if GetDryRun() { - fmt.Println("[dry-run] Would list pending gate reviews") - return nil - } - - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("get working directory: %w", err) - } - - p := pool.NewPool(cwd) - - entries, err := p.ListPendingReview() - if err != nil { - return fmt.Errorf("list pending: %w", err) - } - - return outputGatePending(entries) - }, -} - -func outputGatePending(entries []pool.PoolEntry) error { - switch GetOutput() { - case "json": - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - return enc.Encode(entries) - - case "yaml": - enc := yaml.NewEncoder(os.Stdout) - return enc.Encode(entries) - - default: - return outputGatePendingTable(entries) - } -} - -// entryUrgency returns a human-readable urgency label for a pool entry. -func entryUrgency(e pool.PoolEntry) string { - switch { - case e.ApproachingAutoPromote: - return "HIGH (approaching 24h)" - case e.Age > 12*time.Hour: - return "MEDIUM" - default: - return "LOW" - } -} - -// outputGatePendingTable renders pending gate entries as a formatted table. -func outputGatePendingTable(entries []pool.PoolEntry) error { - if len(entries) == 0 { - fmt.Println("No pending reviews") - return nil - } - - fmt.Printf("Pending Reviews (%d)\n", len(entries)) - fmt.Println("==================") - fmt.Println() - - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - //nolint:errcheck // CLI tabwriter output to stdout, errors unlikely and non-recoverable - fmt.Fprintln(w, "ID\tTIER\tAGE\tUTILITY\tURGENCY") - //nolint:errcheck // CLI tabwriter output to stdout - fmt.Fprintln(w, "--\t----\t---\t-------\t-------") - - for _, e := range entries { - //nolint:errcheck // CLI tabwriter output to stdout - fmt.Fprintf(w, "%s\t%s\t%s\t%.2f\t%s\n", - truncateID(e.Candidate.ID, 16), - e.Candidate.Tier, - e.AgeString, - e.Candidate.Utility, - entryUrgency(e), - ) - } - - if err := w.Flush(); err != nil { - return err - } - - printAutoPromoteWarning(entries) - return nil -} - -// printAutoPromoteWarning prints a warning if any entries are approaching auto-promote. -func printAutoPromoteWarning(entries []pool.PoolEntry) { - fmt.Println() - approaching := 0 - for _, e := range entries { - if e.ApproachingAutoPromote { - approaching++ - } - } - if approaching > 0 { - fmt.Printf("! %d candidate(s) approaching 24h auto-promote threshold\n", approaching) - } -} - -var gateApproveCmd = &cobra.Command{ - Use: "approve ", - Short: "Approve candidate for promotion", - Long: `Approve a bronze-tier candidate for promotion. - -Records reviewer identity and triggers promotion flow. - -Examples: - ao gate approve cand-abc123 - ao gate approve cand-abc123 --note="Good specificity, approved"`, - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - candidateID := args[0] - - if GetDryRun() { - fmt.Printf("[dry-run] Would approve candidate %s", candidateID) - if gateNote != "" { - fmt.Printf(" with note: %s", gateNote) - } - fmt.Println() - return nil - } - - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("get working directory: %w", err) - } - - p := pool.NewPool(cwd) - - // Get reviewer from system user (not spoofable via env) - reviewer := GetCurrentUser() - - if err := p.Approve(candidateID, gateNote, reviewer); err != nil { - return fmt.Errorf("approve candidate: %w", err) - } - - fmt.Printf("Approved: %s\n", candidateID) - fmt.Printf("Reviewer: %s\n", reviewer) - if gateNote != "" { - fmt.Printf("Note: %s\n", gateNote) - } - - // Suggest next step - fmt.Println() - fmt.Printf("To promote: ao pool promote %s\n", candidateID) - - return nil - }, -} - -var gateRejectCmd = &cobra.Command{ - Use: "reject ", - Short: "Reject candidate", - Long: `Reject a candidate with a required reason. - -Records in audit trail for future analysis. - -Examples: - ao gate reject cand-abc123 --reason="Lacks specificity" - ao gate reject cand-abc123 --reason="Duplicate of existing pattern"`, - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - candidateID := args[0] - - if gateReason == "" { - return fmt.Errorf("--reason is required for rejection") - } - - if GetDryRun() { - fmt.Printf("[dry-run] Would reject candidate %s with reason: %s\n", candidateID, gateReason) - return nil - } - - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("get working directory: %w", err) - } - - p := pool.NewPool(cwd) - - // Get reviewer from system user (not spoofable via env) - reviewer := GetCurrentUser() - - if err := p.Reject(candidateID, gateReason, reviewer); err != nil { - return fmt.Errorf("reject candidate: %w", err) - } - - fmt.Printf("Rejected: %s\n", candidateID) - fmt.Printf("Reviewer: %s\n", reviewer) - fmt.Printf("Reason: %s\n", gateReason) - - return nil - }, -} - -var gateBulkApproveCmd = &cobra.Command{ - Use: "bulk-approve", - Short: "Bulk approve silver candidates", - Long: `Approve all silver-tier candidates older than a threshold. - -Silver candidates auto-promote after 24h if not rejected. -This command accelerates the process for reviewed batches. - -Examples: - ao gate bulk-approve - ao gate bulk-approve --older-than=12h - ao gate bulk-approve --older-than=24h --dry-run`, - RunE: func(cmd *cobra.Command, args []string) error { - // Parse older-than duration - threshold := 24 * time.Hour - if gateOlderThan != "" { - var err error - threshold, err = time.ParseDuration(gateOlderThan) - if err != nil { - return fmt.Errorf("invalid duration %q: %w", gateOlderThan, err) - } - } - - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("get working directory: %w", err) - } - - p := pool.NewPool(cwd) - - // Get reviewer from system user (not spoofable via env) - reviewer := GetCurrentUser() - - approved, err := p.BulkApprove(threshold, reviewer, GetDryRun()) - if err != nil { - return fmt.Errorf("bulk approve: %w", err) - } - - if GetDryRun() { - if len(approved) == 0 { - fmt.Println("[dry-run] No candidates match criteria") - } else { - fmt.Printf("[dry-run] Would approve %d candidate(s):\n", len(approved)) - for _, id := range approved { - fmt.Printf(" - %s\n", id) - } - } - return nil - } - - if len(approved) == 0 { - fmt.Println("No candidates matched criteria") - } else { - fmt.Printf("Approved %d candidate(s):\n", len(approved)) - for _, id := range approved { - fmt.Printf(" - %s\n", id) - } - } - - return nil - }, -} - -func init() { - gateCmd.GroupID = "core" - rootCmd.AddCommand(gateCmd) - - // Add subcommands - gateCmd.AddCommand(gatePendingCmd) - gateCmd.AddCommand(gateApproveCmd) - gateCmd.AddCommand(gateRejectCmd) - gateCmd.AddCommand(gateBulkApproveCmd) - - // Add flags - gateApproveCmd.Flags().StringVar(&gateNote, "note", "", "Optional approval note") - gateRejectCmd.Flags().StringVar(&gateReason, "reason", "", "Required rejection reason") - _ = gateRejectCmd.MarkFlagRequired("reason") //nolint:errcheck - - gateBulkApproveCmd.Flags().StringVar(&gateOlderThan, "older-than", "24h", "Age threshold for bulk approval") - gateBulkApproveCmd.Flags().StringVar(&gateTier, "tier", "silver", "Tier to bulk approve (default: silver)") - _ = gateBulkApproveCmd.RegisterFlagCompletionFunc("tier", staticCompletionFunc("bronze", "silver", "gold")) -} diff --git a/cli/cmd/ao/gate_check.go b/cli/cmd/ao/gate_check.go deleted file mode 100644 index 53673edc6..000000000 --- a/cli/cmd/ao/gate_check.go +++ /dev/null @@ -1,171 +0,0 @@ -// practices: [hexagonal-architecture, ddd-bounded-context] -package main - -import ( - "fmt" - "os" - "os/exec" - "strings" - - "github.com/spf13/cobra" - - "github.com/boshu2/agentops/cli/internal/gates" - // Blank import registers the seed checks into gates.Default via init(). - _ "github.com/boshu2/agentops/cli/internal/gates/checks" -) - -// `ao gate check` is the orchestrated gate surface (ag-qidx GA4): it runs the -// declarative BC2 gate registry (cli/internal/gates) in fast (cockpit/pre-push, -// changed-file routed) or full (CI/refinery, all checks) mode, wiring the -// existing productionGateRunner (for Backing check-*.sh) and a git-backed -// ChangedFilesPort. Distinct from `ao gate run `, which runs ONE check. -// -// Mode is the ONLY variable the three consumers pass — hook `--fast`, CI -// `--full`, refinery `--full --json` — so local and CI run the identical binary. - -var ( - gateCheckFast bool - gateCheckFull bool - gateCheckJSON bool - gateCheckGitHub bool - gateCheckFailFast bool - gateCheckScope string - gateCheckWorkflowCoverage bool - gateCheckRequireWorkflowParity bool - gateCheckWorkflowPath string -) - -// gateCheckRegistry is the registry the command runs; a seam so tests can swap -// in an isolated registry instead of the init()-populated Default. -var gateCheckRegistry = gates.Default - -var gateCheckCmd = &cobra.Command{ - Use: "check", - Short: "Run the gate registry (fast cockpit subset or full suite)", - Long: `Run the declarative gate registry. - - ao gate check # fast: only checks affected by changed files (cockpit) - ao gate check --full # full: every check, routing ignored (CI/refinery) - ao gate check --json # machine-readable report (refinery/CI annotations) - -Exit code is the verdict: 0 if no blocking check FAILed; 1 if any did -(non-blocking FAIL and WARN/SKIP are advisory).`, - Args: cobra.NoArgs, - RunE: runGateCheck, -} - -func init() { - f := gateCheckCmd.Flags() - f.BoolVar(&gateCheckFast, "fast", false, "fast cockpit subset routed to changed files (the default; explicit flag for clarity in hooks)") - f.BoolVar(&gateCheckFull, "full", false, "run every check (routing ignored); default is the fast changed-file subset") - f.BoolVar(&gateCheckJSON, "json", false, "emit the machine-readable JSON report") - f.BoolVar(&gateCheckGitHub, "github-annotations", false, "emit GitHub Actions annotations for WARN/FAIL checks") - f.BoolVar(&gateCheckFailFast, "fail-fast", false, "stop after the first blocking failure") - f.StringVar(&gateCheckScope, "scope", "head", "fast-mode changed-file scope: head|staged|worktree|upstream|range:..") - f.BoolVar(&gateCheckWorkflowCoverage, "workflow-coverage", false, "include validate.yml-vs-registry script coverage in the report") - f.BoolVar(&gateCheckRequireWorkflowParity, "require-workflow-parity", false, "fail if validate.yml references scripts missing from the Go gate registry") - f.StringVar(&gateCheckWorkflowPath, "workflow-path", ".github/workflows/validate.yml", "workflow path used by --workflow-coverage and --require-workflow-parity") - gateCmd.AddCommand(gateCheckCmd) -} - -// applyRangeScope validates a range scope ("range:..") and exports -// AGENTOPS_GATE_RANGE so backing collectors (e.g. -// scripts/check-go-command-test-pair.sh) prefer the explicit range over their -// @{upstream}/git-show-HEAD fallback, which is absent or tip-only in a detached -// landing worktree (age-wy2t). It sets the var on the process so every backing -// subprocess inherits it via the ScriptRunner's cmd.Environ(). Non-range scopes -// are a no-op (the env is left untouched). -func applyRangeScope(scope gates.Scope) error { - spec, ok := gates.ScopeRange(scope) - if !ok { - return nil - } - if err := gates.ValidateRangeSpec(spec); err != nil { - return err - } - if err := os.Setenv("AGENTOPS_GATE_RANGE", spec); err != nil { - return fmt.Errorf("set AGENTOPS_GATE_RANGE: %w", err) - } - return nil -} - -// gateCheckMode maps the --full flag to a run tier. -func gateCheckMode(full bool) gates.Tier { - if full { - return gates.Full - } - return gates.Fast -} - -// gateRepoRoot resolves the repository root robustly via `git rev-parse -// --show-toplevel` (correct from any subdir and inside linked worktrees), -// falling back to the cwd-based resolver. The orchestrator joins scripts/ off -// this, so it must be the repo/worktree root, not the caller's cwd. -func gateRepoRoot() (string, error) { - out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() - if err == nil { - if root := strings.TrimSpace(string(out)); root != "" { - return root, nil - } - } - return resolveProjectDir() -} - -func runGateCheck(cmd *cobra.Command, _ []string) error { - root, err := gateRepoRoot() - if err != nil { - return &gateExitError{code: gateExitInternal, msg: fmt.Sprintf("resolve repo root: %v", err)} - } - - scope := gates.Scope(gateCheckScope) - if rerr := applyRangeScope(scope); rerr != nil { - return &gateExitError{code: gateExitInternal, msg: rerr.Error()} - } - - orch := gates.NewOrchestrator( - gateCheckRegistry, - gates.NewScriptRunner(root), - gates.NewGitChangedFiles(root), - root, - ) - report, err := orch.Run(cmd.Context(), gates.RunOptions{ - Mode: gateCheckMode(gateCheckFull), - Scope: scope, - FailFast: gateCheckFailFast, - }) - if err != nil { - return &gateExitError{code: gateExitInternal, msg: err.Error()} - } - if gateCheckWorkflowCoverage || gateCheckRequireWorkflowParity { - coverage, cerr := gates.RegistryWorkflowCoverage(gateCheckRegistry, root, gateCheckWorkflowPath) - if cerr != nil { - return &gateExitError{code: gateExitInternal, msg: cerr.Error()} - } - report.Coverage = coverage - } - - out := cmd.OutOrStdout() - if gateCheckJSON { - raw, jerr := report.JSON() - if jerr != nil { - return &gateExitError{code: gateExitInternal, msg: jerr.Error()} - } - fmt.Fprintln(out, string(raw)) - } else { - report.Human(out) - } - if gateCheckGitHub { - report.GitHubAnnotations(cmd.ErrOrStderr()) - } - - if code := report.ExitCode(); code != 0 { - return &gateExitError{code: code} - } - if gateCheckRequireWorkflowParity && report.Coverage != nil && report.Coverage.MissingBlockingCount > 0 { - return &gateExitError{ - code: gateExitFail, - msg: fmt.Sprintf("workflow parity missing %d blocking script(s)", report.Coverage.MissingBlockingCount), - } - } - return nil -} diff --git a/cli/cmd/ao/gate_check_test.go b/cli/cmd/ao/gate_check_test.go deleted file mode 100644 index efcbb7e1d..000000000 --- a/cli/cmd/ao/gate_check_test.go +++ /dev/null @@ -1,309 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "os" - "path/filepath" - "testing" - - "github.com/spf13/cobra" - - "github.com/boshu2/agentops/cli/internal/gates" - "github.com/boshu2/agentops/cli/internal/ports" -) - -// nativeCheckReason builds a native FAIL/PASS check that carries a reason but no -// LogTail — the shape (changelog.sync, learning.coherence, silent backing -// scripts) that emitted an empty --json log_tail before age-wy2t. -func nativeCheckReason(id string, status ports.GateStatus, reason string) gates.Check { - return gates.Check{ - ID: id, Tiers: gates.Full, Blocking: true, - Run: func(context.Context, gates.RunContext) (ports.GateVerdict, error) { - return ports.GateVerdict{Status: status, Reason: reason}, nil - }, - } -} - -// TestGateCheck_FailJSONHasNonEmptyLogTail is the SLICE 2 acceptance end-to-end: -// `ao gate check --full --json` on a FAILing native check emits a non-empty -// log_tail carrying the reason, so a landing loop never has to re-run + tee-log -// to learn which gate failed and why. -func TestGateCheck_FailJSONHasNonEmptyLogTail(t *testing.T) { - reg := gates.NewRegistry() - if err := reg.Add(nativeCheckReason("n.fail", ports.GateStatusFail, "CHANGELOG.md != docs/CHANGELOG.md")); err != nil { - t.Fatal(err) - } - // A blocking FAIL returns a gateExitError; the JSON is still written first. - out, err := runGateCheckWith(t, reg, true, true) - if err == nil { - t.Fatal("blocking FAIL should return a gateExitError") - } - var parsed struct { - Gates []struct { - Name string `json:"name"` - Status string `json:"status"` - LogTail string `json:"log_tail"` - } `json:"gates"` - } - if uerr := json.Unmarshal([]byte(out), &parsed); uerr != nil { - t.Fatalf("unmarshal %q: %v", out, uerr) - } - if len(parsed.Gates) != 1 { - t.Fatalf("gates = %+v, want one", parsed.Gates) - } - g := parsed.Gates[0] - if g.Status != "FAIL" { - t.Fatalf("status = %q, want FAIL", g.Status) - } - if g.LogTail != "CHANGELOG.md != docs/CHANGELOG.md" { - t.Fatalf("log_tail = %q, want the reason (non-empty detail)", g.LogTail) - } -} - -func TestApplyRangeScope_ExportsEnv(t *testing.T) { - t.Setenv("AGENTOPS_GATE_RANGE", "") // baseline + auto-restore - if err := applyRangeScope("range:origin/main..HEAD"); err != nil { - t.Fatalf("applyRangeScope: %v", err) - } - if got := os.Getenv("AGENTOPS_GATE_RANGE"); got != "origin/main..HEAD" { - t.Fatalf("AGENTOPS_GATE_RANGE = %q, want origin/main..HEAD", got) - } -} - -func TestApplyRangeScope_NonRangeLeavesEnv(t *testing.T) { - t.Setenv("AGENTOPS_GATE_RANGE", "sentinel") - for _, scope := range []gates.Scope{gates.ScopeHead, gates.ScopeStaged, gates.ScopeUpstream} { - if err := applyRangeScope(scope); err != nil { - t.Fatalf("applyRangeScope(%q): %v", scope, err) - } - } - if got := os.Getenv("AGENTOPS_GATE_RANGE"); got != "sentinel" { - t.Fatalf("AGENTOPS_GATE_RANGE = %q, want unchanged sentinel", got) - } -} - -func TestApplyRangeScope_InvalidRangeErrors(t *testing.T) { - t.Setenv("AGENTOPS_GATE_RANGE", "") - if err := applyRangeScope("range:HEAD"); err == nil { - t.Fatal("range without .. should error") - } - if err := applyRangeScope("range:"); err == nil { - t.Fatal("empty range should error") - } -} - -func TestGateRepoRoot_ResolvesToScriptsParent(t *testing.T) { - root, err := gateRepoRoot() - if err != nil { - t.Fatalf("gateRepoRoot: %v", err) - } - if root == "" { - t.Fatal("gateRepoRoot returned empty root") - } - if _, err := os.Stat(filepath.Join(root, "scripts")); err != nil { - t.Errorf("resolved root %q has no scripts/ dir: %v (orchestrator joins scripts/ off this)", root, err) - } -} - -func nativeCheck(id string, status ports.GateStatus, blocking bool) gates.Check { - return gates.Check{ - ID: id, Tiers: gates.Full, Blocking: blocking, - Run: func(context.Context, gates.RunContext) (ports.GateVerdict, error) { - return ports.GateVerdict{Status: status, Reason: "test"}, nil - }, - } -} - -// runGateCheckWith executes runGateCheck against an isolated registry, restoring -// the package globals afterwards. -func runGateCheckWith(t *testing.T, reg *gates.Registry, full, jsonOut bool) (string, error) { - t.Helper() - save := func() func() { - r, fl, j, ff, sc := gateCheckRegistry, gateCheckFull, gateCheckJSON, gateCheckFailFast, gateCheckScope - wc, req, wp := gateCheckWorkflowCoverage, gateCheckRequireWorkflowParity, gateCheckWorkflowPath - return func() { - gateCheckRegistry, gateCheckFull, gateCheckJSON, gateCheckFailFast, gateCheckScope = r, fl, j, ff, sc - gateCheckWorkflowCoverage, gateCheckRequireWorkflowParity, gateCheckWorkflowPath = wc, req, wp - } - }() - defer save() - gateCheckRegistry, gateCheckFull, gateCheckJSON, gateCheckFailFast, gateCheckScope = reg, full, jsonOut, false, "head" - gateCheckWorkflowCoverage, gateCheckRequireWorkflowParity, gateCheckWorkflowPath = false, false, ".github/workflows/validate.yml" - - var buf bytes.Buffer - c := &cobra.Command{} - c.SetOut(&buf) - c.SetContext(context.Background()) - err := runGateCheck(c, nil) - return buf.String(), err -} - -func TestGateCheckMode(t *testing.T) { - if gateCheckMode(true) != gates.Full { - t.Error("--full should map to Full") - } - if gateCheckMode(false) != gates.Fast { - t.Error("default should map to Fast") - } -} - -func TestGateCheck_RegisteredUnderGate(t *testing.T) { - var found bool - for _, c := range gateCmd.Commands() { - if c.Name() == "check" { - found = true - } - } - if !found { - t.Fatal("`ao gate check` not registered under gate") - } -} - -func TestGateCheck_HasModeFlags(t *testing.T) { - for _, name := range []string{ - "fast", - "full", - "json", - "github-annotations", - "fail-fast", - "scope", - "workflow-coverage", - "require-workflow-parity", - "workflow-path", - } { - if gateCheckCmd.Flags().Lookup(name) == nil { - t.Errorf("ao gate check missing --%s flag", name) - } - } -} - -func TestGateCheck_FullJSONReportsVerdicts(t *testing.T) { - reg := gates.NewRegistry() - if err := reg.Add(nativeCheck("n.pass", ports.GateStatusPass, true)); err != nil { - t.Fatal(err) - } - out, err := runGateCheckWith(t, reg, true, true) - if err != nil { - t.Fatalf("runGateCheck: %v", err) - } - var parsed struct { - Run struct { - Mode string `json:"mode"` - Summary struct { - Total, Passed int - } `json:"summary"` - } `json:"run"` - Gates []struct { - Name string `json:"name"` - } `json:"gates"` - } - if err := json.Unmarshal([]byte(out), &parsed); err != nil { - t.Fatalf("unmarshal %q: %v", out, err) - } - if parsed.Run.Mode != "full" { - t.Errorf("mode = %q, want full", parsed.Run.Mode) - } - if parsed.Run.Summary.Total != 1 || parsed.Run.Summary.Passed != 1 { - t.Errorf("summary = %+v, want total=1 passed=1", parsed.Run.Summary) - } - if len(parsed.Gates) != 1 || parsed.Gates[0].Name != "n.pass" { - t.Errorf("gates = %+v, want one n.pass", parsed.Gates) - } -} - -func TestGateCheck_BlockingFailReturnsExitOne(t *testing.T) { - reg := gates.NewRegistry() - if err := reg.Add(nativeCheck("n.fail", ports.GateStatusFail, true)); err != nil { - t.Fatal(err) - } - _, err := runGateCheckWith(t, reg, true, false) - if err == nil { - t.Fatal("blocking FAIL should return a gateExitError, got nil") - } - var ge *gateExitError - if !errors.As(err, &ge) { - t.Fatalf("err = %T, want *gateExitError", err) - } - if ge.ExitCode() != 1 { - t.Errorf("ExitCode = %d, want 1", ge.ExitCode()) - } -} - -func TestGateCheck_RequireWorkflowParityIgnoresDeferredAndAdvisoryMissing(t *testing.T) { - root := t.TempDir() - workflowDir := filepath.Join(root, ".github", "workflows") - if err := os.MkdirAll(workflowDir, 0o755); err != nil { - t.Fatal(err) - } - workflowPath := filepath.Join(workflowDir, "validate.yml") - workflow := []byte(`name: Validate -jobs: - advisory: - runs-on: ubuntu-latest - steps: - - name: advisory missing - continue-on-error: true - run: scripts/lint-evidence-lines.sh 123 - deferred: - runs-on: ubuntu-latest - steps: - - name: ap7 - run: scripts/verify-gate-claim.sh --log /tmp/run.log pr-1 "claim" -`) - if err := os.WriteFile(workflowPath, workflow, 0o644); err != nil { - t.Fatal(err) - } - - reg := gates.NewRegistry() - if err := reg.Add(nativeCheck("n.pass", ports.GateStatusPass, true)); err != nil { - t.Fatal(err) - } - - save := func() func() { - r, fl, j, ff, sc := gateCheckRegistry, gateCheckFull, gateCheckJSON, gateCheckFailFast, gateCheckScope - wc, req, wp := gateCheckWorkflowCoverage, gateCheckRequireWorkflowParity, gateCheckWorkflowPath - return func() { - gateCheckRegistry, gateCheckFull, gateCheckJSON, gateCheckFailFast, gateCheckScope = r, fl, j, ff, sc - gateCheckWorkflowCoverage, gateCheckRequireWorkflowParity, gateCheckWorkflowPath = wc, req, wp - } - }() - defer save() - gateCheckRegistry = reg - gateCheckFull = true - gateCheckJSON = true - gateCheckFailFast = false - gateCheckScope = "head" - gateCheckWorkflowCoverage = true - gateCheckRequireWorkflowParity = true - gateCheckWorkflowPath = workflowPath - - var buf bytes.Buffer - c := &cobra.Command{} - c.SetOut(&buf) - c.SetContext(context.Background()) - if err := runGateCheck(c, nil); err != nil { - t.Fatalf("runGateCheck returned error for deferred/advisory-only parity gaps: %v", err) - } - var parsed struct { - Coverage struct { - MissingBlockingCount int `json:"missing_blocking_count"` - MissingAdvisoryCount int `json:"missing_advisory_count"` - MissingDeferredCount int `json:"missing_deferred_count"` - } `json:"coverage"` - } - if err := json.Unmarshal(buf.Bytes(), &parsed); err != nil { - t.Fatalf("unmarshal %q: %v", buf.String(), err) - } - if parsed.Coverage.MissingBlockingCount != 0 { - t.Fatalf("MissingBlockingCount = %d, want 0", parsed.Coverage.MissingBlockingCount) - } - if parsed.Coverage.MissingAdvisoryCount != 0 { - t.Fatalf("MissingAdvisoryCount = %d, want 0; lint-evidence-lines is deferred by workflow context", parsed.Coverage.MissingAdvisoryCount) - } - if parsed.Coverage.MissingDeferredCount != 2 { - t.Fatalf("MissingDeferredCount = %d, want 2", parsed.Coverage.MissingDeferredCount) - } -} diff --git a/cli/cmd/ao/gate_cmd.go b/cli/cmd/ao/gate_cmd.go deleted file mode 100644 index 01c56e85a..000000000 --- a/cli/cmd/ao/gate_cmd.go +++ /dev/null @@ -1,95 +0,0 @@ -// practices: [hexagonal-architecture, ddd-bounded-context] -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "os" - - "github.com/spf13/cobra" - - "github.com/boshu2/agentops/cli/internal/ports" -) - -// gateCmd is the parent for BC2 GateRunnerPort CLI surfaces. Built -// using the cycle-147 cli-wiring template. -// -// Note: `gate` already existed as a top-level command before this -// session (human review gates). The cobra surface accepts subcommand -// addition without conflict — `ao gate run ` is the new BC2 -// surface; the prior `ao gate ...` subcommands stay unaffected. -var gateRunCmd = &cobra.Command{ - Use: "run ", - Short: "Run a check-*.sh gate via BC2 GateRunnerPort and emit verdict", - Long: `Invoke a check-*.sh gate via the typed BC2 GateRunnerPort -(productionGateRunner, cycle 115) and emit a GateVerdict (JSON). - -The argument is the gate name without the 'check-' prefix -or '.sh' suffix — productionGateRunner resolves to -scripts/check-.sh. - -Useful as a typed alternative to 'bash scripts/check-.sh; echo $?' -for scripts that want structured output (status, reason, log tail). - -Examples: - ao gate run compile-health # runs scripts/check-compile-health.sh - ao gate run three-gap-supergate # runs the supergate - ao gate run xxx-does-not-exist # emits UNKNOWN status`, - Args: cobra.ExactArgs(1), - RunE: runGateRun, -} - -type gateRunOptions struct { - name string - writer io.Writer - runFn func(ctx context.Context, opts gateRunOptions) (ports.GateVerdict, error) -} - -func init() { - // 'gate' parent already exists in cli/cmd/ao/gate.go; just add - // the 'run' subcommand. Use existing gateCmd from that file. - gateCmd.AddCommand(gateRunCmd) -} - -func runGateRun(cmd *cobra.Command, args []string) error { - return gateRunRun(cmd.Context(), gateRunOptions{ - name: args[0], - writer: cmd.OutOrStdout(), - }) -} - -func gateRunRun(ctx context.Context, opts gateRunOptions) error { - if opts.name == "" { - return errors.New("gate run: name required") - } - if opts.writer == nil { - opts.writer = os.Stdout - } - fn := opts.runFn - if fn == nil { - fn = gateRunViaPort - } - verdict, err := fn(ctx, opts) - if err != nil { - return fmt.Errorf("gate run: %w", err) - } - enc := json.NewEncoder(opts.writer) - if err := enc.Encode(verdict); err != nil { - return fmt.Errorf("gate run encode: %w", err) - } - return nil -} - -// gateRunViaPort wires productionGateRunner (cycle 115) rooted at -// the project root. -func gateRunViaPort(ctx context.Context, opts gateRunOptions) (ports.GateVerdict, error) { - cwd, err := resolveProjectDir() - if err != nil { - return ports.GateVerdict{}, err - } - g := newProductionGateRunner(cwd) - return g.Run(ctx, ports.GateRunRequest{Name: ports.GateName(opts.name)}) -} diff --git a/cli/cmd/ao/gate_cmd_test.go b/cli/cmd/ao/gate_cmd_test.go deleted file mode 100644 index 2ea34276a..000000000 --- a/cli/cmd/ao/gate_cmd_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// practices: [tdd] -package main - -import ( - "bytes" - "context" - "errors" - "strings" - "testing" - - "github.com/boshu2/agentops/cli/internal/ports" -) - -func TestGateRun_StubReturnsVerdict(t *testing.T) { - stub := func(_ context.Context, _ gateRunOptions) (ports.GateVerdict, error) { - return ports.GateVerdict{ - Status: ports.GateStatusPass, - Reason: "exit 0", - LogTail: "all checks passed", - }, nil - } - var buf bytes.Buffer - err := gateRunRun(context.Background(), gateRunOptions{ - name: "compile-health", - writer: &buf, - runFn: stub, - }) - if err != nil { - t.Fatal(err) - } - out := buf.String() - if !strings.Contains(out, `"Status":"PASS"`) { - t.Fatalf("missing Status:PASS in output: %s", out) - } - if !strings.Contains(out, `"Reason":"exit 0"`) { - t.Fatalf("missing Reason: %s", out) - } -} - -func TestGateRun_EmptyNameRejected(t *testing.T) { - err := gateRunRun(context.Background(), gateRunOptions{ - name: "", - }) - if err == nil { - t.Fatal("expected error on empty name, got nil") - } - if !strings.Contains(err.Error(), "name required") { - t.Fatalf("error not informative: %v", err) - } -} - -func TestGateRun_FailureWrapped(t *testing.T) { - stub := func(_ context.Context, _ gateRunOptions) (ports.GateVerdict, error) { - return ports.GateVerdict{}, errors.New("subprocess died") - } - err := gateRunRun(context.Background(), gateRunOptions{ - name: "x", - runFn: stub, - }) - if err == nil { - t.Fatal("expected error") - } - if !strings.Contains(err.Error(), "gate run:") { - t.Fatalf("error not wrapped: %v", err) - } -} - -func TestGateRun_UnknownStatusFromMissingGate(t *testing.T) { - // The port contract says missing script → UNKNOWN; verify the - // CLI emits it cleanly. - stub := func(_ context.Context, _ gateRunOptions) (ports.GateVerdict, error) { - return ports.GateVerdict{ - Status: ports.GateStatusUnknown, - Reason: `no script for gate "does-not-exist"`, - }, nil - } - var buf bytes.Buffer - _ = gateRunRun(context.Background(), gateRunOptions{ - name: "does-not-exist", - writer: &buf, - runFn: stub, - }) - if !strings.Contains(buf.String(), `"Status":"UNKNOWN"`) { - t.Fatalf("missing UNKNOWN: %s", buf.String()) - } -} - -func TestGateRun_LogTailIncluded(t *testing.T) { - stub := func(_ context.Context, _ gateRunOptions) (ports.GateVerdict, error) { - return ports.GateVerdict{ - Status: ports.GateStatusFail, - Reason: "exit 1", - LogTail: "ERROR: something broke\nat line 42", - }, nil - } - var buf bytes.Buffer - _ = gateRunRun(context.Background(), gateRunOptions{ - name: "broken", - writer: &buf, - runFn: stub, - }) - if !strings.Contains(buf.String(), "ERROR: something broke") { - t.Fatalf("LogTail not surfaced: %s", buf.String()) - } -} diff --git a/cli/cmd/ao/gate_composition.go b/cli/cmd/ao/gate_composition.go new file mode 100644 index 000000000..3c59809e6 --- /dev/null +++ b/cli/cmd/ao/gate_composition.go @@ -0,0 +1,112 @@ +// practices: [hexagonal-architecture, ddd-bounded-context] +package main + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + gateadapter "github.com/boshu2/agentops/cli/internal/adapters/gate" + gatecommands "github.com/boshu2/agentops/cli/internal/commands/gate" + gateapp "github.com/boshu2/agentops/cli/internal/gate" + "github.com/boshu2/agentops/cli/internal/gates" + _ "github.com/boshu2/agentops/cli/internal/gates/checks" + "github.com/boshu2/agentops/cli/internal/ports" +) + +func init() { + rootCmd.AddCommand(newGateCommand()) +} + +func newGateCommand() *cobra.Command { + module := gatecommands.NewModule(gatecommands.UseCases{ + Review: gateReviewUseCases{}, + Run: gateRunUseCases{}, + Check: gateCheckUseCases{}, + }, gatecommands.HostOptions{ + DryRun: GetDryRun, + OutputFormat: GetOutput, + }) + return module.Command() +} + +type gateReviewUseCases struct{} + +func (gateReviewUseCases) Pending(ctx context.Context, request gateapp.PendingRequest) (gateapp.PendingResult, error) { + service, err := gateReviewService() + if err != nil { + return gateapp.PendingResult{}, err + } + return service.Pending(ctx, request) +} + +func (gateReviewUseCases) Approve(ctx context.Context, input gateapp.ApproveInput) (gateapp.ApproveResult, error) { + service, err := gateReviewService() + if err != nil { + return gateapp.ApproveResult{}, err + } + return service.Approve(ctx, input) +} + +func (gateReviewUseCases) Reject(ctx context.Context, input gateapp.RejectInput) (gateapp.RejectResult, error) { + service, err := gateReviewService() + if err != nil { + return gateapp.RejectResult{}, err + } + return service.Reject(ctx, input) +} + +func (gateReviewUseCases) BulkApprove(ctx context.Context, input gateapp.BulkApproveInput) (gateapp.BulkApproveResult, error) { + service, err := gateReviewService() + if err != nil { + return gateapp.BulkApproveResult{}, err + } + return service.BulkApprove(ctx, input) +} + +type gateRunUseCases struct{} + +func (gateRunUseCases) Execute(ctx context.Context, request gateapp.RunRequest) (ports.GateVerdict, error) { + root, err := resolveProjectDir() + if err != nil { + return ports.GateVerdict{}, fmt.Errorf("gate run: resolve project: %w", err) + } + return (gateapp.RunService{Runner: gateadapter.NewRunner(root)}).Execute(ctx, request) +} + +type gateCheckUseCases struct{} + +func (gateCheckUseCases) Execute(ctx context.Context, request gateapp.CheckRequest) (gateapp.CheckResult, error) { + service, err := gateCheckService() + if err != nil { + return gateapp.CheckResult{}, err + } + return service.Execute(ctx, request) +} + +func gateReviewService() (gateapp.ReviewService, error) { + root, err := resolveProjectDir() + if err != nil { + return gateapp.ReviewService{}, fmt.Errorf("gate: resolve project: %w", err) + } + return gateapp.ReviewService{Port: gateadapter.NewReviewPool(root, GetCurrentUser)}, nil +} + +func gateCheckService() (gateapp.CheckService, error) { + start, err := resolveProjectDir() + if err != nil { + return gateapp.CheckService{}, fmt.Errorf("gate check: resolve project: %w", err) + } + root, err := gateadapter.ResolveRepoRoot(start) + if err != nil { + return gateapp.CheckService{}, err + } + return gateapp.CheckService{ + Registry: gates.Default, + Runner: gates.NewScriptRunner(root), + Files: gates.NewGitChangedFiles(root), + RepoRoot: root, + Runtime: gateadapter.CheckRuntime{}, + }, nil +} diff --git a/cli/cmd/ao/gate_composition_test.go b/cli/cmd/ao/gate_composition_test.go new file mode 100644 index 000000000..5b6a43302 --- /dev/null +++ b/cli/cmd/ao/gate_composition_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "strings" + "testing" +) + +func TestGateCompositionOwnsCompleteCommandTree(t *testing.T) { + command := newGateCommand() + for _, path := range []string{"ao gate approve", "ao gate bulk-approve", "ao gate check", "ao gate pending", "ao gate reject", "ao gate run"} { + args := strings.Fields(strings.TrimPrefix(path, "ao gate ")) + child, remaining, err := command.Find(args) + if err != nil || child == command || len(remaining) != 0 { + t.Fatalf("missing gate child %q: child=%v remaining=%v err=%v", path, child, remaining, err) + } + } +} + +func TestGateCompositionRegistersExactlyOneRootOwner(t *testing.T) { + owners := 0 + for _, command := range rootCmd.Commands() { + if command.Name() == "gate" { + owners++ + } + } + if owners != 1 { + t.Fatalf("gate root owners = %d, want 1", owners) + } +} diff --git a/cli/cmd/ao/gate_runner_adapter.go b/cli/cmd/ao/gate_runner_adapter.go deleted file mode 100644 index 5c354ce1a..000000000 --- a/cli/cmd/ao/gate_runner_adapter.go +++ /dev/null @@ -1,116 +0,0 @@ -// practices: [hexagonal-architecture, ddd-bounded-context] -package main - -import ( - "bytes" - "context" - "fmt" - "os/exec" - "path/filepath" - "strings" - - "github.com/boshu2/agentops/cli/internal/ports" - "github.com/boshu2/agentops/cli/internal/quality" -) - -// productionGateRunner satisfies ports.GateRunnerPort by invoking -// scripts/check-.sh subprocesses and translating exit code + -// stdout into a GateVerdict. -// -// Mapping rules (documented per port contract): -// - exit 0 → PASS (Reason: "exit 0") -// - exit 2 → WARN (some scripts use 2 for advisory mode) -// - exit 75 → SKIP (sysexits EX_TEMPFAIL — convention used by -// gates that decide structural-availability skip) -// - any other non-zero → FAIL -// - script does not exist → UNKNOWN (the adapter chooses the -// OPTIMISTIC interpretation: typo OR newly-introduced gate) -// - empty req.Name → UNKNOWN (port contract requirement) -// -// LogTail is the last 4096 bytes of combined stdout+stderr (port -// contract cap). -type productionGateRunner struct { - repoRoot string -} - -func newProductionGateRunner(repoRoot string) *productionGateRunner { - return &productionGateRunner{repoRoot: repoRoot} -} - -// Run invokes scripts/check-.sh and returns a GateVerdict. -func (g *productionGateRunner) Run(ctx context.Context, req ports.GateRunRequest) (ports.GateVerdict, error) { - if err := ctx.Err(); err != nil { - return ports.GateVerdict{}, err - } - if req.Name == "" { - return ports.GateVerdict{ - Status: ports.GateStatusUnknown, - Reason: "empty GateName", - }, nil - } - if g.repoRoot == "" { - return ports.GateVerdict{}, fmt.Errorf("productionGateRunner: repoRoot required") - } - scriptPath := filepath.Join(g.repoRoot, "scripts", "check-"+string(req.Name)+".sh") - if !quality.FileExists(scriptPath) { - return ports.GateVerdict{ - Status: ports.GateStatusUnknown, - Reason: fmt.Sprintf("no script for gate %q at %s", req.Name, scriptPath), - }, nil - } - - cmd := exec.CommandContext(ctx, "bash", scriptPath) - cmd.Dir = g.repoRoot - if len(req.Env) > 0 { - envList := make([]string, 0, len(req.Env)) - for k, v := range req.Env { - envList = append(envList, k+"="+v) - } - cmd.Env = append(cmd.Environ(), envList...) - } - var out bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &out - runErr := cmd.Run() - - exitCode := 0 - if exitErr, ok := runErr.(*exec.ExitError); ok { - exitCode = exitErr.ExitCode() - } else if runErr != nil { - // Non-exit error (e.g. binary missing, context cancelled mid-run) - return ports.GateVerdict{ - Status: ports.GateStatusUnknown, - Reason: fmt.Sprintf("subprocess error: %v", runErr), - LogTail: tailBytes(out.Bytes(), 4096), - }, nil - } - - verdict := exitCodeToVerdict(exitCode) - verdict.LogTail = tailBytes(out.Bytes(), 4096) - return verdict, nil -} - -// exitCodeToVerdict maps subprocess exit codes to GateVerdict per the -// adapter's documented rules. -func exitCodeToVerdict(code int) ports.GateVerdict { - switch code { - case 0: - return ports.GateVerdict{Status: ports.GateStatusPass, Reason: "exit 0"} - case 2: - return ports.GateVerdict{Status: ports.GateStatusWarn, Reason: "exit 2 (advisory)"} - case 75: - return ports.GateVerdict{Status: ports.GateStatusSkip, Reason: "exit 75 (structural skip)"} - } - return ports.GateVerdict{Status: ports.GateStatusFail, Reason: fmt.Sprintf("exit %d", code)} -} - -// tailBytes returns at most n trailing bytes of b. -func tailBytes(b []byte, n int) string { - if len(b) <= n { - return string(b) - } - return strings.TrimSpace(string(b[len(b)-n:])) -} - -// Compile-time assertion: productionGateRunner satisfies the port. -var _ ports.GateRunnerPort = (*productionGateRunner)(nil) diff --git a/cli/cmd/ao/gate_runner_adapter_test.go b/cli/cmd/ao/gate_runner_adapter_test.go deleted file mode 100644 index 7a9decc9d..000000000 --- a/cli/cmd/ao/gate_runner_adapter_test.go +++ /dev/null @@ -1,140 +0,0 @@ -// practices: [hexagonal-architecture, tdd] -package main - -import ( - "context" - "errors" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/boshu2/agentops/cli/internal/ports" -) - -// Sibling pattern: cycle 114 finding_compiler_adapter_test.go. -// First subprocess-based adapter test — uses temp repo root with a -// fake scripts/check-.sh that exits with a specific code. - -func newTempRepoWithGate(t *testing.T, name string, script string) string { - t.Helper() - root := t.TempDir() - scriptDir := filepath.Join(root, "scripts") - if err := os.MkdirAll(scriptDir, 0o755); err != nil { - t.Fatal(err) - } - scriptPath := filepath.Join(scriptDir, "check-"+name+".sh") - if err := os.WriteFile(scriptPath, []byte("#!/usr/bin/env bash\n"+script+"\n"), 0o755); err != nil { - t.Fatal(err) - } - return root -} - -func TestProductionGateRunner_Exit0IsPass(t *testing.T) { - root := newTempRepoWithGate(t, "ok", `echo all-good; exit 0`) - g := newProductionGateRunner(root) - v, err := g.Run(context.Background(), ports.GateRunRequest{Name: "ok"}) - if err != nil { - t.Fatal(err) - } - if v.Status != ports.GateStatusPass { - t.Fatalf("Status = %s, want PASS", v.Status) - } - if v.Reason == "" { - t.Fatal("Reason must be non-empty") - } -} - -func TestProductionGateRunner_Exit1IsFail(t *testing.T) { - root := newTempRepoWithGate(t, "broken", `echo bad; exit 1`) - g := newProductionGateRunner(root) - v, _ := g.Run(context.Background(), ports.GateRunRequest{Name: "broken"}) - if v.Status != ports.GateStatusFail { - t.Fatalf("Status = %s, want FAIL", v.Status) - } -} - -func TestProductionGateRunner_Exit2IsWarn(t *testing.T) { - root := newTempRepoWithGate(t, "advisory", `echo warn; exit 2`) - g := newProductionGateRunner(root) - v, _ := g.Run(context.Background(), ports.GateRunRequest{Name: "advisory"}) - if v.Status != ports.GateStatusWarn { - t.Fatalf("Status = %s, want WARN", v.Status) - } -} - -func TestProductionGateRunner_Exit75IsSkip(t *testing.T) { - root := newTempRepoWithGate(t, "structural", `echo skip; exit 75`) - g := newProductionGateRunner(root) - v, _ := g.Run(context.Background(), ports.GateRunRequest{Name: "structural"}) - if v.Status != ports.GateStatusSkip { - t.Fatalf("Status = %s, want SKIP", v.Status) - } -} - -func TestProductionGateRunner_MissingScriptIsUnknown(t *testing.T) { - root := t.TempDir() - g := newProductionGateRunner(root) - v, _ := g.Run(context.Background(), ports.GateRunRequest{Name: "does-not-exist"}) - if v.Status != ports.GateStatusUnknown { - t.Fatalf("Status = %s, want UNKNOWN", v.Status) - } -} - -func TestProductionGateRunner_EmptyNameIsUnknown(t *testing.T) { - g := newProductionGateRunner(t.TempDir()) - v, _ := g.Run(context.Background(), ports.GateRunRequest{Name: ""}) - if v.Status != ports.GateStatusUnknown { - t.Fatalf("Status = %s, want UNKNOWN", v.Status) - } - if v.Reason != "empty GateName" { - t.Fatalf("Reason = %q, want %q (port contract)", v.Reason, "empty GateName") - } -} - -func TestProductionGateRunner_LogTailCaptured(t *testing.T) { - root := newTempRepoWithGate(t, "noisy", `echo first; echo second; exit 0`) - g := newProductionGateRunner(root) - v, _ := g.Run(context.Background(), ports.GateRunRequest{Name: "noisy"}) - if v.LogTail == "" { - t.Fatal("LogTail empty") - } - for _, want := range []string{"first", "second"} { - if !strings.Contains(v.LogTail, want) { - t.Fatalf("LogTail missing %q: %q", want, v.LogTail) - } - } -} - -func TestProductionGateRunner_EnvPassedThrough(t *testing.T) { - root := newTempRepoWithGate(t, "envcheck", `echo "GATE_TEST_VAR=$GATE_TEST_VAR"; exit 0`) - g := newProductionGateRunner(root) - v, _ := g.Run(context.Background(), ports.GateRunRequest{ - Name: "envcheck", - Env: map[string]string{"GATE_TEST_VAR": "honored"}, - }) - if !strings.Contains(v.LogTail, "GATE_TEST_VAR=honored") { - t.Fatalf("env not passed through: %q", v.LogTail) - } -} - -func TestProductionGateRunner_EmptyRepoRootErrors(t *testing.T) { - g := newProductionGateRunner("") - _, err := g.Run(context.Background(), ports.GateRunRequest{Name: "x"}) - if err == nil { - t.Fatal("expected error on empty repoRoot, got nil") - } -} - -func TestProductionGateRunner_HonorsContextCancellation(t *testing.T) { - g := newProductionGateRunner(t.TempDir()) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - _, err := g.Run(ctx, ports.GateRunRequest{Name: "x"}) - if err == nil { - t.Fatal("expected cancellation error, got nil") - } - if !errors.Is(err, context.Canceled) { - t.Fatalf("error = %v, want context.Canceled", err) - } -} diff --git a/cli/cmd/ao/gate_test.go b/cli/cmd/ao/gate_test.go deleted file mode 100644 index ced552df1..000000000 --- a/cli/cmd/ao/gate_test.go +++ /dev/null @@ -1,728 +0,0 @@ -// practices: [design-by-contract, continuous-delivery] -package main - -import ( - "os" - "path/filepath" - "testing" - "time" - - "strings" - - "github.com/boshu2/agentops/cli/internal/pool" - "github.com/boshu2/agentops/cli/internal/types" -) - -// TestGatePendingReturnsBronzeCandidates verifies that gate pending -// returns only bronze-tier candidates awaiting human review. -func TestGatePendingReturnsBronzeCandidates(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - // Add candidates of different tiers - bronzeCandidate := types.Candidate{ - ID: "bronze-pending-1", - Tier: types.TierBronze, - Content: "Bronze content awaiting review", - Maturity: types.MaturityProvisional, - } - silverCandidate := types.Candidate{ - ID: "silver-1", - Tier: types.TierSilver, - Content: "Silver content", - } - goldCandidate := types.Candidate{ - ID: "gold-1", - Tier: types.TierGold, - Content: "Gold content", - } - - // Bronze requires human review - bronzeScoring := types.Scoring{GateRequired: true} - silverScoring := types.Scoring{GateRequired: false} - goldScoring := types.Scoring{GateRequired: false} - - if err := p.Add(bronzeCandidate, bronzeScoring); err != nil { - t.Fatalf("failed to add bronze candidate: %v", err) - } - if err := p.Add(silverCandidate, silverScoring); err != nil { - t.Fatalf("failed to add silver candidate: %v", err) - } - if err := p.Add(goldCandidate, goldScoring); err != nil { - t.Fatalf("failed to add gold candidate: %v", err) - } - - // List pending reviews - pending, err := p.ListPendingReview() - if err != nil { - t.Fatalf("ListPendingReview failed: %v", err) - } - - // Should only return bronze candidates - if len(pending) != 1 { - t.Errorf("expected 1 pending review (bronze only), got %d", len(pending)) - } - - if len(pending) > 0 && pending[0].Candidate.ID != "bronze-pending-1" { - t.Errorf("expected bronze-pending-1, got %s", pending[0].Candidate.ID) - } -} - -// TestGateApproveRecordsReview verifies that approving a candidate -// sets HumanReview fields correctly. -func TestGateApproveRecordsReview(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - candidate := types.Candidate{ - ID: "approve-test-1", - Tier: types.TierBronze, - Content: "Content to approve", - } - - if err := p.Add(candidate, types.Scoring{GateRequired: true}); err != nil { - t.Fatalf("failed to add candidate: %v", err) - } - - // Approve the candidate - reviewer := "test-reviewer" - note := "Good specificity, approved" - if err := p.Approve("approve-test-1", note, reviewer); err != nil { - t.Fatalf("Approve failed: %v", err) - } - - // Verify review was recorded - entry, err := p.Get("approve-test-1") - if err != nil { - t.Fatalf("Get failed: %v", err) - } - - if entry.HumanReview == nil { - t.Fatal("HumanReview not set after approval") - } - - if !entry.HumanReview.Reviewed { - t.Error("HumanReview.Reviewed should be true") - } - - if !entry.HumanReview.Approved { - t.Error("HumanReview.Approved should be true") - } - - if entry.HumanReview.Reviewer != reviewer { - t.Errorf("expected reviewer %q, got %q", reviewer, entry.HumanReview.Reviewer) - } - - if entry.HumanReview.Notes != note { - t.Errorf("expected notes %q, got %q", note, entry.HumanReview.Notes) - } - - if entry.HumanReview.ReviewedAt.IsZero() { - t.Error("ReviewedAt timestamp not set") - } -} - -// TestGateRejectRecordsReason verifies that rejecting a candidate -// sets the rejection reason and prevents future promotion. -func TestGateRejectRecordsReason(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - candidate := types.Candidate{ - ID: "reject-test-1", - Tier: types.TierBronze, - Content: "Content to reject", - } - - if err := p.Add(candidate, types.Scoring{GateRequired: true}); err != nil { - t.Fatalf("failed to add candidate: %v", err) - } - - // Reject the candidate - reason := "Too vague, lacks specificity" - reviewer := "test-reviewer" - if err := p.Reject("reject-test-1", reason, reviewer); err != nil { - t.Fatalf("Reject failed: %v", err) - } - - // Verify rejection was recorded - entry, err := p.Get("reject-test-1") - if err != nil { - t.Fatalf("Get failed: %v", err) - } - - if entry.Status != types.PoolStatusRejected { - t.Errorf("expected status rejected, got %s", entry.Status) - } - - if entry.HumanReview == nil { - t.Fatal("HumanReview not set after rejection") - } - - if !entry.HumanReview.Reviewed { - t.Error("HumanReview.Reviewed should be true") - } - - if entry.HumanReview.Approved { - t.Error("HumanReview.Approved should be false for rejection") - } - - if entry.HumanReview.Notes != reason { - t.Errorf("expected reason %q, got %q", reason, entry.HumanReview.Notes) - } -} - -// TestGateApproveAlreadyReviewedReturnsError verifies that trying to -// approve an already-reviewed candidate returns an appropriate error. -func TestGateApproveAlreadyReviewedReturnsError(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - candidate := types.Candidate{ - ID: "already-reviewed-1", - Tier: types.TierBronze, - Content: "Content already reviewed", - } - - if err := p.Add(candidate, types.Scoring{GateRequired: true}); err != nil { - t.Fatalf("failed to add candidate: %v", err) - } - - // First approval should succeed - if err := p.Approve("already-reviewed-1", "First approval", "first-reviewer"); err != nil { - t.Fatalf("First Approve failed: %v", err) - } - - // Second approval should fail with "already reviewed by X" error - err := p.Approve("already-reviewed-1", "Second approval", "second-reviewer") - if err == nil { - t.Fatal("Expected error for already-reviewed candidate, got nil") - } - - expectedMsg := "already reviewed by first-reviewer" - if err.Error() != expectedMsg { - t.Errorf("expected error %q, got %q", expectedMsg, err.Error()) - } -} - -// TestGateStateMachine verifies the state machine transitions: -// pending -> approved -> [24h] -> promoted (silver) -// pending -> rejected (terminal) -func TestGateStateMachine(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - t.Run("pending to approved to promoted", func(t *testing.T) { - candidate := types.Candidate{ - ID: "state-promote-1", - Tier: types.TierBronze, - Type: types.KnowledgeTypeLearning, - Content: "Learning content for promotion", - Maturity: types.MaturityCandidate, - } - - if err := p.Add(candidate, types.Scoring{GateRequired: true}); err != nil { - t.Fatalf("failed to add candidate: %v", err) - } - - // Verify initial state is pending - entry, _ := p.Get("state-promote-1") - if entry.Status != types.PoolStatusPending { - t.Errorf("expected initial status pending, got %s", entry.Status) - } - - // Approve - if err := p.Approve("state-promote-1", "Approved for promotion", "reviewer"); err != nil { - t.Fatalf("Approve failed: %v", err) - } - - // Stage for promotion - if err := p.Stage("state-promote-1", types.TierBronze); err != nil { - t.Fatalf("Stage failed: %v", err) - } - - // Verify staged - entry, _ = p.Get("state-promote-1") - if entry.Status != types.PoolStatusStaged { - t.Errorf("expected status staged, got %s", entry.Status) - } - - // Promote - artifactPath, err := p.Promote("state-promote-1") - if err != nil { - t.Fatalf("Promote failed: %v", err) - } - - if artifactPath == "" { - t.Error("expected artifact path, got empty") - } - - // Verify artifact was created - if _, err := os.Stat(artifactPath); os.IsNotExist(err) { - t.Errorf("artifact not created: %s", artifactPath) - } - }) - - t.Run("pending to rejected is terminal", func(t *testing.T) { - candidate := types.Candidate{ - ID: "state-reject-1", - Tier: types.TierBronze, - Content: "Content to be rejected", - } - - if err := p.Add(candidate, types.Scoring{GateRequired: true}); err != nil { - t.Fatalf("failed to add candidate: %v", err) - } - - // Reject - if err := p.Reject("state-reject-1", "Terminal rejection", "reviewer"); err != nil { - t.Fatalf("Reject failed: %v", err) - } - - // Verify rejected status - entry, _ := p.Get("state-reject-1") - if entry.Status != types.PoolStatusRejected { - t.Errorf("expected status rejected, got %s", entry.Status) - } - - // Verify rejected candidates cannot be staged - err := p.Stage("state-reject-1", types.TierBronze) - if err == nil { - t.Error("Expected error when staging rejected candidate") - } - }) -} - -// TestGatePendingOldestFirst verifies that pending reviews are sorted -// by age with oldest items first. -func TestGatePendingOldestFirst(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - // Add candidates - note: we can't easily control AddedAt time, - // but we verify the sorting logic by checking returned order - candidates := []types.Candidate{ - {ID: "bronze-a", Tier: types.TierBronze, Content: "First"}, - {ID: "bronze-b", Tier: types.TierBronze, Content: "Second"}, - {ID: "bronze-c", Tier: types.TierBronze, Content: "Third"}, - } - - for _, c := range candidates { - if err := p.Add(c, types.Scoring{GateRequired: true}); err != nil { - t.Fatalf("failed to add candidate %s: %v", c.ID, err) - } - // Small delay to ensure different timestamps - time.Sleep(10 * time.Millisecond) - } - - pending, err := p.ListPendingReview() - if err != nil { - t.Fatalf("ListPendingReview failed: %v", err) - } - - if len(pending) != 3 { - t.Fatalf("expected 3 pending, got %d", len(pending)) - } - - // Verify oldest first ordering - first added should be first in list - if pending[0].Candidate.ID != "bronze-a" { - t.Errorf("expected bronze-a first (oldest), got %s", pending[0].Candidate.ID) - } -} - -// TestGateChainEventRecording verifies that gate operations -// are recorded in the chain file for audit purposes. -func TestGateChainEventRecording(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - candidate := types.Candidate{ - ID: "chain-test-1", - Tier: types.TierBronze, - Content: "Content for chain test", - } - - if err := p.Add(candidate, types.Scoring{GateRequired: true}); err != nil { - t.Fatalf("failed to add candidate: %v", err) - } - - // Approve - if err := p.Approve("chain-test-1", "Chain test approval", "chain-reviewer"); err != nil { - t.Fatalf("Approve failed: %v", err) - } - - // Check chain file exists and contains events - chainPath := filepath.Join(tmpDir, ".agents/pool/chain.jsonl") - if _, err := os.Stat(chainPath); os.IsNotExist(err) { - t.Fatal("chain.jsonl not created") - } - - // Read chain events - events, err := p.GetChain() - if err != nil { - t.Fatalf("GetChain failed: %v", err) - } - - if len(events) < 2 { - t.Errorf("expected at least 2 events (add + approve), got %d", len(events)) - } - - // Find approve event - foundApprove := false - for _, e := range events { - if e.Operation == "approve" && e.CandidateID == "chain-test-1" { - foundApprove = true - if e.Reviewer != "chain-reviewer" { - t.Errorf("expected reviewer chain-reviewer, got %s", e.Reviewer) - } - } - } - - if !foundApprove { - t.Error("approve event not found in chain") - } -} - -// TestGateApproveNotExistReturnsError verifies error handling for -// non-existent candidates. -func TestGateApproveNotExistReturnsError(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - // Initialize pool directories - if err := p.Init(); err != nil { - t.Fatalf("Init failed: %v", err) - } - - err := p.Approve("nonexistent-id", "note", "reviewer") - if err == nil { - t.Fatal("Expected error for non-existent candidate, got nil") - } -} - -// TestGateRejectNotExistReturnsError verifies error handling for -// non-existent candidates. -func TestGateRejectNotExistReturnsError(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - // Initialize pool directories - if err := p.Init(); err != nil { - t.Fatalf("Init failed: %v", err) - } - - err := p.Reject("nonexistent-id", "reason", "reviewer") - if err == nil { - t.Fatal("Expected error for non-existent candidate, got nil") - } -} - -// TestGateRejectReasonTooLong verifies that rejection reason is -// limited to MaxReasonLength (1000 chars). -func TestGateRejectReasonTooLong(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - candidate := types.Candidate{ - ID: "reject-long-reason", - Tier: types.TierBronze, - Content: "Content to reject with long reason", - } - - if err := p.Add(candidate, types.Scoring{GateRequired: true}); err != nil { - t.Fatalf("failed to add candidate: %v", err) - } - - // Create a reason that exceeds 1000 chars - longReason := make([]byte, 1001) - for i := range longReason { - longReason[i] = 'x' - } - - err := p.Reject("reject-long-reason", string(longReason), "reviewer") - if err == nil { - t.Fatal("Expected error for reason exceeding 1000 chars, got nil") - } - - expectedMsg := "reason/note exceeds maximum length of 1000 characters" - if err.Error() != expectedMsg { - t.Errorf("expected error %q, got %q", expectedMsg, err.Error()) - } -} - -// TestGateApproveNoteTooLong verifies that approval note is -// limited to MaxReasonLength (1000 chars). -func TestGateApproveNoteTooLong(t *testing.T) { - tmpDir := t.TempDir() - p := pool.NewPool(tmpDir) - - candidate := types.Candidate{ - ID: "approve-long-note", - Tier: types.TierBronze, - Content: "Content to approve with long note", - } - - if err := p.Add(candidate, types.Scoring{GateRequired: true}); err != nil { - t.Fatalf("failed to add candidate: %v", err) - } - - // Create a note that exceeds 1000 chars - longNote := make([]byte, 1001) - for i := range longNote { - longNote[i] = 'x' - } - - err := p.Approve("approve-long-note", string(longNote), "reviewer") - if err == nil { - t.Fatal("Expected error for note exceeding 1000 chars, got nil") - } - - expectedMsg := "reason/note exceeds maximum length of 1000 characters" - if err.Error() != expectedMsg { - t.Errorf("expected error %q, got %q", expectedMsg, err.Error()) - } -} - -// --------------------------------------------------------------------------- -// gate.go — entryUrgency -// --------------------------------------------------------------------------- - -func TestGate_entryUrgency(t *testing.T) { - tests := []struct { - name string - entry pool.PoolEntry - want string - }{ - { - name: "approaching auto promote is HIGH", - entry: pool.PoolEntry{ - ApproachingAutoPromote: true, - Age: 23 * time.Hour, - }, - want: "HIGH (approaching 24h)", - }, - { - name: "over 12h is MEDIUM", - entry: pool.PoolEntry{ - ApproachingAutoPromote: false, - Age: 13 * time.Hour, - }, - want: "MEDIUM", - }, - { - name: "under 12h is LOW", - entry: pool.PoolEntry{ - ApproachingAutoPromote: false, - Age: 6 * time.Hour, - }, - want: "LOW", - }, - { - name: "zero age is LOW", - entry: pool.PoolEntry{ - ApproachingAutoPromote: false, - Age: 0, - }, - want: "LOW", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := entryUrgency(tc.entry) - if got != tc.want { - t.Errorf("entryUrgency() = %q, want %q", got, tc.want) - } - }) - } -} - -// --------------------------------------------------------------------------- -// gate.go — outputGatePending (table mode) -// --------------------------------------------------------------------------- - -func TestGate_outputGatePending_table(t *testing.T) { - oldOutput := output - output = "table" - defer func() { output = oldOutput }() - - t.Run("empty list", func(t *testing.T) { - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - - err := outputGatePending(nil) - - _ = w.Close() - os.Stdout = oldStdout - - if err != nil { - t.Fatalf("outputGatePending: %v", err) - } - - buf := make([]byte, 4096) - n, _ := r.Read(buf) - _ = r.Close() - out := string(buf[:n]) - - if !strings.Contains(out, "No pending reviews") { - t.Errorf("expected 'No pending reviews', got: %s", out) - } - }) - - t.Run("with entries", func(t *testing.T) { - entries := []pool.PoolEntry{ - { - PoolEntry: types.PoolEntry{ - Candidate: types.Candidate{ - ID: "gate-entry-1", - Tier: types.TierBronze, - Utility: 0.55, - }, - }, - AgeString: "18h", - Age: 18 * time.Hour, - ApproachingAutoPromote: false, - }, - { - PoolEntry: types.PoolEntry{ - Candidate: types.Candidate{ - ID: "gate-entry-2", - Tier: types.TierBronze, - Utility: 0.60, - }, - }, - AgeString: "23h", - Age: 23 * time.Hour, - ApproachingAutoPromote: true, - }, - } - - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - - err := outputGatePending(entries) - - _ = w.Close() - os.Stdout = oldStdout - - if err != nil { - t.Fatalf("outputGatePending: %v", err) - } - - buf := make([]byte, 8192) - n, _ := r.Read(buf) - _ = r.Close() - out := string(buf[:n]) - - if !strings.Contains(out, "Pending Reviews (2)") { - t.Errorf("expected 'Pending Reviews (2)', got: %s", out) - } - if !strings.Contains(out, "approaching 24h") { - t.Errorf("expected auto-promote warning, got: %s", out) - } - }) -} - -// --------------------------------------------------------------------------- -// gate.go — outputGatePending (JSON mode) -// --------------------------------------------------------------------------- - -func TestGate_outputGatePending_json(t *testing.T) { - oldOutput := output - output = "json" - defer func() { output = oldOutput }() - - entries := []pool.PoolEntry{ - { - PoolEntry: types.PoolEntry{ - Candidate: types.Candidate{ - ID: "gate-json-1", - Tier: types.TierBronze, - }, - Status: types.PoolStatusPending, - }, - AgeString: "5h", - }, - } - - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - - err := outputGatePending(entries) - - _ = w.Close() - os.Stdout = oldStdout - - if err != nil { - t.Fatalf("outputGatePending: %v", err) - } - - buf := make([]byte, 8192) - n, _ := r.Read(buf) - _ = r.Close() - out := string(buf[:n]) - - if !strings.Contains(out, "gate-json-1") { - t.Errorf("expected JSON with candidate ID, got: %s", out) - } -} - -// --------------------------------------------------------------------------- -// gate.go — printAutoPromoteWarning -// --------------------------------------------------------------------------- - -func TestGate_printAutoPromoteWarning(t *testing.T) { - t.Run("no approaching entries", func(t *testing.T) { - entries := []pool.PoolEntry{ - {ApproachingAutoPromote: false}, - {ApproachingAutoPromote: false}, - } - - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - - printAutoPromoteWarning(entries) - - _ = w.Close() - os.Stdout = oldStdout - - buf := make([]byte, 4096) - n, _ := r.Read(buf) - _ = r.Close() - out := string(buf[:n]) - - if strings.Contains(out, "approaching 24h") { - t.Errorf("should not show warning when no entries approaching, got: %s", out) - } - }) - - t.Run("some approaching entries", func(t *testing.T) { - entries := []pool.PoolEntry{ - {ApproachingAutoPromote: false}, - {ApproachingAutoPromote: true}, - {ApproachingAutoPromote: true}, - } - - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - - printAutoPromoteWarning(entries) - - _ = w.Close() - os.Stdout = oldStdout - - buf := make([]byte, 4096) - n, _ := r.Read(buf) - _ = r.Close() - out := string(buf[:n]) - - if !strings.Contains(out, "2 candidate(s) approaching 24h") { - t.Errorf("expected warning about 2 approaching candidates, got: %s", out) - } - }) -} diff --git a/cli/cmd/ao/refinery.go b/cli/cmd/ao/refinery.go index 5665c61bd..000fcad76 100644 --- a/cli/cmd/ao/refinery.go +++ b/cli/cmd/ao/refinery.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" + gateadapter "github.com/boshu2/agentops/cli/internal/adapters/gate" // Register the seed checks the refinery's full gate runs. _ "github.com/boshu2/agentops/cli/internal/gates/checks" "github.com/boshu2/agentops/cli/internal/refinery" @@ -54,7 +55,7 @@ func init() { } func runRefineryOnce(cmd *cobra.Command, _ []string) error { - root, err := gateRepoRoot() + root, err := refineryRepoRoot() if err != nil { return fmt.Errorf("resolve repo root: %w", err) } @@ -77,10 +78,18 @@ func runRefineryOnce(cmd *cobra.Command, _ []string) error { } func runRefineryRun(cmd *cobra.Command, _ []string) error { - root, err := gateRepoRoot() + root, err := refineryRepoRoot() if err != nil { return fmt.Errorf("resolve repo root: %w", err) } fmt.Fprintf(cmd.OutOrStdout(), "refinery: looping every %s (Ctrl-C to stop)\n", refineryInterval) return refinery.NewProduction(root).Loop(cmd.Context(), refineryInterval) -} \ No newline at end of file +} + +func refineryRepoRoot() (string, error) { + start, err := resolveProjectDir() + if err != nil { + return "", err + } + return gateadapter.ResolveRepoRoot(start) +} diff --git a/cli/docs/COMMANDS.md b/cli/docs/COMMANDS.md index 977cb951a..5755c688b 100644 --- a/cli/docs/COMMANDS.md +++ b/cli/docs/COMMANDS.md @@ -254,7 +254,7 @@ ao gate [command] #### `ao gate approve` -Approve a bronze-tier candidate for promotion. +Approve candidate for promotion ``` ao gate approve [flags] @@ -269,7 +269,7 @@ ao gate approve [flags] #### `ao gate bulk-approve` -Approve all silver-tier candidates older than a threshold. +Bulk approve silver candidates ``` ao gate bulk-approve [flags] @@ -285,7 +285,7 @@ ao gate bulk-approve [flags] #### `ao gate check` -Run the declarative gate registry. +Run the gate registry (fast cockpit subset or full suite) ``` ao gate check [flags] @@ -308,7 +308,7 @@ ao gate check [flags] #### `ao gate pending` -List bronze-tier candidates awaiting human review. +List candidates pending review ``` ao gate pending [flags] @@ -316,7 +316,7 @@ ao gate pending [flags] #### `ao gate reject` -Reject a candidate with a required reason. +Reject candidate ``` ao gate reject [flags] @@ -331,7 +331,7 @@ ao gate reject [flags] #### `ao gate run` -Invoke a check-*.sh gate via the typed BC2 GateRunnerPort +Run a check-*.sh gate via BC2 GateRunnerPort and emit verdict ``` ao gate run [flags] diff --git a/cli/internal/adapters/gate/review_pool.go b/cli/internal/adapters/gate/review_pool.go index 5a4b368cc..282abcaa7 100644 --- a/cli/internal/adapters/gate/review_pool.go +++ b/cli/internal/adapters/gate/review_pool.go @@ -24,14 +24,16 @@ func (adapter *ReviewPool) ListPending(ctx context.Context) ([]gateapp.ReviewEnt if err != nil { return nil, err } - result := make([]gateapp.ReviewEntry, 0, len(entries)) + var result []gateapp.ReviewEntry + if entries != nil { + result = make([]gateapp.ReviewEntry, 0, len(entries)) + } for _, entry := range entries { result = append(result, gateapp.ReviewEntry{ - ID: entry.Candidate.ID, - Tier: string(entry.Candidate.Tier), + PoolEntry: entry.PoolEntry, + FilePath: entry.FilePath, Age: entry.Age, AgeString: entry.AgeString, - Utility: entry.Candidate.Utility, ApproachingAutoPromote: entry.ApproachingAutoPromote, }) } diff --git a/cli/internal/adapters/gate/review_pool_test.go b/cli/internal/adapters/gate/review_pool_test.go index 7ee361b3b..bfab58cab 100644 --- a/cli/internal/adapters/gate/review_pool_test.go +++ b/cli/internal/adapters/gate/review_pool_test.go @@ -22,11 +22,21 @@ func TestReviewPoolListsAndMapsPendingEntries(t *testing.T) { if err != nil { t.Fatalf("ListPending: %v", err) } - if len(entries) != 1 || entries[0].ID != "cand-1" || entries[0].Tier != string(types.TierBronze) || entries[0].Utility != 0.61 { + if len(entries) != 1 || entries[0].Candidate.ID != "cand-1" || entries[0].Candidate.Tier != types.TierBronze || entries[0].Candidate.Utility != 0.61 { t.Fatalf("entries = %+v", entries) } } +func TestReviewPoolPreservesNilPendingSlice(t *testing.T) { + entries, err := NewReviewPool(t.TempDir(), nil).ListPending(context.Background()) + if err != nil { + t.Fatalf("ListPending: %v", err) + } + if entries != nil { + t.Fatalf("entries = %#v, want nil to preserve legacy JSON null", entries) + } +} + func TestReviewPoolDelegatesApproveWithReviewer(t *testing.T) { root := t.TempDir() p := pool.NewPool(root) diff --git a/cli/internal/commands/gate/module.go b/cli/internal/commands/gate/module.go new file mode 100644 index 000000000..74dc233b7 --- /dev/null +++ b/cli/internal/commands/gate/module.go @@ -0,0 +1,429 @@ +package gate + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "text/tabwriter" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + + "github.com/boshu2/agentops/cli/internal/clicontract" + gateapp "github.com/boshu2/agentops/cli/internal/gate" + "github.com/boshu2/agentops/cli/internal/ports" + "github.com/boshu2/agentops/cli/internal/search" +) + +type ReviewUseCases interface { + Pending(context.Context, gateapp.PendingRequest) (gateapp.PendingResult, error) + Approve(context.Context, gateapp.ApproveInput) (gateapp.ApproveResult, error) + Reject(context.Context, gateapp.RejectInput) (gateapp.RejectResult, error) + BulkApprove(context.Context, gateapp.BulkApproveInput) (gateapp.BulkApproveResult, error) +} + +type RunUseCases interface { + Execute(context.Context, gateapp.RunRequest) (ports.GateVerdict, error) +} + +type CheckUseCases interface { + Execute(context.Context, gateapp.CheckRequest) (gateapp.CheckResult, error) +} + +type UseCases struct { + Review ReviewUseCases + Run RunUseCases + Check CheckUseCases +} + +type HostOptions struct { + DryRun func() bool + OutputFormat func() string +} + +type Module struct { + useCases UseCases + host HostOptions +} + +func NewModule(useCases UseCases, host HostOptions) Module { + return Module{useCases: useCases, host: host} +} + +func (Module) Contract() clicontract.CommandContract { + return clicontract.CommandContract{ + ID: "ao.gate", + Profiles: clicontract.ProfileDefault | clicontract.ProfileFlywheel | + clicontract.ProfileLegacy | clicontract.ProfileCombined, + Args: clicontract.ArgsPolicy{Name: "no-args", Validate: cobra.NoArgs}, + Output: clicontract.OutputNone, + Effects: clicontract.EffectFilesystem | clicontract.EffectProcess | clicontract.EffectEnvironment | clicontract.EffectClock, + ExitClasses: map[int]clicontract.ExitClass{ + 0: clicontract.ExitSuccess, + 1: clicontract.ExitFailure, + 2: clicontract.ExitUsage, + }, + } +} + +type ExitError struct { + Code int + Message string +} + +func (failure *ExitError) Error() string { return failure.Message } +func (failure *ExitError) ExitCode() int { return failure.Code } + +func (module Module) Command() *cobra.Command { + command := &cobra.Command{ + Use: "gate", + Short: "Human review gates", + GroupID: "core", + Long: `Manage human review gates for bronze-tier candidates. + +Bronze-tier candidates (score 0.50-0.69) require human review +before promotion. The gate command provides the review interface. + +Examples: + ao gate pending + ao gate approve + ao gate reject --reason="Too vague"`, + } + command.AddCommand( + module.newPendingCommand(), + module.newApproveCommand(), + module.newRejectCommand(), + module.newBulkApproveCommand(), + module.newRunCommand(), + module.newCheckCommand(), + ) + return command +} + +func (module Module) newPendingCommand() *cobra.Command { + return &cobra.Command{ + Use: "pending", + Short: "List candidates pending review", + Long: `List bronze-tier candidates awaiting human review. + +Shows age/urgency with oldest items first. +Highlights items approaching 24h auto-promote threshold. + +Examples: + ao gate pending + ao gate pending --json`, + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if module.useCases.Review == nil { + return fmt.Errorf("gate pending: use case not configured") + } + result, err := module.useCases.Review.Pending(command.Context(), gateapp.PendingRequest{DryRun: module.dryRun()}) + if err != nil { + return err + } + if result.DryRun { + fmt.Fprintln(command.OutOrStdout(), "[dry-run] Would list pending gate reviews") + return nil + } + return renderPending(command, result.Entries, module.outputFormat()) + }, + } +} + +func renderPending(command *cobra.Command, entries []gateapp.ReviewEntry, format string) error { + switch format { + case "json": + encoder := json.NewEncoder(command.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(entries) + case "yaml": + return yaml.NewEncoder(command.OutOrStdout()).Encode(entries) + default: + return renderPendingTable(command, entries) + } +} + +func renderPendingTable(command *cobra.Command, entries []gateapp.ReviewEntry) error { + output := command.OutOrStdout() + if len(entries) == 0 { + fmt.Fprintln(output, "No pending reviews") + return nil + } + fmt.Fprintf(output, "Pending Reviews (%d)\n", len(entries)) + fmt.Fprintln(output, "==================") + fmt.Fprintln(output) + writer := tabwriter.NewWriter(output, 0, 0, 2, ' ', 0) + fmt.Fprintln(writer, "ID\tTIER\tAGE\tUTILITY\tURGENCY") + fmt.Fprintln(writer, "--\t----\t---\t-------\t-------") + for _, entry := range entries { + fmt.Fprintf(writer, "%s\t%s\t%s\t%.2f\t%s\n", + search.TruncateText(entry.Candidate.ID, 16), + entry.Candidate.Tier, + entry.AgeString, + entry.Candidate.Utility, + entry.Urgency, + ) + } + if err := writer.Flush(); err != nil { + return err + } + fmt.Fprintln(output) + approaching := 0 + for _, entry := range entries { + if entry.ApproachingAutoPromote { + approaching++ + } + } + if approaching > 0 { + fmt.Fprintf(output, "! %d candidate(s) approaching 24h auto-promote threshold\n", approaching) + } + return nil +} + +func (module Module) newApproveCommand() *cobra.Command { + var note string + command := &cobra.Command{ + Use: "approve ", + Short: "Approve candidate for promotion", + Long: `Approve a bronze-tier candidate for promotion. + +Records reviewer identity and triggers promotion flow. + +Examples: + ao gate approve cand-abc123 + ao gate approve cand-abc123 --note="Good specificity, approved"`, + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + if module.useCases.Review == nil { + return fmt.Errorf("gate approve: use case not configured") + } + result, err := module.useCases.Review.Approve(command.Context(), gateapp.ApproveInput{CandidateID: args[0], Note: note, DryRun: module.dryRun()}) + if err != nil { + return err + } + output := command.OutOrStdout() + if result.DryRun { + fmt.Fprintf(output, "[dry-run] Would approve candidate %s", result.CandidateID) + if result.Note != "" { + fmt.Fprintf(output, " with note: %s", result.Note) + } + fmt.Fprintln(output) + return nil + } + fmt.Fprintf(output, "Approved: %s\nReviewer: %s\n", result.CandidateID, result.Reviewer) + if result.Note != "" { + fmt.Fprintf(output, "Note: %s\n", result.Note) + } + fmt.Fprintf(output, "\nTo promote: ao pool promote %s\n", result.CandidateID) + return nil + }, + } + command.Flags().StringVar(¬e, "note", "", "Optional approval note") + return command +} + +func (module Module) newRejectCommand() *cobra.Command { + var reason string + command := &cobra.Command{ + Use: "reject ", + Short: "Reject candidate", + Long: `Reject a candidate with a required reason. + +Records in audit trail for future analysis. + +Examples: + ao gate reject cand-abc123 --reason="Lacks specificity" + ao gate reject cand-abc123 --reason="Duplicate of existing pattern"`, + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + if module.useCases.Review == nil { + return fmt.Errorf("gate reject: use case not configured") + } + result, err := module.useCases.Review.Reject(command.Context(), gateapp.RejectInput{CandidateID: args[0], Reason: reason, DryRun: module.dryRun()}) + if err != nil { + return err + } + output := command.OutOrStdout() + if result.DryRun { + fmt.Fprintf(output, "[dry-run] Would reject candidate %s with reason: %s\n", result.CandidateID, result.Reason) + return nil + } + fmt.Fprintf(output, "Rejected: %s\nReviewer: %s\nReason: %s\n", result.CandidateID, result.Reviewer, result.Reason) + return nil + }, + } + command.Flags().StringVar(&reason, "reason", "", "Required rejection reason") + _ = command.MarkFlagRequired("reason") + return command +} + +func (module Module) newBulkApproveCommand() *cobra.Command { + var olderThan, tier string + command := &cobra.Command{ + Use: "bulk-approve", + Short: "Bulk approve silver candidates", + Long: `Approve all silver-tier candidates older than a threshold. + +Silver candidates auto-promote after 24h if not rejected. +This command accelerates the process for reviewed batches. + +Examples: + ao gate bulk-approve + ao gate bulk-approve --older-than=12h + ao gate bulk-approve --older-than=24h --dry-run`, + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if module.useCases.Review == nil { + return fmt.Errorf("gate bulk-approve: use case not configured") + } + result, err := module.useCases.Review.BulkApprove(command.Context(), gateapp.BulkApproveInput{OlderThan: olderThan, Tier: tier, DryRun: module.dryRun()}) + if err != nil { + return err + } + output := command.OutOrStdout() + if result.DryRun { + if len(result.Approved) == 0 { + fmt.Fprintln(output, "[dry-run] No candidates match criteria") + } else { + fmt.Fprintf(output, "[dry-run] Would approve %d candidate(s):\n", len(result.Approved)) + printIDs(output, result.Approved) + } + return nil + } + if len(result.Approved) == 0 { + fmt.Fprintln(output, "No candidates matched criteria") + } else { + fmt.Fprintf(output, "Approved %d candidate(s):\n", len(result.Approved)) + printIDs(output, result.Approved) + } + return nil + }, + } + command.Flags().StringVar(&olderThan, "older-than", "24h", "Age threshold for bulk approval") + command.Flags().StringVar(&tier, "tier", "silver", "Tier to bulk approve (default: silver)") + _ = command.RegisterFlagCompletionFunc("tier", fixedCompletions("bronze", "silver", "gold")) + return command +} + +func printIDs(output interface{ Write([]byte) (int, error) }, ids []string) { + for _, id := range ids { + fmt.Fprintf(output, " - %s\n", id) + } +} + +func (module Module) newRunCommand() *cobra.Command { + return &cobra.Command{ + Use: "run ", + Short: "Run a check-*.sh gate via BC2 GateRunnerPort and emit verdict", + Long: `Invoke a check-*.sh gate via the typed BC2 GateRunnerPort +and emit a GateVerdict (JSON). + +The argument is the gate name without the 'check-' prefix +or '.sh' suffix. The runner resolves scripts/check-.sh. + +Useful as a typed alternative to 'bash scripts/check-.sh; echo $?' +for scripts that want structured output (status, reason, log tail). + +Examples: + ao gate run compile-health + ao gate run three-gap-supergate + ao gate run xxx-does-not-exist`, + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + if module.useCases.Run == nil { + return fmt.Errorf("gate run: use case not configured") + } + verdict, err := module.useCases.Run.Execute(command.Context(), gateapp.RunRequest{Name: args[0]}) + if err != nil { + return err + } + if err := json.NewEncoder(command.OutOrStdout()).Encode(verdict); err != nil { + return fmt.Errorf("gate run encode: %w", err) + } + return nil + }, + } +} + +func (module Module) newCheckCommand() *cobra.Command { + var fast, full, jsonOutput, githubAnnotations, failFast bool + var scope, workflowPath string + var workflowCoverage, requireWorkflowParity bool + command := &cobra.Command{ + Use: "check", + Short: "Run the gate registry (fast cockpit subset or full suite)", + Long: `Run the declarative gate registry. + + ao gate check # fast: only checks affected by changed files (cockpit) + ao gate check --full # full: every check, routing ignored (CI/refinery) + ao gate check --json # machine-readable report (refinery/CI annotations) + +Exit code is the verdict: 0 if no blocking check FAILed; 1 if any did +(non-blocking FAIL and WARN/SKIP are advisory).`, + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + _ = fast + if module.useCases.Check == nil { + return &ExitError{Code: 2, Message: "gate check: use case not configured"} + } + result, err := module.useCases.Check.Execute(command.Context(), gateapp.CheckRequest{ + Full: full, Scope: scope, FailFast: failFast, + WorkflowCoverage: workflowCoverage, RequireWorkflowParity: requireWorkflowParity, WorkflowPath: workflowPath, + }) + if err != nil { + return &ExitError{Code: 2, Message: err.Error()} + } + if jsonOutput { + raw, jsonErr := result.Report.JSON() + if jsonErr != nil { + return &ExitError{Code: 2, Message: jsonErr.Error()} + } + fmt.Fprintln(command.OutOrStdout(), string(raw)) + } else { + result.Report.Human(command.OutOrStdout()) + } + if githubAnnotations { + result.Report.GitHubAnnotations(command.ErrOrStderr()) + } + if result.ExitCode != 0 { + message := "" + if result.WorkflowParityMissing > 0 { + message = fmt.Sprintf("workflow parity missing %d blocking script(s)", result.WorkflowParityMissing) + } + return &ExitError{Code: result.ExitCode, Message: message} + } + return nil + }, + } + flags := command.Flags() + flags.BoolVar(&fast, "fast", false, "fast cockpit subset routed to changed files (the default; explicit flag for clarity in hooks)") + flags.BoolVar(&full, "full", false, "run every check (routing ignored); default is the fast changed-file subset") + flags.BoolVar(&jsonOutput, "json", false, "emit the machine-readable JSON report") + flags.BoolVar(&githubAnnotations, "github-annotations", false, "emit GitHub Actions annotations for WARN/FAIL checks") + flags.BoolVar(&failFast, "fail-fast", false, "stop after the first blocking failure") + flags.StringVar(&scope, "scope", "head", "fast-mode changed-file scope: head|staged|worktree|upstream|range:..") + flags.BoolVar(&workflowCoverage, "workflow-coverage", false, "include validate.yml-vs-registry script coverage in the report") + flags.BoolVar(&requireWorkflowParity, "require-workflow-parity", false, "fail if validate.yml references scripts missing from the Go gate registry") + flags.StringVar(&workflowPath, "workflow-path", ".github/workflows/validate.yml", "workflow path used by --workflow-coverage and --require-workflow-parity") + return command +} + +func (module Module) dryRun() bool { + return module.host.DryRun != nil && module.host.DryRun() +} + +func (module Module) outputFormat() string { + if module.host.OutputFormat == nil { + return "table" + } + return module.host.OutputFormat() +} + +func fixedCompletions(values ...string) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + sorted := append([]string(nil), values...) + sort.Strings(sorted) + return func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return sorted, cobra.ShellCompDirectiveNoFileComp + } +} diff --git a/cli/internal/commands/gate/module_test.go b/cli/internal/commands/gate/module_test.go new file mode 100644 index 000000000..9cc0d4af2 --- /dev/null +++ b/cli/internal/commands/gate/module_test.go @@ -0,0 +1,147 @@ +package gate + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + + gateapp "github.com/boshu2/agentops/cli/internal/gate" + "github.com/boshu2/agentops/cli/internal/gates" + "github.com/boshu2/agentops/cli/internal/ports" + "github.com/boshu2/agentops/cli/internal/types" +) + +type reviewUseCasesStub struct { + pendingResult gateapp.PendingResult + approveResult gateapp.ApproveResult + approveInput gateapp.ApproveInput +} + +func (stub *reviewUseCasesStub) Pending(context.Context, gateapp.PendingRequest) (gateapp.PendingResult, error) { + return stub.pendingResult, nil +} + +func (stub *reviewUseCasesStub) Approve(_ context.Context, input gateapp.ApproveInput) (gateapp.ApproveResult, error) { + stub.approveInput = input + return stub.approveResult, nil +} + +func (*reviewUseCasesStub) Reject(context.Context, gateapp.RejectInput) (gateapp.RejectResult, error) { + return gateapp.RejectResult{}, nil +} + +func (*reviewUseCasesStub) BulkApprove(context.Context, gateapp.BulkApproveInput) (gateapp.BulkApproveResult, error) { + return gateapp.BulkApproveResult{}, nil +} + +type checkUseCasesStub struct{ result gateapp.CheckResult } + +func (stub checkUseCasesStub) Execute(context.Context, gateapp.CheckRequest) (gateapp.CheckResult, error) { + return stub.result, nil +} + +func TestModuleOwnsCompleteGateTreeAndFlags(t *testing.T) { + command := NewModule(UseCases{}, HostOptions{}).Command() + for _, path := range [][]string{{"pending"}, {"approve"}, {"reject"}, {"bulk-approve"}, {"run"}, {"check"}} { + child, remaining, err := command.Find(path) + if err != nil || child == command || len(remaining) != 0 { + t.Fatalf("path=%v child=%v remaining=%v err=%v", path, child, remaining, err) + } + } + flags := map[string][]string{ + "approve": {"note"}, + "reject": {"reason"}, + "bulk-approve": {"older-than", "tier"}, + "check": {"fail-fast", "fast", "full", "github-annotations", "json", "require-workflow-parity", "scope", "workflow-coverage", "workflow-path"}, + } + for name, names := range flags { + child, _, _ := command.Find([]string{name}) + for _, flag := range names { + if child.Flags().Lookup(flag) == nil { + t.Errorf("gate %s missing --%s", name, flag) + } + } + } +} + +func TestPendingJSONPreservesFullPoolEntryWireShape(t *testing.T) { + entry := gateapp.ReviewEntry{PoolEntry: types.PoolEntry{Candidate: types.Candidate{ID: "cand-1", Tier: types.TierBronze}, Status: types.PoolStatusPending}} + command := NewModule(UseCases{ + Review: &reviewUseCasesStub{pendingResult: gateapp.PendingResult{Entries: []gateapp.ReviewEntry{entry}}}, + }, HostOptions{ + OutputFormat: func() string { return "json" }, + }).Command() + var output bytes.Buffer + command.SetOut(&output) + command.SetArgs([]string{"pending"}) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + var rows []map[string]any + if err := json.Unmarshal(output.Bytes(), &rows); err != nil { + t.Fatalf("json=%q err=%v", output.String(), err) + } + candidate, ok := rows[0]["candidate"].(map[string]any) + if !ok || candidate["id"] != "cand-1" || rows[0]["status"] != string(types.PoolStatusPending) { + t.Fatalf("rows=%+v", rows) + } +} + +func TestPendingYAMLPreservesLegacyOperationalKeys(t *testing.T) { + entry := gateapp.ReviewEntry{FilePath: "/tmp/cand-1.json", AgeString: "13h", ApproachingAutoPromote: true} + command := NewModule(UseCases{ + Review: &reviewUseCasesStub{pendingResult: gateapp.PendingResult{Entries: []gateapp.ReviewEntry{entry}}}, + }, HostOptions{OutputFormat: func() string { return "yaml" }}).Command() + var output bytes.Buffer + command.SetOut(&output) + command.SetArgs([]string{"pending"}) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + wire := output.String() + for _, key := range []string{"filepath:", "age:", "agestring:", "approachingautopromote:"} { + if !strings.Contains(wire, key) { + t.Errorf("legacy YAML missing %q in:\n%s", key, wire) + } + } + for _, drift := range []string{"file_path:", "age_string:", "approaching_auto_promote:"} { + if strings.Contains(wire, drift) { + t.Errorf("YAML contains drifted key %q in:\n%s", drift, wire) + } + } +} + +func TestApproveDelegatesClosureLocalFlagsAndRenders(t *testing.T) { + review := &reviewUseCasesStub{approveResult: gateapp.ApproveResult{CandidateID: "cand-1", Note: "good", Reviewer: "reviewer"}} + command := NewModule(UseCases{Review: review}, HostOptions{}).Command() + var output bytes.Buffer + command.SetOut(&output) + command.SetArgs([]string{"approve", "cand-1", "--note", "good"}) + if err := command.Execute(); err != nil { + t.Fatalf("Execute: %v", err) + } + if review.approveInput.CandidateID != "cand-1" || review.approveInput.Note != "good" || !strings.Contains(output.String(), "Approved: cand-1\nReviewer: reviewer") { + t.Fatalf("input=%+v output=%q", review.approveInput, output.String()) + } +} + +func TestCheckRendersJSONBeforeReturningTypedExit(t *testing.T) { + report := &gates.Report{Mode: gates.Full, Scope: gates.ScopeHead, Results: []gates.CheckResult{{ + Check: gates.Check{ID: "n.fail", Blocking: true}, + Verdict: ports.GateVerdict{Status: ports.GateStatusFail, Reason: "failed"}, + }}} + command := NewModule(UseCases{Check: checkUseCasesStub{result: gateapp.CheckResult{Report: report, ExitCode: 1}}}, HostOptions{}).Command() + var output bytes.Buffer + command.SetOut(&output) + command.SetArgs([]string{"check", "--full", "--json"}) + err := command.Execute() + if err == nil { + t.Fatal("expected typed exit") + } + exit, ok := err.(interface{ ExitCode() int }) + if !ok || exit.ExitCode() != 1 || !strings.Contains(output.String(), `"name": "n.fail"`) { + t.Fatalf("err=%T %v output=%q", err, err, output.String()) + } +} diff --git a/cli/internal/gate/review_service.go b/cli/internal/gate/review_service.go index 22a1df1fb..85470ee91 100644 --- a/cli/internal/gate/review_service.go +++ b/cli/internal/gate/review_service.go @@ -4,18 +4,19 @@ import ( "context" "fmt" "time" + + "github.com/boshu2/agentops/cli/internal/types" ) const defaultBulkApproveThreshold = 24 * time.Hour type ReviewEntry struct { - ID string `json:"id" yaml:"id"` - Tier string `json:"tier" yaml:"tier"` - Age time.Duration `json:"-" yaml:"-"` - AgeString string `json:"age,omitempty" yaml:"age,omitempty"` - Utility float64 `json:"utility" yaml:"utility"` - ApproachingAutoPromote bool `json:"approaching_auto_promote,omitempty" yaml:"approaching_auto_promote,omitempty"` - Urgency string `json:"urgency" yaml:"urgency"` + types.PoolEntry + FilePath string `json:"file_path,omitempty"` + Age time.Duration `json:"-"` + AgeString string `json:"age,omitempty"` + ApproachingAutoPromote bool `json:"approaching_auto_promote,omitempty"` + Urgency string `json:"-" yaml:"-"` } type ApproveRequest struct { diff --git a/cli/internal/gate/review_service_test.go b/cli/internal/gate/review_service_test.go index b387cd127..6d5f804b3 100644 --- a/cli/internal/gate/review_service_test.go +++ b/cli/internal/gate/review_service_test.go @@ -41,9 +41,9 @@ func (port *reviewPortSpy) Reviewer() string { return port.reviewer } func TestReviewServicePendingComputesUrgency(t *testing.T) { port := &reviewPortSpy{entries: []ReviewEntry{ - {ID: "high", ApproachingAutoPromote: true}, - {ID: "medium", Age: 13 * time.Hour}, - {ID: "low", Age: time.Hour}, + {ApproachingAutoPromote: true}, + {Age: 13 * time.Hour}, + {Age: time.Hour}, }} result, err := (ReviewService{Port: port}).Pending(context.Background(), PendingRequest{}) if err != nil { From 62e0bcba7ec6f97c1f4bf4065342df8ff4741ecf Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Sun, 12 Jul 2026 01:30:02 -0400 Subject: [PATCH 108/108] chore(gate): seal accepted migration lineage --- cli/testdata/compatibility-baseline/families/gate/lineage.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/testdata/compatibility-baseline/families/gate/lineage.json b/cli/testdata/compatibility-baseline/families/gate/lineage.json index 4a2441004..2aeeb290c 100644 --- a/cli/testdata/compatibility-baseline/families/gate/lineage.json +++ b/cli/testdata/compatibility-baseline/families/gate/lineage.json @@ -6,5 +6,6 @@ "fixture_sha256": "67198187ec7bc117bc9190eca6238ab2c11a2853cfef3c48e89a89af46d65c7f", "ownership_sha256": "0461b6c20f5477fde5b2fef03ecafaea9740c3f732d75092590877d4f0828a96", "scope_sha256": "d286af8b81c202bea8c5a1a85d215d5da5aeedbaaac8f9ba1ebef9246b2cb2ad", - "migration_state": "migrating" + "migration_state": "migrated", + "accepted_sha": "bb6f4b58cfe3efb9489d61512b3bb9af5cfd8d45" }