fix(cli): exact --json empty states (goals history, session rehydrate, scenario list) + eval discoverability (age-gocli-audit-remediation-6fybr.8)

cli/cmd/ao/rehydrate.go

@@ -33,6 +33,14 @@ func runRehydrate(cmd *cobra.Command, _ []string) error {
 	}
 	path, err := pickLatestHandoff(cwd)
 	if err != nil {
+		// Under --json, stdout must be exactly one JSON document (`{}` for the
+		// empty state) so `ao session rehydrate --json | jq` never breaks; the
+		// human hint goes to stderr. Exit 0 either way.
+		if rehydrateJSON {
+			fmt.Fprintln(cmd.ErrOrStderr(), "rehydrate: no handoff found")
+			fmt.Fprintln(cmd.OutOrStdout(), "{}")
+			return nil
+		}
 		fmt.Fprintln(cmd.OutOrStdout(), "rehydrate: no handoff found")
 		return nil
 	}

cli/cmd/ao/session_evidence_test.go

@@ -33,6 +33,40 @@ func TestSessionBootstrapOnlyReportsLocalOrientation(t *testing.T) {
 	}
 }
 
+// TestRehydrateJSONEmptyStateEmitsEmptyObject asserts that --json with no
+// handoff present emits exactly one JSON document `{}` on stdout (jq-safe), with
+// the human hint on stderr and exit 0. RED before the fix: the "no handoff found"
+// prose is printed to stdout regardless of --json, so json.Decoder fails.
+func TestRehydrateJSONEmptyStateEmitsEmptyObject(t *testing.T) {
+	dir := t.TempDir()
+	t.Chdir(dir)
+
+	var stdout, stderr bytes.Buffer
+	readCommand := *rehydrateCmd
+	readCommand.SetOut(&stdout)
+	readCommand.SetErr(&stderr)
+	rehydrateJSON = true
+	t.Cleanup(func() { rehydrateJSON = false })
+	if err := runRehydrate(&readCommand, nil); err != nil {
+		t.Fatalf("rehydrate returned error: %v", err)
+	}
+
+	dec := json.NewDecoder(&stdout)
+	var decoded map[string]any
+	if err := dec.Decode(&decoded); err != nil {
+		t.Fatalf("stdout is not one JSON document: %v (raw stdout: %q)", err, stdout.String())
+	}
+	if len(decoded) != 0 {
+		t.Errorf("empty state: expected {} (0 keys), got %d", len(decoded))
+	}
+	if dec.More() {
+		t.Error("stdout contains more than one JSON document")
+	}
+	if !strings.Contains(stderr.String(), "no handoff found") {
+		t.Errorf("stderr = %q, want the 'no handoff found' hint", stderr.String())
+	}
+}
+
 func TestHandoffAndRehydratePreserveCallerTextWithoutLifecycleState(t *testing.T) {
 	dir := t.TempDir()
 	t.Chdir(dir)

cli/docs/COMMANDS.md

@@ -391,7 +391,7 @@ ao eval outcomes ingest <score.json> [flags]
 
 #### `ao eval run`
 
-Run a deterministic eval suite
+Run a deterministic eval suite.
 
 ```
 ao eval run <suite.json> [flags]

cli/internal/commands/eval/module.go

@@ -180,7 +180,15 @@ func (module Module) outputMode(command *cobra.Command) string {
 
 func (module Module) runCommand() *cobra.Command {
 	options := runOptions{baselineMode: string(aoeval.BaselineModeSkillOn), contextMode: string(aoeval.ContextModeNone)}
-	command := &cobra.Command{Use: "run <suite.json>", Short: "Run a deterministic eval suite", Args: cobra.ExactArgs(1)}
+	command := &cobra.Command{
+		Use:   "run <suite.json>",
+		Short: "Run a deterministic eval suite",
+		Long: "Run a deterministic eval suite.\n\n" +
+			"The suite file must conform to the JSON Schema at schemas/eval-suite.v1.schema.json.\n" +
+			"See evals/agentops-core for working example suites (e.g. evals/agentops-core/rpi-behavior.json).\n\n" +
+			"Example:\n  ao eval run evals/agentops-core/rpi-behavior.json",
+		Args: cobra.ExactArgs(1),
+	}
 	flags := command.Flags()
 	flags.StringVar(&options.output, "out", "", "write eval run record to path")
 	flags.StringVar(&options.runID, "run-id", "", "stable run id to use in the run record")
@@ -197,7 +205,7 @@ func (module Module) runCommand() *cobra.Command {
 	command.RunE = func(command *cobra.Command, args []string) error {
 		result, err := module.useCases.Core.Run(command.Context(), aoeval.CoreRunRequest{SuitePath: args[0], RunID: options.runID, Runtime: options.runtime, OutputPath: options.output, BaselinePath: options.baseline, BaselineMode: options.baselineMode, ContextMode: options.contextMode, ContextOffDir: options.contextOffDir, ContextOnDir: options.contextOnDir, DeltaOut: options.deltaOut})
 		if err != nil {
-			return err
+			return annotateSuiteParseError(err)
 		}
 		return module.renderRun(command, result)
 	}
@@ -651,13 +659,16 @@ func (module Module) scenarioListCommand() *cobra.Command {
 		if err != nil {
 			return err
 		}
+		// stdout is always exactly one JSON document (`[]` for both empty
+		// states) so `ao eval scenario list | jq` never breaks; any human hint
+		// goes to stderr. Exit 0.
 		if result.MissingDirectory {
-			fmt.Fprintln(command.OutOrStdout(), "No holdout directory found. Run 'ao scenario init' first.")
-			return nil
+			fmt.Fprintln(command.ErrOrStderr(), "No holdout directory found. Run 'ao eval scenario init' first.")
+			return writeJSON(command, []aoeval.ScenarioSummary{})
 		}
 		if len(result.Scenarios) == 0 {
-			fmt.Fprintln(command.OutOrStdout(), "No scenarios found.")
-			return nil
+			fmt.Fprintln(command.ErrOrStderr(), "No scenarios found.")
+			return writeJSON(command, []aoeval.ScenarioSummary{})
 		}
 		return writeJSON(command, result.Scenarios)
 	}
@@ -927,6 +938,28 @@ func renderCoverage(command *cobra.Command, label string, missing, required []st
 	}
 }
 
+// suiteSchemaHint cites the authoritative suite schema and a working example so
+// a failed `ao eval run` points the caller at the exact shape to fix.
+const suiteSchemaHint = "see schemas/eval-suite.v1.schema.json for the suite schema and evals/agentops-core for a working example"
+
+// annotateSuiteParseError appends the schema/example citation to the three
+// suite-load failure modes surfaced by cli/internal/eval.LoadSuite — missing
+// file ("read eval suite"), malformed JSON ("decode eval suite"), and
+// schema-invalid ("eval suite validation failed"). Command-layer decoration
+// only; the underlying error text in cli/internal/eval is unchanged.
+func annotateSuiteParseError(err error) error {
+	if err == nil {
+		return nil
+	}
+	msg := err.Error()
+	for _, sig := range []string{"read eval suite", "decode eval suite", "eval suite validation failed"} {
+		if strings.Contains(msg, sig) {
+			return fmt.Errorf("%w (%s)", err, suiteSchemaHint)
+		}
+	}
+	return err
+}
+
 func writeJSON(command *cobra.Command, value any) error {
 	encoder := json.NewEncoder(command.OutOrStdout())
 	encoder.SetIndent("", "  ")

cli/internal/commands/eval/module_test.go

@@ -1,7 +1,11 @@
 package eval
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
 	"strings"
 	"testing"
 
@@ -12,6 +16,101 @@ import (
 	scenarioapp "github.com/boshu2/agentops/cli/internal/scenario"
 )
 
+// scenarioListSpy is a configurable ScenarioUseCases double whose List result
+// the test controls, so both empty-state paths (missing directory / empty list)
+// can be exercised.
+type scenarioListSpy struct{ listResult aoeval.ScenarioListResult }
+
+func (*scenarioListSpy) Add(context.Context, aoeval.ScenarioAddRequest) (*scenarioapp.CreateResult, error) {
+	return &scenarioapp.CreateResult{}, nil
+}
+func (*scenarioListSpy) Init(context.Context) (string, error) { return ".agents/holdout", nil }
+func (spy *scenarioListSpy) List(context.Context, string) (aoeval.ScenarioListResult, error) {
+	return spy.listResult, nil
+}
+func (*scenarioListSpy) Validate(context.Context) (aoeval.ScenarioValidationResult, error) {
+	return aoeval.ScenarioValidationResult{}, nil
+}
+func (*scenarioListSpy) Evaluate(context.Context, aoeval.ScenarioEvaluateRequest) (*aoeval.ScenarioEvaluateReport, error) {
+	return &aoeval.ScenarioEvaluateReport{}, nil
+}
+
+// TestAnnotateSuiteParseError asserts that each of the three suite-load failure
+// modes surfaced by LoadSuite gets the schema + example citation, and that an
+// unrelated runtime error is passed through untouched.
+func TestAnnotateSuiteParseError(t *testing.T) {
+	cases := []struct {
+		name     string
+		in       error
+		wantCite bool
+	}{
+		{name: "missing file", in: fmt.Errorf("read eval suite: open missing.json: no such file or directory"), wantCite: true},
+		{name: "malformed json", in: fmt.Errorf("decode eval suite: invalid character '}' looking for beginning of value"), wantCite: true},
+		{name: "schema invalid", in: fmt.Errorf("eval suite validation failed: schema_version must be 1"), wantCite: true},
+		{name: "unrelated error", in: fmt.Errorf("runtime static: check failed"), wantCite: false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := annotateSuiteParseError(tc.in)
+			hasCite := strings.Contains(got.Error(), "schemas/eval-suite.v1.schema.json") && strings.Contains(got.Error(), "evals/agentops-core")
+			if hasCite != tc.wantCite {
+				t.Fatalf("annotateSuiteParseError(%q) = %q; wantCite=%v", tc.in, got, tc.wantCite)
+			}
+			if !errors.Is(got, tc.in) {
+				t.Errorf("wrapped error must preserve the original: errors.Is == false")
+			}
+		})
+	}
+}
+
+// TestModuleScenarioListJSONEmptyStates asserts that `eval scenario list` emits
+// exactly one JSON document `[]` on stdout in BOTH empty states (missing holdout
+// directory and empty scenario list), routes the human hint to stderr, and names
+// the REAL init command. RED before the fix: empty-state prose is printed to
+// stdout (breaking jq) and the missing-dir hint names 'ao scenario init', which
+// does not exist.
+func TestModuleScenarioListJSONEmptyStates(t *testing.T) {
+	cases := []struct {
+		name       string
+		result     aoeval.ScenarioListResult
+		wantStderr string
+	}{
+		{name: "missing directory", result: aoeval.ScenarioListResult{MissingDirectory: true}, wantStderr: "ao eval scenario init"},
+		{name: "empty list", result: aoeval.ScenarioListResult{}, wantStderr: "No scenarios found"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			spy := &scenarioListSpy{listResult: tc.result}
+			command := NewModule(UseCases{Core: &coreUseCasesSpy{}, Scenario: spy}, HostOptions{}).Command()
+			command.SetArgs([]string{"scenario", "list"})
+			var stdout, stderr bytes.Buffer
+			command.SetOut(&stdout)
+			command.SetErr(&stderr)
+			if err := command.Execute(); err != nil {
+				t.Fatalf("Execute: %v", err)
+			}
+
+			dec := json.NewDecoder(&stdout)
+			var decoded []aoeval.ScenarioSummary
+			if err := dec.Decode(&decoded); err != nil {
+				t.Fatalf("stdout is not one JSON document: %v (raw stdout: %q)", err, stdout.String())
+			}
+			if len(decoded) != 0 {
+				t.Errorf("expected [] (0 scenarios), got %d", len(decoded))
+			}
+			if dec.More() {
+				t.Error("stdout contains more than one JSON document")
+			}
+			if !strings.Contains(stderr.String(), tc.wantStderr) {
+				t.Errorf("stderr = %q, want it to contain %q", stderr.String(), tc.wantStderr)
+			}
+			if strings.Contains(stderr.String(), "ao scenario init") {
+				t.Errorf("stderr names the nonexistent 'ao scenario init': %q", stderr.String())
+			}
+		})
+	}
+}
+
 type coreUseCasesSpy struct {
 	runRequest aoeval.CoreRunRequest
 }

cli/internal/goals/commands.go

@@ -48,6 +48,7 @@ type HistoryOptions struct {
 	JSON        bool
 	HistoryPath string
 	Stdout      io.Writer
+	Stderr      io.Writer
 }
 
 // RunHistory loads and displays goal measurement history.
@@ -58,17 +59,15 @@ func RunHistory(opts HistoryOptions) error {
 	if opts.Stdout == nil {
 		opts.Stdout = os.Stdout
 	}
+	if opts.Stderr == nil {
+		opts.Stderr = os.Stderr
+	}
 
 	entries, err := LoadHistory(opts.HistoryPath)
 	if err != nil {
 		return fmt.Errorf("loading history: %w", err)
 	}
 
-	if len(entries) == 0 {
-		fmt.Fprintln(opts.Stdout, "No history entries found. Run 'ao goals measure' first.")
-		return nil
-	}
-
 	if opts.Since != "" || opts.GoalID != "" {
 		var since time.Time
 		if opts.Since != "" {
@@ -82,11 +81,22 @@ func RunHistory(opts HistoryOptions) error {
 	}
 
 	if opts.JSON {
+		// stdout is exactly one JSON document; the empty store encodes as `[]`
+		// (LoadHistory/QueryHistory always return a non-nil slice). Any human
+		// hint goes to stderr so `ao goals history --json | jq` never breaks.
+		if len(entries) == 0 {
+			fmt.Fprintln(opts.Stderr, "No history entries found. Run 'ao goals measure' first.")
+		}
 		enc := json.NewEncoder(opts.Stdout)
 		enc.SetIndent("", "  ")
 		return enc.Encode(entries)
 	}
 
+	if len(entries) == 0 {
+		fmt.Fprintln(opts.Stdout, "No history entries found. Run 'ao goals measure' first.")
+		return nil
+	}
+
 	fmt.Fprintf(opts.Stdout, "%-20s  %4s  %5s  %7s  %s\n", "TIMESTAMP", "PASS", "TOTAL", "SCORE", "GIT SHA")
 	for _, e := range entries {
 		fmt.Fprintf(opts.Stdout, "%-20s  %4d  %5d  %6.1f%%  %s\n",

cli/internal/goals/goals_history_test.go

@@ -31,6 +31,44 @@ func TestGoalsHistory_NoHistoryFile(t *testing.T) {
 	}
 }
 
+// TestGoalsHistory_JSONEmptyStore_Decoder asserts that --json over an empty
+// store emits exactly one JSON document `[]` on stdout (jq-safe), with any human
+// hint on stderr and exit 0. RED before the fix: empty-state prose is printed to
+// stdout, so json.Decoder fails to parse it.
+func TestGoalsHistory_JSONEmptyStore_Decoder(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+	historyPath := filepath.Join(dir, ".agents/ao/goals/history.jsonl")
+
+	var stdout, stderr bytes.Buffer
+	err := goals.RunHistory(goals.HistoryOptions{
+		JSON:        true,
+		HistoryPath: historyPath,
+		Stdout:      &stdout,
+		Stderr:      &stderr,
+	})
+	if err != nil {
+		t.Fatalf("history returned error: %v", err)
+	}
+
+	dec := json.NewDecoder(&stdout)
+	var decoded []goals.HistoryEntry
+	if err := dec.Decode(&decoded); err != nil {
+		t.Fatalf("stdout is not one JSON document: %v (raw stdout: %q)", err, stdout.String())
+	}
+	if len(decoded) != 0 {
+		t.Errorf("empty store: expected [] (0 entries), got %d", len(decoded))
+	}
+	// Exactly one document: nothing but whitespace after it.
+	if dec.More() {
+		t.Error("stdout contains more than one JSON document")
+	}
+	// The human hint belongs on stderr, never stdout.
+	if !strings.Contains(stderr.String(), "No history entries found") {
+		t.Errorf("stderr = %q, want the 'No history entries found' hint", stderr.String())
+	}
+}
+
 func TestGoalsHistory_WithEntries(t *testing.T) {
 	t.Parallel()
 	dir := t.TempDir()