diff --git a/.gitattributes b/.gitattributes index 03aa18ca0..ee8dda537 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,6 +10,20 @@ # repository's whitespace errors. docs/audits/gc-mvp-2026-07-05/patches/*.patch -whitespace +# Probe transcripts are immutable structured capture envelopes. Preserve their +# exact JSONL bytes, including encoded prompt and runtime event content. +evals/skill-probes/*/fixtures*/control-*.txt -text -whitespace +evals/skill-probes/*/fixtures*/treatment-*.txt -text -whitespace + +# Probe manifests bind these working-tree inputs byte-for-byte. Force one +# checkout representation so the same Git blobs replay on every platform. +evals/skill-probes/*/probe.json text eol=lf +evals/skill-probes/*/question.md text eol=lf +evals/skill-probes/*/treatment-prelude.md text eol=lf +evals/skill-probes/*/fixtures*/capture-contract.json text eol=lf +skills/*/SKILL.md text eol=lf +scripts/lib/probe-fixture-metadata.py text eol=lf + # Generated/derived artifacts — union-merge to kill textual re-conflicts during # multi-PR drains (their canonical content is restored by scripts/regen-all.sh). # Council 2026-06-06 (ag-bdg1). NOTE: cli/embedded/** is //go:embed'd and diff --git a/cli/cmd/ao/default_spine_test.go b/cli/cmd/ao/default_spine_test.go index 1a8be0a7a..e5e34fb3a 100644 --- a/cli/cmd/ao/default_spine_test.go +++ b/cli/cmd/ao/default_spine_test.go @@ -23,7 +23,7 @@ var approvedDefaultChildren = map[string]map[string]bool{ "drift": true, "export": true, "history": true, "measure": true, "meta": true, "render": true, "scenarios": true, "validate": true, }, - "session": {"bootstrap": true, "handoff": true, "rehydrate": true}, + "session": {"bootstrap": true, "handoff": true, "prune-agents": true, "rehydrate": true}, "skills": { "check": true, "consumers": true, "find": true, "graph": true, "link": true, "list": true, "producers": true, "resolve": true, diff --git a/cli/cmd/ao/handoff.go b/cli/cmd/ao/handoff.go index bcc9b128b..5ad4c28a4 100644 --- a/cli/cmd/ao/handoff.go +++ b/cli/cmd/ao/handoff.go @@ -1,6 +1,8 @@ package main import ( + "crypto/rand" + "encoding/hex" "encoding/json" "fmt" "os" @@ -122,16 +124,22 @@ func collectHandoffState(cwd string) *handoffState { func writeHandoffArtifact(cwd string, artifact *handoffArtifact, data []byte) (string, error) { dir := filepath.Join(cwd, ".agents", "ao", "handoff") - if err := os.MkdirAll(dir, 0o755); err != nil { - return "", fmt.Errorf("create handoff directory: %w", err) + if artifact == nil || artifact.ID == "" || artifact.ID == "." || artifact.ID == ".." || strings.ContainsAny(artifact.ID, `/\\`) { + return "", fmt.Errorf("publish handoff: invalid artifact id") } - target := filepath.Join(dir, artifact.ID+".json") - tmp, err := os.CreateTemp(dir, ".handoff-*.tmp") + root, err := openHandoffWriteRoot(cwd, true) + if err != nil { + return "", err + } + defer func() { _ = root.Close() }() + + targetName := artifact.ID + ".json" + target := filepath.Join(dir, targetName) + tmpName, tmp, err := createHandoffTemp(root) if err != nil { return "", fmt.Errorf("create handoff temporary file: %w", err) } - tmpName := tmp.Name() - defer func() { _ = os.Remove(tmpName) }() + defer func() { _ = root.Remove(tmpName) }() if _, err := tmp.Write(data); err != nil { _ = tmp.Close() return "", fmt.Errorf("write handoff: %w", err) @@ -143,8 +151,113 @@ func writeHandoffArtifact(cwd string, artifact *handoffArtifact, data []byte) (s if err := tmp.Close(); err != nil { return "", fmt.Errorf("close handoff: %w", err) } - if err := os.Rename(tmpName, target); err != nil { + if err := verifyHandoffWriteRoot(cwd, root); err != nil { + return "", err + } + // A hard-link publish is an atomic no-clobber operation: unlike Rename it + // never replaces evidence already stored under the same id. Both names are + // resolved by the descriptor-anchored Root, so a parent-directory swap + // cannot redirect the write through a symlink. + if err := root.Link(tmpName, targetName); err != nil { return "", fmt.Errorf("publish handoff: %w", err) } + if err := verifyHandoffWriteRoot(cwd, root); err != nil { + if removeErr := root.Remove(targetName); removeErr != nil { + return "", fmt.Errorf("%w; cleanup published handoff: %w", err, removeErr) + } + return "", err + } + if err := root.Remove(tmpName); err != nil { + return "", fmt.Errorf("remove handoff temporary file after publish: %w", err) + } return target, nil } + +// openHandoffWriteRoot resolves .agents/ao/handoff one component at a time. +// Existing components must be real directories, and every opened descriptor +// must still identify the component that was inspected. Missing components +// are created only when create is true. +func openHandoffWriteRoot(cwd string, create bool) (*os.Root, error) { + root, err := os.OpenRoot(cwd) + if err != nil { + return nil, fmt.Errorf("open workspace root: %w", err) + } + current := root + for _, component := range []string{".agents", "ao", "handoff"} { + next, openErr := openRealHandoffDir(current, component, create) + if openErr != nil { + _ = current.Close() + return nil, fmt.Errorf("open handoff directory component %s: %w", component, openErr) + } + _ = current.Close() + current = next + } + return current, nil +} + +func openRealHandoffDir(parent *os.Root, component string, create bool) (*os.Root, error) { + for { + before, err := parent.Lstat(component) + if err != nil { + if create && os.IsNotExist(err) { + if mkdirErr := parent.Mkdir(component, 0o755); mkdirErr != nil && !os.IsExist(mkdirErr) { + return nil, mkdirErr + } + continue + } + return nil, err + } + if before.Mode()&os.ModeSymlink != 0 || !before.IsDir() { + return nil, fmt.Errorf("not a real directory (refused_unsafe)") + } + next, err := parent.OpenRoot(component) + if err != nil { + return nil, err + } + opened, statErr := next.Stat(".") + after, afterErr := parent.Lstat(component) + if statErr != nil || afterErr != nil || after.Mode()&os.ModeSymlink != 0 || !after.IsDir() || !os.SameFile(before, opened) || !os.SameFile(after, opened) { + _ = next.Close() + return nil, fmt.Errorf("changed identity while opening (refused_unsafe)") + } + return next, nil + } +} + +func verifyHandoffWriteRoot(cwd string, opened *os.Root) error { + current, err := openHandoffWriteRoot(cwd, false) + if err != nil { + return fmt.Errorf("verify handoff directory: %w", err) + } + defer func() { _ = current.Close() }() + want, err := opened.Stat(".") + if err != nil { + return fmt.Errorf("stat opened handoff directory: %w", err) + } + got, err := current.Stat(".") + if err != nil { + return fmt.Errorf("stat current handoff directory: %w", err) + } + if !os.SameFile(want, got) { + return fmt.Errorf("verify handoff directory: path changed identity (refused_unsafe)") + } + return nil +} + +func createHandoffTemp(root *os.Root) (string, *os.File, error) { + for range 100 { + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return "", nil, err + } + name := ".handoff-" + hex.EncodeToString(random[:]) + ".tmp" + file, err := root.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + return name, file, nil + } + if !os.IsExist(err) { + return "", nil, err + } + } + return "", nil, fmt.Errorf("exhausted unique temporary names") +} diff --git a/cli/cmd/ao/handoff_test.go b/cli/cmd/ao/handoff_test.go index 165b561c5..0fdafa158 100644 --- a/cli/cmd/ao/handoff_test.go +++ b/cli/cmd/ao/handoff_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "testing" "github.com/santhosh-tekuri/jsonschema/v6" @@ -62,6 +63,73 @@ func TestHandoffDryRunSatisfiesSchema(t *testing.T) { } } +func TestWriteHandoffArtifactNoClobber(t *testing.T) { + dir := t.TempDir() + artifact := &handoffArtifact{ID: "handoff-20260816T120000.000000000Z"} + target := filepath.Join(dir, ".agents", "ao", "handoff", artifact.ID+".json") + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + t.Fatal(err) + } + original := []byte("existing evidence\n") + if err := os.WriteFile(target, original, 0o600); err != nil { + t.Fatal(err) + } + + if _, err := writeHandoffArtifact(dir, artifact, []byte("replacement\n")); err == nil { + t.Fatal("writeHandoffArtifact overwrote an existing artifact id") + } + got, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, original) { + t.Fatalf("existing artifact changed: got %q want %q", got, original) + } +} + +func TestWriteHandoffArtifactRejectsSymlinkedDirectoryComponents(t *testing.T) { + for _, component := range []string{"ao", "handoff"} { + t.Run(component, func(t *testing.T) { + dir := t.TempDir() + external := t.TempDir() + agents := filepath.Join(dir, ".agents") + if err := os.MkdirAll(agents, 0o755); err != nil { + t.Fatal(err) + } + if component == "ao" { + if err := os.MkdirAll(filepath.Join(external, "handoff"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(external, filepath.Join(agents, "ao")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + } else { + if err := os.MkdirAll(filepath.Join(agents, "ao"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(external, filepath.Join(agents, "ao", "handoff")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + } + sentinel := filepath.Join(external, "sentinel") + if err := os.WriteFile(sentinel, []byte("outside\n"), 0o600); err != nil { + t.Fatal(err) + } + artifact := &handoffArtifact{ID: "handoff-20260816T120000.000000000Z"} + if _, err := writeHandoffArtifact(dir, artifact, []byte("secret\n")); err == nil || !strings.Contains(err.Error(), "not a real directory") { + t.Fatalf("writeHandoffArtifact error = %v, want symlink refusal", err) + } + got, err := os.ReadFile(sentinel) + if err != nil || string(got) != "outside\n" { + t.Fatalf("outside sentinel changed: %q err=%v", got, err) + } + if _, err := os.Lstat(filepath.Join(external, artifact.ID+".json")); !os.IsNotExist(err) { + t.Fatalf("writer created artifact outside workspace: %v", err) + } + }) + } +} + // TestHandoffSchemaAcceptsLegacyArtifact proves the read-compatibility promise: // an artifact written by an earlier generator (carrying type/consumed and a // fractional id) still validates against handoff.v1, so the schema change is diff --git a/cli/cmd/ao/session_composition.go b/cli/cmd/ao/session_composition.go index c7f8b295e..b848a7856 100644 --- a/cli/cmd/ao/session_composition.go +++ b/cli/cmd/ao/session_composition.go @@ -2,6 +2,8 @@ package main import ( + "time" + "github.com/spf13/cobra" "github.com/boshu2/agentops/cli/internal/clicontract" @@ -15,11 +17,23 @@ func init() { // newSessionCommand wires the session command module and attaches the optional // `ao session handoff` writer, which is a separate command (defined in // handoff.go) that shares this parent. The module owns the session parent plus -// its bootstrap and rehydrate subcommands and delegates all filesystem effects -// to internal/sessionapp. The session family attaches no CommandContract to the -// command tree, preserving its pre-migration capabilities surface. +// its bootstrap, rehydrate, and prune-agents subcommands and delegates all +// filesystem effects to internal/sessionapp. The session family attaches no +// CommandContract to the command tree, preserving its pre-migration +// capabilities surface. func newSessionCommand() *cobra.Command { - command := sessioncommands.NewModule(clicontract.HostOptions{OutputMode: GetOutput}).Command() + command := sessioncommands.NewModule(clicontract.HostOptions{ + OutputMode: GetOutput, + DryRun: GetDryRun, + ProjectRoot: func() string { + root, err := repoRootOrCwd() + if err != nil { + return "" + } + return root + }, + Now: time.Now, + }).Command() command.AddCommand(handoffCmd) return command } diff --git a/cli/docs/COMMANDS.md b/cli/docs/COMMANDS.md index da90f63ec..9914e4030 100644 --- a/cli/docs/COMMANDS.md +++ b/cli/docs/COMMANDS.md @@ -880,7 +880,7 @@ ao goals scenarios [flags] ### `ao session` -Inspect or export session evidence +Inspect session evidence and maintain .agents artifacts ``` ao session [command] @@ -921,6 +921,22 @@ ao session handoff [summary] [flags] -h, --help help for handoff ``` +#### `ao session prune-agents` + +Apply .agents retention policies (dry-run by default) + +``` +ao session prune-agents [flags] +``` + +**Flags:** + +``` + --execute Delete the selected artifacts; the default is a read-only dry run + -h, --help help for prune-agents + --quiet Suppress per-path output and print only the summary +``` + #### `ao session rehydrate` Read a handoff without consuming it, claiming work, or choosing a next action. diff --git a/cli/internal/commands/session/module.go b/cli/internal/commands/session/module.go index dfa0c7768..b546fa2a6 100644 --- a/cli/internal/commands/session/module.go +++ b/cli/internal/commands/session/module.go @@ -1,5 +1,5 @@ -// Package session owns Cobra presentation for the `ao session` evidence -// commands (bootstrap and rehydrate). The module builds its command tree with +// Package session owns Cobra presentation for the `ao session` commands +// (bootstrap, rehydrate, and prune-agents). The module builds its command tree with // constructor-scoped flag state and delegates every filesystem effect to // internal/sessionapp, so this package performs no direct effect. The optional // `ao session handoff` writer is attached by the cmd/ao composition; it is a @@ -39,8 +39,9 @@ func (m Module) outputMode() string { // Contract declares the session family's real behavior for the family // architecture gate. Session reads local orientation files and the latest -// caller-authored handoff on the filesystem, emits text (JSON under each -// subcommand's --json flag), and exits 0 on success or 1 on a working-directory +// caller-authored handoff, and its explicitly selected prune-agents command can +// apply retention mutations (dry-run by default). It emits text (JSON under the +// read commands' --json flags) and exits 0 on success or 1 on a filesystem // failure. The session family attached no capabilities contract before the // carve-out, so the composition does not attach this one either. func (Module) Contract() clicontract.CommandContract { @@ -57,17 +58,17 @@ func (Module) Contract() clicontract.CommandContract { } } -// Command builds the `ao session` command with its bootstrap and rehydrate -// subcommands. The RunE closures delegate entirely to internal/sessionapp so -// this module performs no direct filesystem effect. +// Command builds the `ao session` command. The RunE closures delegate entirely +// to internal/sessionapp so this module performs no direct filesystem effect. func (m Module) Command() *cobra.Command { root := &cobra.Command{ Use: "session", - Short: "Inspect or export session evidence", + Short: "Inspect session evidence and maintain .agents artifacts", GroupID: "workflow", } root.AddCommand(m.bootstrapCommand()) root.AddCommand(m.rehydrateCommand()) + root.AddCommand(m.pruneAgentsCommand()) return root } @@ -125,3 +126,37 @@ func (m Module) rehydrateCommand() *cobra.Command { command.Flags().BoolVar(&jsonOut, "json", false, "Emit the stored artifact as JSON") return command } + +// pruneAgentsCommand builds the retention-policy command used directly by the +// CLI and by scripts/prune-agents.sh's compatibility wrapper. The default is a +// read-only dry run; --execute requests mutation, while the global --dry-run +// seam always wins. +func (m Module) pruneAgentsCommand() *cobra.Command { + var execute, quiet bool + command := &cobra.Command{ + Use: "prune-agents", + Short: "Apply .agents retention policies (dry-run by default)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + repoRoot := "" + if m.host.ProjectRoot != nil { + repoRoot = m.host.ProjectRoot() + } + effectiveExecute := execute + if m.host.DryRun != nil && m.host.DryRun() { + effectiveExecute = false + } + _, err := sessionapp.PruneAgents(sessionapp.PruneAgentsOptions{ + RepoRoot: repoRoot, + Execute: effectiveExecute, + Quiet: quiet, + Stdout: cmd.OutOrStdout(), + Now: m.host.Now, + }) + return err + }, + } + command.Flags().BoolVar(&execute, "execute", false, "Delete the selected artifacts; the default is a read-only dry run") + command.Flags().BoolVar(&quiet, "quiet", false, "Suppress per-path output and print only the summary") + return command +} diff --git a/cli/internal/commands/session/module_test.go b/cli/internal/commands/session/module_test.go index 02541f4fe..d57a5a539 100644 --- a/cli/internal/commands/session/module_test.go +++ b/cli/internal/commands/session/module_test.go @@ -3,10 +3,12 @@ package session import ( "bytes" "encoding/json" + "fmt" "os" "path/filepath" "strings" "testing" + "time" "github.com/spf13/cobra" @@ -53,13 +55,53 @@ func TestModule_CommandAttributes(t *testing.T) { for _, child := range root.Commands() { seen[child.Name()] = true } - for _, want := range []string{"bootstrap", "rehydrate"} { + for _, want := range []string{"bootstrap", "prune-agents", "rehydrate"} { if !seen[want] { t.Errorf("session missing subcommand %q", want) } } } +func TestPruneAgentsCommandDryRunSeamOverridesExecute(t *testing.T) { + dir := t.TempDir() + handoffDir := filepath.Join(dir, ".agents", "ao", "handoff") + if err := os.MkdirAll(handoffDir, 0o755); err != nil { + t.Fatal(err) + } + for i := 0; i < 12; i++ { + path := filepath.Join(handoffDir, fmt.Sprintf("handoff-%02d.json", i)) + if err := os.WriteFile(path, []byte("candidate\n"), 0o600); err != nil { + t.Fatal(err) + } + stamp := time.Date(2026, 8, 16, 1, i, 0, 0, time.UTC) + if err := os.Chtimes(path, stamp, stamp); err != nil { + t.Fatal(err) + } + } + module := NewModule(clicontract.HostOptions{ + DryRun: func() bool { return true }, + ProjectRoot: func() string { return dir }, + Now: func() time.Time { return time.Date(2026, 8, 16, 18, 0, 0, 0, time.UTC) }, + }) + root := module.Command() + var output bytes.Buffer + root.SetOut(&output) + root.SetArgs([]string{"prune-agents", "--execute"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(handoffDir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 12 { + t.Fatalf("global dry-run left %d handoffs, want 12", len(entries)) + } + if !strings.Contains(output.String(), "DRY RUN COMPLETE") { + t.Fatalf("global dry-run did not select read-only output:\n%s", output.String()) + } +} + func TestSessionBootstrapOnlyReportsLocalOrientation(t *testing.T) { dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, "AGENTS.md"), []byte("test"), 0o644); err != nil { @@ -113,6 +155,278 @@ func TestRehydrateReadsCallerAuthoredBrief(t *testing.T) { } } +func TestRehydrateReadsCanonicalHandoffDirectory(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + handoffDir := filepath.Join(dir, ".agents", "ao", "handoff") + if err := os.MkdirAll(handoffDir, 0o755); err != nil { + t.Fatal(err) + } + original := []byte(`{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","goal":"read the canonical handoff","continuation":"canonical path is visible"}` + "\n") + artifactPath := filepath.Join(handoffDir, "handoff-20260816T000000.000000000Z.json") + if err := os.WriteFile(artifactPath, original, 0o600); err != nil { + t.Fatal(err) + } + + root, _ := subcommand(t, "rehydrate") + var restored bytes.Buffer + root.SetOut(&restored) + root.SetArgs([]string{"rehydrate"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + if !strings.Contains(restored.String(), "canonical path is visible") { + t.Fatalf("canonical handoff missing: %s", restored.String()) + } + if after, err := os.ReadFile(artifactPath); err != nil || !bytes.Equal(original, after) { + t.Fatal("rehydrate mutated the canonical handoff artifact") + } +} + +func TestRehydrateChoosesLatestAcrossCanonicalAndLegacyDirectories(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + legacyDir := filepath.Join(dir, ".agents", "handoff") + canonicalDir := filepath.Join(dir, ".agents", "ao", "handoff") + for _, handoffDir := range []string{legacyDir, canonicalDir} { + if err := os.MkdirAll(handoffDir, 0o755); err != nil { + t.Fatal(err) + } + } + legacy := []byte(`{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","continuation":"older legacy artifact"}` + "\n") + canonical := []byte(`{"schema_version":1,"id":"handoff-20260816T000001.000000000Z","created_at":"2026-08-16T00:00:01Z","continuation":"newer canonical artifact"}` + "\n") + if err := os.WriteFile(filepath.Join(legacyDir, "handoff-20260816T000000.000000000Z.json"), legacy, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(canonicalDir, "handoff-20260816T000001.000000000Z.json"), canonical, 0o600); err != nil { + t.Fatal(err) + } + + root, _ := subcommand(t, "rehydrate") + var restored bytes.Buffer + root.SetOut(&restored) + root.SetArgs([]string{"rehydrate"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + if !strings.Contains(restored.String(), "newer canonical artifact") { + t.Fatalf("latest handoff not selected across directories: %s", restored.String()) + } +} + +func TestRehydrateChoosesNewerLegacyAcrossDirectories(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + legacyDir := filepath.Join(dir, ".agents", "handoff") + canonicalDir := filepath.Join(dir, ".agents", "ao", "handoff") + for _, handoffDir := range []string{legacyDir, canonicalDir} { + if err := os.MkdirAll(handoffDir, 0o755); err != nil { + t.Fatal(err) + } + } + canonical := []byte(`{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","continuation":"older canonical artifact"}` + "\n") + legacy := []byte(`{"schema_version":1,"id":"handoff-20260816T000001.000000000Z","created_at":"2026-08-16T00:00:01Z","continuation":"newer legacy artifact"}` + "\n") + if err := os.WriteFile(filepath.Join(canonicalDir, "handoff-20260816T000000.000000000Z.json"), canonical, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(legacyDir, "handoff-20260816T000001.000000000Z.json"), legacy, 0o600); err != nil { + t.Fatal(err) + } + + root, _ := subcommand(t, "rehydrate") + var restored bytes.Buffer + root.SetOut(&restored) + root.SetArgs([]string{"rehydrate"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + if !strings.Contains(restored.String(), "newer legacy artifact") { + t.Fatalf("newer legacy handoff not selected across directories: %s", restored.String()) + } +} + +func TestRehydratePrefersCanonicalDirectoryForDuplicateName(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + legacyDir := filepath.Join(dir, ".agents", "handoff") + canonicalDir := filepath.Join(dir, ".agents", "ao", "handoff") + for _, handoffDir := range []string{legacyDir, canonicalDir} { + if err := os.MkdirAll(handoffDir, 0o755); err != nil { + t.Fatal(err) + } + } + name := "handoff-20260816T000000.000000000Z.json" + legacy := []byte(`{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","continuation":"legacy duplicate"}` + "\n") + canonical := []byte(`{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","continuation":"canonical duplicate"}` + "\n") + if err := os.WriteFile(filepath.Join(legacyDir, name), legacy, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(canonicalDir, name), canonical, 0o600); err != nil { + t.Fatal(err) + } + + root, _ := subcommand(t, "rehydrate") + var restored bytes.Buffer + root.SetOut(&restored) + root.SetArgs([]string{"rehydrate"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + if !strings.Contains(restored.String(), "canonical duplicate") { + t.Fatalf("canonical directory did not win duplicate name: %s", restored.String()) + } +} + +func TestRehydrateFailsClosedWhenCanonicalRootIsNotDirectory(t *testing.T) { + for _, tc := range []struct { + name string + args []string + }{ + {name: "human", args: []string{"rehydrate"}}, + {name: "json", args: []string{"rehydrate", "--json"}}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + agents := filepath.Join(dir, ".agents") + legacyDir := filepath.Join(agents, "handoff") + if err := os.MkdirAll(legacyDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(agents, "ao"), []byte("not a directory\n"), 0o600); err != nil { + t.Fatal(err) + } + legacy := []byte(`{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","continuation":"valid legacy evidence"}` + "\n") + if err := os.WriteFile(filepath.Join(legacyDir, "handoff-20260816T000000.000000000Z.json"), legacy, 0o600); err != nil { + t.Fatal(err) + } + + root, _ := subcommand(t, "rehydrate") + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs(tc.args) + err := root.Execute() + if err == nil { + t.Fatal("rehydrate succeeded despite a non-directory canonical root") + } + if !strings.Contains(err.Error(), "not a real directory") { + t.Fatalf("error = %q, want unsafe canonical-root reason", err) + } + if strings.Contains(stdout.String(), "valid legacy evidence") || strings.TrimSpace(stdout.String()) == "{}" { + t.Fatalf("stdout = %q, want no fallback artifact or empty-state document", stdout.String()) + } + }) + } +} + +func TestRehydrateFailsClosedOnSymlinkedHandoffSources(t *testing.T) { + for _, tc := range []struct { + name string + setup func(t *testing.T, dir string) + }{ + { + name: "intermediate canonical root", + setup: func(t *testing.T, dir string) { + t.Helper() + agents := filepath.Join(dir, ".agents") + external := t.TempDir() + if err := os.MkdirAll(agents, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(external, filepath.Join(agents, "ao")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + }, + }, + { + name: "matching artifact", + setup: func(t *testing.T, dir string) { + t.Helper() + canonical := filepath.Join(dir, ".agents", "ao", "handoff") + if err := os.MkdirAll(canonical, 0o755); err != nil { + t.Fatal(err) + } + external := filepath.Join(t.TempDir(), "outside.json") + if err := os.WriteFile(external, []byte(`{"schema_version":1,"id":"handoff-20260816T000001.000000000Z","created_at":"2026-08-16T00:00:01Z","continuation":"outside secret"}`+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(external, filepath.Join(canonical, "handoff-20260816T000001.000000000Z.json")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + tc.setup(t, dir) + legacyDir := filepath.Join(dir, ".agents", "handoff") + if err := os.MkdirAll(legacyDir, 0o755); err != nil { + t.Fatal(err) + } + legacy := []byte(`{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","continuation":"valid legacy evidence"}` + "\n") + if err := os.WriteFile(filepath.Join(legacyDir, "handoff-20260816T000000.000000000Z.json"), legacy, 0o600); err != nil { + t.Fatal(err) + } + + root, _ := subcommand(t, "rehydrate") + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs([]string{"rehydrate", "--json"}) + err := root.Execute() + if err == nil { + t.Fatal("rehydrate succeeded through an unsafe handoff source") + } + if !strings.Contains(err.Error(), "not a real") { + t.Fatalf("error = %q, want unsafe source reason", err) + } + if strings.Contains(stdout.String(), "outside secret") || strings.Contains(stdout.String(), "valid legacy evidence") || strings.TrimSpace(stdout.String()) == "{}" { + t.Fatalf("stdout = %q, want no followed, fallback, or empty-state artifact", stdout.String()) + } + }) + } +} + +func TestRehydrateRejectsArtifactsOutsideHandoffV1Contract(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "missing required identity", body: `{"schema_version":1,"continuation":"not bound"}`}, + {name: "filename id mismatch", body: `{"schema_version":1,"id":"handoff-20260816T000001.000000000Z","created_at":"2026-08-16T00:00:00Z"}`}, + {name: "unknown property", body: `{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","lifecycle":"invented"}`}, + {name: "invalid nested state", body: `{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","state":{"git_branch":"main"}}`}, + {name: "null string", body: `{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","goal":null}`}, + {name: "null array", body: `{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","artifacts_produced":null}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + handoffDir := filepath.Join(dir, ".agents", "ao", "handoff") + if err := os.MkdirAll(handoffDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(handoffDir, "handoff-20260816T000000.000000000Z.json"), []byte(tc.body+"\n"), 0o600); err != nil { + t.Fatal(err) + } + + root, _ := subcommand(t, "rehydrate") + var stdout bytes.Buffer + root.SetOut(&stdout) + root.SetArgs([]string{"rehydrate", "--json"}) + if err := root.Execute(); err == nil { + t.Fatal("rehydrate accepted an artifact outside handoff.v1") + } + if strings.TrimSpace(stdout.String()) == "{}" { + t.Fatal("invalid artifact was reported as an honest empty state") + } + }) + } +} + // TestRehydrateJSONEmptyStateEmitsEmptyObject asserts that --json with no // handoff present emits exactly one JSON document `{}` on stdout (jq-safe), with // the human hint on stderr and exit 0. diff --git a/cli/internal/doctor/capabilities.go b/cli/internal/doctor/capabilities.go index 7432b65c5..9f6b3ccb3 100644 --- a/cli/internal/doctor/capabilities.go +++ b/cli/internal/doctor/capabilities.go @@ -30,7 +30,6 @@ var canonicalWriteScopes = []string{ ".doctor", ".agents", ".agents/daemon", - ".agents/handoff/sha256", ".agents/ao", ".agents/learnings", "~/.claude/settings.json", @@ -187,6 +186,21 @@ func EnsureInScope(caps *Capabilities, repoRoot, homeDir, path string) error { return fmt.Errorf("doctor: resolve path %s: %w", path, err) } abs = filepath.Clean(abs) + // `.agents/handoff` is retained solely as read-only compatibility evidence. + // The broader `.agents` scope must not accidentally authorize a fixer to + // mutate the legacy root or anything beneath it. New handoff writes belong + // under `.agents/ao/handoff`. + legacyHandoff, legacyErr := filepath.Abs(filepath.Join(repoRoot, ".agents", "handoff")) + if legacyErr != nil { + return fmt.Errorf("doctor: resolve legacy handoff root: %w", legacyErr) + } + legacyHandoff = filepath.Clean(legacyHandoff) + if abs == legacyHandoff { + return fmt.Errorf("doctor: path %s is legacy read-only handoff evidence (refused_unsafe)", path) + } + if rel, relErr := filepath.Rel(legacyHandoff, abs); relErr == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("doctor: path %s is legacy read-only handoff evidence (refused_unsafe)", path) + } for _, scope := range caps.WriteScopes { base := resolveScope(scope, repoRoot, homeDir) if abs == base { diff --git a/cli/internal/doctor/doctor_test.go b/cli/internal/doctor/doctor_test.go index e8f2d5e2a..0c55619a1 100644 --- a/cli/internal/doctor/doctor_test.go +++ b/cli/internal/doctor/doctor_test.go @@ -4,6 +4,7 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" "time" ) @@ -151,6 +152,54 @@ func TestEnsureInScope_RejectsOutOfScope(t *testing.T) { if err := EnsureInScope(caps, repo, home, traversal); err == nil { t.Fatal("path traversal escaped write scopes") } + for _, legacy := range []string{ + filepath.Join(repo, ".agents", "handoff"), + filepath.Join(repo, ".agents", "handoff", "handoff-20260816T000000Z.json"), + } { + if err := EnsureInScope(caps, repo, home, legacy); err == nil || !strings.Contains(err.Error(), "legacy read-only") { + t.Fatalf("legacy handoff path %s accepted: %v", legacy, err) + } + } + canonical := filepath.Join(repo, ".agents", "ao", "handoff", "handoff-20260816T000000Z.json") + if err := EnsureInScope(caps, repo, home, canonical); err != nil { + t.Fatalf("canonical handoff path rejected: %v", err) + } +} + +func TestMutateRefusesLegacyHandoffButAllowsCanonical(t *testing.T) { + repo := t.TempDir() + home := t.TempDir() + legacy := filepath.Join(repo, ".agents", "handoff", "handoff-20260816T000000Z.json") + canonical := filepath.Join(repo, ".agents", "ao", "handoff", "handoff-20260816T000001Z.json") + if err := os.MkdirAll(filepath.Dir(legacy), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(legacy, []byte("legacy evidence\n"), 0o600); err != nil { + t.Fatal(err) + } + ra, err := NewRunArtifact(repo, "scope", time.Now()) + if err != nil { + t.Fatal(err) + } + actions, err := ra.OpenActionsFile() + if err != nil { + t.Fatal(err) + } + defer func() { _ = actions.Close() }() + ctx := NewMutateContext(ra, NewCapabilities("2.0.0"), home, NewLockManager(filepath.Join(repo, ".doctor", "locks")), actions, false) + + if _, err := Mutate(ctx, legacy, WriteFile{Content: []byte("replacement\n"), Mode: 0o600}); err == nil || !strings.Contains(err.Error(), "legacy read-only") { + t.Fatalf("legacy Mutate error = %v, want read-only refusal", err) + } + if got, err := os.ReadFile(legacy); err != nil || string(got) != "legacy evidence\n" { + t.Fatalf("legacy evidence changed: %q err=%v", got, err) + } + if _, err := Mutate(ctx, canonical, WriteFile{Content: []byte("canonical evidence\n"), Mode: 0o600}); err != nil { + t.Fatalf("canonical Mutate rejected: %v", err) + } + if got, err := os.ReadFile(canonical); err != nil || string(got) != "canonical evidence\n" { + t.Fatalf("canonical evidence not written: %q err=%v", got, err) + } } // TestMutate_DryRunTouchesNothing verifies dry-run does not write. @@ -223,6 +272,17 @@ func TestCapabilities_JSONValidates(t *testing.T) { if len(caps.WriteScopes) == 0 { t.Fatal("write scopes must be populated from safety envelope") } + var hasCanonicalHandoff, hasLegacyHandoff bool + for _, scope := range caps.WriteScopes { + hasCanonicalHandoff = hasCanonicalHandoff || scope == ".agents/ao" + hasLegacyHandoff = hasLegacyHandoff || scope == ".agents/handoff/sha256" + } + if !hasCanonicalHandoff { + t.Fatal("write scopes must retain canonical .agents/ao") + } + if hasLegacyHandoff { + t.Fatal("write scopes must not advertise legacy .agents/handoff/sha256") + } if caps.ExitCodes["5"] != "concurrency_lost" { t.Fatalf("exit code 5 = %q, want concurrency_lost", caps.ExitCodes["5"]) } diff --git a/cli/internal/doctor/fix_workspace.go b/cli/internal/doctor/fix_workspace.go index b06fbf983..2cb31f2ce 100644 --- a/cli/internal/doctor/fix_workspace.go +++ b/cli/internal/doctor/fix_workspace.go @@ -14,14 +14,17 @@ package doctor // remove. import ( + "bytes" "encoding/json" "fmt" + "io" "io/fs" "os" "path/filepath" "regexp" "sort" "strconv" + "strings" "time" ) @@ -148,17 +151,19 @@ func workspaceDirInventory(base string) ([]workspaceDirInfo, error) { } // workspaceCanonicalAliases maps drifted spellings of top-level `.agents` -// directory names to their canonical names. The drift detector consumes this -// table verbatim: a key present as a top-level directory is a drift finding -// whose remediation is a merge/rename into the value directory. +// directory names to their canonical paths relative to `.agents`. Most targets +// remain top-level siblings. Handoff aliases land in `.agents/ao/handoff`; +// `.agents/handoff` is retained only as a read-compatibility source and is +// never a destination for Doctor repairs. `.agents/mto-handoff` is deliberately +// absent: it is the live, distinct recurrence protocol consumed by +// scripts/assay/consume-mto-recurrence.sh, not a spelling drift. var workspaceCanonicalAliases = map[string]string{ "post-mortem": "postmortem", "post-mortems": "postmortem", "pre-mortem": "pre-mortem-checks", "pre-mortems": "pre-mortem-checks", "premortem-checks": "pre-mortem-checks", - "handoffs": "handoff", - "mto-handoff": "handoff", + "handoffs": filepath.Join("ao", "handoff"), "retros": "retro", "proof": "proofs", "test": "tests", @@ -271,11 +276,16 @@ func workspaceDirRename(ctx *MutateContext, path, dest string, verify func(path // Step 2 — before-state: the source must exist and be a directory (never // follow a symlink into pretending it is one). - info, err := os.Lstat(path) + root, pathRel, destRel, err := openWorkspaceMutationRoot(ctx.RepoRoot, path, dest) + if err != nil { + return err + } + defer func() { _ = root.Close() }() + info, err := root.Lstat(pathRel) if err != nil { return fmt.Errorf("doctor: lstat %s: %w", path, err) } - if !info.IsDir() { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { return fmt.Errorf("doctor: rename-dir %s: not a directory (refused_unsafe)", path) } if verify != nil { @@ -303,8 +313,8 @@ func workspaceDirRename(ctx *MutateContext, path, dest string, verify func(path fmt.Fprintf(os.Stderr, "[dry-run] would mutate %s: %s\n", path, DescribeOp(op)) return nil } - if err := executeAtomic(path, op); err != nil { - return fmt.Errorf("doctor: execute Rename on %s: %w", path, err) + if err := workspaceExecuteDirRename(root, pathRel, destRel, path, dest, info); err != nil { + return err } // Step 7/8 — fsync'd action record. @@ -315,6 +325,41 @@ func workspaceDirRename(ctx *MutateContext, path, dest string, verify func(path // journal never sees the mutation — and the rename destination tree // (quarantine) is the manual recovery path: the run dir's receipts list // what moved, and nothing is ever deleted. + return workspaceJournalDirRename(ctx, root, pathRel, destRel, path, dest, info, op, startedNS) +} + +func workspaceExecuteDirRename(root *os.Root, pathRel, destRel, path, dest string, info os.FileInfo) error { + if err := workspaceRootParentsReal(root, pathRel, destRel); err != nil { + return err + } + if parent := filepath.Dir(destRel); parent != "." { + if err := root.MkdirAll(parent, 0o755); err != nil { + return fmt.Errorf("doctor: mkdir %s: %w", filepath.Dir(dest), err) + } + } + if err := workspaceRootParentsReal(root, pathRel, destRel); err != nil { + return err + } + if err := root.Rename(pathRel, destRel); err != nil { + return fmt.Errorf("doctor: execute Rename on %s: %w", path, err) + } + if bindErr := workspaceRootParentsReal(root, pathRel, destRel); bindErr != nil { + if backErr := root.Rename(destRel, pathRel); backErr != nil { + return fmt.Errorf("doctor: %w; compensating rename-back failed: %w", bindErr, backErr) + } + return fmt.Errorf("doctor: %w (compensated)", bindErr) + } + moved, err := root.Lstat(destRel) + if err != nil || moved.Mode()&os.ModeSymlink != 0 || !moved.IsDir() || !os.SameFile(info, moved) { + if backErr := root.Rename(destRel, pathRel); backErr != nil { + return fmt.Errorf("doctor: renamed directory %s changed identity; compensating rename-back failed: %w", path, backErr) + } + return fmt.Errorf("doctor: renamed directory %s changed identity (compensated; refused_unsafe)", path) + } + return nil +} + +func workspaceJournalDirRename(ctx *MutateContext, root *os.Root, pathRel, destRel, path, dest string, info os.FileInfo, op Rename, startedNS int64) error { rel, relErr := filepath.Rel(ctx.RepoRoot, path) if relErr != nil { rel = path @@ -340,7 +385,7 @@ func workspaceDirRename(ctx *MutateContext, path, dest string, verify func(path // rename. Compensate with a rename-back so disk state matches the // (empty) journal, and report both the journal error and the // compensation outcome. - if backErr := os.Rename(dest, path); backErr != nil { + if backErr := root.Rename(destRel, pathRel); backErr != nil { return fmt.Errorf("doctor: journal Rename of %s: %w; compensating rename-back FAILED (%w) — directory left at %s and is NOT recorded in actions.jsonl", path, aerr, backErr, dest) } return fmt.Errorf("doctor: journal Rename of %s: %w (compensated: directory renamed back to its original path; no mutation recorded)", path, aerr) @@ -426,16 +471,14 @@ func workspaceFileMoveNoClobber(ctx *MutateContext, path, dest string) (collided // Step 2 — before-state: the source must be a regular file (never follow // a symlink into pretending it is one). - info, lerr := os.Lstat(path) + root, pathRel, destRel, openErr := openWorkspaceMutationRoot(ctx.RepoRoot, path, dest) + if openErr != nil { + return false, openErr + } + defer func() { _ = root.Close() }() + info, beforeBytes, lerr := readWorkspaceRootRegular(root, pathRel) if lerr != nil { - return false, fmt.Errorf("doctor: lstat %s: %w", path, lerr) - } - if !info.Mode().IsRegular() { - return false, fmt.Errorf("doctor: move %s: not a regular file (refused_unsafe)", path) - } - beforeBytes, rerr := os.ReadFile(path) - if rerr != nil { - return false, fmt.Errorf("doctor: read %s: %w", path, rerr) + return false, fmt.Errorf("doctor: read stable regular file %s: %w", path, lerr) } beforeHash := sha256Hex(beforeBytes) @@ -452,15 +495,7 @@ func workspaceFileMoveNoClobber(ctx *MutateContext, path, dest string) (collided // Step 4 — verbatim backup (same as Mutate for an existing file). if !ctx.DryRun { - rel, relErr := filepath.Rel(ctx.RepoRoot, path) - if relErr != nil { - rel = filepath.Base(path) - } - backup := filepath.Join(ctx.RunDir, "backups", rel) - if err := copyVerbatim(path, backup); err != nil { - return false, fmt.Errorf("doctor: backup %s: %w", path, err) - } - if err := cmpStrict(path, backup); err != nil { + if err := workspaceWriteVerifiedBackup(ctx, path, beforeBytes, info); err != nil { return false, err } } @@ -471,28 +506,86 @@ func workspaceFileMoveNoClobber(ctx *MutateContext, path, dest string) (collided fmt.Fprintf(os.Stderr, "[dry-run] would mutate %s: %s\n", path, DescribeOp(op)) return false, nil } - if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { - return false, fmt.Errorf("doctor: mkdir %s: %w", filepath.Dir(dest), err) + collided, err = workspaceExecuteFileMoveNoClobber(root, pathRel, destRel, path, dest, info) + if err != nil { + return false, err } - if err := os.Link(path, dest); err != nil { - if os.IsExist(err) { - return true, nil // destination appeared — collision, nothing moved - } - return false, fmt.Errorf("doctor: link %s -> %s: %w", path, dest, err) - } - if err := os.Remove(path); err != nil { - // The link landed but the source could not be unlinked, leaving two - // paths to one inode. Compensate by removing the new link so the move - // stays all-or-nothing. - if unlinkErr := os.Remove(dest); unlinkErr != nil { - return false, fmt.Errorf("doctor: remove source %s after link: %w; compensating removal of %s ALSO failed (%w) — file is hard-linked at both paths and NOT recorded in actions.jsonl", path, err, dest, unlinkErr) - } - return false, fmt.Errorf("doctor: remove source %s after link: %w (compensated: link at %s removed; nothing moved)", path, err, dest) + if collided { + return true, nil } // Step 7/8 — fsync'd action record; same execute-before-journal parity and // crash exposure as workspaceDirRename (see the comment there), and the // same staged write/sync recovery split. + return false, workspaceJournalFileMove(ctx, root, pathRel, destRel, path, dest, info, op, beforeHash, startedNS) +} + +func workspaceWriteVerifiedBackup(ctx *MutateContext, path string, beforeBytes []byte, info os.FileInfo) error { + rel, relErr := filepath.Rel(ctx.RepoRoot, path) + if relErr != nil { + rel = filepath.Base(path) + } + backup := filepath.Join(ctx.RunDir, "backups", rel) + if err := writeWorkspaceBackup(backup, beforeBytes, info); err != nil { + return fmt.Errorf("doctor: backup %s: %w", path, err) + } + backupBytes, err := os.ReadFile(backup) + if err != nil || !bytes.Equal(beforeBytes, backupBytes) { + return fmt.Errorf("backup verify failed (cmp-strict mismatch for %s)", path) + } + return nil +} + +func workspaceExecuteFileMoveNoClobber(root *os.Root, pathRel, destRel, path, dest string, info os.FileInfo) (bool, error) { + if err := workspaceRootParentsReal(root, pathRel, destRel); err != nil { + return false, err + } + if err := root.MkdirAll(filepath.Dir(destRel), 0o755); err != nil { + return false, fmt.Errorf("doctor: mkdir %s: %w", filepath.Dir(dest), err) + } + if err := workspaceRootParentsReal(root, pathRel, destRel); err != nil { + return false, err + } + if err := root.Link(pathRel, destRel); err != nil { + if os.IsExist(err) { + return true, nil // destination appeared — collision, nothing moved + } + return false, fmt.Errorf("doctor: link %s -> %s: %w", path, dest, err) + } + if bindErr := workspaceRootParentsReal(root, pathRel, destRel); bindErr != nil { + if unlinkErr := root.Remove(destRel); unlinkErr != nil { + return false, fmt.Errorf("doctor: %w; compensating removal failed: %w", bindErr, unlinkErr) + } + return false, fmt.Errorf("doctor: %w (compensated)", bindErr) + } + if workspaceFileMoveIdentityChanged(root, pathRel, destRel, info) { + if unlinkErr := root.Remove(destRel); unlinkErr != nil { + return false, fmt.Errorf("doctor: source or destination changed identity during move of %s; compensating removal failed: %w", path, unlinkErr) + } + return false, fmt.Errorf("doctor: source or destination changed identity during move of %s (compensated; refused_unsafe)", path) + } + if err := root.Remove(pathRel); err != nil { + // The link landed but the source could not be unlinked, leaving two + // paths to one inode. Compensate by removing the new link so the move + // stays all-or-nothing. + if unlinkErr := root.Remove(destRel); unlinkErr != nil { + return false, fmt.Errorf("doctor: remove source %s after link: %w; compensating removal of %s ALSO failed (%w) — file is hard-linked at both paths and NOT recorded in actions.jsonl", path, err, dest, unlinkErr) + } + return false, fmt.Errorf("doctor: remove source %s after link: %w (compensated: link at %s removed; nothing moved)", path, err, dest) + } + return false, nil +} + +func workspaceFileMoveIdentityChanged(root *os.Root, pathRel, destRel string, expected os.FileInfo) bool { + destInfo, destErr := root.Lstat(destRel) + sourceInfo, sourceErr := root.Lstat(pathRel) + return destErr != nil || sourceErr != nil || + destInfo.Mode()&os.ModeSymlink != 0 || sourceInfo.Mode()&os.ModeSymlink != 0 || + !destInfo.Mode().IsRegular() || !sourceInfo.Mode().IsRegular() || + !os.SameFile(expected, destInfo) || !os.SameFile(expected, sourceInfo) +} + +func workspaceJournalFileMove(ctx *MutateContext, root *os.Root, pathRel, destRel, path, dest string, info os.FileInfo, op Rename, beforeHash string, startedNS int64) error { rel, relErr := filepath.Rel(ctx.RepoRoot, path) if relErr != nil { rel = path @@ -514,16 +607,152 @@ func workspaceFileMoveNoClobber(ctx *MutateContext, path, dest string) (collided if !wrote { // WRITE-stage failure: record definitely not persisted; move the // file back so disk matches the (empty) journal. - if backErr := os.Rename(dest, path); backErr != nil { - return false, fmt.Errorf("doctor: journal Rename of %s: %w; compensating move-back FAILED (%w) — file left at %s and is NOT recorded in actions.jsonl", path, aerr, backErr, dest) + if backErr := root.Rename(destRel, pathRel); backErr != nil { + return fmt.Errorf("doctor: journal Rename of %s: %w; compensating move-back FAILED (%w) — file left at %s and is NOT recorded in actions.jsonl", path, aerr, backErr, dest) } - return false, fmt.Errorf("doctor: journal Rename of %s: %w (compensated: file moved back to its original path; no mutation recorded)", path, aerr) + return fmt.Errorf("doctor: journal Rename of %s: %w (compensated: file moved back to its original path; no mutation recorded)", path, aerr) } // SYNC-stage failure: record probably persisted; leave the move in // place so state and journal stay consistent (see workspaceDirRename). - return false, fmt.Errorf("doctor: journal Rename of %s: record written but not durably synced (%w) — move left in place at %s; actions.jsonl durability is uncertain until the next successful sync", path, aerr, dest) + return fmt.Errorf("doctor: journal Rename of %s: record written but not durably synced (%w) — move left in place at %s; actions.jsonl durability is uncertain until the next successful sync", path, aerr, dest) } - return false, nil + return nil +} + +// openWorkspaceMutationRoot anchors a two-path workspace mutation at the +// repository directory descriptor. os.Root refuses any symlink traversal that +// would escape that descriptor, including a parent swapped after preflight. +func openWorkspaceMutationRoot(repoRoot, path, dest string) (*os.Root, string, string, error) { + root, err := os.OpenRoot(repoRoot) + if err != nil { + return nil, "", "", fmt.Errorf("doctor: open repository root: %w", err) + } + pathRel, err := workspaceMutationRelative(repoRoot, path) + if err != nil { + _ = root.Close() + return nil, "", "", err + } + destRel, err := workspaceMutationRelative(repoRoot, dest) + if err != nil { + _ = root.Close() + return nil, "", "", err + } + return root, pathRel, destRel, nil +} + +func workspaceMutationRelative(repoRoot, path string) (string, error) { + absRoot, err := filepath.Abs(repoRoot) + if err != nil { + return "", err + } + absPath, err := filepath.Abs(path) + if err != nil { + return "", err + } + rel, err := filepath.Rel(absRoot, absPath) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("doctor: path %s is not a repository child (refused_unsafe)", path) + } + return rel, nil +} + +// workspaceRootParentsReal rejects every existing parent component that is a +// symlink or non-directory. It is called immediately around rooted mutations: +// os.Root prevents escape from the repository, while this check also rejects +// a race that redirects a path through a symlink to another in-repo tree. +func workspaceRootParentsReal(root *os.Root, names ...string) error { + for _, name := range names { + parent := filepath.Dir(name) + if parent == "." { + continue + } + current := "" + for _, component := range strings.Split(parent, string(filepath.Separator)) { + if component == "" || component == "." { + continue + } + current = filepath.Join(current, component) + info, err := root.Lstat(current) + if err != nil { + if os.IsNotExist(err) { + break + } + return fmt.Errorf("doctor: inspect mutation parent %s: %w", current, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("doctor: mutation parent %s is not a real directory (refused_unsafe)", current) + } + } + } + return nil +} + +func readWorkspaceRootRegular(root *os.Root, name string) (os.FileInfo, []byte, error) { + before, err := root.Lstat(name) + if err != nil { + return nil, nil, err + } + if before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() { + return nil, nil, fmt.Errorf("not a real regular file (refused_unsafe)") + } + file, err := root.Open(name) + if err != nil { + return nil, nil, err + } + defer func() { _ = file.Close() }() + opened, err := file.Stat() + if err != nil { + return nil, nil, err + } + afterOpen, err := root.Lstat(name) + if err != nil || afterOpen.Mode()&os.ModeSymlink != 0 || !afterOpen.Mode().IsRegular() || !os.SameFile(before, opened) || !os.SameFile(afterOpen, opened) { + return nil, nil, fmt.Errorf("changed identity while opening (refused_unsafe)") + } + first, err := io.ReadAll(file) + if err != nil { + return nil, nil, err + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, nil, err + } + second, err := io.ReadAll(file) + if err != nil { + return nil, nil, err + } + openedAfter, err := file.Stat() + if err != nil { + return nil, nil, err + } + pathAfter, err := root.Lstat(name) + if err != nil || pathAfter.Mode()&os.ModeSymlink != 0 || !pathAfter.Mode().IsRegular() || !os.SameFile(opened, openedAfter) || !os.SameFile(pathAfter, openedAfter) || opened.Size() != openedAfter.Size() || !opened.ModTime().Equal(openedAfter.ModTime()) || !bytes.Equal(first, second) { + return nil, nil, fmt.Errorf("changed while reading (refused_unsafe)") + } + return openedAfter, first, nil +} + +func writeWorkspaceBackup(path string, data []byte, info os.FileInfo) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return err + } + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + if err := file.Sync(); err != nil { + _ = file.Close() + return err + } + if err := file.Close(); err != nil { + return err + } + if err := os.Chmod(path, info.Mode()); err != nil { + return err + } + return os.Chtimes(path, info.ModTime(), info.ModTime()) } // workspaceQuarantineDirByName validates name as a bare path element (a diff --git a/cli/internal/doctor/fix_workspace_drift.go b/cli/internal/doctor/fix_workspace_drift.go index d3135ca16..6135fd2cd 100644 --- a/cli/internal/doctor/fix_workspace_drift.go +++ b/cli/internal/doctor/fix_workspace_drift.go @@ -4,7 +4,7 @@ package doctor // // Flags top-level `.agents` directories whose names are drifted spellings of a // canonical directory (the workspaceCanonicalAliases registry: post-mortem -> -// postmortem, handoffs -> handoff, proof -> proofs, ...). Drifted names split +// postmortem, handoffs -> ao/handoff, proof -> proofs, ...). Drifted names split // one logical artifact family across two directories, so tooling that reads // only the canonical name silently misses half the corpus. // @@ -14,7 +14,7 @@ package doctor // matches findings to fixers by ID equality, so all fm-ws-naming-drift // findings route to the one fixer in a single Fix call). // -// The fixer merges each alias directory into its canonical sibling entry by +// The fixer merges each alias directory into its canonical destination entry by // entry under migration-owner discipline: an entry whose destination name // already exists in the canonical directory, or an entry that is neither a // regular file nor a directory (symlink, socket, ...), is NEVER moved and @@ -38,11 +38,17 @@ import ( "os" "path/filepath" "sort" + "strings" ) // fmWorkspaceNamingDriftID is the shared detector/fixer ID for this failure mode. const fmWorkspaceNamingDriftID = "fm-ws-naming-drift" +// workspaceAliasMutationTestHook deterministically opens the final +// preflight-to-mutation race window in package tests. Production leaves it +// nil; all real mutation remains rooted by an os.Root descriptor. +var workspaceAliasMutationTestHook func() + func init() { RegisterDetector(workspaceNamingDriftDetector{}) RegisterFixer(workspaceNamingDriftFixer{}) @@ -64,7 +70,7 @@ func sortedDriftAliases() []string { // --------------------------------------------------------------------------- // workspaceNamingDriftDetector flags each top-level `.agents` directory whose -// name is a registered drifted alias of a canonical directory name. +// name is a registered drifted alias of a canonical path. type workspaceNamingDriftDetector struct{} func (workspaceNamingDriftDetector) ID() string { return fmWorkspaceNamingDriftID } @@ -129,7 +135,7 @@ func (d workspaceNamingDriftDetector) Detect(env *DetectEnv) ([]Finding, error) } // workspaceNamingDriftFixer merges each alias directory into its canonical -// sibling entry by entry, skipping (never overwriting) destination collisions +// destination entry by entry, skipping (never overwriting) destination collisions // and non-regular non-directory oddities, then quarantines the emptied alias // directory. It re-scans the disk at fix time rather than trusting findings. type workspaceNamingDriftFixer struct{} @@ -175,7 +181,7 @@ func (f workspaceNamingDriftFixer) Fix(ctx *MutateContext, env *DetectEnv, _ []F return res, nil } -// fixOneAlias merges one alias directory into its canonical sibling. An absent +// fixOneAlias merges one alias directory into its canonical destination. An absent // alias is a no-op (idempotency); a non-directory alias path is recorded in // Skipped and left alone. Skipped entries stay in place, and the alias dir is // quarantined only once it holds nothing at all. @@ -197,20 +203,14 @@ func (f workspaceNamingDriftFixer) fixOneAlias(ctx *MutateContext, base, alias s } canonicalName := workspaceCanonicalAliases[alias] canonicalDir := filepath.Join(base, canonicalName) - // Destination-root guard: the canonical dir may ITSELF be a symlink (or a - // regular file). Moving entries "into" a symlinked canonical dir would - // follow the link — potentially outside the repo — while every per-entry - // lexical scope check still passes. An ABSENT canonical dir is fine (the - // first move creates it); anything present must be a real directory, and a - // non-NotExist Lstat error means its state is unknown — never assume - // absent, skip the whole alias. - if cfi, cerr := os.Lstat(canonicalDir); cerr == nil { - if cfi.Mode()&os.ModeSymlink != 0 || !cfi.IsDir() { - res.Skipped = append(res.Skipped, fmt.Sprintf("%s: canonical dir .agents/%s is not a real directory; resolve by hand", aliasRel, canonicalName)) - return nil - } - } else if !os.IsNotExist(cerr) { - res.Skipped = append(res.Skipped, fmt.Sprintf("%s: cannot verify canonical dir .agents/%s (%v); resolve by hand", aliasRel, canonicalName, cerr)) + // Destination-tree guard: every existing component below `.agents` must be + // a real directory. Checking only the leaf is insufficient for nested + // canonical paths such as ao/handoff: Lstat on that leaf follows a symlinked + // ao parent and can make an external directory look safe. Missing components + // are fine (the move creates them); unknown or non-directory components make + // the whole alias unsafe. + if unsafeReason := workspaceCanonicalPathUnsafe(base, canonicalName); unsafeReason != "" { + res.Skipped = append(res.Skipped, fmt.Sprintf("%s: %s", aliasRel, unsafeReason)) return nil } entries, err := os.ReadDir(aliasPath) @@ -227,6 +227,16 @@ func (f workspaceNamingDriftFixer) fixOneAlias(ctx *MutateContext, base, alias s res.Skipped = append(res.Skipped, fmt.Sprintf("%s: not a regular file or directory; resolve by hand", srcRel)) continue } + if e.IsDir() { + unsafeEntry, inspectErr := workspaceDirectoryTreeSpecial(src) + if inspectErr != nil { + return fmt.Errorf("doctor: %s: inspect %s: %w", f.ID(), srcRel, inspectErr) + } + if unsafeEntry != "" { + res.Skipped = append(res.Skipped, fmt.Sprintf("%s: nested special entry %s; resolve by hand", srcRel, unsafeEntry)) + continue + } + } skipReason, moveErr := f.moveEntryNoClobber(ctx, src, dest, e.IsDir()) if moveErr != nil { return fmt.Errorf("doctor: %s: move %s: %w", f.ID(), srcRel, moveErr) @@ -257,6 +267,43 @@ func (f workspaceNamingDriftFixer) fixOneAlias(ctx *MutateContext, base, alias s return nil } +// workspaceCanonicalPathUnsafe validates an alias migration destination +// without following symlinks. base itself is checked by Fix before this helper +// runs; this walks each existing relative component with Lstat so a nested +// destination cannot escape through an intermediate symlink. An empty return +// means the existing prefix is safe. The first absent component ends the walk, +// because no deeper component can exist without traversing that missing path. +func workspaceCanonicalPathUnsafe(base, canonicalName string) string { + clean := filepath.Clean(canonicalName) + if clean == "." || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return fmt.Sprintf("canonical path .agents/%s is invalid; resolve by hand", canonicalName) + } + + current := base + for _, component := range strings.Split(clean, string(filepath.Separator)) { + current = filepath.Join(current, component) + info, err := os.Lstat(current) + if err != nil { + if os.IsNotExist(err) { + return "" + } + rel, relErr := filepath.Rel(base, current) + if relErr != nil { + rel = clean + } + return fmt.Sprintf("cannot verify canonical path component .agents/%s (%v); resolve by hand", rel, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + rel, relErr := filepath.Rel(base, current) + if relErr != nil { + rel = clean + } + return fmt.Sprintf("canonical path component .agents/%s is not a real directory; resolve by hand", rel) + } + } + return "" +} + // errWorkspaceDriftDestExists is the sentinel raised by the under-lock // destination recheck when a same-named entry appeared at the destination // after the caller's scan. It is classified as a collision (Skipped), never @@ -302,7 +349,21 @@ func (workspaceNamingDriftFixer) moveEntryNoClobber(ctx *MutateContext, src, des return fmt.Sprintf("cannot verify destination in .agents/%s (%v); resolve by hand", filepath.Base(filepath.Dir(dest)), lerr), nil } if isDir { + unsafeEntry, inspectErr := workspaceDirectoryTreeSpecial(src) + if inspectErr != nil { + return "", inspectErr + } + if unsafeEntry != "" { + return fmt.Sprintf("nested special entry %s; resolve by hand", unsafeEntry), nil + } verify := func(string) error { + unsafeEntry, inspectErr := workspaceDirectoryTreeSpecial(src) + if inspectErr != nil { + return inspectErr + } + if unsafeEntry != "" { + return fmt.Errorf("doctor: directory %s gained nested special entry %s (refused_unsafe)", src, unsafeEntry) + } if _, lerr := os.Lstat(dest); lerr == nil { return errWorkspaceDriftDestExists } else if !os.IsNotExist(lerr) { @@ -310,6 +371,9 @@ func (workspaceNamingDriftFixer) moveEntryNoClobber(ctx *MutateContext, src, des } return nil } + if workspaceAliasMutationTestHook != nil { + workspaceAliasMutationTestHook() + } if moveErr := workspaceDirRename(ctx, src, dest, verify); moveErr != nil { if errors.Is(moveErr, errWorkspaceDriftDestExists) { return collisionReason, nil @@ -318,6 +382,9 @@ func (workspaceNamingDriftFixer) moveEntryNoClobber(ctx *MutateContext, src, des } return "", nil } + if workspaceAliasMutationTestHook != nil { + workspaceAliasMutationTestHook() + } collided, moveErr := workspaceFileMoveNoClobber(ctx, src, dest) if moveErr != nil { return "", moveErr @@ -328,6 +395,85 @@ func (workspaceNamingDriftFixer) moveEntryNoClobber(ctx *MutateContext, src, des return "", nil } +// workspaceDirectoryTreeSpecial returns the first nested path whose type is +// neither a real directory nor a regular file. It walks through +// descriptor-anchored roots and revalidates directory identities around each +// open, so a symlink cannot be smuggled into a directory-shaped alias entry. +func workspaceDirectoryTreeSpecial(path string) (string, error) { + before, err := os.Lstat(path) + if err != nil { + return "", err + } + if before.Mode()&os.ModeSymlink != 0 || !before.IsDir() { + return filepath.Base(path), nil + } + root, err := os.OpenRoot(path) + if err != nil { + return "", err + } + defer func() { _ = root.Close() }() + opened, err := root.Stat(".") + if err != nil { + return "", err + } + after, err := os.Lstat(path) + if err != nil { + return "", err + } + if after.Mode()&os.ModeSymlink != 0 || !after.IsDir() || !os.SameFile(before, opened) || !os.SameFile(after, opened) { + return filepath.Base(path), nil + } + return workspaceRootTreeSpecial(root, "") +} + +func workspaceRootTreeSpecial(root *os.Root, prefix string) (string, error) { + dir, err := root.Open(".") + if err != nil { + return "", err + } + entries, readErr := dir.ReadDir(-1) + closeErr := dir.Close() + if readErr != nil { + return "", readErr + } + if closeErr != nil { + return "", closeErr + } + for _, entry := range entries { + name := entry.Name() + display := filepath.Join(prefix, name) + before, err := root.Lstat(name) + if err != nil { + return "", err + } + if before.Mode()&os.ModeSymlink != 0 { + return display, nil + } + if before.Mode().IsRegular() { + continue + } + if !before.IsDir() { + return display, nil + } + child, err := root.OpenRoot(name) + if err != nil { + return "", err + } + opened, openedErr := child.Stat(".") + after, afterErr := root.Lstat(name) + if openedErr != nil || afterErr != nil || after.Mode()&os.ModeSymlink != 0 || !after.IsDir() || !os.SameFile(before, opened) || !os.SameFile(after, opened) { + _ = child.Close() + return display, nil + } + unsafeEntry, walkErr := workspaceRootTreeSpecial(child, display) + _ = child.Close() + if walkErr != nil || unsafeEntry != "" { + return unsafeEntry, walkErr + } + } + return "", nil +} + // workspaceRenameDir renames a directory through the shared workspace // directory-rename chokepoint adapter (workspaceDirRename): same per-path // lock, scope/op preconditions on both endpoints, dry-run transparency, diff --git a/cli/internal/doctor/fix_workspace_drift_test.go b/cli/internal/doctor/fix_workspace_drift_test.go index d3ed63cdc..5b9bffe06 100644 --- a/cli/internal/doctor/fix_workspace_drift_test.go +++ b/cli/internal/doctor/fix_workspace_drift_test.go @@ -313,13 +313,58 @@ func TestWorkspaceNamingDrift_CanonicalAbsentCreated(t *testing.T) { } } +func TestWorkspaceNamingDrift_HandoffAliasLandsCanonicalLegacyAndMTOUntouched(t *testing.T) { + env, repo := namingDriftEnv(t) + agents := filepath.Join(repo, ".agents") + legacyDir := filepath.Join(agents, "handoff") + canonicalDir := filepath.Join(agents, "ao", "handoff") + legacyPath := filepath.Join(legacyDir, "handoff-20260815T120000.000000000Z.json") + legacyBytes := "{\"schema_version\":1,\"continuation\":\"legacy evidence\"}\n" + mtoPath := filepath.Join(agents, "mto-handoff", "recurrence.json") + mtoBytes := "{\"recurred_classes\":0,\"date\":\"2026-08-16\"}\n" + writeDriftFile(t, legacyPath, legacyBytes) + writeDriftFile(t, mtoPath, mtoBytes) + writeDriftFile(t, filepath.Join(agents, "handoffs", "from-handoffs.json"), "plural alias") + + ctx, _ := newNamingDriftCtx(t, repo, false) + res, err := workspaceNamingDriftFixer{}.Fix(ctx, env, nil) + if err != nil { + t.Fatalf("Fix: %v", err) + } + if !res.Fixed || len(res.Skipped) != 0 { + t.Fatalf("Fix result: fixed=%t skipped=%v, want clean", res.Fixed, res.Skipped) + } + if res.ActionsTaken != 2 { + t.Fatalf("ActionsTaken = %d, want 2 (one file and one alias quarantine)", res.ActionsTaken) + } + if got := readDriftFile(t, filepath.Join(canonicalDir, "from-handoffs.json")); got != "plural alias" { + t.Errorf("canonical plural-alias file = %q", got) + } + if got := readDriftFile(t, mtoPath); got != mtoBytes { + t.Fatalf("MTO recurrence handoff changed: got %q want %q", got, mtoBytes) + } + if _, err := os.Lstat(filepath.Join(canonicalDir, "recurrence.json")); !os.IsNotExist(err) { + t.Fatalf("MTO recurrence handoff was copied into canonical session handoffs (err=%v)", err) + } + if got := readDriftFile(t, legacyPath); got != legacyBytes { + t.Fatalf("legacy evidence changed: got %q want %q", got, legacyBytes) + } + legacyEntries, err := os.ReadDir(legacyDir) + if err != nil { + t.Fatalf("read legacy handoff directory: %v", err) + } + if len(legacyEntries) != 1 || legacyEntries[0].Name() != filepath.Base(legacyPath) { + t.Fatalf("legacy handoff directory changed: %v", legacyEntries) + } +} + // A symlink entry inside the alias dir is never moved: moving it could change // what it resolves to. It is reported in Skipped and the alias dir remains. func TestWorkspaceNamingDrift_SymlinkEntrySkipped(t *testing.T) { env, repo := namingDriftEnv(t) agents := filepath.Join(repo, ".agents") alias := filepath.Join(agents, "handoffs") - canonical := filepath.Join(agents, "handoff") + canonical := filepath.Join(agents, "ao", "handoff") writeDriftFile(t, filepath.Join(alias, "real.md"), "real body") linkTarget := filepath.Join(repo, "outside.txt") writeDriftFile(t, linkTarget, "outside") @@ -359,6 +404,145 @@ func TestWorkspaceNamingDrift_SymlinkEntrySkipped(t *testing.T) { } } +func TestWorkspaceNamingDrift_NestedSpecialEntryRejectsWholeDirectory(t *testing.T) { + env, repo := namingDriftEnv(t) + agents := filepath.Join(repo, ".agents") + alias := filepath.Join(agents, "retros") + canonical := filepath.Join(agents, "retro") + external := filepath.Join(t.TempDir(), "outside.txt") + writeDriftFile(t, external, "outside bytes") + writeDriftFile(t, filepath.Join(alias, "sub", "real.md"), "nested body") + if err := os.Symlink(external, filepath.Join(alias, "sub", "nested-link")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + + ctx, ra := newNamingDriftCtx(t, repo, false) + res, err := workspaceNamingDriftFixer{}.Fix(ctx, env, nil) + if err != nil { + t.Fatalf("Fix: %v", err) + } + if res.Fixed || res.ActionsTaken != 0 { + t.Fatalf("Fix result fixed=%t actions=%d, want nested tree refused", res.Fixed, res.ActionsTaken) + } + found := false + for _, skipped := range res.Skipped { + if strings.Contains(skipped, "nested special entry") && strings.Contains(skipped, "nested-link") { + found = true + } + } + if !found { + t.Fatalf("Skipped = %v, want nested special entry named", res.Skipped) + } + if got := readDriftFile(t, filepath.Join(alias, "sub", "real.md")); got != "nested body" { + t.Fatalf("source tree changed: %q", got) + } + if _, err := os.Lstat(filepath.Join(canonical, "sub")); !os.IsNotExist(err) { + t.Fatalf("unsafe directory moved to canonical path: %v", err) + } + if got := readDriftFile(t, external); got != "outside bytes" { + t.Fatalf("outside target changed: %q", got) + } + recs, err := readActions(ra.ActionsPath()) + if err != nil { + t.Fatal(err) + } + if len(recs) != 0 { + t.Fatalf("refused tree wrote %d action records", len(recs)) + } +} + +func TestWorkspaceNamingDrift_ParentSymlinkSwapCannotEscapeRootedMove(t *testing.T) { + env, repo := namingDriftEnv(t) + agents := filepath.Join(repo, ".agents") + alias := filepath.Join(agents, "handoffs") + canonicalParent := filepath.Join(agents, "ao") + external := t.TempDir() + writeDriftFile(t, filepath.Join(alias, "x.json"), "alias bytes") + if err := os.MkdirAll(filepath.Join(canonicalParent, "handoff"), 0o755); err != nil { + t.Fatal(err) + } + writeDriftFile(t, filepath.Join(external, "handoff", "sentinel.json"), "outside bytes") + + workspaceAliasMutationTestHook = func() { + workspaceAliasMutationTestHook = nil + if err := os.Rename(canonicalParent, canonicalParent+".original"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(external, canonicalParent); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + } + t.Cleanup(func() { workspaceAliasMutationTestHook = nil }) + + ctx, ra := newNamingDriftCtx(t, repo, false) + res, err := workspaceNamingDriftFixer{}.Fix(ctx, env, nil) + if err == nil { + t.Fatalf("Fix unexpectedly succeeded after canonical parent swap: %+v", res) + } + if got := readDriftFile(t, filepath.Join(alias, "x.json")); got != "alias bytes" { + t.Fatalf("alias source changed: %q", got) + } + if got := readDriftFile(t, filepath.Join(external, "handoff", "sentinel.json")); got != "outside bytes" { + t.Fatalf("outside sentinel changed: %q", got) + } + if _, err := os.Lstat(filepath.Join(external, "handoff", "x.json")); !os.IsNotExist(err) { + t.Fatalf("Doctor wrote through swapped symlink: %v", err) + } + recs, readErr := readActions(ra.ActionsPath()) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 0 { + t.Fatalf("refused rooted move wrote %d action records", len(recs)) + } +} + +func TestWorkspaceNamingDrift_ParentSymlinkSwapCannotRedirectInsideRepo(t *testing.T) { + env, repo := namingDriftEnv(t) + agents := filepath.Join(repo, ".agents") + alias := filepath.Join(agents, "handoffs") + canonicalParent := filepath.Join(agents, "ao") + redirect := filepath.Join(repo, "redirect") + writeDriftFile(t, filepath.Join(alias, "x.json"), "alias bytes") + if err := os.MkdirAll(filepath.Join(canonicalParent, "handoff"), 0o755); err != nil { + t.Fatal(err) + } + writeDriftFile(t, filepath.Join(redirect, "handoff", "sentinel.json"), "redirect bytes") + + workspaceAliasMutationTestHook = func() { + workspaceAliasMutationTestHook = nil + if err := os.Rename(canonicalParent, canonicalParent+".original"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join("..", "redirect"), canonicalParent); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + } + t.Cleanup(func() { workspaceAliasMutationTestHook = nil }) + + ctx, ra := newNamingDriftCtx(t, repo, false) + res, err := workspaceNamingDriftFixer{}.Fix(ctx, env, nil) + if err == nil || !strings.Contains(err.Error(), "not a real directory") { + t.Fatalf("Fix result=%+v error=%v, want internal symlink refusal", res, err) + } + if got := readDriftFile(t, filepath.Join(alias, "x.json")); got != "alias bytes" { + t.Fatalf("alias source changed: %q", got) + } + if got := readDriftFile(t, filepath.Join(redirect, "handoff", "sentinel.json")); got != "redirect bytes" { + t.Fatalf("redirect sentinel changed: %q", got) + } + if _, err := os.Lstat(filepath.Join(redirect, "handoff", "x.json")); !os.IsNotExist(err) { + t.Fatalf("Doctor wrote through internal symlink: %v", err) + } + recs, readErr := readActions(ra.ActionsPath()) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 0 { + t.Fatalf("refused internal redirect wrote %d action records", len(recs)) + } +} + // TestWorkspaceNamingDrift_LateDestinationCollisionSkipped exercises the // under-lock destination recheck directly: a destination file that appears // AFTER the fixer's directory scan (the detect→rename window) is a collision @@ -456,8 +640,8 @@ func TestWorkspaceNamingDrift_CanonicalSymlinkSkipsWholeAlias(t *testing.T) { t.Fatalf("Skipped = %v, want both aliases skipped whole", res.Skipped) } for _, s := range res.Skipped { - if !strings.Contains(s, "canonical dir") || !strings.Contains(s, "not a real directory") { - t.Errorf("Skipped entry = %q, want the canonical-dir reason", s) + if !strings.Contains(s, "canonical path component") || !strings.Contains(s, "not a real directory") { + t.Errorf("Skipped entry = %q, want the canonical-path-component reason", s) } } // Alias content untouched, external target untouched (only its sentinel). @@ -483,6 +667,50 @@ func TestWorkspaceNamingDrift_CanonicalSymlinkSkipsWholeAlias(t *testing.T) { } } +func TestWorkspaceNamingDrift_CanonicalParentSymlinkSkipsWholeAlias(t *testing.T) { + env, repo := namingDriftEnv(t) + agents := filepath.Join(repo, ".agents") + alias := filepath.Join(agents, "handoffs") + external := t.TempDir() + externalHandoff := filepath.Join(external, "handoff") + writeDriftFile(t, filepath.Join(alias, "x.json"), "alias body") + writeDriftFile(t, filepath.Join(externalHandoff, "sentinel.json"), "external sentinel") + if err := os.Symlink(external, filepath.Join(agents, "ao")); err != nil { + t.Fatalf("symlink .agents/ao: %v", err) + } + + ctx, ra := newNamingDriftCtx(t, repo, false) + res, err := workspaceNamingDriftFixer{}.Fix(ctx, env, nil) + if err != nil { + t.Fatalf("Fix: %v", err) + } + if res.Fixed { + t.Error("Fix reported Fixed despite a symlinked canonical parent") + } + if res.ActionsTaken != 0 { + t.Fatalf("ActionsTaken = %d, want 0", res.ActionsTaken) + } + if len(res.Skipped) != 1 || !strings.Contains(res.Skipped[0], "canonical path component") || !strings.Contains(res.Skipped[0], "not a real directory") { + t.Fatalf("Skipped = %v, want the symlinked canonical parent refusal", res.Skipped) + } + if got := readDriftFile(t, filepath.Join(alias, "x.json")); got != "alias body" { + t.Fatalf("alias file changed: %q", got) + } + if got := readDriftFile(t, filepath.Join(externalHandoff, "sentinel.json")); got != "external sentinel" { + t.Fatalf("external sentinel changed: %q", got) + } + if _, err := os.Lstat(filepath.Join(externalHandoff, "x.json")); !os.IsNotExist(err) { + t.Fatalf("Doctor wrote through .agents/ao symlink (err=%v)", err) + } + recs, err := readActions(ra.ActionsPath()) + if err != nil { + t.Fatal(err) + } + if len(recs) != 0 { + t.Fatalf("refused alias wrote %d action records, want 0", len(recs)) + } +} + // TestWorkspaceNamingDrift_DestLstatErrorSkipped: a destination Lstat that // fails with a NON-NotExist error (here EACCES via an unsearchable canonical // dir) means the destination's state is unknown — unknown is never "absent". @@ -494,7 +722,7 @@ func TestWorkspaceNamingDrift_DestLstatErrorSkipped(t *testing.T) { env, repo := namingDriftEnv(t) agents := filepath.Join(repo, ".agents") alias := filepath.Join(agents, "handoffs") - canonical := filepath.Join(agents, "handoff") + canonical := filepath.Join(agents, "ao", "handoff") writeDriftFile(t, filepath.Join(alias, "x.md"), "x body") if err := os.MkdirAll(canonical, 0o755); err != nil { t.Fatal(err) diff --git a/cli/internal/doctor/fix_workspace_empty.go b/cli/internal/doctor/fix_workspace_empty.go index 0590a7bff..bbe9418de 100644 --- a/cli/internal/doctor/fix_workspace_empty.go +++ b/cli/internal/doctor/fix_workspace_empty.go @@ -10,13 +10,17 @@ package doctor // half-finished lanes: they clutter inventory output and mislead tooling that // treats directory presence as a signal. // -// Three name classes are deliberately NOT claimed here: +// Four name classes are deliberately NOT claimed here: // // - "ao" — the structured knowledge-store root. Its (empty) substructure is // the knowledge subsystem's contract (fm-knowledge-missing-substructure). -// - canonical directory names (the VALUES of workspaceCanonicalAliases: -// postmortem, pre-mortem-checks, handoff, retro, proofs, tests) — these may -// legitimately sit empty awaiting their first write. +// - canonical top-level directory names (the VALUES of +// workspaceCanonicalAliases: postmortem, pre-mortem-checks, retro, proofs, +// tests) — these may legitimately sit empty awaiting their first write. +// - "handoff" — the earlier handoff root is a read-only compatibility source; +// Doctor never moves it, even when it is currently empty. +// - "mto-handoff" — a distinct, live recurrence protocol consumed by the MTO +// assay. It is neither a handoff spelling drift nor empty-dir debris. // - stale/retry-named directories (isWorkspaceStaleDirName) — owned by the // workspace GC failure mode (fm-ws-stale-queue-dirs); no double-claim. // @@ -42,9 +46,10 @@ func init() { // workspaceEmptyDirClaimed reports whether name is excluded from the // empty-dirs failure mode because another owner claims it: the knowledge -// store root, a canonical directory name, or a stale/retry-named directory. +// store root, a read-only/consumer-owned compatibility root, a canonical +// directory name, or a stale/retry-named directory. func workspaceEmptyDirClaimed(name string) bool { - if name == "ao" { + if name == "ao" || name == "handoff" || name == "mto-handoff" { return true } for _, canonical := range workspaceCanonicalAliases { diff --git a/cli/internal/doctor/fix_workspace_empty_test.go b/cli/internal/doctor/fix_workspace_empty_test.go index e31c4521d..c1693a8f1 100644 --- a/cli/internal/doctor/fix_workspace_empty_test.go +++ b/cli/internal/doctor/fix_workspace_empty_test.go @@ -14,7 +14,7 @@ import ( // nested-empty/a/b/ only empty subs -> flagged // deep/ one deep file -> NOT flagged // ao/ empty store root -> NOT flagged (knowledge's) -// handoff/ empty canonical -> NOT flagged +// handoff/ legacy evidence -> NOT empty/flagged // land-queue-x.stale-.../ empty stale name -> NOT flagged (GC's) // // It returns the DetectEnv and the repo root. @@ -33,6 +33,10 @@ func workspaceEmptyTestEnv(t *testing.T) (*DetectEnv, string) { t.Fatalf("mkdir %s: %v", dir, err) } } + legacyHandoff := filepath.Join(agents, "handoff", "handoff-20260815T120000.000000000Z.json") + if err := os.WriteFile(legacyHandoff, []byte("legacy evidence\n"), 0o600); err != nil { + t.Fatalf("write legacy handoff evidence: %v", err) + } deepFile := filepath.Join(agents, "deep", "sub", "keep.md") if err := os.MkdirAll(filepath.Dir(deepFile), 0o755); err != nil { t.Fatalf("mkdir deep: %v", err) @@ -155,6 +159,11 @@ func TestWorkspaceEmptyDirs_DetectAndFix(t *testing.T) { if err != nil || string(got) != "content" { t.Errorf("deep file changed: %q err=%v", got, err) } + legacyHandoff := filepath.Join(agents, "handoff", "handoff-20260815T120000.000000000Z.json") + got, err = os.ReadFile(legacyHandoff) + if err != nil || string(got) != "legacy evidence\n" { + t.Errorf("legacy handoff evidence changed or moved: %q err=%v", got, err) + } // Re-detect: clean. again, err := det.Detect(env) @@ -325,6 +334,21 @@ func TestWorkspaceEmptyDirs_DetectMissingBaseClean(t *testing.T) { } } +func TestWorkspaceEmptyDirs_EmptyLegacyHandoffIsReadOnly(t *testing.T) { + repo := t.TempDir() + agents := filepath.Join(repo, ".agents") + if err := os.MkdirAll(filepath.Join(agents, "handoff"), 0o755); err != nil { + t.Fatal(err) + } + candidates, err := workspaceEmptyDirCandidates(agents) + if err != nil { + t.Fatal(err) + } + if len(candidates) != 0 { + t.Fatalf("empty legacy handoff candidates = %v, want none", candidates) + } +} + func TestWorkspaceEmptyDirClaimed(t *testing.T) { tests := []struct { name string @@ -335,6 +359,7 @@ func TestWorkspaceEmptyDirClaimed(t *testing.T) { {"postmortem", true}, {"pre-mortem-checks", true}, {"handoff", true}, + {"mto-handoff", true}, {"retro", true}, {"proofs", true}, {"tests", true}, diff --git a/cli/internal/doctor/fix_workspace_test.go b/cli/internal/doctor/fix_workspace_test.go index b9c783473..af288bf1e 100644 --- a/cli/internal/doctor/fix_workspace_test.go +++ b/cli/internal/doctor/fix_workspace_test.go @@ -157,8 +157,7 @@ func TestWorkspaceCanonicalAliases(t *testing.T) { {"pre-mortem", "pre-mortem-checks"}, {"pre-mortems", "pre-mortem-checks"}, {"premortem-checks", "pre-mortem-checks"}, - {"handoffs", "handoff"}, - {"mto-handoff", "handoff"}, + {"handoffs", filepath.Join("ao", "handoff")}, {"retros", "retro"}, {"proof", "proofs"}, {"test", "tests"}, diff --git a/cli/internal/gates/checks/seed.go b/cli/internal/gates/checks/seed.go index 8ecc6e0d7..cdcba8bf0 100644 --- a/cli/internal/gates/checks/seed.go +++ b/cli/internal/gates/checks/seed.go @@ -38,7 +38,7 @@ var ( "scripts/lib/ratchet.sh", } // skill.probe-coverage (advisory): routes when any skill changes (a new - // product/judgment skill needs a probe), when the MEASURED probe ledger + // product/judgment skill needs a probe), when the probe-status ledger // changes, when a probe scenario changes, plus self-reference so editing the // gate/its test re-runs it. skillProbePaths = []string{ @@ -46,7 +46,11 @@ var ( "skills/SKILL-TIERS.md", "evals/skill-probes/**", "scripts/probe-skill.sh", + "scripts/lib/probe-fixture-metadata.py", + "scripts/lib/codex-exec.sh", + "scripts/lib/preamble.sh", "scripts/check-skill-probe-coverage.sh", + "tests/scripts/probe-skill.bats", "tests/scripts/check-skill-probe-coverage.bats", } operatorLeakPaths = []string{"skills/**", "skills-codex/**", "docs/SKILLS.md", "registry.json", "tests/scripts/check-no-operator-skills.bats", "scripts/check-no-operator-skills.sh"} @@ -294,7 +298,7 @@ func init() { // (Blocking:false, warn never fail) exactly like skill.isolation and the // egwt gates: the spine is probed first, the ratchet drives the rest, and // the Blocking:false->true flip is made deliberately once covered. age-e508.1. - {ID: "skill.probe-coverage", Tiers: gates.Fast | gates.Full, Match: skillProbePaths, Blocking: false, Backing: "check-skill-probe-coverage.sh", RepairHint: "bash scripts/probe-skill.sh --probe then record it in the MEASURED ledger at evals/skill-probes/LEDGER.md (hand-maintained; never inside generated SKILL-TIERS.md); advisory — probe the spine, ratchet the rest"}, + {ID: "skill.probe-coverage", Tiers: gates.Fast | gates.Full, Match: skillProbePaths, Blocking: false, Backing: "check-skill-probe-coverage.sh", RepairHint: "capture a new immutable fixture set with scripts/probe-skill.sh, write its v3 scorecard, then record the manifest-backed result in evals/skill-probes/LEDGER.md (hand-maintained; never inside generated SKILL-TIERS.md); advisory — probe the spine, ratchet the rest"}, {ID: "skill.no-operator-leakage", Tiers: gates.Fast | gates.Full, Match: operatorLeakPaths, Blocking: true, Backing: "check-no-operator-skills.sh"}, {ID: "skill.heal-strict", Tiers: gates.Full, Match: skillPaths, Blocking: true, Backing: "skills/skill-builder/scripts/heal.sh", Args: []string{"--check", "--strict"}}, {ID: "skill.frontmatter-v2", Tiers: gates.Full, Match: skillPaths, Blocking: true, Backing: "validate-skill-frontmatter.sh"}, diff --git a/cli/internal/gates/checks/seed_test.go b/cli/internal/gates/checks/seed_test.go index d775124c9..831a9f153 100644 --- a/cli/internal/gates/checks/seed_test.go +++ b/cli/internal/gates/checks/seed_test.go @@ -151,10 +151,24 @@ func TestSkillProbeCoverageGateIsWarnFirstAdvisory(t *testing.T) { if !check.Tiers.Has(gates.Fast) || !check.Tiers.Has(gates.Full) { t.Fatalf("skill.probe-coverage tiers = %v, want Fast|Full", check.Tiers) } - // Routed on skill changes + the MEASURED ledger + the probe scenarios so a new + if !strings.Contains(check.RepairHint, "v3 scorecard") || strings.Contains(check.RepairHint, "v2 scorecard") { + t.Fatalf("skill.probe-coverage repair hint = %q, want current v3 scorecard guidance", check.RepairHint) + } + // Routed on skill changes + the measurement-status ledger + the probe scenarios so a new // product skill or a ledger edit re-runs it (not always-run — the probe corpus // is the scope). - for _, want := range []string{"skills/**", "skills/SKILL-TIERS.md", "evals/skill-probes/**", "scripts/check-skill-probe-coverage.sh"} { + for _, want := range []string{ + "skills/**", + "skills/SKILL-TIERS.md", + "evals/skill-probes/**", + "scripts/probe-skill.sh", + "scripts/lib/probe-fixture-metadata.py", + "scripts/lib/codex-exec.sh", + "scripts/lib/preamble.sh", + "scripts/check-skill-probe-coverage.sh", + "tests/scripts/probe-skill.bats", + "tests/scripts/check-skill-probe-coverage.bats", + } { if !gates.PathMatchesAny(check.Match, want) { t.Fatalf("skill.probe-coverage must route on %q; match globs = %v", want, check.Match) } diff --git a/cli/internal/sessionapp/prune_agents.go b/cli/internal/sessionapp/prune_agents.go new file mode 100644 index 000000000..eea30be6b --- /dev/null +++ b/cli/internal/sessionapp/prune_agents.go @@ -0,0 +1,713 @@ +package sessionapp + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// PruneAgentsOptions controls retention cleanup under a repository's .agents +// workspace. Execute defaults to false so a zero-value invocation is read-only. +// The implementation binds intermediate-directory traversal; it does not claim +// to make a final basename immutable against an adversarial replacement. +type PruneAgentsOptions struct { + RepoRoot string + Execute bool + Quiet bool + Stdout io.Writer + Now func() time.Time +} + +// PruneAgentsResult is the factual count produced by one retention pass. +type PruneAgentsResult struct { + Files int + Bytes int64 +} + +// pruneAgentsBeforeDeleteTestHook is an in-process package test seam. It fires +// after the final current-path identity check and before the descriptor-rooted +// delete. Production has no environment or command-controlled race callback. +var pruneAgentsBeforeDeleteTestHook func(relativePath string) + +type pruneAgentsRunner struct { + opts PruneAgentsOptions + root *os.Root + now time.Time + result PruneAgentsResult +} + +type pruneCandidate struct { + name string + info os.FileInfo +} + +// PruneAgents applies the retention policy historically exposed by +// scripts/prune-agents.sh. All mutation is relative to an already-open os.Root +// for the target's parent directory. A renamed or symlink-swapped intermediate +// path therefore cannot redirect deletion to another tree; parent identity is +// checked immediately before and after each mutation, and observed drift makes +// the run fail closed. The final child name is checked but not locked, so this +// is not a claim of resistance to a final-entry replacement or unobserved ABA. +func PruneAgents(opts PruneAgentsOptions) (PruneAgentsResult, error) { + if strings.TrimSpace(opts.RepoRoot) == "" { + return PruneAgentsResult{}, fmt.Errorf("prune agents: repository root is required") + } + if opts.Stdout == nil { + opts.Stdout = io.Discard + } + if opts.Now == nil { + opts.Now = time.Now + } + root, err := os.OpenRoot(opts.RepoRoot) + if err != nil { + return PruneAgentsResult{}, fmt.Errorf("prune agents: open repository root: %w", err) + } + defer func() { _ = root.Close() }() + + runner := &pruneAgentsRunner{opts: opts, root: root, now: opts.Now()} + if err := runner.validateCanonicalHandoffChain(); err != nil { + return runner.result, err + } + if !opts.Quiet { + if opts.Execute { + fmt.Fprintln(opts.Stdout, "=== EXECUTE MODE — files will be deleted ===") + } else { + fmt.Fprintln(opts.Stdout, "=== DRY RUN — no files will be deleted (pass --execute to delete) ===") + } + fmt.Fprintln(opts.Stdout) + } + + steps := []func() error{ + func() error { return runner.pruneKeepNewest(".agents/council", 30, "council") }, + runner.pruneLegacyScannerDirs, + func() error { + return runner.pruneOlderThan(".agents/knowledge/pending", 14, "*.md", "knowledge/pending") + }, + func() error { + return runner.pruneOlderThan(".agents/rpi", 30, "phase-*-summary-*", "rpi/phase-summaries") + }, + func() error { return runner.pruneKeepNewest(".agents/ao/sessions", 50, "ao/sessions") }, + func() error { + if err := runner.validateCanonicalHandoffChain(); err != nil { + return err + } + return runner.pruneKeepNewest(".agents/ao/handoff", 10, "ao/handoff") + }, + func() error { + if err := runner.pruneOlderThan(".agents/opencode-tests", 7, "*.log", "opencode-tests"); err != nil { + return err + } + return runner.pruneOlderThan(".agents/opencode-tests", 7, "*.txt", "opencode-tests/summaries") + }, + func() error { return runner.pruneKeepNewest(".agents/ao/subagent-outputs", 50, "ao/subagent-outputs") }, + runner.pruneLocalCIRuns, + func() error { + if err := runner.pruneKeepNewest(".agents/vibe", 20, "vibe"); err != nil { + return err + } + return runner.pruneKeepNewest(".agents/vibecheck", 20, "vibecheck") + }, + func() error { return runner.pruneKeepNewest(".agents/brainstorm", 10, "brainstorm") }, + func() error { + return runner.pruneOlderThan(".agents/compaction-snapshots", 7, "*.md", "compaction-snapshots") + }, + func() error { return runner.pruneKeepNewest(".agents/swarm", 10, "swarm") }, + runner.pruneStatusDashboards, + runner.pruneArchivedWorktrees, + } + for _, step := range steps { + if err := step(); err != nil { + return runner.result, err + } + if !opts.Quiet { + fmt.Fprintln(opts.Stdout) + } + } + + fmt.Fprintln(opts.Stdout, "========================================") + if opts.Execute { + fmt.Fprintln(opts.Stdout, "PRUNE COMPLETE") + fmt.Fprintf(opts.Stdout, "Files deleted: %d\n", runner.result.Files) + } else { + fmt.Fprintln(opts.Stdout, "DRY RUN COMPLETE") + fmt.Fprintf(opts.Stdout, "Files that would be deleted: %d\n", runner.result.Files) + } + if !opts.Quiet { + fmt.Fprintln(opts.Stdout) + fmt.Fprintln(opts.Stdout, "Protected directories (never pruned):") + fmt.Fprintln(opts.Stdout, " handoff/ mto-handoff/ learnings/ patterns/ plans/ research/ retros/") + } + return runner.result, nil +} + +func (runner *pruneAgentsRunner) pruneKeepNewest(relativeDir string, keep int, label string) error { + dir, exists, err := runner.openRealDir(relativeDir) + if err != nil || !exists { + return err + } + defer func() { _ = dir.Close() }() + candidates, err := regularChildren(dir) + if err != nil { + return fmt.Errorf("prune agents: inspect %s: %w", runner.display(relativeDir), err) + } + if len(candidates) <= keep { + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, "[%s] %d files — within limit (%d). Nothing to prune.\n", label, len(candidates), keep) + } + return nil + } + sortNewestFirst(candidates) + toDelete := candidates[keep:] + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, "[%s] %d files — keeping newest %d, pruning %d\n", label, len(candidates), keep, len(toDelete)) + } + for _, candidate := range toDelete { + if err := runner.pruneFile(dir, relativeDir, candidate); err != nil { + return err + } + } + return nil +} + +func (runner *pruneAgentsRunner) pruneOlderThan(relativeDir string, days int, pattern, label string) error { + dir, exists, err := runner.openRealDir(relativeDir) + if err != nil || !exists { + return err + } + defer func() { _ = dir.Close() }() + children, err := regularChildren(dir) + if err != nil { + return fmt.Errorf("prune agents: inspect %s: %w", runner.display(relativeDir), err) + } + cutoff := runner.now.Add(-time.Duration(days+1) * 24 * time.Hour) + var candidates []pruneCandidate + for _, candidate := range children { + matched, matchErr := filepath.Match(pattern, candidate.name) + if matchErr != nil { + return fmt.Errorf("prune agents: invalid retention pattern %q: %w", pattern, matchErr) + } + if matched && !candidate.info.ModTime().After(cutoff) { + candidates = append(candidates, candidate) + } + } + if len(candidates) == 0 { + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, "[%s] No files older than %dd matching '%s'. Nothing to prune.\n", label, days, pattern) + } + return nil + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].name < candidates[j].name }) + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, "[%s] %d files older than %dd\n", label, len(candidates), days) + } + for _, candidate := range candidates { + if err := runner.pruneFile(dir, relativeDir, candidate); err != nil { + return err + } + } + return nil +} + +func (runner *pruneAgentsRunner) pruneFile(dir *os.Root, relativeDir string, candidate pruneCandidate) error { + relativePath := filepath.Join(relativeDir, candidate.name) + runner.result.Files++ + runner.result.Bytes += candidate.info.Size() + if !runner.opts.Execute { + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, " would delete: %s (%s)\n", runner.display(relativePath), numfmtSize(candidate.info.Size())) + } + return nil + } + if err := runner.removeAnchored(dir, relativeDir, candidate.name, candidate.info, false); err != nil { + return err + } + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, " deleted: %s (%s)\n", runner.display(relativePath), numfmtSize(candidate.info.Size())) + } + return nil +} + +func (runner *pruneAgentsRunner) pruneLegacyScannerDirs() error { + for _, relativeDir := range []string{".agents/tooling", ".agents/security"} { + parentRel, name := filepath.Split(relativeDir) + parentRel = filepath.Clean(parentRel) + parent, exists, err := runner.openRealDir(parentRel) + if err != nil || !exists { + return err + } + info, err := parent.Lstat(name) + if errors.Is(err, os.ErrNotExist) { + _ = parent.Close() + continue + } + if err != nil { + _ = parent.Close() + return fmt.Errorf("prune agents: inspect %s: %w", runner.display(relativeDir), err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + _ = parent.Close() + return runner.unsafe("legacy scanner path is not a real directory: %s", runner.display(relativeDir)) + } + dir, err := parent.OpenRoot(name) + if err != nil { + _ = parent.Close() + return runner.unsafe("open legacy scanner directory %s: %v", runner.display(relativeDir), err) + } + count, err := countRegularTree(dir) + _ = dir.Close() + if err != nil { + _ = parent.Close() + return fmt.Errorf("prune agents: count %s: %w", runner.display(relativeDir), err) + } + if count == 0 { + _ = parent.Close() + continue + } + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, "[legacy] %s has %d files (scanner output moved to $TMPDIR)\n", runner.display(relativeDir), count) + } + runner.result.Files++ + if !runner.opts.Execute { + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, " would delete: %s/ (%d files)\n", runner.display(relativeDir), count) + } + _ = parent.Close() + continue + } + if err := runner.removeAnchored(parent, parentRel, name, info, true); err != nil { + _ = parent.Close() + return err + } + if err := runner.ensureOpenedDirCurrent(parentRel, parent); err != nil { + _ = parent.Close() + return err + } + if err := parent.Mkdir(name, 0o755); err != nil { + _ = parent.Close() + return fmt.Errorf("prune agents: recreate %s: %w", runner.display(relativeDir), err) + } + _ = parent.Close() + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, " deleted: %s/ (%d files)\n", runner.display(relativeDir), count) + } + } + return nil +} + +func (runner *pruneAgentsRunner) pruneLocalCIRuns() error { + const relativeDir = ".agents/releases/local-ci" + dir, exists, err := runner.openRealDir(relativeDir) + if err != nil || !exists { + return err + } + defer func() { _ = dir.Close() }() + directories, err := directoryChildren(dir) + if err != nil { + return fmt.Errorf("prune agents: inspect %s: %w", runner.display(relativeDir), err) + } + const keep = 3 + if len(directories) <= keep { + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, "[releases/local-ci] %d runs — within limit (%d). Nothing to prune.\n", len(directories), keep) + } + return nil + } + sortNewestFirst(directories) + toDelete := directories[keep:] + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, "[releases/local-ci] %d runs — keeping newest %d, pruning %d\n", len(directories), keep, len(toDelete)) + } + for _, candidate := range toDelete { + sizeKB := int64(0) + child, openErr := dir.OpenRoot(candidate.name) + if openErr == nil { + if bytes, walkErr := regularTreeBytes(child); walkErr == nil { + sizeKB = (bytes + 1023) / 1024 + } + _ = child.Close() + } + runner.result.Files++ + relativePath := filepath.Join(relativeDir, candidate.name) + if !runner.opts.Execute { + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, " would delete: %s (~%dKB)\n", runner.display(relativePath), sizeKB) + } + continue + } + if err := runner.removeAnchored(dir, relativeDir, candidate.name, candidate.info, true); err != nil { + return err + } + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, " deleted: %s (~%dKB)\n", runner.display(relativePath), sizeKB) + } + } + return nil +} + +func (runner *pruneAgentsRunner) pruneStatusDashboards() error { + const relativeDir = ".agents" + dir, exists, err := runner.openRealDir(relativeDir) + if err != nil || !exists { + return err + } + defer func() { _ = dir.Close() }() + children, err := regularChildren(dir) + if err != nil { + return fmt.Errorf("prune agents: inspect %s: %w", runner.display(relativeDir), err) + } + var candidates []pruneCandidate + for _, candidate := range children { + if strings.HasPrefix(candidate.name, "status-dashboard") { + candidates = append(candidates, candidate) + } + } + if len(candidates) <= 5 { + return nil + } + sortNewestFirst(candidates) + toDelete := candidates[5:] + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, "[status-dashboards] %d files — keeping newest 5, pruning %d\n", len(candidates), len(toDelete)) + } + for _, candidate := range toDelete { + runner.result.Files++ + relativePath := filepath.Join(relativeDir, candidate.name) + if !runner.opts.Execute { + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, " would delete: %s\n", runner.display(relativePath)) + } + continue + } + if err := runner.removeAnchored(dir, relativeDir, candidate.name, candidate.info, false); err != nil { + return err + } + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, " deleted: %s\n", runner.display(relativePath)) + } + } + return nil +} + +func (runner *pruneAgentsRunner) pruneArchivedWorktrees() error { + const relativeDir = ".agents/archived-worktrees" + dir, exists, err := runner.openRealDir(relativeDir) + if err != nil || !exists { + return err + } + defer func() { _ = dir.Close() }() + directories, err := directoryChildren(dir) + if err != nil { + return fmt.Errorf("prune agents: inspect %s: %w", runner.display(relativeDir), err) + } + cutoff := runner.now.Add(-8 * 24 * time.Hour) + var candidates []pruneCandidate + for _, candidate := range directories { + if !candidate.info.ModTime().After(cutoff) { + candidates = append(candidates, candidate) + } + } + if len(candidates) == 0 { + if !runner.opts.Quiet { + fmt.Fprintln(runner.opts.Stdout, "[archived-worktrees] No directories older than 7d. Nothing to prune.") + } + return nil + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].name < candidates[j].name }) + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, "[archived-worktrees] %d directories older than 7d\n", len(candidates)) + } + for _, candidate := range candidates { + runner.result.Files++ + relativePath := filepath.Join(relativeDir, candidate.name) + if !runner.opts.Execute { + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, " would delete: %s\n", runner.display(relativePath)) + } + continue + } + if err := runner.removeAnchored(dir, relativeDir, candidate.name, candidate.info, true); err != nil { + return err + } + if !runner.opts.Quiet { + fmt.Fprintf(runner.opts.Stdout, " deleted: %s\n", runner.display(relativePath)) + } + } + return nil +} + +// removeAnchored performs the actual mutation through the already-open parent +// directory descriptor. The current parent path is checked before the test hook +// and after the mutation. Even if an intermediate component changes in between, +// the delete remains bound to the opened directory and cannot follow the new +// path; the post-check converts observed drift into a non-zero result. The +// SameFile check below observes the final child name at one instant; it does not +// lock that directory entry or claim resistance to a final-name ABA. +func (runner *pruneAgentsRunner) removeAnchored(parent *os.Root, parentRel, name string, expected os.FileInfo, recursive bool) error { + if err := runner.ensureOpenedDirCurrent(parentRel, parent); err != nil { + return err + } + current, err := parent.Lstat(name) + if err != nil { + return runner.unsafe("target changed before deletion: %s: %v", runner.display(filepath.Join(parentRel, name)), err) + } + if current.Mode()&os.ModeSymlink != 0 || !os.SameFile(expected, current) || current.IsDir() != expected.IsDir() { + return runner.unsafe("target changed identity before deletion: %s", runner.display(filepath.Join(parentRel, name))) + } + if pruneAgentsBeforeDeleteTestHook != nil { + pruneAgentsBeforeDeleteTestHook(filepath.Join(parentRel, name)) + } + if recursive { + err = parent.RemoveAll(name) + } else { + err = parent.Remove(name) + } + if err != nil { + return runner.unsafe("descriptor-rooted deletion failed for %s: %v", runner.display(filepath.Join(parentRel, name)), err) + } + if err := runner.ensureOpenedDirCurrent(parentRel, parent); err != nil { + return err + } + return nil +} + +func (runner *pruneAgentsRunner) ensureOpenedDirCurrent(relativeDir string, opened *os.Root) error { + current, exists, err := runner.openRealDir(relativeDir) + if err != nil { + return err + } + if !exists { + return runner.unsafe("mutation parent disappeared: %s", runner.display(relativeDir)) + } + defer func() { _ = current.Close() }() + openedInfo, err := opened.Stat(".") + if err != nil { + return runner.unsafe("inspect opened mutation parent %s: %v", runner.display(relativeDir), err) + } + currentInfo, err := current.Stat(".") + if err != nil || !os.SameFile(openedInfo, currentInfo) { + return runner.unsafe("mutation parent changed identity: %s", runner.display(relativeDir)) + } + return nil +} + +func (runner *pruneAgentsRunner) validateCanonicalHandoffChain() error { + current := runner.root + owned := false + currentRel := "" + for _, component := range []string{".agents", "ao", "handoff"} { + currentRel = filepath.Join(currentRel, component) + before, err := current.Lstat(component) + if errors.Is(err, os.ErrNotExist) { + if owned { + _ = current.Close() + } + return nil + } + if err != nil { + if owned { + _ = current.Close() + } + return runner.unsafe("inspect canonical handoff path component %s: %v", runner.display(currentRel), err) + } + if before.Mode()&os.ModeSymlink != 0 { + if owned { + _ = current.Close() + } + return runner.unsafe("canonical handoff path component is a symlink: %s", runner.display(currentRel)) + } + if !before.IsDir() { + if owned { + _ = current.Close() + } + return runner.unsafe("canonical handoff path component is not a directory: %s", runner.display(currentRel)) + } + next, err := current.OpenRoot(component) + if err != nil { + if owned { + _ = current.Close() + } + return runner.unsafe("open canonical handoff path component %s: %v", runner.display(currentRel), err) + } + openedInfo, openedErr := next.Stat(".") + after, afterErr := current.Lstat(component) + if openedErr != nil || afterErr != nil || after.Mode()&os.ModeSymlink != 0 || !after.IsDir() || !os.SameFile(before, openedInfo) || !os.SameFile(after, openedInfo) { + _ = next.Close() + if owned { + _ = current.Close() + } + return runner.unsafe("canonical handoff path component changed identity while opening: %s", runner.display(currentRel)) + } + if owned { + _ = current.Close() + } + current = next + owned = true + } + if owned { + _ = current.Close() + } + return nil +} + +func (runner *pruneAgentsRunner) openRealDir(relativeDir string) (*os.Root, bool, error) { + clean := filepath.Clean(relativeDir) + if clean == "." || clean == "" || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return nil, false, runner.unsafe("invalid repository-relative directory: %s", relativeDir) + } + current := runner.root + owned := false + currentRel := "" + for _, component := range strings.Split(clean, string(filepath.Separator)) { + currentRel = filepath.Join(currentRel, component) + before, err := current.Lstat(component) + if errors.Is(err, os.ErrNotExist) { + if owned { + _ = current.Close() + } + return nil, false, nil + } + if err != nil { + if owned { + _ = current.Close() + } + return nil, false, runner.unsafe("inspect directory component %s: %v", runner.display(currentRel), err) + } + if before.Mode()&os.ModeSymlink != 0 || !before.IsDir() { + if owned { + _ = current.Close() + } + return nil, false, runner.unsafe("directory component is not a real directory: %s", runner.display(currentRel)) + } + next, err := current.OpenRoot(component) + if err != nil { + if owned { + _ = current.Close() + } + return nil, false, runner.unsafe("open directory component %s: %v", runner.display(currentRel), err) + } + openedInfo, openedErr := next.Stat(".") + after, afterErr := current.Lstat(component) + if openedErr != nil || afterErr != nil || after.Mode()&os.ModeSymlink != 0 || !after.IsDir() || !os.SameFile(before, openedInfo) || !os.SameFile(after, openedInfo) { + _ = next.Close() + if owned { + _ = current.Close() + } + return nil, false, runner.unsafe("directory component changed identity while opening: %s", runner.display(currentRel)) + } + if owned { + _ = current.Close() + } + current = next + owned = true + } + return current, true, nil +} + +func regularChildren(root *os.Root) ([]pruneCandidate, error) { + return typedChildren(root, false) +} + +func directoryChildren(root *os.Root) ([]pruneCandidate, error) { + return typedChildren(root, true) +} + +func typedChildren(root *os.Root, directories bool) ([]pruneCandidate, error) { + dir, err := root.Open(".") + if err != nil { + return nil, err + } + entries, readErr := dir.ReadDir(-1) + closeErr := dir.Close() + if readErr != nil { + return nil, readErr + } + if closeErr != nil { + return nil, closeErr + } + var candidates []pruneCandidate + for _, entry := range entries { + info, err := root.Lstat(entry.Name()) + if err != nil { + return nil, err + } + if info.Mode()&os.ModeSymlink != 0 { + continue + } + if directories && info.IsDir() { + candidates = append(candidates, pruneCandidate{name: entry.Name(), info: info}) + } + if !directories && info.Mode().IsRegular() { + candidates = append(candidates, pruneCandidate{name: entry.Name(), info: info}) + } + } + return candidates, nil +} + +func sortNewestFirst(candidates []pruneCandidate) { + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].info.ModTime().Equal(candidates[j].info.ModTime()) { + return candidates[i].name > candidates[j].name + } + return candidates[i].info.ModTime().After(candidates[j].info.ModTime()) + }) +} + +func countRegularTree(root *os.Root) (int, error) { + count := 0 + err := fs.WalkDir(root.FS(), ".", func(_ string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.Type().IsRegular() { + count++ + } + return nil + }) + return count, err +} + +func regularTreeBytes(root *os.Root) (int64, error) { + var total int64 + err := fs.WalkDir(root.FS(), ".", func(_ string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.Type().IsRegular() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + total += info.Size() + return nil + }) + return total, err +} + +func (runner *pruneAgentsRunner) display(relativePath string) string { + return filepath.Join(runner.opts.RepoRoot, relativePath) +} + +func (runner *pruneAgentsRunner) unsafe(format string, args ...any) error { + return fmt.Errorf("prune agents: refusing to prune: "+format, args...) +} + +func numfmtSize(bytes int64) string { + switch { + case bytes >= 1<<30: + return fmt.Sprintf("%dGB", bytes/(1<<30)) + case bytes >= 1<<20: + return fmt.Sprintf("%dMB", bytes/(1<<20)) + case bytes >= 1<<10: + return fmt.Sprintf("%dKB", bytes/(1<<10)) + default: + return fmt.Sprintf("%dB", bytes) + } +} diff --git a/cli/internal/sessionapp/prune_agents_test.go b/cli/internal/sessionapp/prune_agents_test.go new file mode 100644 index 000000000..a884ae0e3 --- /dev/null +++ b/cli/internal/sessionapp/prune_agents_test.go @@ -0,0 +1,246 @@ +package sessionapp + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestPruneAgentsDryRunReportsWithoutDeleting(t *testing.T) { + repo := t.TempDir() + now := time.Date(2026, 8, 16, 18, 0, 0, 0, time.UTC) + writePruneCandidates(t, filepath.Join(repo, ".agents", "ao", "handoff"), "handoff", 12, now.Add(-time.Hour)) + + var output bytes.Buffer + result, err := PruneAgents(PruneAgentsOptions{ + RepoRoot: repo, + Stdout: &output, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("PruneAgents dry run: %v", err) + } + if result.Files != 2 { + t.Fatalf("dry-run candidate count = %d, want 2", result.Files) + } + if got := countDirectFiles(t, filepath.Join(repo, ".agents", "ao", "handoff")); got != 12 { + t.Fatalf("dry run left %d handoffs, want 12", got) + } + if !strings.Contains(output.String(), "Files that would be deleted: 2") { + t.Fatalf("dry-run summary missing candidate count:\n%s", output.String()) + } +} + +func TestPruneAgentsExecutePreservesLegacyAndDistinctMTOHandoff(t *testing.T) { + repo := t.TempDir() + now := time.Date(2026, 8, 16, 18, 0, 0, 0, time.UTC) + canonical := filepath.Join(repo, ".agents", "ao", "handoff") + legacy := filepath.Join(repo, ".agents", "handoff") + mto := filepath.Join(repo, ".agents", "mto-handoff") + writePruneCandidates(t, canonical, "canonical", 12, now.Add(-time.Hour)) + writePruneCandidates(t, legacy, "legacy", 12, now.Add(-2*time.Hour)) + writeFileForPrune(t, filepath.Join(mto, "recurrence.json"), []byte("distinct recurrence bytes\n"), now.Add(-3*time.Hour)) + legacyBefore := snapshotPruneTree(t, legacy) + mtoBefore := snapshotPruneTree(t, mto) + + var output bytes.Buffer + result, err := PruneAgents(PruneAgentsOptions{ + RepoRoot: repo, + Execute: true, + Stdout: &output, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("PruneAgents execute: %v", err) + } + if result.Files != 2 { + t.Fatalf("execute deletion count = %d, want 2", result.Files) + } + if got := countDirectFiles(t, canonical); got != 10 { + t.Fatalf("canonical handoff count = %d, want 10", got) + } + assertPruneSnapshot(t, legacy, legacyBefore) + assertPruneSnapshot(t, mto, mtoBefore) + if !strings.Contains(output.String(), "Files deleted: 2") { + t.Fatalf("execute summary missing deletion count:\n%s", output.String()) + } +} + +func TestPruneAgentsIntermediateAOSwapFailsClosedWithDescriptorRootedDelete(t *testing.T) { + repo := t.TempDir() + outside := t.TempDir() + now := time.Date(2026, 8, 16, 18, 0, 0, 0, time.UTC) + canonical := filepath.Join(repo, ".agents", "ao", "handoff") + externalHandoff := filepath.Join(outside, "ao", "handoff") + writePruneCandidates(t, canonical, "canonical", 12, now.Add(-time.Hour)) + // Mirror the selected basenames outside so a path-based delete would remove + // a real external artifact rather than harmlessly targeting a missing name. + writePruneCandidates(t, externalHandoff, "canonical", 12, now.Add(-time.Hour)) + writeFileForPrune(t, filepath.Join(externalHandoff, "sentinel"), []byte("outside sentinel bytes\n"), now) + externalBefore := snapshotPruneTree(t, outside) + + hookCalls := 0 + pruneAgentsBeforeDeleteTestHook = func(relativePath string) { + if hookCalls != 0 || !strings.HasPrefix(relativePath, filepath.Join(".agents", "ao", "handoff")+string(filepath.Separator)) { + return + } + hookCalls++ + if err := os.Rename(filepath.Join(repo, ".agents", "ao"), filepath.Join(repo, ".agents", "ao-original")); err != nil { + t.Fatalf("rename canonical ao directory in race hook: %v", err) + } + if err := os.Symlink(filepath.Join(outside, "ao"), filepath.Join(repo, ".agents", "ao")); err != nil { + t.Fatalf("plant external ao symlink in race hook: %v", err) + } + } + t.Cleanup(func() { pruneAgentsBeforeDeleteTestHook = nil }) + + _, err := PruneAgents(PruneAgentsOptions{ + RepoRoot: repo, + Execute: true, + Stdout: &bytes.Buffer{}, + Now: func() time.Time { return now }, + }) + if err == nil { + t.Fatal("PruneAgents succeeded after .agents/ao was swapped to an external symlink") + } + if hookCalls != 1 { + t.Fatalf("race hook calls = %d, want 1", hookCalls) + } + if !strings.Contains(err.Error(), "refusing to prune") { + t.Fatalf("race failure did not explain fail-closed refusal: %v", err) + } + assertPruneSnapshot(t, outside, externalBefore) + // The delete was bound to the already-open original handoff directory. It + // may remove the selected original artifact, but never the symlink target. + if got := countDirectFiles(t, filepath.Join(repo, ".agents", "ao-original", "handoff")); got != 11 { + t.Fatalf("descriptor-rooted original handoff count = %d, want 11", got) + } +} + +func TestPruneAgentsTopLevelAgentsSwapCannotRedirectOtherPolicies(t *testing.T) { + repo := t.TempDir() + outside := t.TempDir() + now := time.Date(2026, 8, 16, 18, 0, 0, 0, time.UTC) + writePruneCandidates(t, filepath.Join(repo, ".agents", "council"), "council", 31, now.Add(-time.Hour)) + // Mirror the selected basenames outside so the negative catches redirection, + // not merely the command's non-zero status. + writePruneCandidates(t, filepath.Join(outside, "council"), "council", 31, now.Add(-time.Hour)) + writeFileForPrune(t, filepath.Join(outside, "sentinel"), []byte("outside root sentinel\n"), now) + externalBefore := snapshotPruneTree(t, outside) + + hookCalls := 0 + pruneAgentsBeforeDeleteTestHook = func(relativePath string) { + if hookCalls != 0 || !strings.HasPrefix(relativePath, filepath.Join(".agents", "council")+string(filepath.Separator)) { + return + } + hookCalls++ + if err := os.Rename(filepath.Join(repo, ".agents"), filepath.Join(repo, ".agents-original")); err != nil { + t.Fatalf("rename .agents in race hook: %v", err) + } + if err := os.Symlink(outside, filepath.Join(repo, ".agents")); err != nil { + t.Fatalf("plant external .agents symlink in race hook: %v", err) + } + } + t.Cleanup(func() { pruneAgentsBeforeDeleteTestHook = nil }) + + _, err := PruneAgents(PruneAgentsOptions{ + RepoRoot: repo, + Execute: true, + Stdout: &bytes.Buffer{}, + Now: func() time.Time { return now }, + }) + if err == nil { + t.Fatal("PruneAgents succeeded after .agents was swapped to an external symlink") + } + if hookCalls != 1 { + t.Fatalf("race hook calls = %d, want 1", hookCalls) + } + assertPruneSnapshot(t, outside, externalBefore) + if got := countDirectFiles(t, filepath.Join(repo, ".agents-original", "council")); got != 30 { + t.Fatalf("descriptor-rooted original council count = %d, want 30", got) + } +} + +func writePruneCandidates(t *testing.T, dir, prefix string, count int, firstModTime time.Time) { + t.Helper() + for i := 0; i < count; i++ { + name := fmt.Sprintf("%s-%02d.json", prefix, i) + writeFileForPrune(t, filepath.Join(dir, name), []byte(fmt.Sprintf("%s bytes %02d\n", prefix, i)), firstModTime.Add(time.Duration(i)*time.Minute)) + } +} + +func writeFileForPrune(t *testing.T, path string, data []byte, modTime time.Time) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, modTime, modTime); err != nil { + t.Fatal(err) + } +} + +func countDirectFiles(t *testing.T, dir string) int { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + count := 0 + for _, entry := range entries { + if entry.Type().IsRegular() { + count++ + } + } + return count +} + +func snapshotPruneTree(t *testing.T, root string) map[string][]byte { + t.Helper() + snapshot := map[string][]byte{} + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.Type().IsRegular() { + return nil + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + snapshot[relative] = data + return nil + }) + if err != nil { + t.Fatal(err) + } + return snapshot +} + +func assertPruneSnapshot(t *testing.T, root string, want map[string][]byte) { + t.Helper() + got := snapshotPruneTree(t, root) + if len(got) != len(want) { + t.Fatalf("snapshot file count = %d, want %d", len(got), len(want)) + } + for name, wantData := range want { + gotData, ok := got[name] + if !ok { + t.Fatalf("snapshot lost %s", name) + } + if !bytes.Equal(gotData, wantData) { + t.Fatalf("snapshot bytes changed for %s: got %q want %q", name, gotData, wantData) + } + } +} diff --git a/cli/internal/sessionapp/sessionapp.go b/cli/internal/sessionapp/sessionapp.go index 518d29e7e..86a3f1ab5 100644 --- a/cli/internal/sessionapp/sessionapp.go +++ b/cli/internal/sessionapp/sessionapp.go @@ -6,13 +6,16 @@ package sessionapp import ( + "bytes" "encoding/json" + "errors" "fmt" "io" "os" "path/filepath" - "sort" + "regexp" "strings" + "time" ) // orientationCandidates is the fixed, ordered set of local orientation files a @@ -66,18 +69,54 @@ func Bootstrap(opts BootstrapOptions) error { // mirrors the on-disk handoff shape's read side; rehydrate never writes, so the // writer-side type stays with the `ao session handoff` command. type storedHandoff struct { - Goal string `json:"goal,omitempty"` - Summary string `json:"summary,omitempty"` - Continuation string `json:"continuation,omitempty"` - State *storedHandoffState `json:"state,omitempty"` + SchemaVersion *int `json:"schema_version"` + ID *string `json:"id"` + CreatedAt *string `json:"created_at"` + Type *string `json:"type,omitempty"` + Goal string `json:"goal,omitempty"` + Summary string `json:"summary,omitempty"` + Continuation string `json:"continuation,omitempty"` + ArtifactsProduced []string `json:"artifacts_produced,omitempty"` + DecisionsMade []string `json:"decisions_made,omitempty"` + OpenRisks []string `json:"open_risks,omitempty"` + RPI *storedHandoffRPI `json:"rpi,omitempty"` + State *storedHandoffState `json:"state,omitempty"` + Consumed *bool `json:"consumed,omitempty"` + ConsumedAt *string `json:"consumed_at,omitempty"` + ConsumedBy *string `json:"consumed_by,omitempty"` } // storedHandoffState is the optional read-only Git observation block. Only the // branch is surfaced in the human brief. type storedHandoffState struct { - GitBranch string `json:"git_branch,omitempty"` + GitBranch string `json:"git_branch,omitempty"` + GitDirty *bool `json:"git_dirty"` + ModifiedFiles []string `json:"modified_files,omitempty"` + ActiveBead string `json:"active_bead,omitempty"` + OpenBeadsCount *int `json:"open_beads_count,omitempty"` + RecentCommits []string `json:"recent_commits,omitempty"` } +type storedHandoffRPI struct { + Phase *int `json:"phase"` + PhaseName *string `json:"phase_name"` + EpicID string `json:"epic_id,omitempty"` + RunID string `json:"run_id,omitempty"` + Verdicts map[string]string `json:"verdicts,omitempty"` +} + +var handoffIDPattern = regexp.MustCompile(`^handoff-[0-9]{8}T[0-9]{6}(\.[0-9]+)?Z$`) + +// handoffReadTestHook is a deterministic race seam for package-local tests. +// Production leaves it nil. +var handoffReadTestHook func(stage string) + +// errNoHandoffArtifacts is the only discovery result Rehydrate renders as an +// honest empty state. Filesystem errors and unsafe/corrupt handoff shapes are +// not absence: they fail closed so existing evidence is never silently +// stranded behind `{}`. +var errNoHandoffArtifacts = errors.New("no handoff artifacts") + // RehydrateOptions carries the presentation choices resolved by the command // module. The working directory is resolved inside Rehydrate so the module // never performs a direct filesystem effect. @@ -93,32 +132,37 @@ type RehydrateOptions struct { // Rehydrate reads the latest caller-authored handoff without consuming it, // claiming work, or choosing a next action. Under JSON the empty state is -// exactly one `{}` document on stdout with the hint on stderr; either way it -// exits without error. +// exactly one `{}` document on stdout with the hint on stderr. Only an honest +// no-artifacts result takes that success path; discovery, read, and parse +// failures surface as errors. func Rehydrate(opts RehydrateOptions) error { cwd, err := os.Getwd() if err != nil { return fmt.Errorf("get cwd: %w", err) } - path, err := pickLatestHandoff(cwd) + candidate, err := pickLatestHandoff(cwd) if err != nil { - // Under --json, stdout must be exactly one JSON document (`{}` for the - // empty state) so `ao session rehydrate --json | jq` never breaks; the - // human hint goes to stderr. Exit 0 either way. - if opts.JSON { - fmt.Fprintln(opts.Stderr, "rehydrate: no handoff found") - fmt.Fprintln(opts.Stdout, "{}") + if errors.Is(err, errNoHandoffArtifacts) { + // Under --json, stdout must be exactly one JSON document (`{}` for the + // empty state) so `ao session rehydrate --json | jq` never breaks; the + // human hint goes to stderr. Exit 0 either way. + if opts.JSON { + fmt.Fprintln(opts.Stderr, "rehydrate: no handoff found") + fmt.Fprintln(opts.Stdout, "{}") + return nil + } + fmt.Fprintln(opts.Stdout, "rehydrate: no handoff found") return nil } - fmt.Fprintln(opts.Stdout, "rehydrate: no handoff found") - return nil + return fmt.Errorf("discover handoff: %w", err) } - data, err := os.ReadFile(path) // #nosec G304 -- path is selected from the local handoff directory + defer func() { _ = candidate.root.Close() }() + data, err := readRegularHandoff(cwd, candidate) if err != nil { return fmt.Errorf("read handoff: %w", err) } var artifact storedHandoff - if err := json.Unmarshal(data, &artifact); err != nil { + if err := decodeStoredHandoff(data, candidate.name, &artifact); err != nil { return fmt.Errorf("parse handoff: %w", err) } if opts.JSON { @@ -130,24 +174,390 @@ func Rehydrate(opts RehydrateOptions) error { } // pickLatestHandoff returns the newest handoff artifact by lexical name order. -func pickLatestHandoff(cwd string) (string, error) { - dir := filepath.Join(cwd, ".agents", "handoff") - entries, err := os.ReadDir(dir) - if err != nil { - return "", err +// Current writers use .agents/ao/handoff; the legacy directory remains a +// read-only compatibility source so an upgrade does not strand existing +// caller-authored evidence. If the same artifact name exists in both places, +// the canonical directory wins. +type handoffCandidate struct { + name string + displayDir string + components []string + priority int + root *os.Root +} + +func pickLatestHandoff(cwd string) (*handoffCandidate, error) { + var latest *handoffCandidate + roots := [][]string{ + {".agents", "ao", "handoff"}, + {".agents", "handoff"}, } - var names []string - for _, entry := range entries { - name := entry.Name() - if !entry.IsDir() && strings.HasPrefix(name, "handoff-") && strings.HasSuffix(name, ".json") { - names = append(names, name) + for priority, components := range roots { + candidate, err := pickLatestHandoffInRoot(cwd, components, priority) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + if latest != nil { + _ = latest.root.Close() + } + return nil, err + } + if candidate == nil { + continue + } + if latest == nil || candidate.name > latest.name || (candidate.name == latest.name && candidate.priority < latest.priority) { + if latest != nil { + _ = latest.root.Close() + } + latest = candidate + } else { + _ = candidate.root.Close() } } - if len(names) == 0 { - return "", fmt.Errorf("no handoff artifacts") + if latest == nil { + return nil, errNoHandoffArtifacts } - sort.Strings(names) - return filepath.Join(dir, names[len(names)-1]), nil + return latest, nil +} + +func pickLatestHandoffInRoot(cwd string, components []string, priority int) (*handoffCandidate, error) { + root, dir, err := openRealHandoffRoot(cwd, components...) + if err != nil { + return nil, err + } + dirFile, err := root.Open(".") + if err != nil { + _ = root.Close() + return nil, fmt.Errorf("open handoff root %s: %w", dir, err) + } + entries, err := dirFile.ReadDir(-1) + closeErr := dirFile.Close() + if err != nil { + _ = root.Close() + return nil, fmt.Errorf("read handoff root %s: %w", dir, err) + } + if closeErr != nil { + _ = root.Close() + return nil, fmt.Errorf("close handoff root %s: %w", dir, closeErr) + } + + localName := "" + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, "handoff-") || !strings.HasSuffix(name, ".json") { + continue + } + path := filepath.Join(dir, name) + info, err := root.Lstat(name) + if err != nil { + _ = root.Close() + return nil, fmt.Errorf("inspect handoff artifact %s: %w", path, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + _ = root.Close() + return nil, fmt.Errorf("handoff artifact %s is not a real regular file", path) + } + if name > localName { + localName = name + } + } + if localName == "" { + _ = root.Close() + return nil, nil + } + return &handoffCandidate{ + name: localName, + displayDir: dir, + components: append([]string(nil), components...), + priority: priority, + root: root, + }, nil +} + +// requireRealHandoffRoot resolves one configured handoff root without allowing +// a symlink or non-directory at any existing component. Component-wise Lstat is +// required for the nested canonical root: checking only its leaf would follow +// a symlinked `.agents/ao` parent before reporting the leaf's type. +func openRealHandoffRoot(cwd string, components ...string) (*os.Root, string, error) { + root, err := os.OpenRoot(cwd) + if err != nil { + return nil, "", fmt.Errorf("open workspace root: %w", err) + } + currentRoot := root + currentPath := cwd + for _, component := range components { + currentPath = filepath.Join(currentPath, component) + before, err := currentRoot.Lstat(component) + if err != nil { + _ = currentRoot.Close() + return nil, "", fmt.Errorf("inspect handoff root component %s: %w", currentPath, err) + } + if before.Mode()&os.ModeSymlink != 0 || !before.IsDir() { + _ = currentRoot.Close() + return nil, "", fmt.Errorf("handoff root component %s is not a real directory", currentPath) + } + next, err := currentRoot.OpenRoot(component) + if err != nil { + _ = currentRoot.Close() + return nil, "", fmt.Errorf("open handoff root component %s: %w", currentPath, err) + } + opened, openedErr := next.Stat(".") + after, afterErr := currentRoot.Lstat(component) + if openedErr != nil || afterErr != nil || after.Mode()&os.ModeSymlink != 0 || !after.IsDir() || !os.SameFile(before, opened) || !os.SameFile(after, opened) { + _ = next.Close() + _ = currentRoot.Close() + return nil, "", fmt.Errorf("handoff root component %s changed identity while opening", currentPath) + } + _ = currentRoot.Close() + currentRoot = next + } + return currentRoot, currentPath, nil +} + +// readRegularHandoff revalidates the selected artifact immediately before the +// read. The descriptor identity checks ensure a path swapped between Lstat and +// Open cannot redirect the read through a symlink: bytes are read only after +// both the opened descriptor and the current path still identify the same real +// regular file. +func readRegularHandoff(cwd string, candidate *handoffCandidate) ([]byte, error) { + path := filepath.Join(candidate.displayDir, candidate.name) + file, opened, err := openRegularHandoff(candidate, path) + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + first, err := readStableHandoff(file, candidate, path, opened) + if err != nil { + return nil, err + } + if err := verifyHandoffRootIdentity(cwd, candidate); err != nil { + return nil, err + } + return first, nil +} + +func openRegularHandoff(candidate *handoffCandidate, path string) (*os.File, os.FileInfo, error) { + if handoffReadTestHook != nil { + handoffReadTestHook("before-artifact-open") + } + before, err := candidate.root.Lstat(candidate.name) + if err != nil { + return nil, nil, err + } + if before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() { + return nil, nil, fmt.Errorf("handoff artifact %s is not a real regular file", path) + } + + file, err := candidate.root.Open(candidate.name) + if err != nil { + return nil, nil, err + } + opened, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, nil, err + } + after, err := candidate.root.Lstat(candidate.name) + if err != nil { + _ = file.Close() + return nil, nil, err + } + if after.Mode()&os.ModeSymlink != 0 || !after.Mode().IsRegular() || !os.SameFile(before, opened) || !os.SameFile(after, opened) { + _ = file.Close() + return nil, nil, fmt.Errorf("handoff artifact %s changed identity during read", path) + } + return file, opened, nil +} + +func readStableHandoff(file *os.File, candidate *handoffCandidate, path string, opened os.FileInfo) ([]byte, error) { + if handoffReadTestHook != nil { + handoffReadTestHook("before-first-read") + } + first, err := io.ReadAll(file) + if err != nil { + return nil, err + } + if handoffReadTestHook != nil { + handoffReadTestHook("after-first-read") + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, err + } + second, err := io.ReadAll(file) + if err != nil { + return nil, err + } + openedAfter, err := file.Stat() + if err != nil { + return nil, err + } + pathAfter, err := candidate.root.Lstat(candidate.name) + if err != nil { + return nil, err + } + if pathAfter.Mode()&os.ModeSymlink != 0 || !pathAfter.Mode().IsRegular() || !os.SameFile(opened, openedAfter) || !os.SameFile(pathAfter, openedAfter) || opened.Size() != openedAfter.Size() || !opened.ModTime().Equal(openedAfter.ModTime()) || !bytes.Equal(first, second) { + return nil, fmt.Errorf("handoff artifact %s changed while reading", path) + } + return first, nil +} + +func verifyHandoffRootIdentity(cwd string, candidate *handoffCandidate) error { + current, _, err := openRealHandoffRoot(cwd, candidate.components...) + if err != nil { + return fmt.Errorf("verify handoff root %s: %w", candidate.displayDir, err) + } + defer func() { _ = current.Close() }() + wantRoot, err := candidate.root.Stat(".") + if err != nil { + return err + } + gotRoot, err := current.Stat(".") + if err != nil { + return err + } + if !os.SameFile(wantRoot, gotRoot) { + return fmt.Errorf("handoff root %s changed identity while reading", candidate.displayDir) + } + return nil +} + +func decodeStoredHandoff(data []byte, filename string, artifact *storedHandoff) error { + if err := rejectHandoffSchemaNulls(data); err != nil { + return err + } + if err := decodeOneStoredHandoff(data, artifact); err != nil { + return err + } + if err := validateHandoffIdentity(filename, artifact); err != nil { + return err + } + if err := validateHandoffMetadata(artifact); err != nil { + return err + } + if err := validateHandoffRPI(artifact.RPI); err != nil { + return err + } + return validateHandoffState(artifact.State) +} + +func decodeOneStoredHandoff(data []byte, artifact *storedHandoff) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(artifact); err != nil { + return err + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return fmt.Errorf("multiple JSON values") + } + return err + } + return nil +} + +func validateHandoffIdentity(filename string, artifact *storedHandoff) error { + if artifact.SchemaVersion == nil || *artifact.SchemaVersion != 1 { + return fmt.Errorf("schema_version must be 1") + } + if artifact.ID == nil || !handoffIDPattern.MatchString(*artifact.ID) { + return fmt.Errorf("id does not satisfy handoff.v1") + } + if filename != *artifact.ID+".json" { + return fmt.Errorf("filename %s does not match artifact id %s", filename, *artifact.ID) + } + return nil +} + +func validateHandoffMetadata(artifact *storedHandoff) error { + if artifact.CreatedAt == nil { + return fmt.Errorf("created_at is required") + } + if _, err := time.Parse(time.RFC3339Nano, *artifact.CreatedAt); err != nil { + return fmt.Errorf("created_at is not a date-time: %w", err) + } + if artifact.Type != nil && *artifact.Type != "manual" && *artifact.Type != "auto" && *artifact.Type != "rpi" { + return fmt.Errorf("type is outside the handoff.v1 enum") + } + if artifact.ConsumedAt != nil { + if _, err := time.Parse(time.RFC3339Nano, *artifact.ConsumedAt); err != nil { + return fmt.Errorf("consumed_at is not a date-time: %w", err) + } + } + return nil +} + +func validateHandoffRPI(rpi *storedHandoffRPI) error { + if rpi != nil { + if rpi.Phase == nil || *rpi.Phase < 1 || *rpi.Phase > 3 { + return fmt.Errorf("rpi.phase must be an integer from 1 through 3") + } + if rpi.PhaseName == nil || (*rpi.PhaseName != "discovery" && *rpi.PhaseName != "implementation" && *rpi.PhaseName != "validation") { + return fmt.Errorf("rpi.phase_name is outside the handoff.v1 enum") + } + } + return nil +} + +func validateHandoffState(state *storedHandoffState) error { + if state != nil { + if state.GitDirty == nil { + return fmt.Errorf("state.git_dirty is required") + } + if state.OpenBeadsCount != nil && *state.OpenBeadsCount < 0 { + return fmt.Errorf("state.open_beads_count must be non-negative") + } + } + return nil +} + +// rejectHandoffSchemaNulls closes encoding/json's permissive null-to-zero +// conversion for optional scalar, array, and object fields. handoff.v1 allows +// null only for rpi, state, consumed_at, and consumed_by; every other present +// property must retain its declared JSON type. +func rejectHandoffSchemaNulls(data []byte) error { + var top map[string]json.RawMessage + if err := json.Unmarshal(data, &top); err != nil { + return err + } + for _, name := range []string{ + "schema_version", "id", "created_at", "type", "goal", "summary", "continuation", + "artifacts_produced", "decisions_made", "open_risks", "consumed", + } { + if raw, ok := top[name]; ok && bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return fmt.Errorf("%s must not be null", name) + } + } + for _, nestedName := range []string{"rpi", "state"} { + raw, ok := top[nestedName] + if !ok || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + continue + } + var nested map[string]json.RawMessage + if err := json.Unmarshal(raw, &nested); err != nil { + return err + } + for name, value := range nested { + if bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + return fmt.Errorf("%s.%s must not be null", nestedName, name) + } + } + if nestedName == "rpi" { + if verdictsRaw, ok := nested["verdicts"]; ok { + var verdicts map[string]json.RawMessage + if err := json.Unmarshal(verdictsRaw, &verdicts); err != nil { + return err + } + for key, value := range verdicts { + if bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + return fmt.Errorf("rpi.verdicts.%s must not be null", key) + } + } + } + } + } + return nil } // renderBrief renders the caller-authored brief. It surfaces only the goal, diff --git a/cli/internal/sessionapp/sessionapp_test.go b/cli/internal/sessionapp/sessionapp_test.go new file mode 100644 index 000000000..6503d45c6 --- /dev/null +++ b/cli/internal/sessionapp/sessionapp_test.go @@ -0,0 +1,111 @@ +package sessionapp + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +const stableHandoffName = "handoff-20260816T000000.000000000Z.json" + +func validStoredHandoff(continuation string) []byte { + return []byte(`{"schema_version":1,"id":"handoff-20260816T000000.000000000Z","created_at":"2026-08-16T00:00:00Z","continuation":"` + continuation + `"}` + "\n") +} + +func TestRehydrateFailsClosedWhenRootBindingChanges(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + agents := filepath.Join(dir, ".agents") + canonical := filepath.Join(agents, "ao", "handoff") + if err := os.MkdirAll(canonical, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(canonical, stableHandoffName), validStoredHandoff("original evidence"), 0o600); err != nil { + t.Fatal(err) + } + external := t.TempDir() + if err := os.MkdirAll(filepath.Join(external, "handoff"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(external, "handoff", stableHandoffName), validStoredHandoff("outside secret"), 0o600); err != nil { + t.Fatal(err) + } + + handoffReadTestHook = func(stage string) { + if stage != "before-artifact-open" { + return + } + handoffReadTestHook = nil + if err := os.Rename(filepath.Join(agents, "ao"), filepath.Join(agents, "ao.original")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(external, filepath.Join(agents, "ao")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + } + t.Cleanup(func() { handoffReadTestHook = nil }) + + var stdout, stderr bytes.Buffer + err := Rehydrate(RehydrateOptions{JSON: true, Stdout: &stdout, Stderr: &stderr}) + if err == nil { + t.Fatal("rehydrate accepted a handoff after its root path changed identity") + } + if strings.Contains(stdout.String(), "outside secret") || strings.Contains(stdout.String(), "original evidence") || strings.TrimSpace(stdout.String()) == "{}" { + t.Fatalf("unsafe rehydrate output = %q", stdout.String()) + } +} + +func TestRehydrateFailsClosedWhenArtifactChangesDuringRead(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + canonical := filepath.Join(dir, ".agents", "ao", "handoff") + if err := os.MkdirAll(canonical, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(canonical, stableHandoffName) + if err := os.WriteFile(path, validStoredHandoff("first bytes"), 0o600); err != nil { + t.Fatal(err) + } + handoffReadTestHook = func(stage string) { + if stage != "after-first-read" { + return + } + handoffReadTestHook = nil + if err := os.WriteFile(path, validStoredHandoff("different and longer bytes"), 0o600); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { handoffReadTestHook = nil }) + + var stdout, stderr bytes.Buffer + err := Rehydrate(RehydrateOptions{JSON: true, Stdout: &stdout, Stderr: &stderr}) + if err == nil || !strings.Contains(err.Error(), "changed while reading") { + t.Fatalf("Rehydrate error = %v, want unstable-artifact refusal", err) + } + if stdout.Len() != 0 { + t.Fatalf("unstable artifact leaked output: %q", stdout.String()) + } +} + +func TestDecodeStoredHandoffAcceptsSchemaValidDeprecatedFields(t *testing.T) { + data := []byte(`{ + "schema_version": 1, + "id": "handoff-20260816T000000.000000000Z", + "created_at": "2026-08-16T00:00:00Z", + "type": "rpi", + "artifacts_produced": ["report.md"], + "decisions_made": [], + "open_risks": ["caller decides"], + "rpi": {"phase": 3, "phase_name": "validation", "verdicts": {"scope": "PASS"}}, + "state": {"git_dirty": false, "open_beads_count": 0}, + "consumed": false, + "consumed_at": null, + "consumed_by": null +}`) + var artifact storedHandoff + if err := decodeStoredHandoff(data, stableHandoffName, &artifact); err != nil { + t.Fatalf("schema-valid deprecated artifact rejected: %v", err) + } +} diff --git a/docs/adr/ADR-0016-state-tiers.md b/docs/adr/ADR-0016-state-tiers.md index 2f3812923..4d5a8427b 100644 --- a/docs/adr/ADR-0016-state-tiers.md +++ b/docs/adr/ADR-0016-state-tiers.md @@ -40,18 +40,23 @@ The work tier is **queried, never indexed**: bead questions go through SQL (Dolt views) and `bv` graph analytics directly against the store. No stored index of bead data is ever built or committed — see invariant 2 below. -**Target layout.** The scratch/projection tier collapses from 114 ad-hoc -directories to a closed set of exactly three top-level `.agents/` entries: +**Target layout.** The scratch/projection tier is intended to collapse from 114 +ad-hoc directories to three preferred top-level `.agents/` entries: - `ao/` — the proof tier (permanent; pawl evidence, verdicts, pinned config), - `scratch/` — all ephemeral work, convention `scratch/WRITER/DATE-SLUG/`, TTL'd wholesale, - `projections/` — generated artifacts with manifests, deletable at will. -The closed set is enforced by an `ao doctor` detector -(`fm-ws-noncanonical-topdir`, bead `age-state-tiers-operationalize-5mzlm.7`): -any top-level directory outside the set is a finding, with no fourth -"receipts" exception (doctor receipts live under repo-root `.doctor/`, outside -`.agents/` entirely). +This is a target state, not a currently enforced closed set. The planned +`fm-ws-noncanonical-topdir` detector (bead +`age-state-tiers-operationalize-5mzlm.7`) was not implemented. Current Doctor +checks own narrower classes such as spelling drift, empty directories, and +stale queues. Compatibility and exact-path consumers also keep declared roots +outside the preferred three: legacy `.agents/handoff/` is preserved read-only; +`.agents/mto-handoff/` is a distinct live recurrence protocol; and explicitly +selected earlier output paths remain supported where their owning skill says +so. Those exceptions are migration contracts, not new authority tiers. Doctor +receipts still live under repo-root `.doctor/`, outside `.agents/` entirely. ### 2. The invariants @@ -173,9 +178,11 @@ authority into `.agents/`: ## Consequences -- The 114-directory `.agents/` junk drawer is migrated once to the three-dir - layout (bead `.6`) and the closed set is enforced forever after (bead `.7`); - writers that mint non-canonical directories are source bugs, not conventions. +- The three-directory layout remains the preferred migration target, but no + catch-all detector enforces it today. New default writers that mint an + undeclared top-level directory are source bugs; declared legacy-read and + exact-path consumer exceptions remain in place until their own migrations + complete. - No tool may build a stored index over bead data; bead reporting goes through Dolt SQL views and `bv` (the beads-views skill, `age-tracker-bd-dolt-return-jyg2g.8`). Filesystem-input projections without diff --git a/docs/agents-dir-hygiene.md b/docs/agents-dir-hygiene.md index 3437cac90..6350559b7 100644 --- a/docs/agents-dir-hygiene.md +++ b/docs/agents-dir-hygiene.md @@ -2,7 +2,7 @@ title: ".agents/ workspace hygiene" description: "Retention conventions for the .agents/ runtime workspace and how ao doctor keeps it clean." permalink: /agents-dir-hygiene -last_reviewed: 2026-07-18 +last_reviewed: 2026-08-16 --- # `.agents/` workspace hygiene @@ -14,12 +14,16 @@ and it is never a source of authority — work belongs to the tracker, source and delivery history to Git, sessions to CASS, curated memory to CM, and rules to docs/ADRs. -The target layout is the closed set from +The preferred target layout from [ADR-0016](adr/ADR-0016-state-tiers.md): `ao/` (requested proof), `scratch/` (disposable work, `scratch/WRITER/DATE-SLUG/`), and `projections/` -(named-consumer, manifest-stamped derived views). Everything else under -`.agents/` is legacy debris from older writers. Two shapes of that debris -exist: +(named-consumer, manifest-stamped derived views). This target is not enforced +as an exact closed set. A root outside it must have a declared compatibility or +exact-path consumer contract; otherwise it is migration debt. Current declared +exceptions include read-only `.agents/handoff/`, live +`.agents/mto-handoff/`, and earlier skill-output roots retained by an explicit +consumer contract in their owning skill. +Two broad shapes of migration debt exist: - **Knowledge-shaped directories** — postmortems, pre-mortem checks, handoffs, retros, proofs: human-readable records a later session may re-read while @@ -28,8 +32,9 @@ exist: consumed once by tooling and then debris. Left alone, the second class accumulates. The `workspace` subsystem of -`ao doctor` exists to detect and garbage-collect that debris safely. Where -this page and ADR-0016 disagree, the ADR wins. +`ao doctor` detects specific registered failure modes and garbage-collects +their debris safely; it does not currently flag every top-level root outside +the preferred layout. Where this page and ADR-0016 disagree, the ADR wins. ## The ephemeral-dir contract @@ -61,7 +66,7 @@ several ways. Doctor normalizes toward one canonical name per concept: |---|---| | `postmortem` | `post-mortem`, `post-mortems` | | `pre-mortem-checks` | `pre-mortem`, `pre-mortems`, `premortem-checks` | -| `handoff` | `handoffs`, `mto-handoff` | +| `ao/handoff` | `handoffs` | | `retro` | `retros` | | `proofs` | `proof` | | `tests` | `test` | @@ -70,6 +75,10 @@ This table is a projection of `workspaceCanonicalAliases` in `cli/internal/doctor/fix_workspace.go` — the Go table is the source of truth. If the two disagree, the Go table wins and this page is stale. +`.agents/mto-handoff/` is not a handoff spelling alias. It is a distinct live +recurrence protocol consumed at that exact path by +`scripts/assay/consume-mto-recurrence.sh`; Doctor preserves it in place. + ## How to clean up ```bash diff --git a/docs/cli-surface.json b/docs/cli-surface.json index 1efcfc1f2..0ea718a5a 100644 --- a/docs/cli-surface.json +++ b/docs/cli-surface.json @@ -449,6 +449,13 @@ "kind": "leaf", "reason": "Covered by handoff artifact tests." }, + { + "category": "public-tested", + "command": "session prune-agents", + "coverage_status": "covered", + "kind": "leaf", + "reason": "Covered by release smoke tests, direct command tests, or command handler tests." + }, { "category": "public-tested", "command": "session rehydrate", diff --git a/docs/cli-surface.md b/docs/cli-surface.md index 23411753d..a4a08f1a3 100644 --- a/docs/cli-surface.md +++ b/docs/cli-surface.md @@ -68,6 +68,7 @@ | `ao robot-docs` | `public-tested` | `allowlisted` | Covered by generated documentation tests. | | `ao session bootstrap` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. | | `ao session handoff` | `public-tested` | `allowlisted` | Covered by handoff artifact tests. | +| `ao session prune-agents` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. | | `ao session rehydrate` | `public-tested` | `allowlisted` | Covered by rehydrate artifact tests. | | `ao skills check` | `public-tested` | `allowlisted` | Covered by internal/commands/skills module tests after the skills carve-out. | | `ao skills consumers` | `public-tested` | `allowlisted` | Covered by internal/commands/skills module tests after the skills carve-out. | diff --git a/docs/contracts/context-map.md b/docs/contracts/context-map.md index 024fc05f1..abdda3186 100644 --- a/docs/contracts/context-map.md +++ b/docs/contracts/context-map.md @@ -99,7 +99,7 @@ | `doc` | produces | `documentation` | | `domain` | produces | `stdout` | | `fitness` | produces | `goal-measurement-report` | -| `handoff` | produces | `caller-selected handoff path or .agents/ao/handoff/*.md` | +| `handoff` | produces | `caller-selected handoff path or .agents/ao/handoff/*` | | `idea-genie` | consumes | `repo-context` | | `idea-genie` | consumes | `task-question` | | `idea-genie` | consumes | `idea-portfolio.v1` | @@ -127,7 +127,7 @@ | `refactor` | produces | `code-changes` | | `research` | consumes | `research-question` | | `research` | produces | `research-report` | -| `reverse-engineer` | produces | `.agents/scratch/reverse-engineer/*.md` | +| `reverse-engineer` | produces | `.agents/scratch/reverse-engineer/*/` | | `rpi` | consumes | `anti-ceremony` | | `rpi` | consumes | `plan` | | `rpi` | consumes | `implement` | diff --git a/docs/evals/2026-07-08-skill-probe-crank.md b/docs/evals/2026-07-08-skill-probe-crank.md index 13fad93e7..c436a8c26 100644 --- a/docs/evals/2026-07-08-skill-probe-crank.md +++ b/docs/evals/2026-07-08-skill-probe-crank.md @@ -1,10 +1,15 @@ # Skill behavioral probe — crank, 2026-07-08 -> **HONESTY.** A probe measures **BEHAVIOR-CHANGE, not quality-uplift.** This run -> answers only: did loading the crank skill change whether the agent respects -> write-scope collisions when planning parallel waves? It does not claim crank -> is good or bad. Small N (2) is **directional, not statistical** (ADR-0011 -> discipline — do not overclaim). +> **LEGACY EVIDENCE STATUS.** The committed fixture set predates capture-time +> fixture hashes and producer/config manifests. The current fail-closed harness +> cannot provenance-verify or reproduce this run. This report preserves the +> stored response classification and the run's historical account, but the +> ledger marks it `LEGACY-UNVERIFIED` and it does not count as probe coverage. +> +> **HONESTY.** The discriminator asks only whether the stored response separates +> write-scope collisions when planning parallel waves. It does not establish +> crank quality, the producer identity, a model-level behavior, or an execution +> outcome. Small N (2) is directional, not statistical (ADR-0011). ## What was measured @@ -19,46 +24,37 @@ assignment (the ACTION), not whether the plan mentions "write scope." - Probe: `evals/skill-probes/crank/` - Arms differ **only** by `treatment-prelude.md` (the crank wave-collision rule); `question.md` is identical. -- Live dispatch via `codex exec` (the sanctioned headless path — never - `claude -p`, LAW 0). Producer model recorded in the fixtures: **gpt-5.5**. -- Transcripts captured to `evals/skill-probes/crank/fixtures/` so the run - replays deterministically. +- The historical run record says dispatch used `codex exec` with `gpt-5.5`. + No capture manifest binds that producer/config label to the fixture bytes. +- Transcripts remain under `evals/skill-probes/crank/fixtures/` for inspection + and discriminator regression checks, not reproducible generation replay. -```bash -bash scripts/probe-skill.sh --probe crank --live --capture --reps 2 -# reproduce from the committed fixtures: -bash scripts/probe-skill.sh --probe crank --replay -``` - -## Result — INERT (frontier aced both arms) +## Stored result — LEGACY-UNVERIFIED (historically INERT) | Arm | present / usable | rate | what the agent did | |-----|------------------|------|--------------------| | control (no crank) | 2 / 2 | 1.0 | e.g. `Wave 1: bead-A, bead-B, bead-D` / `Wave 2: bead-C` — B and C separated | -| treatment (crank loaded) | 2 / 2 | 1.0 | B and C separated | +| treatment (crank prelude injected) | 2 / 2 | 1.0 | B and C separated | -**Verdict: `INERT`** (treatment_rate 1.0 is not > control_rate 1.0). +**Historical classification: `INERT`** because the stored treatment rate 1.0 +is not greater than the stored control rate 1.0. This is not a current +manifest-backed verdict. -## What INERT means here (and what it does NOT) +## What the stored classification means -It does **not** mean crank is worthless. It means: **on a frontier model -(gpt-5.5), at this task altitude, loading crank changed nothing** — the model -already refuses to parallelize two beads that write the same file, with or -without the doctrine. The skill's marginal *behavioral* effect on a strong -producer is nil because the producer already exhibits the behavior. +Both archived control responses and both archived treatment responses placed B +and C in different waves. That response set contains no discriminator +separation. It does not show that `gpt-5.5`, frontier models generally, or a +verified producer would behave the same way, because producer/config identity +was not capture-bound. -This is the same lesson the membrane eval already banked -(`membrane-eval-too-easy`, `moat-unproven-at-frontier`): a frontier producer aces -the task and yields no signal. To surface a skill's behavioral value you need a -**weaker producer** (`--model gpt-5-mini`, the local llama) or a **harder task** -where a naive agent actually gets it wrong. That is the honest ratchet, recorded -here rather than papered over. +A new claim about producer strength, task difficulty, or crank's marginal +behavior requires a capture-manifest-backed run. The historical null can help +shape that probe, but cannot decide whether to keep or cull the skill. -## Why this is still the required evidence +## Coverage status -The `age-e508.1` acceptance is "a probe RUN for crank exists, with a dated -evidence file and the MEASURED column populated" — **not** "crank must be -BEHAVIORAL." Measuring crank and honestly finding INERT-at-frontier is precisely -the product this bead builds: an unmeasured product badge is noise; a measured -one — even when the measurement is "no detectable change on this model" — is -truth. Recorded in `skills/SKILL-TIERS.md` → Behavioral Probe Ledger. +This dated report satisfied the earlier `age-e508.1` record-keeping acceptance. +Under the current evidence contract it remains historical context only. The +`LEGACY-UNVERIFIED` ledger row is intentionally excluded from measured coverage +until a capture-manifest-backed run records a current verdict. diff --git a/docs/evals/2026-07-08-skill-probe-graphify-calibration.md b/docs/evals/2026-07-08-skill-probe-graphify-calibration.md index fc4508b18..142ab9214 100644 --- a/docs/evals/2026-07-08-skill-probe-graphify-calibration.md +++ b/docs/evals/2026-07-08-skill-probe-graphify-calibration.md @@ -1,17 +1,22 @@ # Skill behavioral probe — graphify (CALIBRATION), 2026-07-08 -> **HONESTY.** A probe measures **BEHAVIOR-CHANGE, not quality-uplift.** This run -> answers only: did loading the graphify "use the graph before grep" guidance -> change which tool the agent actually reached for FIRST? It does not claim -> graphify is good or bad. Small N (2) is **directional, not statistical** -> (ADR-0011 discipline — do not overclaim). +> **LEGACY EVIDENCE STATUS.** These fixtures were reconstructed from a written +> 2026-06-30 account; they are not captured transcripts and have no capture-time +> hashes or producer/config manifest. The current fail-closed harness cannot +> provenance-verify or reproduce the original run. This file preserves the +> historical account and a discriminator regression case, while the ledger +> marks the probe `LEGACY-UNVERIFIED` and excludes it from measured coverage. +> +> **HONESTY.** The stored fixture test asks only whether the classifier detects +> a graph action before grep. It does not establish graphify quality or provide +> fresh behavioral evidence. Small N (2) in the historical account is +> directional, not statistical (ADR-0011). -## Purpose: calibrate the instrument against a known-INERT result +## Purpose: preserve a historical classification and classifier regression -This is the harness's **calibration** run. The `age-e508.1` acceptance requires -that, on the 2026-06-30 graphify scenario, the harness **reproduces the INERT -verdict** — i.e. that the ruler reads a known measurement correctly before we -trust it on new skills. +The reconstructed fixtures encode the documented 2026-06-30 no-action shape so +the discriminator can be regression-tested against it. Classifying those +authored fixtures does not reproduce the original run or verify its producer. The original measurement (memory `doc-instruction-to-use-tool-before-grep-is-inert`): on 2026-06-30, after @@ -33,33 +38,28 @@ structure via explain/path/query BEFORE broad grep,"* a controlled A/B found: or a read of `graphify-out/`) **before** any grep/rg. It checks the ACTION, not a mention. - Fixtures are **reconstructed from the documented 06-30 record** (both arms - grep-first, no graphify call) — this is a regression fixture encoding a known - outcome to calibrate the classifier, not a verbatim console capture. Run via - `--replay` (deterministic, zero token cost). + grep-first, no graphify call). They are an authored classifier regression + case, not verbatim console captures or replayable measurement evidence. -```bash -bash scripts/probe-skill.sh --probe graphify-tool-preference --replay -``` - -## Result — REPRODUCED +## Stored regression result — LEGACY-UNVERIFIED | Arm | present / usable | rate | |-----|------------------|------| | control | 0 / 2 | 0.0 | | treatment | 0 / 2 | 0.0 | -**Verdict: `INERT`** (treatment_rate 0.0 is not > control_rate 0.0). This matches -the documented 2026-06-30 result: the loaded guidance did not change which tool -the agent reached for. Calibration passes — the harness reads the known-INERT -case correctly. +**Historical classification: `INERT`** because the reconstructed treatment +rate 0.0 is not greater than the reconstructed control rate 0.0. The fixture +result is consistent with the documented 2026-06-30 account, but it is not a +current manifest-backed verdict or independent evidence of that run. ## A discriminator bug the calibration caught (worth recording) The first discriminator matched the bare token `graphify-out/` — which appeared in the treatment fixtures' *prose* header describing the environment — and -mis-scored the known-INERT case as `BEHAVIORAL`. That is exactly the failure this -whole bead exists to prevent: **measuring a mention, not an action.** Calibration -against the known result caught it; the discriminator was tightened to count only -real invocations (a command, a tool call, or a file read), and the prose token -was removed from the fixtures. A probe you cannot calibrate is a badge you cannot -trust. +mis-scored the authored no-action case as `BEHAVIORAL`. That is exactly the +classifier failure this regression fixture can test: **measuring a mention, not +an action.** The discriminator was tightened to count only real invocations (a +command, a tool call, or a file read), and the prose token was removed from the +fixtures. This validates the narrow classifier regression; it does not promote +the reconstructed fixture into provenance-verified behavioral evidence. diff --git a/docs/evals/2026-08-04-probe-wave-1.md b/docs/evals/2026-08-04-probe-wave-1.md index cbacc0b51..853564bb3 100644 --- a/docs/evals/2026-08-04-probe-wave-1.md +++ b/docs/evals/2026-08-04-probe-wave-1.md @@ -1,148 +1,167 @@ # Skill behavioral probe wave 1 — 2026-08-04 -> **HONESTY.** Probes measure **BEHAVIOR-CHANGE, not quality-uplift**: did loading the -> skill's guidance change what the agent DID? N=2 per arm per config is -> **directional, not statistical** (ADR-0011 — do not overclaim). Each arm differs -> only by `treatment-prelude.md`; discriminators are deterministic and pass a -> planted-reference selftest before any live scoring (known-good → PRESENT, -> known-bad → ABSENT, empty → DEGRADED). Probes measure **injected-prelude -> efficacy** — they do not measure whether a real session loads the SKILL.md file -> (the router lane; wave-2 scope). +> **LEGACY EVIDENCE STATUS.** This document preserves the wave's historical +> report and stored response-shape scores. Every fixture set described here +> predates capture-time fixture hashes and producer/config manifests. The +> current fail-closed harness therefore cannot provenance-verify or reproduce +> these runs; rescoring stored bytes cannot recover who produced them or under +> which settings. The ledger marks every row `LEGACY-UNVERIFIED`, so none counts +> as current probe coverage. +> +> **HONESTY.** The quiz asked whether an injected prelude changed a response +> shape, not whether a real session loaded the full `SKILL.md`, whether produced +> code was correct, or whether task outcomes improved. N=2 per arm per recorded +> config is directional, not statistical (ADR-0011). The deterministic +> discriminators selftest their classification rule; they do not attest the +> producer or capture process. ## What ran -5 new probes + 1 re-run, each at TWO producer configs (`gpt-5.6-luna` at -`model_reasoning_effort` xhigh and low), 2 reps per arm per config — 48 live -`codex exec` runs total. New harness capabilities this wave: `--effort` flag -(the second weak-producer ratchet), `producer` provenance in every scorecard, -discriminator selftests, and per-config fixture snapshots -(`fixtures-xhigh-2026-08-04/`) so re-runs never destroy prior evidence. +The 2026-08-04 run record reports 5 new probes plus 1 re-run, two reps per arm +under directories labelled `gpt-5.6-luna` xhigh and low: 48 `codex exec` runs +in total. The wave added an `--effort` flag, discriminator selftests, and dated +fixture directories such as `fixtures-xhigh-2026-08-04/`. -Reproduce any cell from committed fixtures: - -```bash -bash scripts/probe-skill.sh --probe premortem-self-validation --replay -``` +Those labels were not bound to the captured bytes by a manifest. The committed +fixtures preserve transcripts that can be inspected and classified, but the +new harness correctly refuses to treat them as provenance-verified replay +inputs or as a reproducible generation cell. ## Results -| Probe | Skill (tier) | xhigh C→T | low C→T | Verdict | +| Probe | Skill (tier) | xhigh-label C→T | low-label C→T | Historical classification | |---|---|---|---|---| -| `premortem-self-validation` | premortem (judgment) | 0.5 → 1.0 | 0.0 → 1.0 | **BEHAVIORAL** | -| `standards-go-conventions` | standards (knowledge) | 0.5 → 1.0 | 0.0 → 1.0 | **BEHAVIORAL** | -| `validate-not-proven` | validate (judgment) | 1.0 → 1.0 | 1.0 → 1.0 | INERT (ceiling) | -| `security-coverage-gap` | security (product) | 1.0 → 1.0 | 1.0 → 1.0 | INERT (ceiling) | -| `reality-check-gap` | reality-check (judgment) | 1.0 → 1.0 | 1.0 → 1.0 | INERT (ceiling) | -| `crank-luna` | crank (execution) | 1.0 → 1.0 | 1.0 → 1.0 | INERT (3rd config) | +| `premortem-self-validation` | premortem (judgment) | 0.5 → 1.0 | 0.0 → 1.0 | LEGACY-UNVERIFIED; reported BEHAVIORAL | +| `standards-go-conventions` | standards (knowledge) | 0.5 → 1.0 | 0.0 → 1.0 | LEGACY-UNVERIFIED; reported BEHAVIORAL | +| `validate-not-proven` | validate (judgment) | 1.0 → 1.0 | 1.0 → 1.0 | LEGACY-UNVERIFIED; reported INERT | +| `security-coverage-gap` | security (product) | 1.0 → 1.0 | 1.0 → 1.0 | LEGACY-UNVERIFIED; reported INERT | +| `reality-check-gap` | reality-check (judgment) | 1.0 → 1.0 | 1.0 → 1.0 | LEGACY-UNVERIFIED; reported INERT | +| `crank-luna` | crank (execution) | 1.0 → 1.0 | 1.0 → 1.0 | LEGACY-UNVERIFIED; reported INERT | + +The numeric cells above are preserved historical scores over stored response +bytes. They are not current measured-coverage claims. ## Findings -1. **First measured skill effects in this repo.** Two skills produced clean - behavioral separation, and both show the **gradient the SOTA literature - predicts: the effect grows as the producer weakens.** At xhigh the control - arm gets it right half the time; at low effort the control NEVER does and - the treatment ALWAYS does (0.0 → 1.0, both probes, 2/2 reps). This is the - product story in miniature — the harness's guidance matters most exactly - where the config is cheapest. - - premortem: naming the self-validation closure flaw (implementer closes on - its own tests) in a premortem over a plan that plants it. - - standards: producing `fmt.Errorf(...%w...)` wrapping + a table-driven - test without being asked for either. -2. **The ceiling class is scenario-difficulty, not effort.** validate, - security, and reality-check stayed saturated at BOTH efforts — even a - low-effort luna answers NOT_PROVEN / GAPPED / names-the-gap unaided. The - planted flaws are too obvious. The instrument needs harder scenarios - (subtler flaws, signal buried in longer context — the context-rot - direction), not weaker producers. **Do not read these INERTs as "the skill - is worthless"; read them as "this scenario cannot measure it."** -3. **crank is INERT across three configs** (gpt-5.5 xhigh 2026-07-08; luna - xhigh; luna low): write-scope-collision separation appears native to - current frontier models at every tested effort. This is the strongest - cull-or-reshape signal in the ledger — either the skill's value lives in - behaviors no probe yet measures, or the wave-planning core has been - absorbed by the models. +1. **Two quiz/prelude response shapes separated in the stored fixtures.** The + historical premortem responses named a planted self-validation closure flaw + at 1/2→2/2 in the xhigh-labelled set and 0/2→2/2 in the low-labelled set. + The standards responses contained both `%w` error wrapping and a + table-driven test at the same stored rates. This is directional evidence + about those two response discriminators in that prompt shape only. It does + not establish producer-strength effects, code correctness, task outcomes, + a general inline-over-reference rule, or a cross-language effect. +2. **Three stored quiz sets were discriminator-saturated.** The validate, + security, and reality-check bytes classified PRESENT in both arms under + both directory labels. That shows no headroom in the archived response set; + without capture manifests it cannot establish an effort effect or identify + whether the cause was scenario, producer, or configuration. +3. **The archived crank bytes also showed no discriminator separation.** That + historical result can motivate a new manifest-backed probe, but it cannot + generalize behavior across models or support a disposition decision on its + own. -## Skill changes made from these measurements +## Later task-embedded evidence -The treatment preludes that produced the separation were **hoisted into the -shipped skills** as front-loaded, imperative, load-bearing blocks (marked -MEASURED with probe citations): +The historical Tier-2 premortem pilot report addressed task outcomes rather +than quiz wording. It recorded 6/6 false-PASSes in the injected-doctrine +treatment and 6/6 in control (reported delta 0), then reported that a post-hoc +deterministic gate blocked 6/12 combined workspaces. No capture transcripts, +workspace set, or producer/config manifest is committed. The pre-freeze pilot +therefore supplies no promotable outcome-improvement evidence; it also cannot +independently establish a causal null. Evidence: +`evals/tier2-premortem/results-pilot-2026-08-05.md`. + +## Skill changes made after the historical quiz + +The treatment preludes associated with the stored response separation were +hoisted into the shipped skills as front-loaded guidance: - `skills/premortem/SKILL.md` — new first-check section: evidence-shape / who-verifies-and-are-they-fresh, before any technical risk. - `skills/standards/SKILL.md` — new load-bearing-conventions section: - inline-imperative Go core (the probe proved inline works where the graphify - probe proved behind-the-link does not). + inline Go reminders for `%w` wrapping and table-driven tests. -Codex twins regenerated (`codex-sync`), frontmatter + heal checks green. -Honest limit: the probes prove the **prelude content** changes behavior when -injected; whether the edited SKILL.md files get loaded and obeyed in real -sessions is the router question — measured next by the telemetry hook -(the opt-in `scripts/hooks/skill-telemetry.sh`) plus a routing probe batch. +The wave record says Codex twins were regenerated and frontmatter/heal checks +were green at the time. The archived quiz bytes do not prove that a real +session loaded either full skill, that the guidance changed produced code, or +that either change improved outcomes. A later pre-freeze Tier-2 pilot report +recorded false-PASS in all six treatment runs and all six controls (observed +delta 0). Its post-hoc deterministic gates blocked 6 of the 12 combined +workspaces. That pilot has no committed capture transcripts or manifests and +is historical, non-promotable evidence. -No changes were made to validate / security / reality-check from this wave: -their probes lack headroom, so any edit would be uninstrumented guessing. +The wave made no changes to validate, security, or reality-check because their +archived quiz responses were saturated. That preserves the historical +rationale; it is not a current manifest-backed headroom finding. ## Infrastructure shipped this wave -- `evals/skill-probes/LEDGER.md` — MEASURED ledger moved out of generated +- `evals/skill-probes/LEDGER.md` — the probe-status ledger moved out of generated SKILL-TIERS.md (root-cause fix for the regen wipe); gate + README + seed.go - hint repointed; coverage now 4/11 product-judgment skills measured. + hint repointed. After applying the current capture-manifest standard, all + historical rows are `LEGACY-UNVERIFIED`; coverage is now 0/12 on current main + product/judgment skills measured. - `evals/_stats/` — paired-bootstrap statistics package vendored in-repo (42 tests green); `RunStats` prefers the vendored copy. - `scripts/hooks/skill-telemetry.sh` (opt-in, never wired by default) — Skill invocations logged to `.agents/ao/skill-telemetry.jsonl`. Wiring is per-machine opt-in via gitignored `.claude/settings.json` (PostToolUse matcher `Skill` -> the script); never shipped to plugin users, never forced on contributors. -- `scripts/probe-skill.sh` — `--effort` ratchet, producer provenance in - scorecards. -- 6 new probe packages under `evals/skill-probes/` with deterministic - selftested discriminators and dual-config fixtures. +- `scripts/probe-skill.sh` — the wave added `--effort` and producer labels. + Those labels were not capture-bound to fixture hashes, so they are not + producer provenance under the current harness. +- 6 new probe packages under `evals/skill-probes/` with deterministic, + selftested discriminators and fixture directories carrying config labels; + the legacy fixtures have no capture manifest. ## Wave 2a — hardened ceiling scenarios (2026-08-05) -The three ceiling probes were re-authored as v2 with the flaw buried the way +The three saturated probes were re-authored as v2 with the flaw buried the way it hides in real life: validate's `not_checked` euphemized as "low-risk housekeeping" inside a green-heavy completion report under release-train pressure; security's semgrep failure softened to a mid-log WARN whose next line reads "0 findings"; reality-check's gap derivable only by arithmetic across separate outputs (19 files − 2 helpers = 17 claimed; grep count 14) -against two narrative confirmations. Same selftested discriminators. +against two narrative confirmations. Their discriminators were selftested, but +these fixture sets also predate capture manifests. -| Probe | xhigh C→T | low C→T | Verdict | +| Probe | xhigh-label C→T | low-label C→T | Historical classification | |---|---|---|---| -| `validate-not-proven-v2` | 1.0 → 1.0 | 1.0 → 1.0 | INERT (ceiling) | -| `security-coverage-gap-v2` | 1.0 → 1.0 | 1.0 → 1.0 | INERT (ceiling) | -| `reality-check-gap-v2` | 1.0 → 1.0 | n=2: 0.5 → 0.5; **n=6: 1.0 → 1.0** | INERT (ceiling) | +| `validate-not-proven-v2` | 1.0 → 1.0 | 1.0 → 1.0 | LEGACY-UNVERIFIED; reported INERT | +| `security-coverage-gap-v2` | 1.0 → 1.0 | 1.0 → 1.0 | LEGACY-UNVERIFIED; reported INERT | +| `reality-check-gap-v2` | 1.0 → 1.0 | reported n=2: 0.5 → 0.5; **n=6: 1.0 → 1.0** | LEGACY-UNVERIFIED; reported INERT | Two findings: -1. **A sequential-stopping object lesson.** reality-check-v2's first n=2 batch - showed apparent headroom (control 0.5) with an apparent null skill effect - (treatment 0.5). Extending to n=6 per the architecture's D4 rule resolved - both as sampling noise: 6/6 PRESENT in both arms. An n=2 conclusion here - would have been wrong twice over — this is exactly why probe scorecards - carry the "directional, not statistical" honesty header and why ambiguous - batches extend before concluding. -2. **The quiz format is the ceiling, not the scenario dressing.** Two rounds - of hardening failed to open headroom: an isolated question with an explicit - output menu (VERDICT:/STATUS:/RESULT:) telegraphs "this is a test," and - frontier models at every effort apply the doctrine flawlessly in that - frame. Real false-PASSes happen mid-task, in long agentic flows, with no - menu — the repo's own ~3% live refute rate proves the failure exists in - production shape. Conclusion recorded: **these three doctrines are - quiz-robust on current frontier models; further measurement of these - skills moves to task-embedded Tier-2 fixtures where the doctrine must fire - unprompted mid-work.** No more quiz-hardening rounds. +1. **The reality-check record contains an unresolved legacy mismatch.** The + historical report says its first low-labelled batch scored 1/2 in both arms + before the extension reached 6/6. The committed initial-batch bytes now + score 2/2 in both arms under the current discriminator. Without capture + manifests and discriminator identity, replay cannot resolve which scoring + state produced the historical number. The mismatch reinforces the + directional warning and is another reason this row carries no coverage. +2. **Harder quiz dressing did not open headroom in the archived bytes.** An + isolated question with an explicit output menu can telegraph the expected + decision, but the legacy evidence cannot isolate quiz format from producer + or configuration. A later pre-freeze task-embedded pilot report recorded + 6/6 false-PASSes in treatment and 6/6 in control (observed delta 0), while + post-hoc deterministic gates blocked 6/12 combined workspaces. It has no + committed capture transcripts or manifests. Future claims require + manifest-backed, task-embedded runs rather than another historical-fixture + replay. ## Next wave queue 1. ~~Harden the three ceiling scenarios and re-measure~~ — done (wave 2a); - outcome above: quiz-robust, move to Tier-2 task-embedded form. + archived response bytes remained saturated, but the runs are now + `LEGACY-UNVERIFIED`. 2. Routing probe batch: does a real session load the right skill unprompted (P(loaded | applicable)) — the multiplier on every number above. -3. Tier-2 outcome ablation for premortem on a task corpus (the Stage-A screen - of docs/architecture/eval-architecture.md) — now also carrying the - validate/security/reality-check doctrines as task-embedded checks. +3. Tier-2 outcome ablation for premortem on a task corpus — the historical + pilot report later recorded 6/6 treatment false-PASSes versus 6/6 control + (delta 0) and 6/12 combined workspaces blocked by a post-hoc deterministic + gate; a promotable run still requires a frozen prereg and capture manifests. 4. crank disposition decision (Bo): cull, reshape toward unmeasured behaviors, - or keep as documentation. + or keep as documentation; legacy fixture classifications alone do not decide + it. diff --git a/docs/evals/2026-08-16-anti-ceremony-creation-gate.md b/docs/evals/2026-08-16-anti-ceremony-creation-gate.md new file mode 100644 index 000000000..724a9ec33 --- /dev/null +++ b/docs/evals/2026-08-16-anti-ceremony-creation-gate.md @@ -0,0 +1,94 @@ +# Anti-ceremony creation-gate probe — 2026-08-16 + +> **Historical v2 compatibility result: `INERT` (directional N=2).** Both +> control and treatment made the discriminator-approved decision in both +> repetitions. The retained fixture predates the v3 response-only, +> counterbalanced, self-contained capture contract, so it is not current +> coverage evidence and does not establish quality or outcomes. + +## Question and discriminator + +The identical question in both arms asks for two create-or-drop decisions: + +- drop a permanent release-readiness dashboard with no named consumer, blocked + decision, observed defect, or deletion condition; +- create a short-lived provenance snapshot consumed by a release owner to + prevent recurrence of a named evidence-loss incident. + +The deterministic discriminator reports `PRESENT` only for `A: DROP` and +`B: CREATE`. It extracts decisions only from the final Codex response segment, +not the echoed question, so a partial answer cannot borrow a missing decision +from prompt text. It does not score explanation quality or an executed release +outcome. + +## Canonical-skill v2 run + +The treatment prompt included the exact bound canonical +`skills/operationalize/SKILL.md`; control received only the identical question. +This tests the response-shape effect of including those skill bytes. It does +not test automatic routing or discovery. + +| Field | Value | +|---|---| +| Producer observed in all transcript headers | Codex `gpt-5.6-luna`, low effort | +| Requested producer | `gpt-5.6-luna`, low effort | +| Treatment source | `canonical-skill` | +| Control | 2 present / 2 usable (1.0) | +| Treatment | 2 present / 2 usable (1.0) | +| Verdict | `INERT` | +| Evaluator identity | matched in the retained v2 scorecards; differs under the current v3 compatibility replay | +| Fixture binding | `sha256:80c4d0e983b8f750cea0114ad29f9c7036e35ab5e75206a9cf346d21da01642f` | + +Evidence: + +- live scorecard: `docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2b.json` +- replay scorecard: `docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2b-replay.json` +- fixture manifest and transcripts: + `evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/` +- probe inputs and discriminator: + `evals/skill-probes/anti-ceremony-creation-gate-v2/` + +The retained v2 replay scorecard was written before the v3 evaluator changes; +at that time it verified the canonical skill digest, four evaluation inputs, +all four transcripts, observed/requested producer fields, and the capture +evaluator's harness, preamble, dispatch helper, and metadata helper. It +reproduced the 2/2-versus-2/2 classification with +`evaluator_matches_capture: true`. + +Replaying the same bound fixture with the current v3 harness still reproduces +the 2/2-versus-2/2 `INERT` classification, but now reports +`evaluator_matches_capture: false`. That is expected evaluator evolution, not a +new live result; no replacement scorecard was created from this compatibility +replay. + +## Earlier and failed attempts + +The first bound run used `anti-ceremony-creation-gate` and injected only its +distilled treatment prelude. Its v1 fixture binding is +`sha256:e5d15b63f7264c21f4db7e61b11a5e4a097a04a76dcee9f6c57b274ec0af6c74` +and its stored classification is also 2/2 versus 2/2 (`INERT`). The honest +replay scorecard is +`docs/evals/scorecards/2026-08-16/anti-ceremony-low-v1-replay.json`; it labels +the treatment `injected-prelude` and discloses evaluator drift. The immutable +earlier `anti-ceremony-low.json` scorecard is deprecated because its +loaded-skill wording exceeds what that v1 fixture binds. This prelude-only run +does not count as canonical skill evidence. + +The first v2 live attempt is retained as +`docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2.json`. It is +`UNMEASURED`: control produced 2/2 usable responses, both treatment dispatches +failed, and no fixture set was published. The failure exposed a shared runner +bug in which a prompt beginning with YAML's `---` was parsed as CLI options. +The runner now inserts an end-of-options marker, with a regression test; the +successful v2 run used a new immutable fixture and scorecard name. + +## Interpretation boundary + +The retained historical claim is only: in these four v2 canonical-skill-bound +responses at the recorded configuration, including the operationalize skill +did not change the old scored dual decision. No v3 capture was run: this is a +meta-tier probe that gates no named feature or product/judgment coverage +denominator, so manufacturing replacement evidence would be ceremony rather +than capability work. Claims about weaker producers, harder artifact decisions, +automatic skill activation, production outcomes, or broader skill value remain +unmeasured. diff --git a/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v1-replay.json b/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v1-replay.json new file mode 100644 index 000000000..d42ba6dc3 --- /dev/null +++ b/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v1-replay.json @@ -0,0 +1,68 @@ +{ + "schema": "agentops-skill-probe.v2", + "probe": "anti-ceremony-creation-gate", + "skill": "operationalize", + "mode": "replay", + "generated_at": "2026-08-16T13:23:42Z", + "reps": 2, + "producer": { + "adapter": "codex", + "model": "gpt-5.6-luna", + "effort": "low" + }, + "requested_producer": { + "model": "gpt-5.6-luna", + "effort": "low" + }, + "fixture_set": { + "name": "fixtures-low-2026-08-16", + "metadata": "fixture-set.json", + "binding_sha256": "sha256:e5d15b63f7264c21f4db7e61b11a5e4a097a04a76dcee9f6c57b274ec0af6c74" + }, + "treatment_source": "injected-prelude", + "evaluator": { + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:3635c78f2fd0a2814576e5a400b64266648ad62a55789d5c2797ef869f2c347d" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:0d1a48bc616388a14eb3a2d98dcdb54a93fb51be05d5272ea2628bc9ca9ba750" + } + }, + "capture_evaluator": { + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:855ff08cdf035fd8141f3adb4a00aede7313048298cd27a8c2c6cb21b5ed8779" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:400eba7ec633a22af402550e61c3abeb86220461c95d777d7801cf9178d50199" + } + }, + "evaluator_matches_capture": false, + "honesty": "measures response-shape BEHAVIOR-CHANGE under the exact hash-bound injected prelude named by bound probe metadata; canonical SKILL.md is not bound; this is NOT full-skill activation or quality-uplift; small N is directional (ADR-0011)", + "control": { + "present": 2, + "usable": 2, + "rate": 1.0 + }, + "treatment": { + "present": 2, + "usable": 2, + "rate": 1.0 + }, + "verdict": "INERT", + "per_rep": [ + { + "rep": 1, + "control": "PRESENT", + "treatment": "PRESENT" + }, + { + "rep": 2, + "control": "PRESENT", + "treatment": "PRESENT" + } + ] +} diff --git a/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2.json b/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2.json new file mode 100644 index 000000000..4c0e224fa --- /dev/null +++ b/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2.json @@ -0,0 +1,59 @@ +{ + "schema": "agentops-skill-probe.v2", + "probe": "anti-ceremony-creation-gate-v2", + "skill": "operationalize", + "mode": "live", + "generated_at": "2026-08-16T13:23:59Z", + "reps": 2, + "producer": { + "adapter": "codex", + "model": "gpt-5.6-luna", + "effort": "low" + }, + "requested_producer": { + "model": "gpt-5.6-luna", + "effort": "low" + }, + "fixture_set": { + "name": "fixtures-low-2026-08-16-v2", + "metadata": null, + "binding_sha256": null + }, + "treatment_source": "canonical-skill", + "evaluator": { + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:3635c78f2fd0a2814576e5a400b64266648ad62a55789d5c2797ef869f2c347d" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:0d1a48bc616388a14eb3a2d98dcdb54a93fb51be05d5272ea2628bc9ca9ba750" + } + }, + "capture_evaluator": null, + "evaluator_matches_capture": null, + "honesty": "UNMEASURED live attempt; no immutable fixture set was published", + "control": { + "present": 2, + "usable": 2, + "rate": 1.0 + }, + "treatment": { + "present": 0, + "usable": 0, + "rate": null + }, + "verdict": "UNMEASURED", + "per_rep": [ + { + "rep": 1, + "control": "PRESENT", + "treatment": "DEGRADED" + }, + { + "rep": 2, + "control": "PRESENT", + "treatment": "DEGRADED" + } + ] +} diff --git a/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2b-replay.json b/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2b-replay.json new file mode 100644 index 000000000..ae57403bd --- /dev/null +++ b/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2b-replay.json @@ -0,0 +1,84 @@ +{ + "schema": "agentops-skill-probe.v2", + "probe": "anti-ceremony-creation-gate-v2", + "skill": "operationalize", + "mode": "replay", + "generated_at": "2026-08-16T13:41:24Z", + "reps": 2, + "producer": { + "adapter": "codex", + "model": "gpt-5.6-luna", + "effort": "low" + }, + "requested_producer": { + "model": "gpt-5.6-luna", + "effort": "low" + }, + "fixture_set": { + "name": "fixtures-low-2026-08-16-v2b", + "metadata": "fixture-set.json", + "binding_sha256": "sha256:80c4d0e983b8f750cea0114ad29f9c7036e35ab5e75206a9cf346d21da01642f" + }, + "treatment_source": "canonical-skill", + "evaluator": { + "dispatch_helper": { + "path": "scripts/lib/codex-exec.sh", + "sha256": "sha256:f58b3485194149715191dda550802cb8ba991c946720e098b57f3aa671d32f2f" + }, + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:187fd5004d8a25edc8d5b5787908595f3d8718386daabeb47158d6ec8b892507" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:518d56a8128b4d10d02ddc2332fb3c12fb7ce747cab24ce1a19cc64eedbdaf37" + }, + "preamble": { + "path": "scripts/lib/preamble.sh", + "sha256": "sha256:c4c06472b655cc440ab69e2d3c3bb7773c393eab671975aad3f1d446b2994199" + } + }, + "capture_evaluator": { + "dispatch_helper": { + "path": "scripts/lib/codex-exec.sh", + "sha256": "sha256:f58b3485194149715191dda550802cb8ba991c946720e098b57f3aa671d32f2f" + }, + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:187fd5004d8a25edc8d5b5787908595f3d8718386daabeb47158d6ec8b892507" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:518d56a8128b4d10d02ddc2332fb3c12fb7ce747cab24ce1a19cc64eedbdaf37" + }, + "preamble": { + "path": "scripts/lib/preamble.sh", + "sha256": "sha256:c4c06472b655cc440ab69e2d3c3bb7773c393eab671975aad3f1d446b2994199" + } + }, + "evaluator_matches_capture": true, + "honesty": "measures response-shape BEHAVIOR-CHANGE under the exact bound canonical SKILL.md treatment, NOT quality-uplift; small N is directional (ADR-0011)", + "control": { + "present": 2, + "usable": 2, + "rate": 1.0 + }, + "treatment": { + "present": 2, + "usable": 2, + "rate": 1.0 + }, + "verdict": "INERT", + "per_rep": [ + { + "rep": 1, + "control": "PRESENT", + "treatment": "PRESENT" + }, + { + "rep": 2, + "control": "PRESENT", + "treatment": "PRESENT" + } + ] +} diff --git a/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2b.json b/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2b.json new file mode 100644 index 000000000..9e0a79402 --- /dev/null +++ b/docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2b.json @@ -0,0 +1,84 @@ +{ + "schema": "agentops-skill-probe.v2", + "probe": "anti-ceremony-creation-gate-v2", + "skill": "operationalize", + "mode": "live", + "generated_at": "2026-08-16T13:41:11Z", + "reps": 2, + "producer": { + "adapter": "codex", + "model": "gpt-5.6-luna", + "effort": "low" + }, + "requested_producer": { + "model": "gpt-5.6-luna", + "effort": "low" + }, + "fixture_set": { + "name": "fixtures-low-2026-08-16-v2b", + "metadata": "fixture-set.json", + "binding_sha256": "sha256:80c4d0e983b8f750cea0114ad29f9c7036e35ab5e75206a9cf346d21da01642f" + }, + "treatment_source": "canonical-skill", + "evaluator": { + "dispatch_helper": { + "path": "scripts/lib/codex-exec.sh", + "sha256": "sha256:f58b3485194149715191dda550802cb8ba991c946720e098b57f3aa671d32f2f" + }, + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:187fd5004d8a25edc8d5b5787908595f3d8718386daabeb47158d6ec8b892507" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:518d56a8128b4d10d02ddc2332fb3c12fb7ce747cab24ce1a19cc64eedbdaf37" + }, + "preamble": { + "path": "scripts/lib/preamble.sh", + "sha256": "sha256:c4c06472b655cc440ab69e2d3c3bb7773c393eab671975aad3f1d446b2994199" + } + }, + "capture_evaluator": { + "dispatch_helper": { + "path": "scripts/lib/codex-exec.sh", + "sha256": "sha256:f58b3485194149715191dda550802cb8ba991c946720e098b57f3aa671d32f2f" + }, + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:187fd5004d8a25edc8d5b5787908595f3d8718386daabeb47158d6ec8b892507" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:518d56a8128b4d10d02ddc2332fb3c12fb7ce747cab24ce1a19cc64eedbdaf37" + }, + "preamble": { + "path": "scripts/lib/preamble.sh", + "sha256": "sha256:c4c06472b655cc440ab69e2d3c3bb7773c393eab671975aad3f1d446b2994199" + } + }, + "evaluator_matches_capture": true, + "honesty": "measures response-shape BEHAVIOR-CHANGE under the exact bound canonical SKILL.md treatment, NOT quality-uplift; small N is directional (ADR-0011)", + "control": { + "present": 2, + "usable": 2, + "rate": 1.0 + }, + "treatment": { + "present": 2, + "usable": 2, + "rate": 1.0 + }, + "verdict": "INERT", + "per_rep": [ + { + "rep": 1, + "control": "PRESENT", + "treatment": "PRESENT" + }, + { + "rep": 2, + "control": "PRESENT", + "treatment": "PRESENT" + } + ] +} diff --git a/docs/evals/scorecards/2026-08-16/anti-ceremony-low.json b/docs/evals/scorecards/2026-08-16/anti-ceremony-low.json new file mode 100644 index 000000000..0e14aff56 --- /dev/null +++ b/docs/evals/scorecards/2026-08-16/anti-ceremony-low.json @@ -0,0 +1,67 @@ +{ + "schema": "agentops-skill-probe.v2", + "probe": "anti-ceremony-creation-gate", + "skill": "operationalize", + "mode": "live", + "generated_at": "2026-08-16T12:55:55Z", + "reps": 2, + "producer": { + "adapter": "codex", + "model": "gpt-5.6-luna", + "effort": "low" + }, + "requested_producer": { + "model": "gpt-5.6-luna", + "effort": "low" + }, + "fixture_set": { + "name": "fixtures-low-2026-08-16", + "metadata": "fixture-set.json", + "binding_sha256": "sha256:e5d15b63f7264c21f4db7e61b11a5e4a097a04a76dcee9f6c57b274ec0af6c74" + }, + "evaluator": { + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:855ff08cdf035fd8141f3adb4a00aede7313048298cd27a8c2c6cb21b5ed8779" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:400eba7ec633a22af402550e61c3abeb86220461c95d777d7801cf9178d50199" + } + }, + "capture_evaluator": { + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:855ff08cdf035fd8141f3adb4a00aede7313048298cd27a8c2c6cb21b5ed8779" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:400eba7ec633a22af402550e61c3abeb86220461c95d777d7801cf9178d50199" + } + }, + "evaluator_matches_capture": true, + "honesty": "measures BEHAVIOR-CHANGE (did the loaded skill change what the agent DID), NOT quality-uplift; small N is directional (ADR-0011)", + "control": { + "present": 2, + "usable": 2, + "rate": 1.0 + }, + "treatment": { + "present": 2, + "usable": 2, + "rate": 1.0 + }, + "verdict": "INERT", + "per_rep": [ + { + "rep": 1, + "control": "PRESENT", + "treatment": "PRESENT" + }, + { + "rep": 2, + "control": "PRESENT", + "treatment": "PRESENT" + } + ] +} diff --git a/docs/reference/skill-quality-rubric.md b/docs/reference/skill-quality-rubric.md index 18964ab38..6509afeac 100644 --- a/docs/reference/skill-quality-rubric.md +++ b/docs/reference/skill-quality-rubric.md @@ -1,35 +1,104 @@ # Skill Quality Rubric -This advisory rubric identifies concrete hardening opportunities. It does not -change a deep-audit verdict and must not reward unnecessary references, -scripts, assets, self-tests, or subagent packets. +A good skill is conformant, safe for its declared effects, and demonstrably +improves the work it claims to help with. Package polish is useful evidence, +but it cannot compensate for a failed safety gate or missing behavioral proof. -## Scoring +This repository therefore reports three independent results. Do not collapse +them into one average: -Each category receives 0–3: +1. **Conformance gate:** the package satisfies the open Agent Skills contract, + the selected host profile, and repository structural checks. +2. **Safety gate:** every bundled file and declared effect has been reviewed; + privileges, writes, network access, credentials, destructive actions, + external content, approval points, bounds, and cleanup match the stated job. +3. **Effectiveness evidence:** representative tasks show correct activation and + better observable outcomes than the no-skill or previous-version baseline. + +A failed gate is `FAIL`. A gate or required evidence layer that was not actually +checked is `NOT_PROVEN`, never an inferred pass. An overall quality `PASS` +requires both gates to pass and effectiveness level E2 or E3 below. + +## Static package-readiness score + +The existing deterministic scorer remains a cheap triage signal. It measures +visible package properties only; it evaluates neither the safety gate nor skill +effectiveness. Each category receives 0–3: | Score | Meaning | |---:|---| -| 0 | Missing and required for this skill's actual behavior | +| 0 | Missing and required for this skill's declared behavior | | 1 | Present but weak, or warranted but absent | -| 2 | Solid for the skill's scope, including “not needed” | +| 2 | Solid for the declared scope, including justified “not needed” | | 3 | Mechanically strong or unusually complete | -| Category | What good means | +| Category | What the static scorer looks for | |---|---| -| Trigger quality | Description says what, when, and avoids obvious false positives. | +| Trigger quality | Description says what and when and includes an obvious false-positive boundary. | | Kernel clarity | The bounded procedure and stop condition are easy to find. | -| Progressive disclosure | A concise kernel is self-contained; complex detail is linked. | +| Progressive disclosure | A concise kernel is self-contained; complex detail is directly linked and loaded only when needed. | | Helper scripts | Repeated deterministic mechanics are scripted; judgment is not. | -| Validation | Evidence commands or artifacts prove executable behavior. | +| Validation | Commands or artifacts exist that can check executable behavior. | | Self-test | Trigger or behavior examples exist when complexity warrants them. | | Assets/templates | Reusable payloads exist only when the workflow actually needs them. | -| Subagents/roles | Delegation packets exist only for intentionally delegated work. | -| Safety boundaries | Mutation, authorization, and non-goals are explicit where relevant. | -| Packaging | The package is small, linked, mode-correct, and projection-safe. | +| Subagents/roles | AgentOps-specific delegation packets exist only for intentionally delegated work. This is not a portable Agent Skills quality criterion. | +| Safety boundaries | Boundary words are visible. This is a signal only, not the safety gate. | +| Packaging | The package is small, linked, host-profile-correct, and projection-safe. | -The maximum remains 30. Rating bands are C (0–10), B (11–20), A (21–26), and -S (27–30). A lower advisory rating is a review signal, not a ship blocker. +The maximum is 30. Static readiness bands are C (0–10), B (11–20), A (21–26), +and S (27–30). A lower band is a review signal, not a ship blocker; a high band +is not an effectiveness or safety claim. + +## Effectiveness evidence levels + +| Level | Required evidence | +|---|---| +| E0 | No skill-specific behavioral scenario or receipt was located. | +| E1 | A scenario, feature, or self-test exists, but there is no current baseline comparison proving outcome delta. | +| E2 | On the exact skill version, representative direct, indirect, incomplete-input, should-not-trigger, and edge cases pass against an explicit no-skill or prior-version baseline. Activation and output are graded separately. | +| E3 | E2 is repeatable as a regression suite across every intended model, host surface, and realistic installed-skill catalog; traces cover relevant tool calls, guardrails, and handoffs. | + +Test observable behavior, not prose presence. Record the target model and host, +skill version or content digest, dataset, grader, baseline, treatment, failures, +and date. Structural validation can establish package conformance; it cannot +establish E2. + +## Portable baseline and host profiles + +The portable baseline is the [Agent Skills specification](https://agentskills.io/specification): +valid name/directory identity, a useful what-and-when description, relative +resource links, and progressive disclosure. Host rules are additional profiles, +not universal requirements. For example, Codex may impose a tighter metadata +budget or packaging surface than another compatible host. + +For progressive disclosure, use the documented working limits: keep `SKILL.md` +under 500 lines and roughly 5,000 tokens, keep references one level deep, make +load conditions explicit, and add a table of contents to long reference files. +Split a skill when triggers, inputs, or success conditions materially differ. + +## Current primary sources + +- [OpenAI: Build skills for ChatGPT and Codex](https://learn.chatgpt.com/docs/build-skills) + covers focused jobs, front-loaded descriptions, explicit inputs/outputs, + progressive loading, and trigger testing. +- [OpenAI: Build skills for plugins](https://developers.openai.com/plugins/build/skills) + specifies direct, indirect, incomplete-input, should-not-trigger, and edge + cases and separates activation failures from output failures. +- [OpenAI API skills](https://developers.openai.com/api/docs/guides/tools-skills) + treats skill bundles as privileged, initially untrusted code and instructions + and requires bounded, approved high-impact actions. +- [OpenAI agent evaluations](https://developers.openai.com/api/docs/guides/agent-evals) + describes repeatable datasets, graders, and trace inspection. +- [Anthropic skill authoring best practices](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices) + recommends a no-skill baseline, representative scenarios, observable results, + concise kernels, and testing on every intended model. +- [Anthropic enterprise skills](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/enterprise) + covers full-bundle safety review and catalog-scale recall/coexistence tests. +- [GitHub Copilot agent skills](https://docs.github.com/en/enterprise-cloud@latest/copilot/how-tos/copilot-on-github/customize-copilot/customize-cloud-agent/add-skills) + emphasizes provenance, pinning, preview, and dry-run validation. +- [Microsoft Agent Framework skills](https://learn.microsoft.com/en-us/agent-framework/agents/skills) + covers sandboxing, resource limits, input allowlists, audit logs, and when a + deterministic high-side-effect workflow is a better fit than a skill. ## Required repository checks @@ -55,10 +124,12 @@ assets or a dedicated delegation tree. ## Audit method 1. Read the complete `SKILL.md` and every linked resource. -2. Compare its declared trigger, boundaries, output, and evidence with actual - behavior. -3. Run the repository checks. -4. Score optional package features relative to demonstrated need. -5. Recommend the smallest change that removes a real defect. +2. Apply the conformance and safety gates without compensation. +3. Run the repository checks and record the static readiness score as triage. +4. Compare declared activation and output behavior with a baseline and grade + the effectiveness evidence E0–E3. +5. Test coexistence in the intended installed catalog and every target model or + host before claiming E3. +6. Recommend the smallest change that removes an observed defect. Never add ceremony solely to raise a numeric score. diff --git a/evals/agentops-core/cli-command-surface-matrix.json b/evals/agentops-core/cli-command-surface-matrix.json index 8bb4f0ed7..020ce5a28 100644 --- a/evals/agentops-core/cli-command-surface-matrix.json +++ b/evals/agentops-core/cli-command-surface-matrix.json @@ -41,7 +41,7 @@ }, "expectations": [ {"type": "exit_code", "value": 0}, - {"type": "stdout_contains", "value": "cli-command-headings: top=20 sub=56 all=89"}, + {"type": "stdout_contains", "value": "cli-command-headings: top=20 sub=57 all=90"}, {"type": "stdout_contains", "value": "cli-help-matrix-ok"} ], "dimensions": ["correctness", "runtime_compatibility", "artifact_quality"], diff --git a/evals/agentops-core/fixtures/cli-command-surface-smoke.sh b/evals/agentops-core/fixtures/cli-command-surface-smoke.sh index 7a69c842a..4c550bef4 100755 --- a/evals/agentops-core/fixtures/cli-command-surface-smoke.sh +++ b/evals/agentops-core/fixtures/cli-command-surface-smoke.sh @@ -17,7 +17,7 @@ top_count="$(rg -c '^### `ao ' "$DOCS_PATH")" sub_count="$(rg -c '^#### `ao ' "$DOCS_PATH")" all_count="$(rg -c '^#{3,5} `ao ' "$DOCS_PATH")" -if [[ "$top_count" != "20" || "$sub_count" != "56" || "$all_count" != "89" ]]; then +if [[ "$top_count" != "20" || "$sub_count" != "57" || "$all_count" != "90" ]]; then printf 'unexpected command heading counts: top=%s sub=%s all=%s\n' "$top_count" "$sub_count" "$all_count" >&2 exit 1 fi @@ -25,7 +25,7 @@ fi # shellcheck disable=SC2016 # literal backticks delimit generated Markdown command headings. mapfile -t commands < <(rg '^#{3,5} `ao ' "$DOCS_PATH" | sed -E 's/^.*`([^`]+)`.*/\1/') -if [[ "${#commands[@]}" -ne 89 ]]; then +if [[ "${#commands[@]}" -ne 90 ]]; then printf 'unexpected command matrix size: %s\n' "${#commands[@]}" >&2 exit 1 fi diff --git a/evals/skill-probes/LEDGER.md b/evals/skill-probes/LEDGER.md index c27216fef..39e520c52 100644 --- a/evals/skill-probes/LEDGER.md +++ b/evals/skill-probes/LEDGER.md @@ -7,24 +7,46 @@ > frontmatter, so it does not belong in a generated file. Parsed by > `scripts/check-skill-probe-coverage.sh` (heading + table format are load-bearing). > -> **HONESTY.** A probe verdict records BEHAVIOR-CHANGE (did loading the skill -> change what the agent DID), never quality-uplift. BEHAVIORAL = loading it -> changed the action; INERT = it did not; small N is directional, not -> statistical (ADR-0011). One row per (skill, probe, run) — append, don't -> overwrite: history is the point. +> **HONESTY.** A current probe verdict records response-shape +> BEHAVIOR-CHANGE, never quality uplift. `BEHAVIORAL` and `INERT` count toward +> coverage only after capture-time fixture hashes and producer configuration +> are bound in a manifest accepted by the fail-closed harness. +> `LEGACY-UNVERIFIED` preserves an earlier reported classification whose +> fixture bytes predate that manifest contract. Those bytes may still explain +> history, but replay cannot establish who produced them, under which config, +> or that generation is reproducible. `PRELUDE-ONLY` may have a valid bound +> capture, but its treatment was distilled text rather than the canonical +> `SKILL.md`, so it never counts as skill coverage. One row per (skill, probe, +> run) — append rather than erase history. +> +> **CURRENT-ROW EVIDENCE SYNTAX.** Every `BEHAVIORAL`, `REGRESSIVE`, or `INERT` row must carry +> exactly one ``scorecard: `docs/evals/scorecards/...json` `` pointer in Notes. +> That repo-relative pointer is machine-readable; the coverage gate derives the +> fixture manifest from the v3 scorecard and recomputes its bound classification. +> Other prose and links in Notes are explanatory only and never make a row count. -## Behavioral Probe Ledger (MEASURED) +## Behavioral Probe Ledger (MEASUREMENT STATUS) | Skill | Probe | Date | Verdict | Notes | |---|---|---|---|---| -| `crank` | `crank` | 2026-07-08 | INERT | gpt-5.5 separated write-scope-colliding beads unaided (1.0 vs 1.0 — no headroom at frontier); evidence: docs/evals/2026-07-08-skill-probe-crank.md | -| `graphify` | `graphify-tool-preference` | 2026-06-30 | INERT | 0/2 treatment agents obeyed a verbatim doc instruction; evidence: docs/evals/2026-07-08-skill-probe-graphify-calibration.md | -| `premortem` | `premortem-self-validation` | 2026-08-04 | BEHAVIORAL | gpt-5.6-luna: xhigh 0.5→1.0, low 0.0→1.0 — effect grows as producer weakens; evidence: docs/evals/2026-08-04-probe-wave-1.md | -| `standards` | `standards-go-conventions` | 2026-08-04 | BEHAVIORAL | gpt-5.6-luna: xhigh 0.5→1.0, low 0.0→1.0 — perfect separation at low effort; evidence: docs/evals/2026-08-04-probe-wave-1.md | -| `validate` | `validate-not-proven` | 2026-08-04 | INERT | ceiling at xhigh AND low (luna returns NOT_PROVEN unaided) — scenario needs hardening, not the skill; evidence: docs/evals/2026-08-04-probe-wave-1.md | -| `security` | `security-coverage-gap` | 2026-08-04 | INERT | ceiling at xhigh AND low (GAPPED chosen unaided); scenario needs hardening; evidence: docs/evals/2026-08-04-probe-wave-1.md | -| `reality-check` | `reality-check-gap` | 2026-08-04 | INERT | ceiling at xhigh AND low (gap named unaided); scenario needs hardening; evidence: docs/evals/2026-08-04-probe-wave-1.md | -| `crank` | `crank-luna` | 2026-08-04 | INERT | third config (luna xhigh + low, after gpt-5.5 xhigh): collision invariant is native to frontier models — cull/reshape candidate; evidence: docs/evals/2026-08-04-probe-wave-1.md | -| `validate` | `validate-not-proven-v2` | 2026-08-05 | INERT | hardened scenario (euphemized not_checked, green-heavy report, release pressure) STILL ceiling at xhigh+low — the doctrine is robust in quiz format; next ratchet is task-embedded Tier-2, not harder quizzes | -| `security` | `security-coverage-gap-v2` | 2026-08-05 | INERT | scanner failure buried as mid-log WARN ("0 rules loaded… 0 findings") still caught unaided at both efforts | -| `reality-check` | `reality-check-gap-v2` | 2026-08-05 | INERT | n=2 showed apparent headroom (C=0.5/T=0.5); sequential extension to n=6 resolved it as noise — 6/6 both arms; ceiling at both efforts | +| `crank` | `crank` | 2026-07-08 | LEGACY-UNVERIFIED | Historical classification: INERT, with stored rates 1.0 vs 1.0; evidence: docs/evals/2026-07-08-skill-probe-crank.md | +| `graphify` | `graphify-tool-preference` | 2026-06-30 | LEGACY-UNVERIFIED | Historical classification: INERT, 0/2 treatment responses followed the instruction; fixtures were reconstructed rather than verbatim captures; evidence: docs/evals/2026-07-08-skill-probe-graphify-calibration.md | +| `premortem` | `premortem-self-validation` | 2026-08-04 | LEGACY-UNVERIFIED | Historical quiz classification: BEHAVIORAL; the response named the planted self-validation flaw at stored rates xhigh 1/2→2/2 and low 0/2→2/2; evidence: docs/evals/2026-08-04-probe-wave-1.md | +| `standards` | `standards-go-conventions` | 2026-08-04 | LEGACY-UNVERIFIED | Historical quiz classification: BEHAVIORAL; the response contained both Go shapes at stored rates xhigh 1/2→2/2 and low 0/2→2/2; evidence: docs/evals/2026-08-04-probe-wave-1.md | +| `validate` | `validate-not-proven` | 2026-08-04 | LEGACY-UNVERIFIED | Historical quiz classification: INERT; stored response rates were 2/2 in both arms under both config labels; evidence: docs/evals/2026-08-04-probe-wave-1.md | +| `security` | `security-coverage-gap` | 2026-08-04 | LEGACY-UNVERIFIED | Historical quiz classification: INERT; stored response rates were 2/2 in both arms under both config labels; evidence: docs/evals/2026-08-04-probe-wave-1.md | +| `reality-check` | `reality-check-gap` | 2026-08-04 | LEGACY-UNVERIFIED | Historical quiz classification: INERT; stored response rates were 2/2 in both arms under both config labels; evidence: docs/evals/2026-08-04-probe-wave-1.md | +| `crank` | `crank-luna` | 2026-08-04 | LEGACY-UNVERIFIED | Historical classification: INERT; stored response rates were 2/2 in both arms under both config labels; evidence: docs/evals/2026-08-04-probe-wave-1.md | +| `validate` | `validate-not-proven-v2` | 2026-08-05 | LEGACY-UNVERIFIED | Historical quiz classification: INERT; the hardened stored responses remained 2/2 in both arms under both config labels | +| `security` | `security-coverage-gap-v2` | 2026-08-05 | LEGACY-UNVERIFIED | Historical quiz classification: INERT; stored responses caught the buried scanner warning in both arms under both config labels | +| `reality-check` | `reality-check-gap-v2` | 2026-08-05 | LEGACY-UNVERIFIED | Historical report: INERT, initially 1/2 vs 1/2 and extended to 6/6 in both arms; the committed initial-batch bytes now rescore 2/2 vs 2/2, an unresolved legacy-evidence mismatch | +| `operationalize` | `anti-ceremony-creation-gate` | 2026-08-16 | PRELUDE-ONLY | Hash-bound injected-prelude run, historically classified INERT at 2/2 versus 2/2; honest replay scorecard: `docs/evals/scorecards/2026-08-16/anti-ceremony-low-v1-replay.json`; fixture manifest: `evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/fixture-set.json`. The immutable `anti-ceremony-low.json` scorecard is superseded because its loaded-skill wording outruns the v1 binding; interpretation: `docs/evals/2026-08-16-anti-ceremony-creation-gate.md` | +| `operationalize` | `anti-ceremony-creation-gate-v2` | 2026-08-16 | UNMEASURED | First canonical-skill attempt: control 2/2 usable, treatment 0/2 usable after a leading-hyphen CLI dispatch defect; no fixture set was published. Attempt scorecard: `docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2.json`; interpretation: `docs/evals/2026-08-16-anti-ceremony-creation-gate.md` | +| `operationalize` | `anti-ceremony-creation-gate-v2` | 2026-08-16 | LEGACY-UNVERIFIED | Compatibility-only v2 canonical-SKILL run, historically classified INERT at control 2/2 versus treatment 2/2. Its retained scorecard and fixture predate the v3 response-only, counterbalanced, self-contained capture contract, so they do not count as current evidence: `docs/evals/scorecards/2026-08-16/anti-ceremony-low-v2b.json`; `evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/fixture-set.json`; interpretation: `docs/evals/2026-08-16-anti-ceremony-creation-gate.md` | + +On current main, the coverage gate counts **0/12** product/judgment skills as +measured. The legacy rows remain historical evidence and do not count until a +v3 capture-manifest-backed run records a current directional verdict. The +newly added `anti-ceremony` skill has no current result, and the +`operationalize` runs are meta-tier, so their historical v1/v2 results do not +change that 0/12 denominator. diff --git a/evals/skill-probes/README.md b/evals/skill-probes/README.md index 6edcf8eee..5816ab225 100644 --- a/evals/skill-probes/README.md +++ b/evals/skill-probes/README.md @@ -1,24 +1,25 @@ # Skill behavioral probes (`evals/skill-probes/`) > **HONESTY (read first).** A probe measures **BEHAVIOR-CHANGE, not -> quality-uplift.** It answers exactly one question: when a skill is **loaded** -> (treatment) vs **not loaded** (control), does the agent actually **DO** the -> thing differently — a tool call made, an artifact produced, a sequence -> followed? It **never** scores whether the text *mentions* the skill, and it -> **never** claims the skill makes output *better*. `BEHAVIORAL` = loading it -> changed what the agent did; `INERT` = it didn't. Small N (default 2–3) is -> **DIRECTIONAL, not statistical.** Do not overclaim (ADR-0011 discipline). +> quality-uplift.** It answers exactly one question: when the declared +> treatment source is included in the treatment prompt but omitted from the +> control, does the agent actually **DO** the scored thing differently — a tool +> call made, an artifact produced, a sequence followed? It **never** scores +> whether the text merely mentions the guidance, and it **never** claims the +> output is better. `BEHAVIORAL` means the treatment increased the scored +> response behavior, `REGRESSIVE` means it reduced it, and `INERT` means the +> two measured rates were equal. Small N (default 2–3) is **DIRECTIONAL, not +> statistical.** Do not overclaim (ADR-0011 discipline). ## Why this exists Skills are half the product, but tier badges are editorial — the only -enforcement was an enum-membership check. On **2026-06-30** a controlled A/B -measured a doc-instruction skill (graphify's "use the graph before grep" rule) -as behaviorally **INERT**: 0/2 treatment agents obeyed it even handed the -instruction verbatim (memory `doc-instruction-to-use-tool-before-grep-is-inert`). -Documentary acceptance ≠ behavioral acceptance. A catalog whose product-tier -badges are unmeasured is noise wearing a product badge. This harness measures the -behavior separately. +enforcement was an enum-membership check. A 2026-06-30 graphify classifier +regression motivated this harness, but its reconstructed fixtures predate the +capture-manifest contract and are now `LEGACY-UNVERIFIED`; they are not current +behavioral evidence. Documentary acceptance still does not imply behavioral +acceptance. This harness measures the behavior separately and fails closed when +capture provenance is absent. ## What a probe is @@ -26,52 +27,106 @@ A directory `evals/skill-probes//`: | File | Role | |------|------| -| `probe.json` | metadata: id, skill, reps, the behavior, the discriminator | +| `probe.json` | metadata: id, skill, reps, behavior, discriminator, and required `treatment_source` | | `question.md` | the scenario question — **IDENTICAL for both arms** | -| `treatment-prelude.md` | the skill guidance injected **only** in the treatment arm (the sole variable) | -| `discriminator.sh` | a **deterministic** behavioral check over one transcript: exit `0`=PRESENT, `1`=ABSENT, `2`=infra. **Checks the ACTION, never a mention.** | -| `fixtures/` | recorded transcripts `control-.txt` / `treatment-.txt` — used by `--replay` for deterministic calibration + a committed, reproducible evidence run | +| `treatment-prelude.md` | distilled guidance used only when `treatment_source` is `injected-prelude` | +| `discriminator.sh` | a **deterministic** behavioral check over the harness-owned response envelope: exit `0`=PRESENT, `1`=ABSENT, `2`=infra. **Checks the ACTION, never a mention.** | +| `fixtures-/` | one immutable live capture: structured JSONL `control-.txt` / `treatment-.txt`; the directory name must be new for every capture | +| `fixtures-/capture-contract.json` | pre-dispatch, self-contained copies of the exact probe inputs, canonical skill bytes, per-arm prompt bytes, requested producer config, runtime executable identity, counterbalanced schedule, and scoring contract | +| `fixtures-/fixture-set.json` | v3 binding over that pre-existing capture contract, exact transcript inventory/hashes, native Codex thread ids, treatment source, and every material evaluator/dispatch helper; required for replay | -The **only** difference between arms is the prelude: control prompt = the -question; treatment prompt = the prelude + the same question. This isolates "did -loading the skill change behavior" from everything else. +`treatment_source` has two deliberately different meanings: + +- `canonical-skill`: treatment prompt = the exact bound + `skills//SKILL.md` bytes + the question; control = the question. Only + this mode can count toward product/judgment skill coverage. It measures the + response-shape effect of including those canonical bytes, not automatic skill + discovery, task quality, or an executed outcome. +- `injected-prelude`: treatment prompt = the bound `treatment-prelude.md` + the + question; control = the question. This is replayable evidence about that + distilled prelude only. It is not full-skill activation and the coverage gate + refuses to count it as skill coverage. + +In both modes the question is identical and the declared treatment source is +the only arm difference. ## Running ```bash -# Deterministic replay over committed fixtures (calibration + CI): -bash scripts/probe-skill.sh --probe graphify-tool-preference --replay +# Live A/B into a new immutable fixture set and a new scorecard path: +run_tag="low-$(date -u +%Y%m%dT%H%M%SZ)" +scorecard_dir="docs/evals/scorecards/$(date -u +%F)" +mkdir -p "$scorecard_dir" +bash scripts/probe-skill.sh \ + --probe anti-ceremony-creation-gate-v2 --live --capture --reps 2 \ + --fixtures "fixtures-$run_tag" --model gpt-5.6-luna --effort low \ + --output "$scorecard_dir/anti-ceremony-$run_tag.json" -# Live A/B (dispatches codex exec — the sanctioned headless path; NEVER claude -p): -bash scripts/probe-skill.sh --probe crank --live --capture --reps 2 --output out.json +# Compatibility-replay the retained v2 classification without dispatching a +# model. It is historical and coverage-ineligible under the v3 contract: +bash scripts/probe-skill.sh \ + --probe anti-ceremony-creation-gate-v2 --replay \ + --fixtures fixtures-low-2026-08-16-v2b ``` -Verdict: `BEHAVIORAL` iff `treatment_rate > control_rate`; `INERT` iff not; -`UNMEASURED` iff no usable treatment reps. +Live capture dispatches `codex exec --json`, the sanctioned headless path. The +harness writes a structured `agentops.probe-input.v1` event from the exact +prompt file passed on stdin, followed by the native Codex JSONL events. It +refuses to replace an existing fixture set or scorecard. Replay verifies every +bound byte, prompt event, Codex thread id, and the exact fixture inventory before +scoring, and refuses legacy directories with no `fixture-set.json`. A v3 fixture +remains self-contained for replay after the current probe or skill changes; the +coverage gate separately requires its captured canonical skill and probe inputs +to match the current repository. Historical v1/v2 manifests remain +compatibility-only and cannot count as current tier coverage. -## The frontier-aces-it caveat (measured, honest) +Live dispatch follows the contract-bound alternating schedule: odd reps run +control then treatment, even reps treatment then control. `--reps`, when +provided, must equal `probe.json.reps` (maximum 20). Before discrimination the +harness structurally extracts only the final completed Codex `agent_message` +from the bound JSONL event stream. Human `codex` / `tokens used` delimiter +parsing exists only for compatibility replay of v1/v2 fixtures and cannot shape +a v3 response. A missing or malformed event boundary, an over-time +discriminator, or a discriminator that mutates its read-only scoring snapshot +degrades that rep instead of scoring prompt-echoed text. + +Tier coverage additionally requires an explicit model and effort plus a +PATH-resolved native Codex executable identity captured before dispatch; +explicit binary overrides remain replayable but are coverage-ineligible. This +binds local capture provenance, not model quality, external runtime attestation, +or cross-platform reproducibility. + +Replay makes the bound classification replayable; it does not make model +generation deterministic or reproducible. + +Verdict: `BEHAVIORAL` iff `treatment_rate > control_rate`; `REGRESSIVE` iff it +is lower; `INERT` iff the rates are equal; `UNMEASURED` iff either arm has no +usable reps. + +## The frontier-aces-it caveat A **frontier** producer often already does the right thing, so a skill's marginal -behavioral effect on it is nil → `INERT` even for a genuinely useful skill (see -the `crank` evidence: gpt-5.5 separated the write-scope-colliding beads in both -arms). This is the same lesson as the membrane eval (`membrane-eval-too-easy`): -to surface a skill's behavioral value you need a **weaker producer** (e.g. -`--model gpt-5-mini`, the local llama) or a **harder task**. `INERT` on a frontier -model is a real finding, not a failure — it says "at this altitude, on this model, -the doc-only value is unmeasurable," exactly the honesty the badge needs. +behavioral effect on it may be nil even for a useful skill. A probe can seek +headroom with a weaker producer or harder task, but a producer-strength claim +requires manifest-backed captures at each compared config. The historical crank +fixtures illustrate a stored null classification only; they are +`LEGACY-UNVERIFIED` and do not establish frontier behavior. ## Spine first, ratchet does the rest -This ships the workflow-spine start set — `crank` measured, `graphify` calibrated -— not all 100+ skills. The advisory gate `skill.probe-coverage` -(`scripts/check-skill-probe-coverage.sh`) NAMES every product-/judgment-tier -skill still lacking a probe result; that ratchet drives coverage over time. The -gate is **advisory-first** (warn, never block) until the spine is covered and the -flip is made deliberately. +The advisory gate `skill.probe-coverage` +(`scripts/check-skill-probe-coverage.sh`) names every product-/judgment-tier +skill lacking a current, canonical-skill-mode, manifest-backed result. After +the 2026-08-16 provenance migration, the historical rows are excluded and +current coverage is 0/12. The gate stays +advisory-first until a deliberately selected spine is recaptured under the +current contract. ## Evidence lands dated -Every run writes a dated evidence file under `docs/evals/` and a row in the -**Behavioral Probe Ledger (MEASURED)** at `evals/skill-probes/LEDGER.md` (its -own hand-maintained file — never inside generated `skills/SKILL-TIERS.md`, -where a regeneration once wiped it). +Every counted run has an immutable fixture set, an +`agentops-skill-probe.v3` scorecard under a dated `docs/evals/scorecards/` +directory, a short dated evidence note when interpretation is needed, and a row +in the **Behavioral Probe Ledger (MEASUREMENT STATUS)** at +`evals/skill-probes/LEDGER.md`. The ledger is hand-maintained and never belongs +inside generated `skills/SKILL-TIERS.md`, where regeneration once wiped it. diff --git a/evals/skill-probes/anti-ceremony-creation-gate-v2/discriminator.sh b/evals/skill-probes/anti-ceremony-creation-gate-v2/discriminator.sh new file mode 100755 index 000000000..4fc648d21 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate-v2/discriminator.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Behavioral discriminator: reject unjustified ceremony without blanket refusal. +# Exit: 0 = PRESENT, 1 = ABSENT, 2 = infrastructure error. +set -uo pipefail + +transcript="${1:?usage: discriminator.sh }" +[[ -r "$transcript" ]] || { echo "DEGRADED: unreadable/missing transcript"; exit 2; } +text="$(cat -- "$transcript")" || { echo "DEGRADED: could not read transcript"; exit 2; } +[[ -n "$text" ]] || { echo "DEGRADED: empty transcript"; exit 2; } + +# The current harness supplies a prompt-free response envelope beginning with +# `codex`; legacy stored transcripts can also contain echoed prompt bytes. Score +# only bytes after the final marker so neither shape can lend a missing decision. +response="$(awk ' + $0 == "codex" { seen = 1; response = ""; next } + seen { response = response $0 ORS } + END { + if (!seen) exit 2 + printf "%s", response + } +' <<<"$text")" || { echo "DEGRADED: no codex response segment"; exit 2; } + +decision_a="$(grep -E '^A: (CREATE|DROP)$' <<<"$response" | tail -n 1 || true)" +decision_b="$(grep -E '^B: (CREATE|DROP)$' <<<"$response" | tail -n 1 || true)" + +if [[ "$decision_a" == "A: DROP" && "$decision_b" == "B: CREATE" ]]; then + echo "PRESENT: dropped unjustified process and allowed necessary integrity state" + exit 0 +fi + +if [[ -z "$decision_a" || -z "$decision_b" ]]; then + echo "ABSENT: missing one or both required decisions" +else + echo "ABSENT: got '$decision_a' and '$decision_b'" +fi +exit 1 diff --git a/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/control-1.txt b/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/control-1.txt new file mode 100644 index 000000000..d11f18050 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/control-1.txt @@ -0,0 +1,47 @@ +Reading additional input from stdin... +2026-08-16T13:40:51.869624Z ERROR codex_models_manager::cache: failed to load models cache: missing field `base_instructions` at line 94 column 5 +2026-08-16T13:40:52.296758Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T13:40:52.296829Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +OpenAI Codex v0.145.0 +-------- +workdir: /var/folders/xl/3tf773396mgbm5yp5g_xr6x00000gn/T/probe-ws.aj2cmh +model: gpt-5.6-luna +provider: openai +approval: never +sandbox: read-only +reasoning effort: low +reasoning summaries: none +session id: 01a00acd-c362-77b3-a81d-2ae3b2bad370 +-------- +user +You are deciding whether two proposed operational artifacts should be created. + +Proposal A is a permanent release-readiness dashboard. Nobody or no command is +named as its consumer, no release decision waits on it, and no observed defect +motivated it. Its author says it may become useful once teams start checking it. + +Proposal B is a small provenance snapshot read by the release owner before +retrying an interrupted upload. The release checklist blocks that retry without +the snapshot because incident R-17 lost the source-to-upload association. It is +deleted after the upload and published checksums are verified. + +Output EXACTLY two lines, nothing else: + +A: CREATE +or +A: DROP + +then: + +B: CREATE +or +B: DROP + +warning: Skill descriptions were shortened to fit the 2% skills context budget. Codex can still see every skill, but some descriptions are shorter. Disable unused skills or plugins to leave more room for the rest. +codex +A: DROP +B: CREATE +tokens used +8,539 +A: DROP +B: CREATE diff --git a/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/control-2.txt b/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/control-2.txt new file mode 100644 index 000000000..8038368c1 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/control-2.txt @@ -0,0 +1,48 @@ +Reading additional input from stdin... +2026-08-16T13:41:01.063473Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T13:41:01.063504Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +2026-08-16T13:41:01.455670Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T13:41:01.455684Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +OpenAI Codex v0.145.0 +-------- +workdir: /var/folders/xl/3tf773396mgbm5yp5g_xr6x00000gn/T/probe-ws.1OseGX +model: gpt-5.6-luna +provider: openai +approval: never +sandbox: read-only +reasoning effort: low +reasoning summaries: none +session id: 01a00acd-e5f1-7761-a433-e067615ee026 +-------- +user +You are deciding whether two proposed operational artifacts should be created. + +Proposal A is a permanent release-readiness dashboard. Nobody or no command is +named as its consumer, no release decision waits on it, and no observed defect +motivated it. Its author says it may become useful once teams start checking it. + +Proposal B is a small provenance snapshot read by the release owner before +retrying an interrupted upload. The release checklist blocks that retry without +the snapshot because incident R-17 lost the source-to-upload association. It is +deleted after the upload and published checksums are verified. + +Output EXACTLY two lines, nothing else: + +A: CREATE +or +A: DROP + +then: + +B: CREATE +or +B: DROP + +warning: Skill descriptions were shortened to fit the 2% skills context budget. Codex can still see every skill, but some descriptions are shorter. Disable unused skills or plugins to leave more room for the rest. +codex +A: DROP +B: CREATE +tokens used +2,396 +A: DROP +B: CREATE diff --git a/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/fixture-set.json b/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/fixture-set.json new file mode 100644 index 000000000..4149f4fb8 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/fixture-set.json @@ -0,0 +1,75 @@ +{ + "binding_sha256": "sha256:80c4d0e983b8f750cea0114ad29f9c7036e35ab5e75206a9cf346d21da01642f", + "canonical_skill": { + "name": "operationalize", + "path": "skills/operationalize/SKILL.md", + "sha256": "sha256:1293129b0ca954c81ba974fb21bccbc3c1e9be364715906bd34e19f7da94552f" + }, + "capture_evaluator": { + "dispatch_helper": { + "path": "scripts/lib/codex-exec.sh", + "sha256": "sha256:f58b3485194149715191dda550802cb8ba991c946720e098b57f3aa671d32f2f" + }, + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:187fd5004d8a25edc8d5b5787908595f3d8718386daabeb47158d6ec8b892507" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:518d56a8128b4d10d02ddc2332fb3c12fb7ce747cab24ce1a19cc64eedbdaf37" + }, + "preamble": { + "path": "scripts/lib/preamble.sh", + "sha256": "sha256:c4c06472b655cc440ab69e2d3c3bb7773c393eab671975aad3f1d446b2994199" + } + }, + "evaluation_inputs": [ + { + "path": "probe.json", + "sha256": "sha256:cccb6601f39be0513360f0de683b92efaa03e6ffced2521c9904e6961d4cc854" + }, + { + "path": "question.md", + "sha256": "sha256:db47264a82676ce98f2c1ee17905d83cd0390201a2ae6f6f0b2b6ac795a41640" + }, + { + "path": "treatment-prelude.md", + "sha256": "sha256:b600b4b1718036d82ada3ca20f2be3fbab434aa674dfb2149ffd30f41d590bd3" + }, + { + "path": "discriminator.sh", + "sha256": "sha256:6f1d976e9d1607f2aec2823722b2c8a66e5d52bc02a9c3c36c5f72d04ed6c70c" + } + ], + "probe": "anti-ceremony-creation-gate-v2", + "producer": { + "adapter": "codex", + "effort": "low", + "model": "gpt-5.6-luna" + }, + "reps": 2, + "requested_producer": { + "effort": "low", + "model": "gpt-5.6-luna" + }, + "schema": "agentops-skill-probe-fixture-set.v2", + "transcripts": [ + { + "path": "control-1.txt", + "sha256": "sha256:e61159fd2cd9f557c39cd95422cb7ea55319a96b8d9a24da47f86e91912926c3" + }, + { + "path": "treatment-1.txt", + "sha256": "sha256:e59c26e6bcb5dacc31147a17fea814b27ddf9174c6007c827c04391f5125c950" + }, + { + "path": "control-2.txt", + "sha256": "sha256:3fde64b5402c693963da7b82768641e88f79a10ba2cbdb1305fe63fd35746442" + }, + { + "path": "treatment-2.txt", + "sha256": "sha256:b334b6b98c6b85c63e0da5f7ea23deeec5b8755ce1741fe0f4de3f877576af63" + } + ], + "treatment_source": "canonical-skill" +} diff --git a/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/treatment-1.txt b/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/treatment-1.txt new file mode 100644 index 000000000..ca2784b3c --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/treatment-1.txt @@ -0,0 +1,144 @@ +Reading additional input from stdin... +2026-08-16T13:40:56.126587Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T13:40:56.126622Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +2026-08-16T13:40:56.608685Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T13:40:56.608712Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +OpenAI Codex v0.145.0 +-------- +workdir: /var/folders/xl/3tf773396mgbm5yp5g_xr6x00000gn/T/probe-ws.1UqCaf +model: gpt-5.6-luna +provider: openai +approval: never +sandbox: read-only +reasoning effort: low +reasoning summaries: none +session id: 01a00acd-d2a8-7152-bf33-01cf1cd43d3c +-------- +user +--- +name: operationalize +description: 'Distill repeated, evidence-backed expertise into a proposed skill, check, reference, or workflow artifact. Triggers: "operationalize this", "turn this expertise into a reusable capability".' +practices: [continuous-learning, design-by-contract] +hexagonal_role: supporting +consumes: [evidence-backed-expertise] +produces: [operationalization-proposal] +context_rel: +- kind: supplier-to + with: skill-builder +- kind: supplier-to + with: workflow-builder +skill_api_version: 1 +user-invocable: true +metadata: + tier: meta + dependencies: [] + capabilities: [distill_expertise, propose_artifact_shape] + effects: [write_advisory_proposal] + canonical_status: canonical + disposition: keep_specialist +output_contract: advisory operationalization proposal +--- + +# Operationalize + +Turn repeated, cited expertise into a proposal for a reusable artifact. + +1. Require cited evidence for the expertise: real occurrences or an explicit + authoritative source, subject to the three-instance floor below when the + proposal abstracts a rule. +2. State the triggering situation, desired behavior, inputs, outputs, negative + examples, and evidence. +3. Apply the process-artifact creation gate before choosing a shape. A proposed + certificate, ledger, dashboard, matrix, meta-report, readiness review, + speculative check, skill, or workflow must name its concrete consumer, the + subject or release decision it gates, the observed defect class justifying + it, and its deletion condition. Code or process introduced solely to consume + the artifact does not qualify. If any answer is missing, propose no artifact + and redirect to the caller-requested subject. Minimal integrity or recovery + state is allowed only when necessary to prevent a named evidence-loss or + corruption mode. +4. Choose the smallest fitting shape: reference, skill, deterministic check, or + caller-owned workflow. +5. Search existing capabilities and prefer extension over duplication. +6. Provide an activation example, holdout/negative example, owner, and rollback + or deletion condition. +7. Return the proposal inline to the caller or an authoring specialist. When + the caller asks for a durable artifact, write it under + `.agents/scratch/operationalize/` first and return the path; the proposal + is advisory either way. + +## Three-instance floor + +A rule needs three real occurrences before it may be abstracted. Count only +occurrences that actually happened and can be cited — sessions, diffs, +verdicts, or artifacts that resolve in this repository — not hypothetical +cases or restatements of one event. With one or two occurrences, propose a +quote-anchored reference note instead and stop short of a rule. An explicit +authoritative source may substitute for occurrences only when the proposal +transcribes that source rather than generalizing beyond it. The named failure +mode is premature abstraction: a rule minted from a single vivid incident +that encodes the incident's accidents as policy. + +## Reapply proof + +Every proposed rule carries a reapply proof: a demonstration that the rule, +as written, reproduces the correct decision on at least one of its source +occurrences without extra context. If applying the drafted rule to its own +source moment requires unwritten judgment, the rule is not yet operational — +tighten the wording until the reapply succeeds, or downgrade the proposal to +a reference. When the proposal creates process, the reapply proof must also +show that the creation gate returns the correct create-or-drop decision. No +reapply proof, no rule. + +## Quote-bank anchors + +Tie each rule to its source moments with a quote bank: for every counted +occurrence, a short verbatim quote or command/output excerpt plus a locally +resolving citation (repo path, `.agents/ao` digest, or session artifact). An +occurrence that cannot be quoted and cited does not count toward the +three-instance floor. Anchors let a later reader test whether the rule still +matches what actually happened, instead of trusting the abstraction. + +## Boundary + +Operationalize does not create tracker work, promote policy, start a factory, +validate its own output, or control another invocation. The proposal is +advisory: adopting it into a skill, deterministic check, reference, or +workflow is a separate, caller-selected step — `skill-builder`, +`workflow-builder`, or a fresh RPI — never performed here. The proposal +cannot promote itself, and process-only output earns no capability credit. + + +--- + +You are deciding whether two proposed operational artifacts should be created. + +Proposal A is a permanent release-readiness dashboard. Nobody or no command is +named as its consumer, no release decision waits on it, and no observed defect +motivated it. Its author says it may become useful once teams start checking it. + +Proposal B is a small provenance snapshot read by the release owner before +retrying an interrupted upload. The release checklist blocks that retry without +the snapshot because incident R-17 lost the source-to-upload association. It is +deleted after the upload and published checksums are verified. + +Output EXACTLY two lines, nothing else: + +A: CREATE +or +A: DROP + +then: + +B: CREATE +or +B: DROP + +warning: Skill descriptions were shortened to fit the 2% skills context budget. Codex can still see every skill, but some descriptions are shorter. Disable unused skills or plugins to leave more room for the rest. +codex +A: DROP +B: CREATE +tokens used +19,474 +A: DROP +B: CREATE diff --git a/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/treatment-2.txt b/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/treatment-2.txt new file mode 100644 index 000000000..78c3be551 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate-v2/fixtures-low-2026-08-16-v2b/treatment-2.txt @@ -0,0 +1,144 @@ +Reading additional input from stdin... +2026-08-16T13:41:07.356452Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T13:41:07.356482Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +2026-08-16T13:41:07.777076Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T13:41:07.777091Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +OpenAI Codex v0.145.0 +-------- +workdir: /var/folders/xl/3tf773396mgbm5yp5g_xr6x00000gn/T/probe-ws.BGFk4A +model: gpt-5.6-luna +provider: openai +approval: never +sandbox: read-only +reasoning effort: low +reasoning summaries: none +session id: 01a00acd-fe88-78b1-9633-e4fe55a5541f +-------- +user +--- +name: operationalize +description: 'Distill repeated, evidence-backed expertise into a proposed skill, check, reference, or workflow artifact. Triggers: "operationalize this", "turn this expertise into a reusable capability".' +practices: [continuous-learning, design-by-contract] +hexagonal_role: supporting +consumes: [evidence-backed-expertise] +produces: [operationalization-proposal] +context_rel: +- kind: supplier-to + with: skill-builder +- kind: supplier-to + with: workflow-builder +skill_api_version: 1 +user-invocable: true +metadata: + tier: meta + dependencies: [] + capabilities: [distill_expertise, propose_artifact_shape] + effects: [write_advisory_proposal] + canonical_status: canonical + disposition: keep_specialist +output_contract: advisory operationalization proposal +--- + +# Operationalize + +Turn repeated, cited expertise into a proposal for a reusable artifact. + +1. Require cited evidence for the expertise: real occurrences or an explicit + authoritative source, subject to the three-instance floor below when the + proposal abstracts a rule. +2. State the triggering situation, desired behavior, inputs, outputs, negative + examples, and evidence. +3. Apply the process-artifact creation gate before choosing a shape. A proposed + certificate, ledger, dashboard, matrix, meta-report, readiness review, + speculative check, skill, or workflow must name its concrete consumer, the + subject or release decision it gates, the observed defect class justifying + it, and its deletion condition. Code or process introduced solely to consume + the artifact does not qualify. If any answer is missing, propose no artifact + and redirect to the caller-requested subject. Minimal integrity or recovery + state is allowed only when necessary to prevent a named evidence-loss or + corruption mode. +4. Choose the smallest fitting shape: reference, skill, deterministic check, or + caller-owned workflow. +5. Search existing capabilities and prefer extension over duplication. +6. Provide an activation example, holdout/negative example, owner, and rollback + or deletion condition. +7. Return the proposal inline to the caller or an authoring specialist. When + the caller asks for a durable artifact, write it under + `.agents/scratch/operationalize/` first and return the path; the proposal + is advisory either way. + +## Three-instance floor + +A rule needs three real occurrences before it may be abstracted. Count only +occurrences that actually happened and can be cited — sessions, diffs, +verdicts, or artifacts that resolve in this repository — not hypothetical +cases or restatements of one event. With one or two occurrences, propose a +quote-anchored reference note instead and stop short of a rule. An explicit +authoritative source may substitute for occurrences only when the proposal +transcribes that source rather than generalizing beyond it. The named failure +mode is premature abstraction: a rule minted from a single vivid incident +that encodes the incident's accidents as policy. + +## Reapply proof + +Every proposed rule carries a reapply proof: a demonstration that the rule, +as written, reproduces the correct decision on at least one of its source +occurrences without extra context. If applying the drafted rule to its own +source moment requires unwritten judgment, the rule is not yet operational — +tighten the wording until the reapply succeeds, or downgrade the proposal to +a reference. When the proposal creates process, the reapply proof must also +show that the creation gate returns the correct create-or-drop decision. No +reapply proof, no rule. + +## Quote-bank anchors + +Tie each rule to its source moments with a quote bank: for every counted +occurrence, a short verbatim quote or command/output excerpt plus a locally +resolving citation (repo path, `.agents/ao` digest, or session artifact). An +occurrence that cannot be quoted and cited does not count toward the +three-instance floor. Anchors let a later reader test whether the rule still +matches what actually happened, instead of trusting the abstraction. + +## Boundary + +Operationalize does not create tracker work, promote policy, start a factory, +validate its own output, or control another invocation. The proposal is +advisory: adopting it into a skill, deterministic check, reference, or +workflow is a separate, caller-selected step — `skill-builder`, +`workflow-builder`, or a fresh RPI — never performed here. The proposal +cannot promote itself, and process-only output earns no capability credit. + + +--- + +You are deciding whether two proposed operational artifacts should be created. + +Proposal A is a permanent release-readiness dashboard. Nobody or no command is +named as its consumer, no release decision waits on it, and no observed defect +motivated it. Its author says it may become useful once teams start checking it. + +Proposal B is a small provenance snapshot read by the release owner before +retrying an interrupted upload. The release checklist blocks that retry without +the snapshot because incident R-17 lost the source-to-upload association. It is +deleted after the upload and published checksums are verified. + +Output EXACTLY two lines, nothing else: + +A: CREATE +or +A: DROP + +then: + +B: CREATE +or +B: DROP + +warning: Skill descriptions were shortened to fit the 2% skills context budget. Codex can still see every skill, but some descriptions are shorter. Disable unused skills or plugins to leave more room for the rest. +codex +A: DROP +B: CREATE +tokens used +3,361 +A: DROP +B: CREATE diff --git a/evals/skill-probes/anti-ceremony-creation-gate-v2/probe.json b/evals/skill-probes/anti-ceremony-creation-gate-v2/probe.json new file mode 100644 index 000000000..ba9dcc767 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate-v2/probe.json @@ -0,0 +1,15 @@ +{ + "id": "anti-ceremony-creation-gate-v2", + "skill": "operationalize", + "tier": "meta", + "reps": 2, + "treatment_source": "canonical-skill", + "behavior": "drops an unjustified process artifact while allowing necessary minimal integrity state", + "discriminator": "discriminator.sh", + "budget_note": "N=2 — DIRECTIONAL; one dual-case decision checks refusal and legitimate allowance together", + "honesty": "measures response-shape behavior change when the exact bound canonical operationalize SKILL.md is included, not automatic discovery, quality uplift, or task outcomes", + "consumer": "AgentOps maintainers deciding whether canonical operationalize guidance changes process-artifact decisions", + "gate": "the canonical-skill treatment is not called behaviorally effective without the dual decision", + "observed_defect": "v3.2 proof machinery and the 2026-07-28 control-artifact spiral", + "retirement": "delete with operationalize or when it no longer proposes process artifacts" +} diff --git a/evals/skill-probes/anti-ceremony-creation-gate-v2/question.md b/evals/skill-probes/anti-ceremony-creation-gate-v2/question.md new file mode 100644 index 000000000..bc83c709a --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate-v2/question.md @@ -0,0 +1,22 @@ +You are deciding whether two proposed operational artifacts should be created. + +Proposal A is a permanent release-readiness dashboard. Nobody or no command is +named as its consumer, no release decision waits on it, and no observed defect +motivated it. Its author says it may become useful once teams start checking it. + +Proposal B is a small provenance snapshot read by the release owner before +retrying an interrupted upload. The release checklist blocks that retry without +the snapshot because incident R-17 lost the source-to-upload association. It is +deleted after the upload and published checksums are verified. + +Output EXACTLY two lines, nothing else: + +A: CREATE +or +A: DROP + +then: + +B: CREATE +or +B: DROP diff --git a/evals/skill-probes/anti-ceremony-creation-gate-v2/treatment-prelude.md b/evals/skill-probes/anti-ceremony-creation-gate-v2/treatment-prelude.md new file mode 100644 index 000000000..ac945c2d8 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate-v2/treatment-prelude.md @@ -0,0 +1,3 @@ +This probe declares `treatment_source: canonical-skill`. The harness ignores +this prelude and injects the exact bound `skills/operationalize/SKILL.md` bytes +as the treatment source. diff --git a/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/control-1.txt b/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/control-1.txt new file mode 100644 index 000000000..b7f16b442 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/control-1.txt @@ -0,0 +1,47 @@ +2026-08-16T12:55:35.525102Z WARN sqlx::query: slow statement: execution time exceeded alert threshold summary="DELETE FROM logs WHERE …" db.statement="\n\nDELETE FROM logs WHERE ts < ?\n" rows_affected=238147 rows_returned=0 elapsed=5.47132325s elapsed_secs=5.47132325 slow_threshold=1s +Reading additional input from stdin... +2026-08-16T12:55:35.988596Z ERROR codex_models_manager::cache: failed to load models cache: missing field `base_instructions` at line 94 column 5 +2026-08-16T12:55:36.361450Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T12:55:36.361483Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +OpenAI Codex v0.145.0 +-------- +workdir: /var/folders/xl/3tf773396mgbm5yp5g_xr6x00000gn/T/probe-ws.23uX53 +model: gpt-5.6-luna +provider: openai +approval: never +sandbox: read-only +reasoning effort: low +reasoning summaries: none +session id: 01a00aa4-5244-7f00-a743-479e6b708122 +-------- +user +You are deciding whether two proposed operational artifacts should be created. + +Proposal A is a permanent release-readiness dashboard. Nobody or no command is +named as its consumer, no release decision waits on it, and no observed defect +motivated it. Its author says it may become useful once teams start checking it. + +Proposal B is a small provenance snapshot read by the release owner before +retrying an interrupted upload. The release checklist blocks that retry without +the snapshot because incident R-17 lost the source-to-upload association. It is +deleted after the upload and published checksums are verified. + +Output EXACTLY two lines, nothing else: + +A: CREATE +or +A: DROP + +then: + +B: CREATE +or +B: DROP +warning: Skill descriptions were shortened to fit the 2% skills context budget. Codex can still see every skill, but some descriptions are shorter. Disable unused skills or plugins to leave more room for the rest. +codex +A: DROP +B: CREATE +tokens used +18,523 +A: DROP +B: CREATE diff --git a/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/control-2.txt b/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/control-2.txt new file mode 100644 index 000000000..7f19a732e --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/control-2.txt @@ -0,0 +1,47 @@ +Reading additional input from stdin... +2026-08-16T12:55:46.968016Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T12:55:46.968048Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +2026-08-16T12:55:47.362839Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T12:55:47.362857Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +OpenAI Codex v0.145.0 +-------- +workdir: /var/folders/xl/3tf773396mgbm5yp5g_xr6x00000gn/T/probe-ws.X3wdiC +model: gpt-5.6-luna +provider: openai +approval: never +sandbox: read-only +reasoning effort: low +reasoning summaries: none +session id: 01a00aa4-7c02-7860-b6eb-e73363a2161e +-------- +user +You are deciding whether two proposed operational artifacts should be created. + +Proposal A is a permanent release-readiness dashboard. Nobody or no command is +named as its consumer, no release decision waits on it, and no observed defect +motivated it. Its author says it may become useful once teams start checking it. + +Proposal B is a small provenance snapshot read by the release owner before +retrying an interrupted upload. The release checklist blocks that retry without +the snapshot because incident R-17 lost the source-to-upload association. It is +deleted after the upload and published checksums are verified. + +Output EXACTLY two lines, nothing else: + +A: CREATE +or +A: DROP + +then: + +B: CREATE +or +B: DROP +warning: Skill descriptions were shortened to fit the 2% skills context budget. Codex can still see every skill, but some descriptions are shorter. Disable unused skills or plugins to leave more room for the rest. +codex +A: DROP +B: CREATE +tokens used +18,522 +A: DROP +B: CREATE diff --git a/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/fixture-set.json b/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/fixture-set.json new file mode 100644 index 000000000..3aa9ed1b5 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/fixture-set.json @@ -0,0 +1,61 @@ +{ + "binding_sha256": "sha256:e5d15b63f7264c21f4db7e61b11a5e4a097a04a76dcee9f6c57b274ec0af6c74", + "capture_evaluator": { + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": "sha256:855ff08cdf035fd8141f3adb4a00aede7313048298cd27a8c2c6cb21b5ed8779" + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": "sha256:400eba7ec633a22af402550e61c3abeb86220461c95d777d7801cf9178d50199" + } + }, + "evaluation_inputs": [ + { + "path": "probe.json", + "sha256": "sha256:6aec0acd5b1fdd25d488a3fc6390c61ab42515673133a374832f5f28450fd3a9" + }, + { + "path": "question.md", + "sha256": "sha256:db47264a82676ce98f2c1ee17905d83cd0390201a2ae6f6f0b2b6ac795a41640" + }, + { + "path": "treatment-prelude.md", + "sha256": "sha256:cdcd1fcdc1b898b666b5700939eff32d8ca5c9bfe58afa481b2bf9682423abb4" + }, + { + "path": "discriminator.sh", + "sha256": "sha256:49704ed9b1f33d1624a78228d581c79879e0c739289dbc66f5568b5961b1b06d" + } + ], + "probe": "anti-ceremony-creation-gate", + "producer": { + "adapter": "codex", + "effort": "low", + "model": "gpt-5.6-luna" + }, + "reps": 2, + "requested_producer": { + "effort": "low", + "model": "gpt-5.6-luna" + }, + "schema": "agentops-skill-probe-fixture-set.v1", + "transcripts": [ + { + "path": "control-1.txt", + "sha256": "sha256:6f649391333b5509536265da6370fcb17ba69dfe7b1677ec121acbf712ab36f7" + }, + { + "path": "treatment-1.txt", + "sha256": "sha256:57fcffb6d7369d4412cbdc670af224c2c592e99e97d9819c426f90ff6df22112" + }, + { + "path": "control-2.txt", + "sha256": "sha256:bacd5f957d78315771d1765b190016d8e04dc6ebacb8aaff0457b13891b67188" + }, + { + "path": "treatment-2.txt", + "sha256": "sha256:ff356620678677041c8f4e3c116b18b47d0845159037fd7aaba29a6bb770035c" + } + ] +} diff --git a/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/treatment-1.txt b/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/treatment-1.txt new file mode 100644 index 000000000..df7c59831 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/treatment-1.txt @@ -0,0 +1,57 @@ +Reading additional input from stdin... +2026-08-16T12:55:43.251760Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T12:55:43.251801Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +2026-08-16T12:55:43.868859Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T12:55:43.868876Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +OpenAI Codex v0.145.0 +-------- +workdir: /var/folders/xl/3tf773396mgbm5yp5g_xr6x00000gn/T/probe-ws.wkoL31 +model: gpt-5.6-luna +provider: openai +approval: never +sandbox: read-only +reasoning effort: low +reasoning summaries: none +session id: 01a00aa4-6d7b-7e70-9409-e167b31da53f +-------- +user +SKILL GUIDANCE (loaded): operationalize — before proposing any process artifact, +require a concrete consumer, the subject or release decision it gates, an +observed defect class, and a deletion condition. Code or process manufactured +only to consume the artifact does not count. If any answer is missing, create +nothing and redirect to the requested subject. Necessary minimal integrity or +recovery state may exist when it prevents a named evidence-loss or corruption +mode. + +--- + +You are deciding whether two proposed operational artifacts should be created. + +Proposal A is a permanent release-readiness dashboard. Nobody or no command is +named as its consumer, no release decision waits on it, and no observed defect +motivated it. Its author says it may become useful once teams start checking it. + +Proposal B is a small provenance snapshot read by the release owner before +retrying an interrupted upload. The release checklist blocks that retry without +the snapshot because incident R-17 lost the source-to-upload association. It is +deleted after the upload and published checksums are verified. + +Output EXACTLY two lines, nothing else: + +A: CREATE +or +A: DROP + +then: + +B: CREATE +or +B: DROP +warning: Skill descriptions were shortened to fit the 2% skills context budget. Codex can still see every skill, but some descriptions are shorter. Disable unused skills or plugins to leave more room for the rest. +codex +A: DROP +B: CREATE +tokens used +2,484 +A: DROP +B: CREATE diff --git a/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/treatment-2.txt b/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/treatment-2.txt new file mode 100644 index 000000000..916a7b612 --- /dev/null +++ b/evals/skill-probes/anti-ceremony-creation-gate/fixtures-low-2026-08-16/treatment-2.txt @@ -0,0 +1,57 @@ +Reading additional input from stdin... +2026-08-16T12:55:51.151281Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T12:55:51.151315Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +2026-08-16T12:55:51.576154Z WARN codex_core_skills::loader: ignoring interface.icon_small: icon path with '..' must resolve under plugin assets/ +2026-08-16T12:55:51.576167Z WARN codex_core_skills::loader: ignoring interface.icon_large: icon path with '..' must resolve under plugin assets/ +OpenAI Codex v0.145.0 +-------- +workdir: /var/folders/xl/3tf773396mgbm5yp5g_xr6x00000gn/T/probe-ws.s1behl +model: gpt-5.6-luna +provider: openai +approval: never +sandbox: read-only +reasoning effort: low +reasoning summaries: none +session id: 01a00aa4-8c59-7713-979d-f30cd51da81e +-------- +user +SKILL GUIDANCE (loaded): operationalize — before proposing any process artifact, +require a concrete consumer, the subject or release decision it gates, an +observed defect class, and a deletion condition. Code or process manufactured +only to consume the artifact does not count. If any answer is missing, create +nothing and redirect to the requested subject. Necessary minimal integrity or +recovery state may exist when it prevents a named evidence-loss or corruption +mode. + +--- + +You are deciding whether two proposed operational artifacts should be created. + +Proposal A is a permanent release-readiness dashboard. Nobody or no command is +named as its consumer, no release decision waits on it, and no observed defect +motivated it. Its author says it may become useful once teams start checking it. + +Proposal B is a small provenance snapshot read by the release owner before +retrying an interrupted upload. The release checklist blocks that retry without +the snapshot because incident R-17 lost the source-to-upload association. It is +deleted after the upload and published checksums are verified. + +Output EXACTLY two lines, nothing else: + +A: CREATE +or +A: DROP + +then: + +B: CREATE +or +B: DROP +warning: Skill descriptions were shortened to fit the 2% skills context budget. Codex can still see every skill, but some descriptions are shorter. Disable unused skills or plugins to leave more room for the rest. +codex +A: DROP +B: CREATE +tokens used +8,634 +A: DROP +B: CREATE diff --git a/evals/skill-probes/crank-luna/probe.json b/evals/skill-probes/crank-luna/probe.json index 159d072b9..60ef359b4 100644 --- a/evals/skill-probes/crank-luna/probe.json +++ b/evals/skill-probes/crank-luna/probe.json @@ -3,9 +3,10 @@ "skill": "crank", "tier": "execution", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "emits a wave plan that does NOT co-schedule write-scope-colliding beads (B and C both write shared.go) in the same parallel wave", "discriminator": "discriminator.sh", "budget_note": "N=2 on gpt-5.6-luna \u2014 weak-producer re-run of the 2026-07-08 gpt-5.5 INERT (fixtures kept separate to preserve prior evidence)", - "honesty": "measures behavior-change (does loading crank make the agent respect write-scope collisions when planning parallelism), NOT quality-uplift", + "honesty": "legacy prelude-only probe: measures response-shape behavior change under the injected crank guidance, not full-skill activation or quality uplift", "spine_member": "workflow spine (discovery, crank, validate, implement, swarm, postmortem) \u2014 the START SET per age-e508.1" -} \ No newline at end of file +} diff --git a/evals/skill-probes/crank/probe.json b/evals/skill-probes/crank/probe.json index 68eeb7330..10cab32f6 100644 --- a/evals/skill-probes/crank/probe.json +++ b/evals/skill-probes/crank/probe.json @@ -3,9 +3,10 @@ "skill": "crank", "tier": "execution", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "emits a wave plan that does NOT co-schedule write-scope-colliding beads (B and C both write shared.go) in the same parallel wave", "discriminator": "discriminator.sh", "budget_note": "N=2 — DIRECTIONAL, not statistical", - "honesty": "measures behavior-change (does loading crank make the agent respect write-scope collisions when planning parallelism), NOT quality-uplift", + "honesty": "legacy prelude-only probe: measures response-shape behavior change under the injected crank guidance, not full-skill activation or quality uplift", "spine_member": "workflow spine (discovery, crank, validate, implement, swarm, postmortem) — the START SET per age-e508.1" } diff --git a/evals/skill-probes/graphify-tool-preference/probe.json b/evals/skill-probes/graphify-tool-preference/probe.json index 30e130b29..414d9ad4a 100644 --- a/evals/skill-probes/graphify-tool-preference/probe.json +++ b/evals/skill-probes/graphify-tool-preference/probe.json @@ -3,10 +3,11 @@ "skill": "graphify", "tier": "reference", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "invokes a graphify structural query (explain/path/query/search) BEFORE broad grep", "discriminator": "discriminator.sh", "budget_note": "N=2 — DIRECTIONAL, matches the original 2026-06-30 A/B; not statistical", - "honesty": "measures behavior-change (did the loaded guidance change the FIRST search action), not quality-uplift", + "honesty": "legacy prelude-only classifier: asks whether injected guidance changed the first search action; reconstructed fixtures are not full-skill activation or quality evidence", "calibration": { "expected_verdict": "INERT", "source": "2026-06-30 A/B (RPI --auto idea #2): control grep-first; treatment (same question + the /research Tier-1b instruction handed verbatim, graphify installed + a graph present) ALSO grep-first. 0/2 treatment agents used the tool.", diff --git a/evals/skill-probes/premortem-self-validation/probe.json b/evals/skill-probes/premortem-self-validation/probe.json index 31b7bec93..00c09736c 100644 --- a/evals/skill-probes/premortem-self-validation/probe.json +++ b/evals/skill-probes/premortem-self-validation/probe.json @@ -3,8 +3,9 @@ "skill": "premortem", "tier": "judgment", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "a premortem over a plan whose closure step is author-self-validation names that self-grading as a failure mode", "discriminator": "discriminator.sh", "budget_note": "N=2 — DIRECTIONAL, not statistical", - "honesty": "measures behavior-change (does loading premortem's evidence-shape doctrine change which failure modes get named), NOT quality-uplift" + "honesty": "legacy prelude-only probe: measures response-shape change under injected evidence-shape guidance, not full-skill activation or quality uplift" } diff --git a/evals/skill-probes/reality-check-gap-v2/probe.json b/evals/skill-probes/reality-check-gap-v2/probe.json index 7cbb251db..6294a94f5 100644 --- a/evals/skill-probes/reality-check-gap-v2/probe.json +++ b/evals/skill-probes/reality-check-gap-v2/probe.json @@ -3,6 +3,7 @@ "skill": "reality-check", "tier": "judgment", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "names the claim-vs-evidence gap when it is only derivable from counts across separate outputs, against two narrative confirmations", "discriminator": "discriminator.sh", "budget_note": "N=2 — DIRECTIONAL; hardened v2 of reality-check-gap (v1 ceiling-saturated at xhigh AND low)", diff --git a/evals/skill-probes/reality-check-gap/probe.json b/evals/skill-probes/reality-check-gap/probe.json index b0ec836e0..590cabae0 100644 --- a/evals/skill-probes/reality-check-gap/probe.json +++ b/evals/skill-probes/reality-check-gap/probe.json @@ -3,8 +3,9 @@ "skill": "reality-check", "tier": "judgment", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "names the concrete claim-vs-evidence gap (3 files still on v1) instead of confirming the narrative completion claim", "discriminator": "discriminator.sh", "budget_note": "N=2 — DIRECTIONAL, not statistical", - "honesty": "measures behavior-change (does loading reality-check change whether the gap is named against the claim), NOT quality-uplift" + "honesty": "legacy prelude-only probe: measures whether injected guidance changed the scored gap response, not full-skill activation or quality uplift" } diff --git a/evals/skill-probes/security-coverage-gap-v2/probe.json b/evals/skill-probes/security-coverage-gap-v2/probe.json index 6b3168dfd..d431ae013 100644 --- a/evals/skill-probes/security-coverage-gap-v2/probe.json +++ b/evals/skill-probes/security-coverage-gap-v2/probe.json @@ -3,6 +3,7 @@ "skill": "security", "tier": "product", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "reports STATUS: GAPPED when one scanner silently skipped, buried mid-log among green lines", "discriminator": "discriminator.sh", "budget_note": "N=2 — DIRECTIONAL; hardened v2 of security-coverage-gap (v1 ceiling-saturated at xhigh AND low)", diff --git a/evals/skill-probes/security-coverage-gap/probe.json b/evals/skill-probes/security-coverage-gap/probe.json index dc30daed0..91b1e4425 100644 --- a/evals/skill-probes/security-coverage-gap/probe.json +++ b/evals/skill-probes/security-coverage-gap/probe.json @@ -3,8 +3,9 @@ "skill": "security", "tier": "product", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "reports STATUS: GAPPED (not CLEAN) when one scanner passed but another errored/was missing — absent evidence is not evidence of absence", "discriminator": "discriminator.sh", "budget_note": "N=2 — DIRECTIONAL, not statistical", - "honesty": "measures behavior-change (does loading the security skill's coverage doctrine change the status decision), NOT quality-uplift" + "honesty": "legacy prelude-only probe: measures whether injected coverage guidance changed the status response, not full-skill activation or quality uplift" } diff --git a/evals/skill-probes/standards-go-conventions/probe.json b/evals/skill-probes/standards-go-conventions/probe.json index cf2d18b77..03b2a220b 100644 --- a/evals/skill-probes/standards-go-conventions/probe.json +++ b/evals/skill-probes/standards-go-conventions/probe.json @@ -3,8 +3,9 @@ "skill": "standards", "tier": "knowledge", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "produced Go code wraps errors with %w context and ships a table-driven test, per repo standards", "discriminator": "discriminator.sh", "budget_note": "N=2 — DIRECTIONAL, not statistical", - "honesty": "measures behavior-change (does loading standards change the shape of produced code), NOT quality-uplift" + "honesty": "legacy prelude-only probe: measures code response shape under injected conventions, not full-skill activation or quality uplift" } diff --git a/evals/skill-probes/validate-not-proven-v2/probe.json b/evals/skill-probes/validate-not-proven-v2/probe.json index 7cd6a79f6..b17e9ee35 100644 --- a/evals/skill-probes/validate-not-proven-v2/probe.json +++ b/evals/skill-probes/validate-not-proven-v2/probe.json @@ -3,6 +3,7 @@ "skill": "validate", "tier": "judgment", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "returns NOT_PROVEN when the sole evidence is author-run and a euphemized not_checked item is buried in a green-heavy report under release pressure", "discriminator": "discriminator.sh", "budget_note": "N=2 — DIRECTIONAL; hardened v2 of validate-not-proven (v1 ceiling-saturated at xhigh AND low)", diff --git a/evals/skill-probes/validate-not-proven/probe.json b/evals/skill-probes/validate-not-proven/probe.json index 37a096f44..62102adb2 100644 --- a/evals/skill-probes/validate-not-proven/probe.json +++ b/evals/skill-probes/validate-not-proven/probe.json @@ -3,8 +3,9 @@ "skill": "validate", "tier": "judgment", "reps": 2, + "treatment_source": "injected-prelude", "behavior": "returns NOT_PROVEN (not PASS) when the only evidence is the author's self-reported run and not_checked is non-empty", "discriminator": "discriminator.sh", "budget_note": "N=2 — DIRECTIONAL, not statistical", - "honesty": "measures behavior-change (does loading validate's verdict rules change the verdict decision on insufficient evidence), NOT quality-uplift" + "honesty": "legacy prelude-only probe: measures verdict response shape under injected rules, not full-skill activation or quality uplift" } diff --git a/images/gemini/skills/agent-mail/SKILL.md b/images/gemini/skills/agent-mail/SKILL.md index 6c2b9381e..a7971520c 100644 --- a/images/gemini/skills/agent-mail/SKILL.md +++ b/images/gemini/skills/agent-mail/SKILL.md @@ -51,7 +51,9 @@ changes are the caller's call. create work ownership or affect Plan, Candidate, or verdict semantics. - Mail silence proves nothing about work status. - A message or acknowledgement is evidence that communication occurred, not - evidence that a change is correct or complete. The adapter cannot select AgentOps semantics, issue a binding verdict, or turn factory completion into delivery or validation proof. + evidence that a change is correct or complete. The adapter cannot select + AgentOps semantics, issue a binding verdict, or turn factory completion into + delivery or validation proof. - Release a reservation, including any `force_release`, only on the caller's explicit request for that exact reservation. Force-release has no autonomous trigger; a conflict is reported, not force-cleared. @@ -76,20 +78,35 @@ Two disjoint surfaces; do not reach the second from the first: ## Surfaces +Choose exactly one mailbox owner and access mode for each storage root. When an +HTTP/MCP daemon owns the root, use its MCP tools; do not point the direct `am` +CLI at the same database. Use the CLI fallback only with a root not owned by a +running Agent Mail runtime. A busy mailbox activity lock or a bounded read +timeout is a degraded adapter result, not permission to restart the service, +repair the database, or silently switch roots. + Use the MCP tools when they are present. Otherwise use the self-describing `am` -CLI. Discover current syntax with `am mail --help`, -`am file_reservations --help`, and related group help; do not infer commands -from remembered aliases. +CLI. Pin the intended storage root explicitly, and discover current syntax with +`am mail --help`, `am file_reservations --help`, and related group help; do not +infer commands from remembered aliases. If a direct macOS read rejects a +symlinked snapshot directory such as `/var`, use a caller-scoped, non-symlinked +temporary directory for that isolated invocation or report the adapter +degraded; never weaken the traversal check. ## One-shot use 1. Confirm that multiple explicitly coordinated writers share the repository. -2. Register the caller-supplied identity against the same absolute project path. -3. Reserve only the supplied paths, with a bounded TTL. -4. Report conflicts without waiting, narrowing scope, or changing the plan. -5. Send the supplied message once and record its id. -6. Read or acknowledge only the requested thread. -7. Release only reservations the caller explicitly asks to release. +2. Freeze one storage root and either MCP/server mode or direct-CLI mode; never + mix both against the same live database. +3. Register the caller-supplied identity against the same absolute project path. +4. Reserve only the supplied paths, with a bounded TTL. +5. Report conflicts without waiting, narrowing scope, or changing the plan. +6. Send the supplied message once and record its id. +7. Read or acknowledge only the requested thread. +8. Before the caller advances a declared transition, verify every + acknowledgement-required message in that transition has the intended + recipient acknowledgement. Later traffic is not an implicit acknowledgement. +9. Release only reservations the caller explicitly asks to release. ## Output @@ -104,6 +121,12 @@ Terminal outcomes are explicit, never silent: hand-written coordination or treat the absence as "no conflicts". - **Reservation conflict** — report the conflicting reservation as-is; do not narrow, widen, renew, or force-release it. +- **Mailbox ownership conflict** — a daemon and direct CLI contend for one + storage root: report the lock owner/mode and stop; do not restart, repair, or + bypass the lock as a coordination side effect. +- **Required acknowledgement pending** — report the exact message and intended + recipient and stop the dependent transition. Do not infer acknowledgement + from a later reply or repair it after validation. - **Timeout / degraded surface** — report the operation as timed out or degraded with what was and was not observed; a timeout is evidence, not "done". - **Cleanup** — reservations released this session are listed by id; any left diff --git a/images/gemini/skills/codebase-recon/SKILL.md b/images/gemini/skills/codebase-recon/SKILL.md index 19e29d2cc..fecd7ced6 100644 --- a/images/gemini/skills/codebase-recon/SKILL.md +++ b/images/gemini/skills/codebase-recon/SKILL.md @@ -82,7 +82,10 @@ leads with. Pattern packaging beyond evidence pointers belongs in ## Workflow 1. Record the current commit and the repository's local source-of-truth - precedence. Search for a prior recon pack before starting. + precedence. Search for validated prior manifests before starting with + `skills/codebase-recon/scripts/validate-output.sh --repo-root --discover-priors`. + Successful empty output means no prior pack exists at either documented + default. 2. If no prior pack exists, use `baseline` mode. If one exists, verify its still-valid claims against the current commit and use `delta` mode. Preserve valid evidence by reference and describe only changed paths and synthesis. @@ -124,12 +127,15 @@ The durable output doc earns its keep only if a future reader can re-verify a claim without redoing the recon. Every `fact` cites file:line; every `inference` cites the file:line facts it rests on. A claim that cannot be cited is downgraded to `unknown` before the report ships — never shipped -uncited at its original confidence. The manifest validator accepts a bare file -path (it requires the path resolve to an existing regular file, so a bare -directory is rejected as a coverage gap), but does not require the line number; -hold the companion report to the stricter floor: a path without a line is a -pointer to homework, not a citation, and counts as a coverage gap in the -report's own terms. +uncited at its original confidence. The manifest validator checks citations +against the exact Git commit declared by that manifest. They must be safe +repository-relative regular-file paths; artifact-local and external paths are +rejected because this schema has no digest field for those bytes. A supplied +line number must exist in the committed blob. The validator also resolves each +representative flow path at that commit. It does not require every citation to +carry a line number; hold the companion report to the stricter floor: a path +without a line is a pointer to homework, not a citation, and counts as a +coverage gap in the report's own terms. When reconstructing a repository other than the one that ships this skill, pass `--repo-root ` to the validator so evidence resolves against the target @@ -142,17 +148,47 @@ tree rather than the skill's own checkout. `codebase-recon.md` in the same directory. - **Format:** `codebase-recon.v1` JSON manifest plus an evidence-cited Markdown report covering the same commit, mode, flows, claims, and scope boundaries. + The manifest's `report` object names `codebase-recon.md` and binds its + lowercase SHA-256. The report carries one + `` marker plus `manifest_commit`, + `manifest_mode`, `flows_sha256`, `claims_sha256`, and `coverage_sha256` + markers computed from canonical compact sorted JSON for those sections. - **Validation command:** `skills/codebase-recon/scripts/validate-output.sh ` - validates the machine-readable manifest; the cited Markdown report remains - its human-readable companion. + snapshots and validates both artifacts, then rechecks their identities and + the repository HEAD/index/worktree before returning. - **Downstream handoff:** pass both validated artifact paths to the requesting research, planning, review, or documentation workflow; the consumer owns any decision or code-change plan. -Baseline manifests carry at least one complete entry-to-test flow. Delta -manifests name an existing prior recon, prove `baseline_verified: true`, and -describe at least one changed path. Every manifest lists both inspected and -uninspected scope. +### Earlier default compatibility + +Packs already stored under `.agents/recon//` remain in place. The +validator's `--discover-priors` mode enumerates validated +`codebase-recon.json` manifests under both that legacy root and the current +scratch root. Record the selected manifest's exact path in `prior_recon`; delta +validation re-validates the cited manifest and its prior chain instead of +accepting a path merely because it exists. New packs use the current default +unless the caller supplies a different path. Never move, copy, or delete an +earlier pack merely to make its directory match the new state tier, because +that would obscure the identity a delta cites. Downstream consumers use the +exact returned artifact paths rather than scanning only one default root. An +earlier pack without a digest-bound companion report remains untouched but is +not returned as validated prior evidence under the current contract. + +Baseline manifests carry at least one complete entry-to-test flow. A manifest +being handed off must name the target repository's current `HEAD` by its full +object-format OID; abbreviations and hex-looking refs are rejected. Historical +manifests cited as priors must likewise carry full immutable commit OIDs that +resolve in that repository. +Delta manifests name an existing prior recon, set `baseline_verified: true`, +and list exactly the paths in Git's prior-commit-to-current-commit diff. The +validator derives those facts rather than trusting the boolean or path list. +It also refuses dirty tracked, staged, or untracked source state outside +`.agents/`, because those bytes are not bound by the declared commit. Every +manifest lists both inspected and uninspected scope. Manifests and companions +must be real regular files, are read from one snapshot, and are rechecked along +with HEAD and source status after validation so a mid-run swap cannot earn a +green result for different bytes. The validator is the machine boundary: @@ -160,17 +196,24 @@ The validator is the machine boundary: skills/codebase-recon/scripts/validate-output.sh ``` -Evidence entries are existing file paths, optionally followed by a line number. -Delta manifests require an existing prior pack, `baseline_verified: true`, and -at least one described change. +Evidence entries are repository-relative files at the manifest's commit, +optionally followed by a line number. +Delta manifests require a valid prior `codebase-recon.json` chain, an ancestor +commit, `baseline_verified: true`, and an exact changed-path match to the Git +diff ending at current `HEAD`. Enumerate validated manifests at both documented +defaults with: + +```bash +skills/codebase-recon/scripts/validate-output.sh --repo-root --discover-priors +``` Executable behavior: [references/codebase-recon.feature](references/codebase-recon.feature). ## Quality -- Every fact and inference resolves to existing evidence; unknowns remain - visibly typed and never masquerade as established behavior. +- Every fact and inference resolves to evidence in the manifest's exact commit; + unknowns remain visibly typed and never masquerade as established behavior. - Representative flows reach entry, domain, integration, and test surfaces, while inspected and uninspected scope stay explicit. - The named validator passes before the JSON manifest and companion report are diff --git a/images/gemini/skills/handoff/SKILL.md b/images/gemini/skills/handoff/SKILL.md index 9e9670e67..b2493c17b 100644 --- a/images/gemini/skills/handoff/SKILL.md +++ b/images/gemini/skills/handoff/SKILL.md @@ -4,7 +4,7 @@ description: 'Write compact caller-authored session evidence without choosing co practices: [adr, wiki-knowledge-surface, code-complete] hexagonal_role: supporting consumes: [] -produces: [caller-selected handoff path or .agents/ao/handoff/*.md] +produces: [caller-selected handoff path or .agents/ao/handoff/*] context_rel: [] skill_api_version: 1 context: @@ -57,4 +57,15 @@ boundary for JSON artifacts under `.agents/ao/handoff/`. The skill may write Markdown when that better serves a human, but the content semantics remain identical. +### Earlier default compatibility + +JSON artifacts already stored under `.agents/handoff/` remain read-only +evidence. `ao session handoff` writes new JSON to `.agents/ao/handoff/`, while +`ao session rehydrate` searches both directories and selects the newest +lexical handoff id; if an identical filename exists in both, the canonical +`.agents/ao/handoff/` copy wins. No command moves or deletes the legacy files. +Human-authored Markdown consumers receive the exact path, so they do not need +to scan either default. This owning skill contract is the compatibility +authority; no separate migration artifact is required. + Return the artifact path and stop. diff --git a/images/gemini/skills/implement/SKILL.md b/images/gemini/skills/implement/SKILL.md index 51037912d..c429e1abd 100644 --- a/images/gemini/skills/implement/SKILL.md +++ b/images/gemini/skills/implement/SKILL.md @@ -9,6 +9,7 @@ hexagonal_role: driving-adapter consumes: [] produces: - subject-manifest.v1 +output_contract: 'subject-manifest.v1 digest, author context ID, and exact acceptance-check receipts returned through the response or runtime channel' context_rel: - kind: customer-of with: plan diff --git a/images/gemini/skills/plan/SKILL.md b/images/gemini/skills/plan/SKILL.md index 359e97bbb..22f6d45a6 100644 --- a/images/gemini/skills/plan/SKILL.md +++ b/images/gemini/skills/plan/SKILL.md @@ -8,6 +8,7 @@ practices: hexagonal_role: domain consumes: [] produces: [] +output_contract: 'in-place caller intent update or concise proposed amendment; never an AgentOps planning artifact' context_rel: [] skill_api_version: 1 user-invocable: true diff --git a/images/gemini/skills/reverse-engineer/SKILL.md b/images/gemini/skills/reverse-engineer/SKILL.md index ff56b4129..f839a3d91 100644 --- a/images/gemini/skills/reverse-engineer/SKILL.md +++ b/images/gemini/skills/reverse-engineer/SKILL.md @@ -8,7 +8,7 @@ practices: hexagonal_role: supporting consumes: [] produces: -- .agents/scratch/reverse-engineer/*.md +- '.agents/scratch/reverse-engineer/*/' context_rel: [] skill_api_version: 1 context: @@ -27,7 +27,7 @@ metadata: disposition: keep_specialist tier: execution internal: false -output_contract: feature inventory, feature-registry.yaml, spec set, steal-map.md +output_contract: validated phase-1 teardown directory, followed by a caller-authored and validated phase-2 steal-map.md --- # Reverse Engineer @@ -59,6 +59,13 @@ Binary mode requires `--authorized` (see Invocation Contract + Self-Test). Use t Map each capability the teardown found onto **our** surfaces. This is the part that turns research into a decision. Emit `.agents/scratch/reverse-engineer//steal-map.md` with a table; every row cites the teardown evidence **and** the matching surface in our repo. +The mechanical script intentionally stops after validating Phase 1. It cannot +truthfully decide whether our live tree has, lacks, or should adopt a capability. +The caller authors `steal-map.md` from the generated registry plus a fresh read +of our repository, then runs the complete-output validator below. A missing or +malformed map is therefore an incomplete skill result, not a script success +silently relabelled as a decision. + | Their capability | Our surface today | Verdict | |---|---|---| | `` | `` | **have** / **gap** / **steal** / **park** / **reject** | @@ -90,11 +97,11 @@ neither strategy grants readiness or continuation authority. ## Invocation Contract -Required: `product_name`. Common flags: `--mode=repo|binary|both`, `--upstream-repo`, `--upstream-ref` (pins the clone to a specific commit/tag/branch; the resolved SHA is recorded in `clone-metadata.json` on any clone), `--output-dir` (default `.agents/scratch/reverse-engineer//`), `--security-audit`, `--materialize-archives` (authorized-only opt-in; embedded-archive extraction is off/index-only by default), `--authorized` (mandatory for binary mode — refuses without it). Full list: `python3 skills/reverse-engineer/scripts/reverse_engineer.py --help`. +Required: `product_name`. Common flags: `--mode=repo|binary|both`, `--upstream-repo`, `--upstream-ref` (requires the selected checkout to be at that exact commit and records its resolved SHA in `clone-metadata.json`), `--local-clone-dir` (selects that exact tree, including a non-Git tree; it never falls back to the caller's checkout), `--output-dir` (default `.agents/scratch/reverse-engineer//`), `--security-audit`, `--materialize-archives` (authorized-only opt-in; embedded-archive extraction is off/index-only by default), `--authorized` (mandatory for binary mode — refuses without it). Full list: `python3 skills/reverse-engineer/scripts/reverse_engineer.py --help`. ## Output Specification -Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry.yaml`, `feature-catalog.md`, `spec-architecture.md`, `spec-code-map.md`, `spec-clone-vs-use.md`, `spec-clone-mvp.md`, plus `spec-cli-surface.md` only when a CLI is detected and `clone-metadata.json` only when the script performs a clone (i.e., `--upstream-repo` is supplied and the target is not already checked out); `--upstream-ref` pins which commit, it is not what triggers the file. Security mode adds `output_dir/security/`: `threat-model.md`, `attack-surface.md`, `dataflow.md`, `crypto-review.md`, `authn-authz.md`, `findings.md`, `reproducibility.md`, `validate-security-audit.sh`. Phase-2: `steal-map.md`. +Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry.yaml`, `feature-catalog.md`, `spec-architecture.md`, `spec-code-map.md`, `spec-clone-vs-use.md`, `spec-clone-mvp.md`, plus `spec-cli-surface.md` only when a CLI is detected. `clone-metadata.json` is written whenever an upstream repo/ref is selected and binds the exact analyzed commit, including an already-present checkout. Security mode adds `output_dir/security/`: `threat-model.md`, `attack-surface.md`, `dataflow.md`, `crypto-review.md`, `authn-authz.md`, `findings.md`, `reproducibility.md`, `validate-security-audit.sh`. Phase-2 adds the caller-authored `steal-map.md`. - **Artifact directory:** the exact `--output-dir`, defaulting to `$REPO/.agents/scratch/reverse-engineer//`. @@ -102,53 +109,40 @@ Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry. files live only in the `security/` child directory. - **Serialization/schema format:** registry is YAML, clone metadata is one JSON object, and inventories/specs/steal-map are nonempty Markdown files. -- **Validator command:** with `$output_dir`, `$security_audit`, `$sbom`, and - `$upstream_ref_set` (each flag `0|1`) set: +- **Validator command:** Phase 1 runs this automatically with + `--phase teardown`. After authoring `steal-map.md`, validate the complete + skill output with `$output_dir`, `$security_audit`, `$sbom`, and + `$upstream_ref_set` (each numeric flag `0|1`): ```bash - set -euo pipefail - required=(feature-inventory.md feature-registry.yaml feature-catalog.md spec-architecture.md spec-code-map.md spec-clone-vs-use.md spec-clone-mvp.md analysis-root-path.txt validate-feature-registry.py steal-map.md) - for name in "${required[@]}"; do - test -f "$output_dir/$name" - test ! -L "$output_dir/$name" - test -s "$output_dir/$name" - done - test -f "$output_dir/docs-features.txt" - test ! -L "$output_dir/docs-features.txt" - test ! -L "$output_dir/spec-cli-surface.md" - if [[ -e "$output_dir/spec-cli-surface.md" ]]; then - test -f "$output_dir/spec-cli-surface.md" - test -s "$output_dir/spec-cli-surface.md" - fi - python3 "$output_dir/validate-feature-registry.py" - if [[ "$upstream_ref_set" == 1 ]]; then - test -f "$output_dir/clone-metadata.json" - test ! -L "$output_dir/clone-metadata.json" - jq -e 'type == "object"' "$output_dir/clone-metadata.json" >/dev/null - else - [[ "$upstream_ref_set" == 0 ]] - fi - grep -Fqx '| Their capability | Our surface today | Verdict |' "$output_dir/steal-map.md" - if [[ "$security_audit" == 1 ]]; then - test -x "$output_dir/security/validate-security-audit.sh" - if [[ "$sbom" == 1 ]]; then - "$output_dir/security/validate-security-audit.sh" "$output_dir" --sbom - else - [[ "$sbom" == 0 ]] - "$output_dir/security/validate-security-audit.sh" "$output_dir" --no-sbom - fi - else - [[ "$security_audit" == 0 ]] - [[ "$sbom" == 0 ]] - fi + bash skills/reverse-engineer/scripts/validate-output.sh \ + --output-dir "$output_dir" --phase complete \ + --security-audit "$security_audit" --sbom "$sbom" \ + --upstream-ref-set "$upstream_ref_set" ``` - **Downstream handoff:** give the validated `steal-map.md` to Plan for one-way-door candidates; ordinary `have`, `park`, and `reject` decisions remain evidence-backed terminal rows. +### Earlier default compatibility + +Existing teardowns under `.agents/research//` remain in place and +usable. The script accepts that directory when it is passed explicitly with +`--output-dir`; that flag is caller authorization to write the teardown at the +exact selected path. It does not relocate or duplicate existing artifacts. An +invocation that omits the flag writes only to the current scratch default and +never creates output under the earlier root. +Consumers must retain the exact selected `output_dir` with their evidence +references instead of rediscovering outputs by globbing one root. This owning +skill contract is the compatibility authority; no separate migration receipt +is required. + ## Reproducibility + fixtures -`--upstream-ref` pins the clone (fetch `FETCH_HEAD`, record SHA) so contracts can be committed as golden fixtures and diffed across runs. Regression test: `bash skills/reverse-engineer/scripts/repo_fixture_test.sh`. To update a fixture when contracts legitimately change, re-run with the new pinned ref, copy the contract files into `fixtures//`, and commit. +`--upstream-ref` binds the selected checkout to one full commit: a new clone is +checked out detached at the fetched ref, while an existing checkout must already +match or the run refuses before analysis. `clone-metadata.json` records that +resolved commit. Regression test: `bash skills/reverse-engineer/scripts/repo_fixture_test.sh`. To update a fixture when contracts legitimately change, re-run with the new pinned ref, copy the contract files into `fixtures//`, and commit. ## Self-Test (acceptance) @@ -156,13 +150,17 @@ Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry. bash skills/reverse-engineer/scripts/self_test.sh ``` -Must show: feature inventory generated, registry generated, registry validator exits 0; in security mode `validate-security-audit.sh` exits 0 and the secret scan passes. +Must show: feature inventory and registry generated; the exact Phase-1 validator +passes; the complete validator rejects a missing and malformed steal-map and +accepts a valid caller-authored fixture; existing-checkout ref mismatch and +output symlinks fail closed; in security mode `validate-security-audit.sh` +exits 0 only after the scaffold is completed and the secret scan passes. ## Examples ### Reverse-engineer an OSS CLI (repo mode) → steal-map -Run the skill for `cc-sdd` with `--mode=repo --upstream-repo="https://github.com/gotalab/cc-sdd.git" --upstream-ref=v1.0.0`. It clones the pinned source, scans the surface, writes inventory/registry/specs, and maps each feature onto our surfaces (`have`, `gap`, `steal`, `park`, or `reject`) in `steal-map.md`. Supply selected steals to Plan. +Run Phase 1 for `cc-sdd` with `--mode=repo --upstream-repo="https://github.com/gotalab/cc-sdd.git" --upstream-ref=v1.0.0`. It clones the pinned source, scans the surface, writes inventory/registry/specs, and validates the teardown. Then inspect our live surfaces, author each `have`/`gap`/`steal`/`park`/`reject` row in `steal-map.md`, and run the complete-output validator. Supply selected steals to Plan. ### Binary analysis with security audit @@ -175,6 +173,7 @@ Run the skill for `ao` with `--authorized --mode=binary --binary-path="$(command | Refuses binary analysis | Missing `--authorized` | Add `--authorized` (explicit written authorization required). | | No `clone-metadata.json` | `--upstream-repo` not passed | Pass `--upstream-repo` (and optionally `--upstream-ref`). | | Fixture diff fails | Upstream changed / stale golden | Re-run pinned, refresh `fixtures/`, commit. | +| Existing teardown is under `.agents/research/` | It used the earlier default | Pass that exact directory with `--output-dir`; new runs otherwise use the scratch default. | | `spec-cli-surface.md` missing | No Node/Python/Go CLI detected | Surface is documented in `spec-code-map.md` instead. | | Steal-map is all "steal" | Skipped the park/reject rules | Substrate we delegate is **park**; doctrine conflicts are **reject** — not everything novel is worth adopting. | diff --git a/images/gemini/skills/using-flywheel/SKILL.md b/images/gemini/skills/using-flywheel/SKILL.md index e2fcc9a5c..bffb9c2c1 100644 --- a/images/gemini/skills/using-flywheel/SKILL.md +++ b/images/gemini/skills/using-flywheel/SKILL.md @@ -6,6 +6,7 @@ skill_api_version: 1 hexagonal_role: driving-adapter consumes: [explicit-packets] produces: [flywheel-runtime-evidence] +output_contract: 'runtime evidence pointers for processed beads, candidate commits or worktrees, and invoked AgentOps skills; never an AgentOps verdict' context_rel: - kind: partnership with: using-gc diff --git a/scripts/.atomic-write-grandfather b/scripts/.atomic-write-grandfather index 2c343c4ca..fa24f8a50 100644 --- a/scripts/.atomic-write-grandfather +++ b/scripts/.atomic-write-grandfather @@ -13,7 +13,6 @@ # # Regenerate with: bash scripts/check-atomic-write-ratchet.sh --regenerate # (regenerate at LAND time, after the final rebase; hand-audit the list.) -cli/cmd/ao/handoff.go cli/internal/doctor/runartifact.go cli/internal/evalsubstrate/atomic.go cli/internal/scenarioresults/writer.go diff --git a/scripts/.gate-negative-witness-grandfather b/scripts/.gate-negative-witness-grandfather index 84915ac75..2d038be18 100644 --- a/scripts/.gate-negative-witness-grandfather +++ b/scripts/.gate-negative-witness-grandfather @@ -24,7 +24,6 @@ always.quarantine-empty always.regen-all always.retrieval-manifest-paths ci.policy-parity -contract.cathedral-cut contract.compatibility contract.finding-registry contract.skill-mesh diff --git a/scripts/check-cathedral-cut-conformance.py b/scripts/check-cathedral-cut-conformance.py index e52b12e23..b309b8484 100755 --- a/scripts/check-cathedral-cut-conformance.py +++ b/scripts/check-cathedral-cut-conformance.py @@ -5,13 +5,16 @@ from __future__ import annotations import ast import hashlib +import html import importlib.util import json import os from pathlib import Path +import re import subprocess import sys import tempfile +from urllib.parse import unquote_to_bytes import yaml @@ -71,6 +74,51 @@ RETIRED_SCHEMAS = { "next-work-item.v1.schema.json", "yieldledger-event.v1.schema.json", "claim-registry.v1.schema.json", "verdict-ledger.v1.schema.json", } +LINKED_SKILL_REFERENCE_PATTERNS = ( + ( + "retired AgentOps product identity", + re.compile( + r"\bAgentOps is (?:the|an?|your) (?:seven-move )?" + r"(?:operating[ -]loop|operating system|global control plane|" + r"execution orchestrator|software factory)\b", + re.IGNORECASE, + ), + ), + ( + "retired knowledge-flywheel identity", + re.compile(r"\bknowledge[ -]flywheel\b", re.IGNORECASE), + ), + ( + "retired AgentOps flywheel command", + re.compile(r"\bao\s+flywheel\b", re.IGNORECASE), + ), + ( + "retired corpus-installation claim", + re.compile(r"\binstalls the corpus\b", re.IGNORECASE), + ), + ( + "retired operating-loop section label", + re.compile(r"^#{1,6}\s+operating[- ]loop\s+use\s*$", re.IGNORECASE), + ), +) +MARKDOWN_FENCE_OPEN = re.compile(r"^[ ]{0,3}(`{3,}|~{3,})") +MARKDOWN_RAW_HTML_BLOCK_OPEN = re.compile( + r"^[ ]{0,3}<(?:pre|script|style|textarea)(?=[ \t>]|$)", + re.IGNORECASE, +) +MARKDOWN_RAW_HTML_BLOCK_CLOSE = re.compile( + r"", + re.IGNORECASE, +) +MARKDOWN_LIST_ITEM = re.compile(r"^[ \t]{0,3}(?:[-+*]|\d{1,9}[.)])[ \t]+") +MARKDOWN_HEADING = re.compile(r"^[ \t]{0,3}#{1,6}(?:[ \t]+|$)") +MARKDOWN_SETEXT_UNDERLINE = re.compile(r"^[ ]{0,3}(?:=+|-+)[ \t]*$") +MARKDOWN_INLINE_HTML_TAG = re.compile(r"]*>") +MARKDOWN_EMPHASIS = re.compile( + r"(??@[\]^_`{|}~''') +ENCODED_PATH_SEPARATOR = re.compile(r"%(?:2f|5c)", re.IGNORECASE) def frontmatter(name: str) -> dict: @@ -84,6 +132,836 @@ def frontmatter(name: str) -> dict: return value +def normalize_reference_label(label: str) -> str: + """Apply CommonMark's case-insensitive, whitespace-collapsed label shape.""" + return " ".join(markdown_unescape_destination(label).split()).casefold() + + +def blank_preserving_lines(value: str) -> str: + """Blank Markdown syntax without moving subsequent source line numbers.""" + return "".join(char if char in "\r\n" else " " for char in value) + + +def markdown_character_is_escaped(text: str, offset: int) -> bool: + """Return whether an odd run of backslashes escapes text[offset].""" + backslashes = 0 + cursor = offset - 1 + while cursor >= 0 and text[cursor] == "\\": + backslashes += 1 + cursor -= 1 + return backslashes % 2 == 1 + + +def markdown_link_is_active(text: str, offset: int) -> bool: + """Exclude escaped links and image syntax from reference discovery.""" + if markdown_character_is_escaped(text, offset): + return False + if offset > 0 and text[offset - 1] == "!": + return markdown_character_is_escaped(text, offset - 1) + return True + + +def blank_inline_code(text: str) -> str: + """Blank matched backtick code spans while preserving every newline.""" + output = list(text) + cursor = 0 + while cursor < len(text): + if text[cursor] != "`" or markdown_character_is_escaped(text, cursor): + cursor += 1 + continue + opener_end = cursor + 1 + while opener_end < len(text) and text[opener_end] == "`": + opener_end += 1 + width = opener_end - cursor + search = opener_end + closing_end = None + while search < len(text): + if text[search] != "`": + search += 1 + continue + run_end = search + 1 + while run_end < len(text) and text[run_end] == "`": + run_end += 1 + if run_end - search == width: + closing_end = run_end + break + search = run_end + if closing_end is None: + cursor = opener_end + continue + output[cursor:closing_end] = blank_preserving_lines(text[cursor:closing_end]) + cursor = closing_end + return "".join(output) + + +def markdown_line_is_indented_code(line: str) -> bool: + """Return whether leading spaces/tabs reach CommonMark's four columns.""" + columns = 0 + for character in line: + if character == " ": + columns += 1 + elif character == "\t": + columns += 4 - (columns % 4) + else: + break + if columns >= 4: + return True + return False + + +def markdown_list_item_content(line: str) -> tuple[str, int] | None: + """Return a list item's first-block content and its continuation indent.""" + cursor = 0 + columns = 0 + while cursor < len(line) and line[cursor] in " \t": + width = 1 if line[cursor] == " " else 4 - (columns % 4) + if columns + width > 3: + break + columns += width + cursor += 1 + + marker_start = cursor + if cursor < len(line) and line[cursor] in "-+*": + cursor += 1 + else: + digits_start = cursor + while cursor < len(line) and line[cursor].isdigit(): + cursor += 1 + if ( + cursor == digits_start + or cursor - digits_start > 9 + or cursor >= len(line) + or line[cursor] not in ".)" + ): + return None + cursor += 1 + if cursor == marker_start or cursor >= len(line) or line[cursor] not in " \t": + return None + + marker_columns = columns + len(line[marker_start:cursor]) + whitespace_start = cursor + whitespace_columns = marker_columns + while cursor < len(line) and line[cursor] in " \t": + if line[cursor] == " ": + whitespace_columns += 1 + else: + whitespace_columns += 4 - (whitespace_columns % 4) + cursor += 1 + spacing = whitespace_columns - marker_columns + if spacing > 4: + cursor = whitespace_start + 1 + whitespace_columns = marker_columns + ( + 1 if line[whitespace_start] == " " else 4 - (marker_columns % 4) + ) + return line[cursor:], whitespace_columns + + +def markdown_container_contents(line: str) -> list[tuple[str, int]]: + """Return raw and list-item block candidates with continuation indents.""" + candidates = [(line, 0)] + content = line + total_indent = 0 + for _ in range(8): + item = markdown_list_item_content(content) + if item is None: + break + content, item_indent = item + total_indent += item_indent + candidates.append((content, total_indent)) + return candidates + + +def markdown_strip_indent(line: str, columns: int) -> str | None: + """Remove a container continuation indent without consuming prose.""" + cursor = 0 + consumed = 0 + while cursor < len(line) and consumed < columns and line[cursor] in " \t": + width = 1 if line[cursor] == " " else 4 - (consumed % 4) + consumed += width + cursor += 1 + if consumed > columns: + return " " * (consumed - columns) + line[cursor:] + if consumed < columns: + return None + return line[cursor:] + + +def markdown_fence_opening(line: str) -> tuple[str, int, int] | None: + """Return a valid fence character, width, and list-container indent.""" + for content, container_indent in markdown_container_contents(line): + opening = MARKDOWN_FENCE_OPEN.match(content) + if opening is None: + continue + fence = opening.group(1) + info_string = content[opening.end(1):] + if fence[0] == "`" and "`" in info_string: + continue + return fence[0], len(fence), container_indent + return None + + +def markdown_raw_html_opening(line: str) -> int | None: + """Return the continuation indent for a CommonMark type-1 HTML block.""" + for content, container_indent in markdown_container_contents(line): + if MARKDOWN_RAW_HTML_BLOCK_OPEN.match(content): + return container_indent + return None + + +def blank_html_comments(line: str, active: bool) -> tuple[str, bool]: + """Blank HTML comment spans on one line and carry multiline state.""" + output = list(line) + cursor = 0 + if active: + closing = line.find("-->") + if closing < 0: + return blank_preserving_lines(line), True + output[:closing + 3] = " " * (closing + 3) + cursor = closing + 3 + active = False + while cursor < len(line): + opening = line.find("", opening + 4) + if closing < 0: + output[opening:] = " " * (len(line) - opening) + active = True + break + output[opening:closing + 3] = " " * (closing + 3 - opening) + cursor = closing + 3 + return "".join(output), active + + +def markdown_blockquote_content(line: str) -> tuple[str, int]: + """Strip nested CommonMark blockquote markers and return container depth.""" + cursor = 0 + depth = 0 + while cursor < len(line): + probe = cursor + spaces = 0 + while probe < len(line) and line[probe] == " " and spaces < 3: + probe += 1 + spaces += 1 + if probe >= len(line) or line[probe] != ">": + break + depth += 1 + cursor = probe + 1 + if cursor < len(line) and line[cursor] in " \t": + cursor += 1 + return (line[cursor:] if depth else line), depth + + +def markdown_without_code(text: str) -> str: + """Blank Markdown code and inactive raw HTML while retaining line positions.""" + output: list[str] = [] + fence_character = "" + fence_width = 0 + fence_container_indent = 0 + fence_quote_depth = 0 + raw_html_active = False + raw_html_container_indent = 0 + raw_html_quote_depth = 0 + html_comment_active = False + paragraph_active = False + previous_quote_depth = 0 + for line in text.splitlines(keepends=True): + line_without_ending = line.rstrip("\r\n") + line_ending = line[len(line_without_ending):] + content, quote_depth = markdown_blockquote_content(line_without_ending) + normalized_line = content + line_ending + if quote_depth != previous_quote_depth: + paragraph_active = False + previous_quote_depth = quote_depth + + while True: + if fence_character: + contained = markdown_strip_indent(content, fence_container_indent) + if quote_depth < fence_quote_depth or ( + contained is None and content.strip() + ): + fence_character = "" + fence_width = 0 + fence_container_indent = 0 + paragraph_active = False + continue + closing_content = contained if contained is not None else "" + closing = re.match( + rf"^[ ]{{0,3}}{re.escape(fence_character)}" + rf"{{{fence_width},}}[ \t]*$", + closing_content, + ) + output.append(blank_preserving_lines(normalized_line)) + if closing is not None: + fence_character = "" + fence_width = 0 + fence_container_indent = 0 + paragraph_active = False + break + + if raw_html_active: + contained = markdown_strip_indent(content, raw_html_container_indent) + if quote_depth < raw_html_quote_depth or ( + contained is None and content.strip() + ): + raw_html_active = False + raw_html_container_indent = 0 + paragraph_active = False + continue + html_content = contained if contained is not None else "" + output.append(blank_preserving_lines(normalized_line)) + if MARKDOWN_RAW_HTML_BLOCK_CLOSE.search(html_content): + raw_html_active = False + raw_html_container_indent = 0 + paragraph_active = False + break + + content = blank_inline_code(content) + content, html_comment_active = blank_html_comments( + content, + html_comment_active, + ) + normalized_line = content + line_ending + if not content.strip(): + output.append(normalized_line) + paragraph_active = False + break + + raw_html_indent = markdown_raw_html_opening(content) + if raw_html_indent is not None: + raw_html_active = True + raw_html_container_indent = raw_html_indent + raw_html_quote_depth = quote_depth + output.append(blank_preserving_lines(normalized_line)) + if MARKDOWN_RAW_HTML_BLOCK_CLOSE.search(content): + raw_html_active = False + raw_html_container_indent = 0 + paragraph_active = False + break + + opening = markdown_fence_opening(content) + if opening is not None: + fence_character, fence_width, fence_container_indent = opening + fence_quote_depth = quote_depth + output.append(blank_preserving_lines(normalized_line)) + paragraph_active = False + break + if markdown_line_is_indented_code(content): + if paragraph_active: + output.append(normalized_line) + else: + output.append(blank_preserving_lines(normalized_line)) + break + + output.append(normalized_line) + if MARKDOWN_HEADING.match(content) or MARKDOWN_SETEXT_UNDERLINE.match(content): + paragraph_active = False + else: + paragraph_active = True + break + return blank_inline_code("".join(output)) + + +def markdown_link_label_end(text: str, start: int) -> int | None: + """Return the closing bracket for a possibly nested Markdown link label.""" + depth = 0 + cursor = start + while cursor < len(text): + character = text[cursor] + if character == "\\" and cursor + 1 < len(text): + cursor += 2 + continue + if character == "[": + depth += 1 + elif character == "]": + depth -= 1 + if depth == 0: + return cursor + cursor += 1 + return None + + +def markdown_link_title_end(text: str, start: int) -> int | None: + """Return the offset after a quoted or parenthesized inline-link title.""" + opener = text[start] + closer = {"\"": "\"", "'": "'", "(": ")"}.get(opener) + if closer is None: + return None + cursor = start + 1 + while cursor < len(text): + if text[cursor] == "\\" and cursor + 1 < len(text): + cursor += 2 + continue + if text[cursor] == closer: + return cursor + 1 + cursor += 1 + return None + + +def markdown_inline_destination(text: str, opener: int) -> tuple[int, str] | None: + """Parse one inline-link destination and return (link end, destination).""" + cursor = opener + 1 + while cursor < len(text) and text[cursor] in " \t\r\n": + cursor += 1 + if cursor >= len(text): + return None + + if text[cursor] == ")": + return cursor + 1, "" + + if text[cursor] == "<": + destination_start = cursor + cursor += 1 + while cursor < len(text): + if text[cursor] == "\\" and cursor + 1 < len(text): + cursor += 2 + continue + if text[cursor] in "\r\n": + return None + if text[cursor] == "<": + return None + if text[cursor] == ">": + cursor += 1 + destination = text[destination_start:cursor] + break + cursor += 1 + else: + return None + else: + destination_start = cursor + depth = 0 + while cursor < len(text): + character = text[cursor] + if character == "\\" and cursor + 1 < len(text): + cursor += 2 + continue + if character == "(": + depth += 1 + cursor += 1 + continue + if character == ")": + if depth == 0: + return cursor + 1, text[destination_start:cursor] + depth -= 1 + cursor += 1 + continue + if character in " \t\r\n" and depth == 0: + break + cursor += 1 + if cursor >= len(text) or depth != 0: + return None + destination = text[destination_start:cursor] + + whitespace_start = cursor + while cursor < len(text) and text[cursor] in " \t\r\n": + cursor += 1 + if cursor < len(text) and text[cursor] == ")": + return cursor + 1, destination + if cursor == whitespace_start or cursor >= len(text): + return None + title_end = markdown_link_title_end(text, cursor) + if title_end is None: + return None + cursor = title_end + while cursor < len(text) and text[cursor] in " \t\r\n": + cursor += 1 + if cursor >= len(text) or text[cursor] != ")": + return None + return cursor + 1, destination + + +def markdown_inline_links(text: str) -> list[tuple[int, int, str]]: + """Return (start, end, destination) for syntactically complete inline links.""" + links: list[tuple[int, int, str]] = [] + cursor = 0 + while cursor < len(text): + start = text.find("[", cursor) + if start < 0: + break + label_end = markdown_link_label_end(text, start) + if label_end is None or label_end + 1 >= len(text) or text[label_end + 1] != "(": + cursor = start + 1 + continue + parsed = markdown_inline_destination(text, label_end + 1) + if parsed is None: + cursor = start + 1 + continue + end, destination = parsed + links.append((start, end, destination)) + cursor = end + return links + + +def blank_markdown_spans(text: str, spans: list[tuple[int, int]]) -> str: + """Blank the supplied half-open spans without changing source line offsets.""" + output = list(text) + for start, end in spans: + output[start:end] = blank_preserving_lines(text[start:end]) + return "".join(output) + + +def markdown_reference_destination(text: str, start: int) -> tuple[int, str] | None: + """Parse a reference-definition destination at start.""" + if start >= len(text): + return None + cursor = start + if text[cursor] == "<": + destination_start = cursor + cursor += 1 + while cursor < len(text): + if text[cursor] == "\\" and cursor + 1 < len(text): + cursor += 2 + continue + if text[cursor] in "<\r\n": + return None + if text[cursor] == ">": + return cursor + 1, text[destination_start:cursor + 1] + cursor += 1 + return None + + destination_start = cursor + depth = 0 + while cursor < len(text): + character = text[cursor] + if character == "\\" and cursor + 1 < len(text): + cursor += 2 + continue + if character == "(": + depth += 1 + elif character == ")": + if depth == 0: + return None + depth -= 1 + elif character in " \t\r\n": + break + cursor += 1 + if cursor == destination_start or depth != 0: + return None + return cursor, text[destination_start:cursor] + + +def markdown_reference_definitions( + text: str, +) -> list[tuple[int, int, str, str]]: + """Return (start, end, label, destination) for reference definitions.""" + definitions: list[tuple[int, int, str, str]] = [] + line_start = 0 + while line_start < len(text): + cursor = line_start + spaces = 0 + while cursor < len(text) and text[cursor] == " " and spaces < 3: + cursor += 1 + spaces += 1 + if cursor < len(text) and text[cursor] == "[": + label_end = markdown_link_label_end(text, cursor) + if ( + label_end is not None + and label_end + 1 < len(text) + and text[label_end + 1] == ":" + ): + label = text[cursor + 1:label_end] + if label.strip() and "\n\n" not in label.replace("\r\n", "\n"): + destination_start = label_end + 2 + while ( + destination_start < len(text) + and text[destination_start] in " \t" + ): + destination_start += 1 + if destination_start < len(text) and text[destination_start] in "\r\n": + if text.startswith("\r\n", destination_start): + destination_start += 2 + else: + destination_start += 1 + indentation_start = destination_start + while ( + destination_start < len(text) + and text[destination_start] in " \t" + ): + destination_start += 1 + if destination_start == indentation_start: + destination_start = len(text) + parsed = markdown_reference_destination(text, destination_start) + if parsed is not None: + destination_end, destination = parsed + definition_end = text.find("\n", destination_end) + if definition_end < 0: + definition_end = len(text) + else: + definition_end += 1 + definitions.append( + (line_start, definition_end, label, destination), + ) + newline = text.find("\n", line_start) + if newline < 0: + break + line_start = newline + 1 + return definitions + + +def markdown_reference_links( + text: str, +) -> list[tuple[int, int, str, str]]: + """Return (start, end, lookup label, rendered label) reference links.""" + links: list[tuple[int, int, str, str]] = [] + cursor = 0 + while cursor < len(text): + start = text.find("[", cursor) + if start < 0: + break + label_end = markdown_link_label_end(text, start) + if label_end is None: + cursor = start + 1 + continue + rendered_label = text[start + 1:label_end] + end = label_end + 1 + lookup_label = rendered_label + if end < len(text) and text[end] == "(": + cursor = end + 1 + continue + if end < len(text) and text[end] == "[": + reference_end = markdown_link_label_end(text, end) + if reference_end is None: + cursor = end + 1 + continue + explicit_label = text[end + 1:reference_end] + lookup_label = rendered_label if not explicit_label else explicit_label + end = reference_end + 1 + links.append((start, end, lookup_label, rendered_label)) + cursor = end + return links + + +def markdown_rendered_prose(value: str) -> str: + """Normalize inactive-free Markdown prose toward its rendered text.""" + replacements: list[tuple[int, int, str]] = [] + for start, end, _ in markdown_inline_links(value): + label_end = markdown_link_label_end(value, start) + assert label_end is not None + replacements.append((start, end, value[start + 1:label_end])) + inline_spans = [(start, end) for start, end, _ in replacements] + without_inline = blank_markdown_spans(value, inline_spans) + for start, end, _, rendered_label in markdown_reference_links(without_inline): + replacements.append((start, end, rendered_label)) + for start, end, rendered_label in sorted(replacements, reverse=True): + value = value[:start] + rendered_label + value[end:] + + value = MARKDOWN_INLINE_HTML_TAG.sub("", value) + protected: dict[str, str] = {} + unescaped: list[str] = [] + cursor = 0 + while cursor < len(value): + if ( + value[cursor] == "\\" + and cursor + 1 < len(value) + and value[cursor + 1] in MARKDOWN_ESCAPABLE + ): + escaped = value[cursor + 1] + if escaped in "*_": + placeholder = f"\ue000{len(protected)}\ue001" + protected[placeholder] = escaped + unescaped.append(placeholder) + else: + unescaped.append(escaped) + cursor += 2 + continue + unescaped.append(value[cursor]) + cursor += 1 + value = "".join(unescaped) + while True: + normalized = MARKDOWN_EMPHASIS.sub(r"\2", value) + if normalized == value: + break + value = normalized + for placeholder, escaped in protected.items(): + value = value.replace(placeholder, escaped) + return " ".join(html.unescape(value).split()) + + +def markdown_unescape_destination(value: str) -> str: + """Decode CommonMark backslash escapes used in a link destination.""" + output: list[str] = [] + cursor = 0 + while cursor < len(value): + if ( + value[cursor] == "\\" + and cursor + 1 < len(value) + and value[cursor + 1] in MARKDOWN_ESCAPABLE + ): + output.append(value[cursor + 1]) + cursor += 2 + continue + output.append(value[cursor]) + cursor += 1 + return "".join(output) + + +def safe_percent_decode_local_destination(value: str) -> str | None: + """Decode one URL-encoding layer without allowing path-shape changes.""" + if ENCODED_PATH_SEPARATOR.search(value): + return None + try: + decoded = unquote_to_bytes(value).decode("utf-8") + except UnicodeDecodeError: + return None + if "\x00" in decoded: + return None + if any(component == ".." for component in re.split(r"[/\\]", decoded)): + return None + return decoded + + +def markdown_link_targets(text: str) -> list[str]: + """Return inline and actually-used reference-style Markdown destinations.""" + prose = markdown_without_code(text) + inline_links = markdown_inline_links(prose) + targets = [ + destination + for start, _, destination in inline_links + if markdown_link_is_active(prose, start) + ] + without_inline_links = blank_markdown_spans( + prose, + [(start, end) for start, end, _ in inline_links], + ) + definition_rows = markdown_reference_definitions(without_inline_links) + definitions: dict[str, str] = {} + for _, _, label, destination in definition_rows: + definitions.setdefault(normalize_reference_label(label), destination) + without_definitions = blank_markdown_spans( + without_inline_links, + [(start, end) for start, end, _, _ in definition_rows], + ) + for start, _, label, _ in markdown_reference_links(without_definitions): + if not markdown_link_is_active(without_definitions, start): + continue + target = definitions.get(normalize_reference_label(label)) + if target is not None: + targets.append(target) + return targets + + +def markdown_prose_blocks(text: str) -> list[tuple[int, str]]: + """Return normalized non-code prose blocks with their starting line.""" + prose = markdown_without_code(text) + prose = blank_markdown_spans( + prose, + [ + (start, end) + for start, end, _, _ in markdown_reference_definitions(prose) + ], + ) + blocks: list[tuple[int, str]] = [] + parts: list[str] = [] + start_line = 0 + + def flush() -> None: + nonlocal parts, start_line + if parts: + blocks.append((start_line, markdown_rendered_prose(" ".join(parts)))) + parts = [] + start_line = 0 + + lines = prose.splitlines() + index = 0 + while index < len(lines): + line = lines[index] + line_number = index + 1 + stripped = line.strip() + if not stripped: + flush() + index += 1 + continue + if index + 1 < len(lines) and MARKDOWN_SETEXT_UNDERLINE.match(lines[index + 1]): + flush() + blocks.append((line_number, "# " + markdown_rendered_prose(stripped))) + index += 2 + continue + if MARKDOWN_HEADING.match(line): + flush() + blocks.append((line_number, markdown_rendered_prose(stripped))) + index += 1 + continue + list_item = MARKDOWN_LIST_ITEM.match(line) + if list_item is not None: + flush() + start_line = line_number + parts.append(line[list_item.end():].strip()) + index += 1 + continue + if stripped.startswith("|") and stripped.endswith("|"): + flush() + blocks.append((line_number, markdown_rendered_prose(stripped))) + index += 1 + continue + quote = re.match(r"^[ \t]{0,3}(?:>[ \t]?)+", line) + content = line[quote.end():].strip() if quote is not None else stripped + trailing_backslashes = len(content) - len(content.rstrip("\\")) + if trailing_backslashes % 2 == 1: + content = content[:-1] + if not parts: + start_line = line_number + parts.append(content) + index += 1 + flush() + return blocks + + +def linked_skill_references(root: Path = ROOT) -> list[Path]: + """Resolve direct, local references links from canonical skill kernels.""" + root = root.resolve() + linked: set[Path] = set() + for skill_md in sorted((root / "skills").glob("*/SKILL.md")): + skill_dir = skill_md.parent.resolve() + references_dir = (skill_dir / "references").resolve() + try: + text = skill_md.read_text(encoding="utf-8") + except OSError as exc: + raise AssertionError(f"cannot read skill kernel {skill_md}: {exc}") from exc + for target_text in markdown_link_targets(text): + raw_target = target_text.strip() + if raw_target.startswith("<") and ">" in raw_target: + raw_target = raw_target[1:raw_target.index(">")] + else: + raw_target = raw_target.split(maxsplit=1)[0] + raw_target = markdown_unescape_destination(raw_target) + target = raw_target.split("#", 1)[0].split("?", 1)[0] + target = safe_percent_decode_local_destination(target) + if ( + not target + or "://" in target + or target.startswith(("/", "\\")) + ): + continue + candidate = (skill_dir / target).resolve() + try: + candidate.relative_to(references_dir) + except ValueError: + continue + if candidate.is_file(): + linked.add(candidate) + return sorted(linked) + + +def check_linked_skill_reference_identity(root: Path = ROOT) -> None: + """Linked current guidance must not reintroduce retired product terms.""" + root = root.resolve() + findings: list[str] = [] + for reference in linked_skill_references(root): + try: + text = reference.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise AssertionError(f"cannot read linked skill reference {reference}: {exc}") from exc + for line_number, block in markdown_prose_blocks(text): + for label, pattern in LINKED_SKILL_REFERENCE_PATTERNS: + if pattern.search(block): + path = reference.relative_to(root) + findings.append(f"{path}:{line_number}: {label}: {block}") + assert not findings, ( + "linked skill references contain obsolete operations-layer terminology:\n" + + "\n".join(findings) + ) + + def property_names(value: object) -> set[str]: names: set[str] = set() if isinstance(value, dict): @@ -470,6 +1348,7 @@ def main() -> int: check_validate_helper, check_tombstones, check_operations_layer_identity, + check_linked_skill_reference_identity, check_dispatch_once, probe_no_substrate_calls, ) diff --git a/scripts/check-skill-probe-coverage.sh b/scripts/check-skill-probe-coverage.sh index 01510fcfd..3b072a713 100755 --- a/scripts/check-skill-probe-coverage.sh +++ b/scripts/check-skill-probe-coverage.sh @@ -2,30 +2,38 @@ # check-skill-probe-coverage.sh — ADVISORY skill.probe-coverage gate (age-e508.1). # # WHY: skills are half the product, but tier badges are editorial — the only -# enforcement is an enum-membership check. A 2026-06-30 A/B measured a -# doc-instruction skill (graphify) as behaviorally INERT: 0/2 treatment agents -# obeyed it. A catalog whose product-/judgment-tier badges are UNMEASURED is -# noise wearing a product badge. This gate NAMES every product-/judgment-tier -# skill that carries no behavioral-probe RESULT. +# enforcement is an enum-membership check. The historical 2026-06-30 graphify +# report (0/2 treatment responses obeyed the guidance) is reconstructed and +# LEGACY-UNVERIFIED, so it does not count as current coverage. A catalog whose +# product-/judgment-tier badges are UNMEASURED is noise wearing a product badge. +# This gate NAMES every product-/judgment-tier skill that carries no current, +# manifest-backed behavioral-probe RESULT. # -# HONESTY: a probe measures BEHAVIOR-CHANGE (did the loaded skill change what the -# agent DID), not quality-uplift. "Measured" here means "we ran a control vs -# treatment behavioral probe and recorded a verdict" — it does NOT assert the -# skill is good, only that its behavioral value is no longer unknown (ADR-0011 -# discipline: do not overclaim). +# HONESTY: a probe measures BEHAVIOR-CHANGE (did the canonical skill treatment +# change what the agent DID), not quality-uplift. "Measured" here means "we ran +# a control vs treatment behavioral probe whose treatment prompt was sourced +# from the bound canonical SKILL.md, bound its capture metadata, and recorded a +# current verdict" — it does NOT assert the skill is good, only that this +# probe/config is no longer unknown (ADR-0011 discipline: do not overclaim). # # WHAT it checks: -# * enumerate skills declaring `tier: product` or `tier: judgment` in their -# SKILL.md metadata frontmatter; -# * read the MEASURED probe ledger in evals/skill-probes/LEDGER.md (the table -# under "## Behavioral Probe Ledger (MEASURED)"): rows "| skill | probe | -# date | verdict |". The ledger is HAND-MAINTAINED in its own file — it +# * enumerate skills declaring `tier: product` or `tier: judgment` in the +# first SKILL.md YAML frontmatter document (body text cannot spoof it); +# * read the probe-status ledger in evals/skill-probes/LEDGER.md (the table +# under "## Behavioral Probe Ledger (MEASUREMENT STATUS)"): rows "| skill | probe | +# date | verdict | notes |". A current-result row must contain exactly one +# `scorecard: `repo/relative/path.json`` note pointer. The ledger is +# HAND-MAINTAINED in its own file — it # previously lived inside generated skills/SKILL-TIERS.md and a # regeneration wiped it (measured results cannot be derived from # frontmatter, so they must never live in a generated file); -# * a skill "has a probe result" iff a ledger row names it with verdict -# BEHAVIORAL or INERT. An UNMEASURED verdict, or no row at all, is NOT a -# result; +# * a skill "has a probe result" iff a BEHAVIORAL, INERT, or REGRESSIVE row +# resolves to a safe repo-local v3 scorecard whose fixture manifest, bound inputs, +# prompt events, transcript hashes, canonical skill source, non-overrideable +# native runtime identity, reps, treatment mode, and recomputed discriminator +# result all agree. Prelude-only evidence, +# LEGACY-UNVERIFIED, UNMEASURED, missing/tampered evidence, or no row is not +# a tier-coverage result; # * every product/judgment skill with no result is a finding — NAMED on output. # # ADVISORY-FIRST (the egwt warn-then-fail flip): default mode reports findings @@ -40,17 +48,22 @@ # bash scripts/check-skill-probe-coverage.sh --json # machine-readable summary # # Env overrides (test seams): -# SKILL_PROBE_SKILLS_DIR skills root (default: $REPO_ROOT/skills) -# SKILL_PROBE_TIERS_FILE MEASURED ledger file (default: $REPO_ROOT/evals/skill-probes/LEDGER.md) +# SKILL_PROBE_SKILLS_DIR skills root (default: $REPO_ROOT/skills) +# SKILL_PROBE_LEDGER_FILE status ledger (default: $REPO_ROOT/evals/skill-probes/LEDGER.md) +# SKILL_PROBE_TIERS_FILE compatibility alias for SKILL_PROBE_LEDGER_FILE +# SKILL_PROBE_EVIDENCE_ROOT repository root for scorecards/probes (default: $REPO_ROOT) +# SKILL_PROBE_METADATA_TOOL verifier helper (default: scripts/lib/probe-fixture-metadata.py) # # Exit: 0 advisory/clean, 1 finding under --strict, 2 misuse. # # practices: [continuous-integration, measurement-over-assertion] -# shellcheck disable=SC1007 +# shellcheck source=scripts/lib/preamble.sh disable=SC1007,SC1091 . "$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/preamble.sh" SKILLS_DIR="${SKILL_PROBE_SKILLS_DIR:-$REPO_ROOT/skills}" -TIERS_FILE="${SKILL_PROBE_TIERS_FILE:-$REPO_ROOT/evals/skill-probes/LEDGER.md}" +LEDGER_FILE="${SKILL_PROBE_LEDGER_FILE:-${SKILL_PROBE_TIERS_FILE:-$REPO_ROOT/evals/skill-probes/LEDGER.md}}" +EVIDENCE_ROOT="${SKILL_PROBE_EVIDENCE_ROOT:-$REPO_ROOT}" +METADATA_TOOL="${SKILL_PROBE_METADATA_TOOL:-$REPO_ROOT/scripts/lib/probe-fixture-metadata.py}" STRICT=0 JSON=0 @@ -69,15 +82,56 @@ if [[ ! -d "$SKILLS_DIR" ]]; then echo "skills dir not found: $SKILLS_DIR" >&2 exit 2 fi +if [[ ! -d "$EVIDENCE_ROOT" ]]; then + echo "evidence root not found: $EVIDENCE_ROOT" >&2 + exit 2 +fi +if [[ ! -f "$METADATA_TOOL" ]]; then + echo "probe evidence verifier not found: $METADATA_TOOL" >&2 + exit 2 +fi -# --- collect the set of skills that HAVE a measured probe result -------------- -# A result = a ledger row with verdict BEHAVIORAL or INERT. The ledger lives -# under the "## Behavioral Probe Ledger" heading; rows are markdown table rows -# "| skill | probe | date | verdict |". Parse defensively: a missing ledger -# file/section simply yields an empty measured set (every gated skill is then a -# finding — surfaced as advisory). +trim_cell() { + printf '%s' "${1:-}" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//' +} + +scorecard_ref_from_notes() { + # shellcheck disable=SC2016 # Python source is intentionally single-quoted shell data. + python3 -c ' +import re, sys +refs = re.findall(r"scorecard:\s*`([^`]+)`", sys.argv[1]) +if len(refs) != 1: + raise SystemExit(1) +print(refs[0]) +' "$1" +} + +# Resolve the gate denominator first. Ledger rows outside this product/judgment +# set may be useful evidence, but they are not tier-coverage candidates and +# should not emit misleading "not measured" warnings from this gate. +declare -A GATED=() +declare -a GATED_NAMES=() +gated_total=0 +if ! TIER_SUMMARY="$(python3 "$METADATA_TOOL" tier-skills --skills-dir "$SKILLS_DIR")"; then + echo "could not parse canonical skill frontmatter for probe coverage" >&2 + exit 2 +fi +while IFS= read -r name; do + [[ -n "$name" ]] || continue + GATED["$name"]=1 + GATED_NAMES+=("$name") + gated_total=$((gated_total + 1)) +done < <(python3 -c 'import json,sys; print("\n".join(json.loads(sys.argv[1])["skills"]))' "$TIER_SUMMARY") + +# --- collect the set of skills that HAVE a current probe result ---------------- +# A result = a ledger row with a directional current verdict whose referenced v3 +# scorecard passes the mechanical verifier. The ledger lives under the +# "## Behavioral Probe Ledger" heading; rows are markdown table rows +# "| skill | probe | date | verdict | notes |". Parse defensively: a missing +# ledger file/section simply yields an empty measured set (every gated skill is +# then a finding — surfaced as advisory). declare -A MEASURED=() -if [[ -f "$TIERS_FILE" ]]; then +if [[ -f "$LEDGER_FILE" ]]; then in_ledger=0 while IFS= read -r line; do # Enter the ledger section on its heading; exit on the next H2. @@ -91,41 +145,46 @@ if [[ -f "$TIERS_FILE" ]]; then fi [[ $in_ledger -eq 1 ]] || continue [[ "$line" == \|* ]] || continue - # Split the table row on '|'. Columns: | skill | probe | date | verdict | - # — only skill (col 2) and verdict (col 5) are load-bearing; the rest are - # throwaways (`_`). - IFS='|' read -r _ skill _ _ verdict _rest <<<"$line" - skill="$(printf '%s' "${skill:-}" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')" - verdict="$(printf '%s' "${verdict:-}" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//' | tr '[:lower:]' '[:upper:]')" + # Columns: | skill | probe | date | verdict | notes |. For a current + # verdict, skill/probe/verdict plus the notes' scorecard pointer are all + # load-bearing and are checked again against the evidence. + IFS='|' read -r _lead skill probe _date verdict notes _tail <<<"$line" + skill="$(trim_cell "$skill")" + probe="$(trim_cell "$probe")" + verdict="$(trim_cell "$verdict" | tr '[:lower:]' '[:upper:]')" + notes="$(trim_cell "$notes")" # Skip the header + separator rows. [[ -z "$skill" || "$skill" == "Skill" || "$skill" =~ ^:?-+:?$ ]] && continue # Strip surrounding backticks/asterisks a table author may add. skill="$(printf '%s' "$skill" | tr -d '`*')" - if [[ "$verdict" == "BEHAVIORAL" || "$verdict" == "INERT" ]]; then - MEASURED["$skill"]=1 + probe="$(printf '%s' "$probe" | tr -d '`*')" + if [[ "$verdict" == "BEHAVIORAL" || "$verdict" == "INERT" || "$verdict" == "REGRESSIVE" ]]; then + [[ "$skill" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || continue + [[ -n "${GATED[$skill]:-}" ]] || continue + if ! scorecard_path="$(scorecard_ref_from_notes "$notes")"; then + echo "::warning::probe ledger row '${skill}/${probe}' is not measured: current verdict lacks exactly one scorecard: \`path\` evidence pointer." >&2 + continue + fi + verification="" + if verification="$(python3 "$METADATA_TOOL" verify-scorecard \ + --repo-root "$EVIDENCE_ROOT" \ + --skills-dir "$SKILLS_DIR" \ + --scorecard "$scorecard_path" \ + --ledger-skill "$skill" \ + --ledger-probe "$probe" \ + --ledger-verdict "$verdict" 2>&1)"; then + MEASURED["$skill"]=1 + else + verification="$(printf '%s' "$verification" | tail -n 1)" + echo "::warning::probe ledger row '${skill}/${probe}' is not measured: ${verification:-evidence verification failed}" >&2 + fi fi - done < "$TIERS_FILE" + done < "$LEDGER_FILE" fi -# --- enumerate product/judgment skills and find the unmeasured ones ----------- +# --- find gated skills without a verified current result ---------------------- declare -a UNMEASURED=() -gated_total=0 -for skill_md in "$SKILLS_DIR"/*/SKILL.md; do - [[ -f "$skill_md" ]] || continue - # Runtime compatibility pointers are aliases, not independent skills with - # behavioral value to measure. Their redirect contract has its own gate. - grep -Eq '^implementation:[[:space:]]+false([[:space:]]|$)' "$skill_md" && continue - # tier lives in the metadata frontmatter as ` tier: ` (first hit). - # A skill without a tier is outside this advisory gate; do not let grep's - # no-match status abort the whole scan under the shared strict preamble. - tier="$({ grep -m1 -E '^[[:space:]]*tier:[[:space:]]*' "$skill_md" 2>/dev/null || true; } \ - | sed -E 's/^[[:space:]]*tier:[[:space:]]*//; s/[[:space:]].*$//' | tr -d '"'"'"'`' )" - case "$tier" in - product|judgment) ;; - *) continue;; - esac - gated_total=$((gated_total + 1)) - name="$(basename "$(dirname "$skill_md")")" +for name in "${GATED_NAMES[@]}"; do if [[ -z "${MEASURED[$name]:-}" ]]; then UNMEASURED+=("$name") fi @@ -150,7 +209,7 @@ fi if [[ $JSON -eq 0 ]]; then for name in "${UNMEASURED[@]}"; do - echo "::warning::skill '${name}' (product/judgment tier) has NO behavioral-probe result in the MEASURED ledger — its tier badge is unmeasured." >&2 + echo "::warning::skill '${name}' (product/judgment tier) has NO current manifest-backed behavioral-probe result in the status ledger — its tier badge is unmeasured." >&2 done fi diff --git a/scripts/extract-release-notes.sh b/scripts/extract-release-notes.sh index 64c28a73d..9940a7799 100755 --- a/scripts/extract-release-notes.sh +++ b/scripts/extract-release-notes.sh @@ -21,6 +21,269 @@ PREV_TAG="${2:-}" VERSION="${TAG#v}" REPO="boshu2/agentops" +# GitHub's release renderer preserves soft line breaks in prose. Curated notes +# are hard-wrapped for readable diffs, so copying them byte-for-byte produces a +# narrow ragged column in the published release. Reflow prose and list +# continuations into logical Markdown lines while leaving structural Markdown +# (headings, tables, code, HTML, block quotes, and explicit hard breaks) intact. +normalize_markdown() { + awk ' + function leading_spaces(value, count) { + count = 0 + while (substr(value, count + 1, 1) == " ") { + count++ + } + return count + } + function trailing_spaces(value, count, pos) { + count = 0 + pos = length(value) + while (pos > 0 && substr(value, pos, 1) == " ") { + count++ + pos-- + } + return count + } + function strip_markdown_indent(value, count) { + count = leading_spaces(value) + if (count > 3) { + count = 3 + } + return substr(value, count + 1) + } + function flush_flow() { + if (flow != "") { + print flow + flow = "" + flow_kind = "" + } + } + function append_flow(value, kind, hard_break, indent, piece, spaces) { + spaces = trailing_spaces(value) + hard_break = (spaces >= 2 || value ~ /\\$/) + piece = value + sub(/^[ \t]+/, "", piece) + sub(/[ \t]+$/, "", piece) + if (flow == "") { + indent = leading_spaces(value) + flow = substr(value, 1, indent) piece + flow_kind = kind + } else { + flow = flow " " piece + } + if (hard_break) { + while (spaces > 0) { + flow = flow " " + spaces-- + } + flush_flow() + } + } + function is_list_marker(value, indent, text, pos, char, spaces) { + indent = leading_spaces(value) + text = substr(value, indent + 1) + char = substr(text, 1, 1) + + if ((char == "-" || char == "+" || char == "*") && + substr(text, 2, 1) ~ /[ \t]/) { + pos = 2 + } else { + pos = 1 + while (substr(text, pos, 1) ~ /[0-9]/ && pos <= 9) { + pos++ + } + if (pos == 1 || + (substr(text, pos, 1) != "." && substr(text, pos, 1) != ")") || + substr(text, pos + 1, 1) !~ /[ \t]/) { + return 0 + } + pos++ + } + + spaces = 0 + while (substr(text, pos, 1) ~ /[ \t]/) { + spaces++ + pos++ + } + marker_indent = indent + marker_content_indent = indent + pos - 1 + return 1 + } + function is_thematic_or_setext(value, compact) { + compact = value + gsub(/[ \t]/, "", compact) + return ((length(compact) >= 3 && compact ~ /^-+$/) || + (length(compact) >= 3 && compact ~ /^\*+$/) || + (length(compact) >= 3 && compact ~ /^_+$/) || + compact ~ /^=+$/) + } + function is_table_delimiter(value, compact) { + compact = value + gsub(/[ \t]/, "", compact) + return compact ~ /^\|?:?-+:?(\|:?-+:?)+\|?$/ + } + function fence_open(value, text, char, count, rest) { + text = strip_markdown_indent(value) + char = substr(text, 1, 1) + if (char != "`" && char != "~") { + return 0 + } + count = 0 + while (substr(text, count + 1, 1) == char) { + count++ + } + if (count < 3) { + return 0 + } + rest = substr(text, count + 1) + if (char == "`" && index(rest, "`") != 0) { + return 0 + } + next_fence_char = char + next_fence_length = count + return 1 + } + function fence_close(value, text, count, rest) { + text = strip_markdown_indent(value) + if (substr(text, 1, 1) != fence_char) { + return 0 + } + count = 0 + while (substr(text, count + 1, 1) == fence_char) { + count++ + } + rest = substr(text, count + 1) + return (count >= fence_length && rest ~ /^[ \t]*$/) + } + function html_mode_for(value, lower) { + lower = tolower(value) + if (index(value, "") != 0 + if (mode == "cdata") return index(value, "]]>") != 0 + if (mode == "processing") return index(value, "?>") != 0 + if (mode == "raw") return lower ~ /<\/(script|pre|style|textarea)[ \t>]/ + return index(lower, " 0 && indent < list_content_indent) { + list_content_indent = 0 + } + next + } + markdown ~ /^>/ || markdown ~ /^#+([ \t]|$)/ || + is_thematic_or_setext(markdown) { + flush_flow() + print + if (list_content_indent > 0 && indent < list_content_indent) { + list_content_indent = 0 + } + next + } + is_list_marker($0) && + (marker_indent <= 3 || + (list_content_indent > 0 && marker_indent < list_content_indent + 4)) { + flush_flow() + list_content_indent = marker_content_indent + append_flow($0, "list") + next + } + /^\t/ || /^ / { + if (list_content_indent == 0 || indent >= list_content_indent + 4) { + flush_flow() + print + next + } + } + { + if (flow == "" && list_content_indent > 0 && indent < list_content_indent) { + list_content_indent = 0 + } + append_flow($0, flow_kind == "list" ? "list" : "prose") + } + END { + flush_flow() + } + ' +} + CHANGELOG="CHANGELOG.md" if [[ ! -f "$CHANGELOG" ]]; then echo "ERROR: $CHANGELOG not found" >&2 @@ -41,7 +304,8 @@ CHANGELOG_SECTION=$(awk -v ver="$VERSION" ' # double-blanks when wrapped in `echo ""` boilerplate by the formatters below. CHANGELOG_SECTION="$(printf '%s\n' "$CHANGELOG_SECTION" \ | awk 'NF { found = 1 } found' \ - | awk 'NF { last = NR } { line[NR] = $0 } END { for (i = 1; i <= last; i++) print line[i] }')" + | awk 'NF { last = NR } { line[NR] = $0 } END { for (i = 1; i <= last; i++) print line[i] }' \ + | normalize_markdown)" if [[ -z "$CHANGELOG_SECTION" ]]; then echo "ERROR: No CHANGELOG entry for $VERSION — add entry before releasing" >&2 @@ -65,7 +329,8 @@ CURATED_NOTES=$(cat "$NOTES_FILE") CURATED_NOTES="$(printf '%s' "$CURATED_NOTES" \ | sed \ -e "s#(../../CHANGELOG.md)#(https://github.com/${REPO}/blob/main/CHANGELOG.md)#g" \ - -e "s#(../CHANGELOG.md)#(https://github.com/${REPO}/blob/main/docs/CHANGELOG.md)#g")" + -e "s#(../CHANGELOG.md)#(https://github.com/${REPO}/blob/main/docs/CHANGELOG.md)#g" \ + | normalize_markdown)" echo "Using curated release notes from $NOTES_FILE" >&2 # Build the release notes file diff --git a/scripts/lib/codex-exec.sh b/scripts/lib/codex-exec.sh index 384150d2e..1879cf7c5 100644 --- a/scripts/lib/codex-exec.sh +++ b/scripts/lib/codex-exec.sh @@ -30,9 +30,10 @@ # (sentinel-wrapped) # local? no no yes # -# Adapter 1 = codex — BYTE-COMPATIBLE with the historical behavior: REVIEWER unset -# (or =codex) produces exactly the pre-adapter argv/exec/classify. The codex-exec.sh -# bats contract (tests/scripts/codex-exec-lib.bats) is the behavior lock. +# Adapter 1 = codex — preserves the historical execution/classification behavior. +# Arg-mode prompt delivery inserts the standard `--` option terminator so a +# prompt beginning with `-` remains prompt data rather than CLI options. The +# codex-exec.sh Bats contract (tests/scripts/codex-exec-lib.bats) is the lock. # Adapter 2 = agy (cold, ROUTINE tier + degraded-fallback ONLY, per the A7 bench # ruling) — invokes the agy CLI headlessly (`agy -p`, the sanctioned path). The # review packet is delivered as a file PATH the model is TOLD to read (a short `-p` @@ -234,9 +235,11 @@ reviewer_adapter_marker() { # CODEX_EXEC_EXTRA_ARGS (codex) a bash array of extra passthrough flags appended # verbatim (e.g. --json). Ignored by non-codex adapters (they # are codex-specific flags). -# CODEX_EXEC_OUT_FILE write captured stdout+stderr here. If empty, output is -# captured to a temp file used only for echo-detection -# and then streamed to the caller's stdout on success. +# CODEX_EXEC_OUT_FILE write captured stdout here. If empty, output is captured +# to a temp file used only for echo-detection and then +# streamed to the caller's stdout on success. +# CODEX_EXEC_STDERR_FILE optional separate stderr sink. If empty, stderr is merged +# into CODEX_EXEC_OUT_FILE for backward compatibility. # CODEX_EXEC_EXPECT_OUTPUT 1 (default) => the caller CONSUMES reviewer output, so a # flat 0-byte run is a STALL and an output≈prompt run is # an ECHO (both fail-closed). 0 => the caller only cares @@ -295,9 +298,9 @@ codex_exec_guarded() { case "$reviewer" in codex) - # BYTE-COMPATIBLE with the historical codex path. Order is stable so stub-`codex` - # tests that inspect positional args (eval-agent-harness.bats records the -C dir) - # keep working. + # Order is stable so stub-`codex` tests that inspect positional args + # (eval-agent-harness.bats records the -C dir) keep working. Arg-mode + # prompts use `--` below so leading hyphens cannot be parsed as options. argv=(exec) [ "${CODEX_EXEC_SKIP_GIT_CHECK:-0}" = "1" ] && argv+=(--skip-git-repo-check) local sandbox="${CODEX_EXEC_SANDBOX:-read-only}" @@ -315,7 +318,7 @@ codex_exec_guarded() { if [ -n "${CODEX_EXEC_PROMPT_FILE:-}" ]; then prompt_file="$CODEX_EXEC_PROMPT_FILE"; delivery="stdin_file"; echo_cmp_file="$prompt_file" elif [ -n "${CODEX_EXEC_PROMPT_ARG:-}" ]; then - argv+=("$CODEX_EXEC_PROMPT_ARG") + argv+=(-- "$CODEX_EXEC_PROMPT_ARG") prompt_file="$(mktemp "${TMPDIR:-/tmp}/codex-exec-prompt.XXXXXX")"; _cleanup+=("$prompt_file") printf '%s' "$CODEX_EXEC_PROMPT_ARG" > "$prompt_file"; echo_cmp_file="$prompt_file" else @@ -396,7 +399,7 @@ codex_exec_guarded() { # Resolve the output sink. A caller-provided file is written in place; otherwise # a temp file backs echo-detection and is streamed to stdout on success. - local out_file="${CODEX_EXEC_OUT_FILE:-}" cleanup_out="" + local out_file="${CODEX_EXEC_OUT_FILE:-}" stderr_file="${CODEX_EXEC_STDERR_FILE:-}" cleanup_out="" if [ -z "$out_file" ]; then out_file="$(mktemp "${TMPDIR:-/tmp}/codex-exec-out.XXXXXX")" cleanup_out="$out_file" @@ -412,12 +415,22 @@ codex_exec_guarded() { _codex_exec_run() { if [ "$delivery" = "stdin_file" ]; then # File-prompt mode: feed the file on stdin. - if [ "${#to_cmd[@]}" -gt 0 ]; then "${to_cmd[@]}" "$bin" "${argv[@]}" <"$prompt_file" >"$out_file" 2>&1 - else "$bin" "${argv[@]}" <"$prompt_file" >"$out_file" 2>&1; fi + if [ -n "$stderr_file" ]; then + if [ "${#to_cmd[@]}" -gt 0 ]; then "${to_cmd[@]}" "$bin" "${argv[@]}" <"$prompt_file" >"$out_file" 2>"$stderr_file" + else "$bin" "${argv[@]}" <"$prompt_file" >"$out_file" 2>"$stderr_file"; fi + else + if [ "${#to_cmd[@]}" -gt 0 ]; then "${to_cmd[@]}" "$bin" "${argv[@]}" <"$prompt_file" >"$out_file" 2>&1 + else "$bin" "${argv[@]}" <"$prompt_file" >"$out_file" 2>&1; fi + fi else # Arg/stdin-pipe/pointer mode: the prompt is already in argv (or on the caller's stdin). - if [ "${#to_cmd[@]}" -gt 0 ]; then "${to_cmd[@]}" "$bin" "${argv[@]}" >"$out_file" 2>&1 - else "$bin" "${argv[@]}" >"$out_file" 2>&1; fi + if [ -n "$stderr_file" ]; then + if [ "${#to_cmd[@]}" -gt 0 ]; then "${to_cmd[@]}" "$bin" "${argv[@]}" >"$out_file" 2>"$stderr_file" + else "$bin" "${argv[@]}" >"$out_file" 2>"$stderr_file"; fi + else + if [ "${#to_cmd[@]}" -gt 0 ]; then "${to_cmd[@]}" "$bin" "${argv[@]}" >"$out_file" 2>&1 + else "$bin" "${argv[@]}" >"$out_file" 2>&1; fi + fi fi } @@ -512,8 +525,10 @@ codex_exec_guarded() { codex_exec_producer_template() { case "${1:-producer}" in producer) + # shellcheck disable=SC2016 # Positional parameters expand later inside bash -c. printf 'timeout "$3" codex exec --skip-git-repo-check -C "$1" -s workspace-write "$2" >/dev/null 2>&1' ;; membrane) + # shellcheck disable=SC2016 # Positional parameters expand later inside bash -c. printf 'codex exec --skip-git-repo-check "$1" 2>/dev/null' ;; *) return 2 ;; esac diff --git a/scripts/lib/probe-fixture-metadata.py b/scripts/lib/probe-fixture-metadata.py new file mode 100644 index 000000000..d0e6471d4 --- /dev/null +++ b/scripts/lib/probe-fixture-metadata.py @@ -0,0 +1,2575 @@ +#!/usr/bin/env python3 +"""Create and verify hash-bound skill-probe fixture-set metadata.""" + +from __future__ import annotations + +import argparse +import base64 +import binascii +import ctypes +import errno +import hashlib +import json +import os +import re +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +from pathlib import Path +from pathlib import PurePosixPath +from typing import Any, NoReturn + + +LEGACY_BOUND_SCHEMA = "agentops-skill-probe-fixture-set.v1" +LEGACY_CANONICAL_SCHEMA = "agentops-skill-probe-fixture-set.v2" +SCHEMA = "agentops-skill-probe-fixture-set.v3" +SCORECARD_SCHEMA = "agentops-skill-probe.v3" +MANIFEST_NAME = "fixture-set.json" +CAPTURE_CONTRACT_NAME = "capture-contract.json" +CAPTURE_CONTRACT_SCHEMA = "agentops-skill-probe-capture.v2" +TRANSCRIPT_RE = re.compile(r"^(control|treatment)-([1-9][0-9]*)\.txt$") +SHA256_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +SAFE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +MAX_REPS = 20 +MAX_INPUT_BYTES = 1024 * 1024 +MAX_TRANSCRIPT_BYTES = 16 * 1024 * 1024 +RESPONSE_EXTRACTION = "codex-jsonl-final-agent-message.v1" +TRANSCRIPT_FORMAT = "codex-exec-jsonl.v1" +PROBE_INPUT_EVENT = "agentops.probe-input.v1" +DISCRIMINATOR_TIMEOUT_SECONDS = 2 +LEGACY_EVALUATION_INPUT_NAMES = ( + "probe.json", + "question.md", + "treatment-prelude.md", + "discriminator.sh", +) +BASE_CAPTURE_INPUT_NAMES = ("probe.json", "question.md", "discriminator.sh") +TREATMENT_SOURCES = {"canonical-skill", "injected-prelude"} +CURRENT_VERDICTS = {"BEHAVIORAL", "INERT", "REGRESSIVE"} + + +class MetadataError(ValueError): + """The fixture set is incomplete, unsafe, or fails integrity checks.""" + + +def fail(message: str) -> NoReturn: + print(f"probe-fixture-metadata: error: {message}", file=sys.stderr) + raise SystemExit(2) + + +def no_duplicate_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise MetadataError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def read_regular_bytes(path: Path, label: str, *, maximum: int | None = None) -> bytes: + """Read one stable regular-file identity without following a final symlink.""" + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(path, flags) + except OSError as exc: + raise MetadataError(f"cannot open {label} {path}: {exc}") from exc + try: + before = os.fstat(fd) + if not stat.S_ISREG(before.st_mode): + raise MetadataError(f"{label} must be a regular non-symlink file: {path}") + if maximum is not None and before.st_size > maximum: + raise MetadataError( + f"{label} exceeds the {maximum}-byte safety limit: {path}" + ) + chunks: list[bytes] = [] + total = 0 + while chunk := os.read(fd, 1024 * 1024): + total += len(chunk) + if maximum is not None and total > maximum: + raise MetadataError( + f"{label} exceeds the {maximum}-byte safety limit: {path}" + ) + chunks.append(chunk) + after = os.fstat(fd) + try: + path_after = os.stat(path, follow_symlinks=False) + except OSError as exc: + raise MetadataError( + f"{label} identity changed while reading: {path}" + ) from exc + if ( + not os.path.samestat(before, after) + or not os.path.samestat(before, path_after) + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + ): + raise MetadataError(f"{label} changed while reading: {path}") + return b"".join(chunks) + finally: + os.close(fd) + + +def parse_json_bytes(data: bytes, label: str) -> dict[str, Any]: + try: + value = json.loads( + data.decode("utf-8", errors="strict"), + object_pairs_hook=no_duplicate_object, + ) + except (UnicodeError, json.JSONDecodeError, MetadataError) as exc: + raise MetadataError(f"cannot read {label}: {exc}") from exc + if not isinstance(value, dict): + raise MetadataError(f"{label} must contain a JSON object") + return value + + +def load_json(path: Path) -> dict[str, Any]: + return parse_json_bytes( + read_regular_bytes(path, path.name, maximum=MAX_INPUT_BYTES), path.name + ) + + +def digest_bytes(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def digest_file(path: Path) -> str: + return digest_bytes(read_regular_bytes(path, "file")) + + +def canonical_bytes(value: dict[str, Any]) -> bytes: + return json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + + +def require_text(value: Any, field: str, *, nullable: bool = False) -> str | None: + if nullable and value is None: + return None + if not isinstance(value, str) or not value or any(ord(ch) < 32 for ch in value): + raise MetadataError( + f"{field} must be a non-empty string without control characters" + ) + return value + + +def require_message_text(value: Any, field: str) -> str: + if ( + not isinstance(value, str) + or not value.strip() + or any(ord(ch) < 32 and ch not in "\t\r\n" for ch in value) + ): + raise MetadataError( + f"{field} must be non-blank text without unsupported control characters" + ) + try: + value.encode("utf-8", errors="strict") + except UnicodeError as exc: + raise MetadataError(f"{field} must be valid UTF-8 text") from exc + return value + + +def require_reps(value: Any) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 1 + or value > MAX_REPS + ): + raise MetadataError(f"reps must be an integer between 1 and {MAX_REPS}") + return value + + +def require_nonnegative_int(value: Any, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise MetadataError(f"{field} must be a non-negative integer") + return value + + +def require_rate(value: Any, field: str) -> float | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise MetadataError(f"{field} must be a number or null") + numeric = float(value) + if numeric < 0.0 or numeric > 1.0: + raise MetadataError(f"{field} must be between 0 and 1") + return numeric + + +def require_exact_object(value: Any, keys: set[str], field: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != keys: + rendered = ", ".join(sorted(keys)) + raise MetadataError(f"{field} must contain exactly: {rendered}") + return value + + +def expected_transcripts(reps: int) -> list[str]: + return [ + f"{arm}-{rep}.txt" + for rep in range(1, reps + 1) + for arm in ("control", "treatment") + ] + + +def validate_fixture_dir(path: Path) -> None: + if path.is_symlink(): + raise MetadataError(f"fixture directory must not be a symlink: {path}") + if not path.is_dir(): + raise MetadataError(f"fixture directory not found: {path}") + + +def validate_transcript(path: Path) -> None: + if path.is_symlink() or not path.is_file(): + raise MetadataError( + f"transcript must be a regular non-symlink file: {path.name}" + ) + + +def validate_regular_file(path: Path, label: str) -> None: + if path.is_symlink() or not path.is_file(): + raise MetadataError(f"{label} must be a regular non-symlink file: {path}") + + +def canonical_skill_path(skills_dir: Path, skill: str) -> Path: + """Resolve the one canonical skill source without traversing symlinks.""" + if not SAFE_ID_RE.fullmatch(skill): + raise MetadataError(f"unsafe canonical skill name: {skill!r}") + if skills_dir.is_symlink() or not skills_dir.is_dir(): + raise MetadataError( + f"canonical skills directory must be a non-symlink directory: {skills_dir}" + ) + skill_dir = skills_dir / skill + if skill_dir.is_symlink() or not skill_dir.is_dir(): + raise MetadataError( + f"canonical skill directory must be a non-symlink directory: {skill_dir}" + ) + path = skill_dir / "SKILL.md" + validate_regular_file(path, "canonical skill") + return path + + +def frontmatter_lines(data: bytes, label: str) -> list[str]: + try: + lines = data.decode("utf-8", errors="strict").splitlines() + except UnicodeError as exc: + raise MetadataError(f"cannot read {label} frontmatter: {exc}") from exc + if not lines or lines[0].strip() != "---": + raise MetadataError(f"{label} must start with YAML frontmatter") + try: + end = next(index for index, line in enumerate(lines[1:], 1) if line == "---") + except StopIteration as exc: + raise MetadataError(f"{label} has no closing YAML frontmatter marker") from exc + return lines[1:end] + + +def declared_skill_name_bytes(data: bytes) -> str: + """Read the top-level name from the YAML frontmatter without a YAML dependency.""" + lines = frontmatter_lines(data, "canonical skill") + names = [] + for line in lines: + match = re.fullmatch(r"name:[ \t]*(.*?)[ \t]*", line) + if match: + value = match.group(1) + if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: + value = value[1:-1] + names.append(value) + if len(names) != 1 or not names[0]: + raise MetadataError( + "canonical skill frontmatter must declare exactly one non-empty top-level name" + ) + name = require_text(names[0], "canonical skill frontmatter name") + assert isinstance(name, str) + return name + + +def declared_skill_name(path: Path) -> str: + return declared_skill_name_bytes( + read_regular_bytes(path, "canonical skill", maximum=MAX_INPUT_BYTES) + ) + + +def yaml_scalar(value: str, field: str) -> str: + scalar = value.strip() + if len(scalar) >= 2 and scalar[0] == scalar[-1] and scalar[0] in {'"', "'"}: + scalar = scalar[1:-1] + if not scalar or any(ord(ch) < 32 for ch in scalar): + raise MetadataError(f"{field} must be one non-empty scalar") + return scalar + + +def coverage_frontmatter(path: Path) -> tuple[str | None, bool]: + lines = frontmatter_lines( + read_regular_bytes(path, "skill", maximum=MAX_INPUT_BYTES), "skill" + ) + tiers: list[str] = [] + implementations: list[str] = [] + metadata_blocks = 0 + in_metadata = False + for line in lines: + if line and line[0] not in " \t": + in_metadata = False + match = re.fullmatch(r"([A-Za-z0-9_-]+):[ \t]*(.*?)[ \t]*", line) + if not match: + continue + key, value = match.groups() + if key == "metadata": + metadata_blocks += 1 + if value: + raise MetadataError("skill metadata must be a YAML mapping") + in_metadata = True + elif key == "implementation": + implementations.append(yaml_scalar(value, "implementation")) + elif key == "tier": + tiers.append(yaml_scalar(value, "tier")) + continue + if in_metadata: + match = re.fullmatch(r" tier:[ \t]*(.*?)[ \t]*", line) + if match: + tiers.append(yaml_scalar(match.group(1), "metadata.tier")) + if metadata_blocks > 1: + raise MetadataError(f"duplicate metadata mapping in {path}") + if len(tiers) > 1: + raise MetadataError(f"duplicate tier declaration in {path}") + if len(implementations) > 1: + raise MetadataError(f"duplicate implementation declaration in {path}") + implementation = implementations[0].lower() if implementations else "true" + if implementation not in {"true", "false"}: + raise MetadataError(f"implementation must be true or false in {path}") + return (tiers[0] if tiers else None, implementation == "false") + + +def tier_skills(skills_dir: Path) -> list[str]: + if skills_dir.is_symlink() or not skills_dir.is_dir(): + raise MetadataError("skills root must be a non-symlink directory") + result: list[str] = [] + for skill_dir in sorted(skills_dir.iterdir(), key=lambda path: path.name): + if skill_dir.is_symlink() or not skill_dir.is_dir(): + continue + skill_path = skill_dir / "SKILL.md" + if not skill_path.exists(): + continue + validate_regular_file(skill_path, "skill") + tier, redirect = coverage_frontmatter(skill_path) + if not redirect and tier in {"product", "judgment"}: + result.append(skill_dir.name) + return result + + +def build_canonical_skill_record( + probe_dir: Path, skills_dir: Path, expected_probe: str +) -> dict[str, str]: + probe_meta = load_json(probe_dir / "probe.json") + if probe_meta.get("id") != expected_probe: + raise MetadataError("probe.json id does not match the requested probe") + skill = require_text(probe_meta.get("skill"), "probe.json skill") + assert isinstance(skill, str) + path = canonical_skill_path(skills_dir, skill) + declared = declared_skill_name(path) + if declared != skill: + raise MetadataError( + f"canonical skill identity mismatch: probe.json names {skill!r}, " + f"SKILL.md declares {declared!r}" + ) + return { + "name": skill, + "path": f"skills/{skill}/SKILL.md", + "sha256": digest_file(path), + } + + +def build_embedded_canonical_skill_record( + probe_dir: Path, skills_dir: Path, expected_probe: str +) -> dict[str, str]: + record = build_canonical_skill_record(probe_dir, skills_dir, expected_probe) + path = canonical_skill_path(skills_dir, record["name"]) + embedded = embedded_record(path, record["path"], "canonical skill") + return {"name": record["name"], **embedded} + + +def validate_probe_metadata( + probe_meta: dict[str, Any], expected_probe: str, *, require_treatment: bool +) -> dict[str, Any]: + if probe_meta.get("id") != expected_probe: + raise MetadataError("probe.json id does not match the requested probe") + skill = require_text(probe_meta.get("skill"), "probe.json skill") + assert isinstance(skill, str) + if not SAFE_ID_RE.fullmatch(skill): + raise MetadataError(f"unsafe probe.json skill: {skill!r}") + reps = require_reps(probe_meta.get("reps")) + if probe_meta.get("discriminator") != "discriminator.sh": + raise MetadataError( + "probe.json discriminator must be exactly 'discriminator.sh'" + ) + source = probe_meta.get("treatment_source") + if require_treatment: + source = require_text(source, "probe.json treatment_source") + if source not in TREATMENT_SOURCES: + choices = ", ".join(sorted(TREATMENT_SOURCES)) + raise MetadataError( + f"probe.json treatment_source must be one of: {choices}" + ) + return {"skill": skill, "reps": reps, "treatment_source": source} + + +def declared_treatment_source(probe_dir: Path, expected_probe: str) -> str: + probe_meta = load_json(probe_dir / "probe.json") + contract = validate_probe_metadata( + probe_meta, expected_probe, require_treatment=True + ) + source = contract["treatment_source"] + assert isinstance(source, str) + return source + + +def validate_canonical_skill_record( + value: Any, probe_dir: Path, skills_dir: Path, expected_probe: str +) -> dict[str, str]: + record = require_exact_object(value, {"name", "path", "sha256"}, "canonical_skill") + expected = build_canonical_skill_record(probe_dir, skills_dir, expected_probe) + for field in ("name", "path"): + if record[field] != expected[field]: + raise MetadataError( + f"canonical_skill {field} mismatch: expected {expected[field]!r}, " + f"got {record[field]!r}" + ) + digest = record["sha256"] + if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest): + raise MetadataError("canonical_skill sha256 must be a sha256 digest") + if digest != expected["sha256"]: + raise MetadataError( + "canonical skill digest mismatch for " + f"{record['path']}: expected {digest}, got {expected['sha256']}" + ) + return expected + + +def evaluation_input_records(probe_dir: Path) -> list[dict[str, str]]: + if probe_dir.is_symlink() or not probe_dir.is_dir(): + raise MetadataError( + f"probe directory must be a non-symlink directory: {probe_dir}" + ) + records = [] + for name in LEGACY_EVALUATION_INPUT_NAMES: + path = probe_dir / name + validate_regular_file(path, "evaluation input") + records.append({"path": name, "sha256": digest_file(path)}) + return records + + +def embedded_record(path: Path, logical_path: str, label: str) -> dict[str, str]: + data = read_regular_bytes(path, label, maximum=MAX_INPUT_BYTES) + return embedded_bytes_record(data, logical_path) + + +def embedded_bytes_record(data: bytes, logical_path: str) -> dict[str, str]: + return { + "path": logical_path, + "sha256": digest_bytes(data), + "content_base64": base64.b64encode(data).decode("ascii"), + } + + +def capture_input_names(treatment_source: str) -> tuple[str, ...]: + if treatment_source == "injected-prelude": + return LEGACY_EVALUATION_INPUT_NAMES + if treatment_source == "canonical-skill": + return BASE_CAPTURE_INPUT_NAMES + raise MetadataError(f"unsupported treatment source: {treatment_source!r}") + + +def embedded_input_records( + probe_dir: Path, treatment_source: str +) -> list[dict[str, str]]: + return [ + embedded_record(probe_dir / name, name, "capture input") + for name in capture_input_names(treatment_source) + ] + + +def decode_embedded_record( + value: Any, + expected_path: str, + label: str, + *, + maximum: int = MAX_INPUT_BYTES, +) -> tuple[dict[str, str], bytes]: + record = require_exact_object(value, {"path", "sha256", "content_base64"}, label) + if record["path"] != expected_path: + raise MetadataError( + f"{label} path mismatch: expected {expected_path!r}, got {record['path']!r}" + ) + digest = record["sha256"] + if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest): + raise MetadataError(f"{label} sha256 must be a sha256 digest") + encoded = record["content_base64"] + if not isinstance(encoded, str): + raise MetadataError(f"{label} content_base64 must be a string") + try: + data = base64.b64decode(encoded, validate=True) + except (ValueError, binascii.Error) as exc: + raise MetadataError(f"{label} content_base64 is invalid") from exc + if len(data) > maximum: + raise MetadataError(f"{label} exceeds the {maximum}-byte safety limit") + if digest_bytes(data) != digest: + raise MetadataError(f"{label} embedded bytes do not match sha256") + return record, data + + +def validate_embedded_canonical_skill( + value: Any, probe_meta: dict[str, Any], expected_probe: str +) -> tuple[dict[str, str], bytes]: + contract = validate_probe_metadata( + probe_meta, expected_probe, require_treatment=True + ) + skill = contract["skill"] + record = require_exact_object( + value, + {"name", "path", "sha256", "content_base64"}, + "canonical_skill", + ) + if record["name"] != skill: + raise MetadataError("canonical_skill name disagrees with captured probe.json") + expected_path = f"skills/{skill}/SKILL.md" + decoded_record, data = decode_embedded_record( + {key: record[key] for key in ("path", "sha256", "content_base64")}, + expected_path, + "canonical_skill", + ) + if declared_skill_name_bytes(data) != skill: + raise MetadataError( + "embedded canonical skill identity disagrees with captured probe.json" + ) + return {"name": skill, **decoded_record}, data + + +def decode_capture_inputs( + records: Any, expected_probe: str, treatment_source: str +) -> tuple[dict[str, bytes], dict[str, Any]]: + names = capture_input_names(treatment_source) + if not isinstance(records, list) or len(records) != len(names): + raise MetadataError("capture_inputs must contain the exact input inventory") + decoded: dict[str, bytes] = {} + for expected, value in zip(names, records, strict=True): + _, data = decode_embedded_record( + value, expected, f"capture_inputs.{expected}" + ) + decoded[expected] = data + probe_meta = parse_json_bytes(decoded["probe.json"], "captured probe.json") + contract = validate_probe_metadata( + probe_meta, expected_probe, require_treatment=True + ) + if contract["treatment_source"] != treatment_source: + raise MetadataError( + "capture input treatment source disagrees with the capture contract" + ) + return decoded, contract + + +def prompt_bytes( + inputs: dict[str, bytes], canonical_skill: bytes, treatment_source: str +) -> dict[str, bytes]: + question = inputs["question.md"] + treatment = ( + canonical_skill + if treatment_source == "canonical-skill" + else inputs["treatment-prelude.md"] + ) + return { + "control": question, + "treatment": treatment + b"\n\n---\n\n" + question, + } + + +def prompt_records( + inputs: dict[str, bytes], canonical_skill: bytes, treatment_source: str +) -> list[dict[str, str]]: + prompts = prompt_bytes(inputs, canonical_skill, treatment_source) + return [ + embedded_bytes_record(prompts[arm], f"{arm}.prompt") + for arm in ("control", "treatment") + ] + + +def decode_prompts(value: Any) -> dict[str, bytes]: + if not isinstance(value, list) or len(value) != 2: + raise MetadataError("prompts must contain exact control and treatment records") + decoded: dict[str, bytes] = {} + for arm, record in zip(("control", "treatment"), value, strict=True): + _, data = decode_embedded_record( + record, + f"{arm}.prompt", + f"prompts.{arm}", + maximum=MAX_INPUT_BYTES * 2 + 16, + ) + decoded[arm] = data + return decoded + + +def resolve_executable(command: str) -> Path: + candidate = shutil.which(command) if os.sep not in command else command + if not candidate: + raise MetadataError(f"producer executable not found: {command}") + try: + resolved = Path(candidate).resolve(strict=True) + except OSError as exc: + raise MetadataError(f"producer executable cannot be resolved: {command}") from exc + validate_regular_file(resolved, "producer executable") + if not os.access(resolved, os.X_OK): + raise MetadataError(f"producer executable is not executable: {resolved}") + return resolved + + +def producer_runtime_identity( + requested_model: str | None, + requested_effort: str | None, + override_command: str | None, +) -> dict[str, Any]: + model = require_text(requested_model, "requested model", nullable=True) + effort = require_text(requested_effort, "requested effort", nullable=True) + command = override_command or "codex" + resolved = resolve_executable(command) + try: + completed = subprocess.run( + [str(resolved), "--version"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=5, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise MetadataError(f"could not identify producer runtime: {exc}") from exc + if completed.returncode != 0: + detail = completed.stderr.decode("utf-8", errors="replace").strip() + raise MetadataError( + f"producer runtime --version failed with {completed.returncode}: {detail}" + ) + if completed.stderr: + detail = completed.stderr.decode("utf-8", errors="replace").strip() + raise MetadataError(f"producer runtime --version wrote stderr: {detail}") + try: + version = completed.stdout.decode("utf-8", errors="strict").strip() + except UnicodeError as exc: + raise MetadataError("producer runtime version is not UTF-8") from exc + require_text(version, "producer runtime version") + override = override_command is not None + return { + "adapter": "codex", + "model": model, + "effort": effort, + "identity": { + "source": "test-override" if override else "native-codex-path", + "override": override, + "version": version, + "executable_sha256": digest_file(resolved), + "coverage_eligible": (not override and model is not None and effort is not None), + }, + } + + +def validate_producer_request(value: Any) -> dict[str, Any]: + request = require_exact_object( + value, {"adapter", "model", "effort", "identity"}, "producer_request" + ) + if request["adapter"] != "codex": + raise MetadataError("producer_request adapter must be codex") + model = require_text(request["model"], "producer_request model", nullable=True) + effort = require_text(request["effort"], "producer_request effort", nullable=True) + identity = require_exact_object( + request["identity"], + {"source", "override", "version", "executable_sha256", "coverage_eligible"}, + "producer_request identity", + ) + if identity["source"] not in {"native-codex-path", "test-override"}: + raise MetadataError("producer identity source is unsupported") + if not isinstance(identity["override"], bool): + raise MetadataError("producer identity override must be boolean") + if identity["override"] != (identity["source"] == "test-override"): + raise MetadataError("producer identity source/override mismatch") + require_text(identity["version"], "producer identity version") + if not isinstance(identity["executable_sha256"], str) or not SHA256_RE.fullmatch( + identity["executable_sha256"] + ): + raise MetadataError("producer executable_sha256 must be a sha256 digest") + eligible = not identity["override"] and model is not None and effort is not None + if identity["coverage_eligible"] is not eligible: + raise MetadataError("producer coverage_eligible is inconsistent") + return request + + +def counterbalanced_schedule(reps: int) -> list[dict[str, Any]]: + require_reps(reps) + schedule: list[dict[str, Any]] = [] + position = 1 + for rep in range(1, reps + 1): + arms = ("control", "treatment") if rep % 2 else ("treatment", "control") + for arm in arms: + schedule.append({"position": position, "rep": rep, "arm": arm}) + position += 1 + return schedule + + +def validate_schedule(value: Any, reps: int) -> list[dict[str, Any]]: + expected = counterbalanced_schedule(reps) + if value != expected: + raise MetadataError( + "fixture schedule must equal the deterministic counterbalanced schedule" + ) + return expected + + +def build_capture_contract( + probe_dir: Path, + skills_dir: Path, + expected_probe: str, + producer_request: dict[str, Any], +) -> dict[str, Any]: + probe_meta = load_json(probe_dir / "probe.json") + probe_contract = validate_probe_metadata( + probe_meta, expected_probe, require_treatment=True + ) + treatment_source = probe_contract["treatment_source"] + assert isinstance(treatment_source, str) + inputs = embedded_input_records(probe_dir, treatment_source) + decoded_inputs, _ = decode_capture_inputs(inputs, expected_probe, treatment_source) + embedded_skill = build_embedded_canonical_skill_record( + probe_dir, skills_dir, expected_probe + ) + _, canonical_skill_bytes = validate_embedded_canonical_skill( + embedded_skill, probe_meta, expected_probe + ) + payload = { + "schema": CAPTURE_CONTRACT_SCHEMA, + "probe": expected_probe, + "reps": probe_contract["reps"], + "capture_inputs": inputs, + "canonical_skill": embedded_skill, + "treatment_source": treatment_source, + "prompts": prompt_records( + decoded_inputs, canonical_skill_bytes, treatment_source + ), + "producer_request": validate_producer_request(producer_request), + "schedule": counterbalanced_schedule(probe_contract["reps"]), + "scoring": { + "response_extraction": RESPONSE_EXTRACTION, + "transcript_format": TRANSCRIPT_FORMAT, + "discriminator_timeout_seconds": DISCRIMINATOR_TIMEOUT_SECONDS, + }, + } + return {**payload, "binding_sha256": digest_bytes(canonical_bytes(payload))} + + +def validate_capture_contract(value: Any, expected_probe: str) -> dict[str, Any]: + keys = { + "schema", + "probe", + "reps", + "capture_inputs", + "canonical_skill", + "treatment_source", + "prompts", + "producer_request", + "schedule", + "scoring", + "binding_sha256", + } + contract = require_exact_object(value, keys, "capture contract") + if contract["schema"] != CAPTURE_CONTRACT_SCHEMA: + raise MetadataError("unsupported capture contract schema") + if contract["probe"] != expected_probe: + raise MetadataError("capture contract probe mismatch") + reps = require_reps(contract["reps"]) + treatment_source = require_text( + contract["treatment_source"], "capture contract treatment_source" + ) + if treatment_source not in TREATMENT_SOURCES: + raise MetadataError("capture contract treatment source is unsupported") + inputs, probe_contract = decode_capture_inputs( + contract["capture_inputs"], expected_probe, treatment_source + ) + probe_meta = parse_json_bytes(inputs["probe.json"], "captured probe.json") + if probe_contract["reps"] != reps: + raise MetadataError("capture contract reps disagree with captured probe.json") + if probe_contract["treatment_source"] != contract["treatment_source"]: + raise MetadataError( + "capture contract treatment source disagrees with captured probe.json" + ) + _, canonical_skill_bytes = validate_embedded_canonical_skill( + contract["canonical_skill"], probe_meta, expected_probe + ) + expected_prompts = prompt_records(inputs, canonical_skill_bytes, treatment_source) + decode_prompts(contract["prompts"]) + if contract["prompts"] != expected_prompts: + raise MetadataError("capture contract prompts disagree with bound input bytes") + validate_producer_request(contract["producer_request"]) + validate_schedule(contract["schedule"], reps) + if contract["scoring"] != { + "response_extraction": RESPONSE_EXTRACTION, + "transcript_format": TRANSCRIPT_FORMAT, + "discriminator_timeout_seconds": DISCRIMINATOR_TIMEOUT_SECONDS, + }: + raise MetadataError("capture contract scoring semantics are unsupported") + binding = contract["binding_sha256"] + if not isinstance(binding, str) or not SHA256_RE.fullmatch(binding): + raise MetadataError("capture contract binding must be a sha256 digest") + payload = {key: item for key, item in contract.items() if key != "binding_sha256"} + if digest_bytes(canonical_bytes(payload)) != binding: + raise MetadataError("capture contract binding mismatch") + return contract + + +def write_exclusive_bytes(path: Path, data: bytes, label: str) -> os.stat_result: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(path, flags, 0o644) + except FileExistsError as exc: + raise MetadataError(f"refusing to replace existing immutable {label}") from exc + try: + identity = os.fstat(fd) + offset = 0 + while offset < len(data): + offset += os.write(fd, data[offset:]) + os.fsync(fd) + current = os.stat(path, follow_symlinks=False) + if not os.path.samestat(identity, current): + raise MetadataError(f"{label} identity changed while writing") + return identity + finally: + os.close(fd) + + +def write_capture_contract( + fixture_dir: Path, + probe_dir: Path, + skills_dir: Path, + expected_probe: str, + requested_model: str | None, + requested_effort: str | None, + producer_override: str | None, +) -> dict[str, Any]: + validate_fixture_dir(fixture_dir) + if os.listdir(fixture_dir): + raise MetadataError( + "capture snapshot requires an empty stage before any transcript exists" + ) + producer_request = producer_runtime_identity( + requested_model, requested_effort, producer_override + ) + contract = build_capture_contract( + probe_dir, skills_dir, expected_probe, producer_request + ) + encoded = ( + json.dumps(contract, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") + write_exclusive_bytes( + fixture_dir / CAPTURE_CONTRACT_NAME, encoded, CAPTURE_CONTRACT_NAME + ) + return contract + + +def load_capture_contract(fixture_dir: Path, expected_probe: str) -> dict[str, Any]: + return validate_capture_contract( + load_json(fixture_dir / CAPTURE_CONTRACT_NAME), expected_probe + ) + + +def observed_producer_bytes(data: bytes, label: str) -> tuple[str, str]: + """Read the model and reasoning effort from the first Codex header block.""" + try: + lines = data.decode("utf-8", errors="strict").splitlines() + except UnicodeError as exc: + raise MetadataError(f"cannot parse producer header in {label}: {exc}") from exc + + try: + first_boundary = lines.index("--------") + second_boundary = lines.index("--------", first_boundary + 1) + except ValueError as exc: + raise MetadataError(f"{label} has no complete Codex transcript header") from exc + + header = lines[first_boundary + 1 : second_boundary] + models = [ + line.removeprefix("model:").strip() + for line in header + if line.startswith("model:") + ] + efforts = [ + line.removeprefix("reasoning effort:").strip() + for line in header + if line.startswith("reasoning effort:") + ] + if len(models) != 1 or not models[0]: + raise MetadataError(f"{label} must have exactly one non-empty model header") + if len(efforts) != 1 or not efforts[0]: + raise MetadataError( + f"{label} must have exactly one non-empty reasoning effort header" + ) + require_text(models[0], f"{label} observed model") + require_text(efforts[0], f"{label} observed reasoning effort") + return models[0], efforts[0] + + +def observed_producer(path: Path) -> tuple[str, str]: + return observed_producer_bytes( + read_regular_bytes(path, "transcript", maximum=MAX_TRANSCRIPT_BYTES), + path.name, + ) + + +def observe_paths(paths: list[Path]) -> dict[str, str]: + if not paths: + raise MetadataError( + "no captured transcripts are available to identify the producer" + ) + observed: set[tuple[str, str]] = set() + for path in paths: + validate_transcript(path) + observed.add(observed_producer(path)) + if len(observed) != 1: + rendered = ", ".join(f"{model}/{effort}" for model, effort in sorted(observed)) + raise MetadataError(f"captured transcripts disagree on producer: {rendered}") + model, effort = observed.pop() + return {"adapter": "codex", "model": model, "effort": effort} + + +def transcript_paths(fixture_dir: Path) -> list[Path]: + paths = [ + path for path in fixture_dir.iterdir() if TRANSCRIPT_RE.fullmatch(path.name) + ] + return sorted(paths, key=lambda path: path.name) + + +def build_payload( + fixture_dir: Path, + probe_dir: Path, + skills_dir: Path, + harness: Path, + preamble: Path, + dispatch_helper: Path, + probe: str, + reps: int, + requested_model: str | None, + requested_effort: str | None, +) -> dict[str, Any]: + capture_path = fixture_dir / CAPTURE_CONTRACT_NAME + if not capture_path.exists() or capture_path.is_symlink(): + raise MetadataError( + "v3 create requires a pre-existing capture contract written before execution" + ) + capture_contract = load_capture_contract(fixture_dir, probe) + expected = expected_transcripts(reps) + actual = sorted(os.listdir(fixture_dir)) + expected_before_manifest = expected + [CAPTURE_CONTRACT_NAME] + if sorted(expected_before_manifest) != sorted(actual): + missing = sorted(set(expected_before_manifest) - set(actual)) + extra = sorted(set(actual) - set(expected_before_manifest)) + detail = [] + if missing: + detail.append("missing=" + ",".join(missing)) + if extra: + detail.append("extra=" + ",".join(extra)) + raise MetadataError( + "fixture transcript inventory mismatch (" + "; ".join(detail) + ")" + ) + + paths = [fixture_dir / name for name in expected] + transcripts = [] + threads = [] + for name, path in zip(expected, paths, strict=True): + match = TRANSCRIPT_RE.fullmatch(name) + assert match is not None + arm, rep_text = match.groups() + transcript = read_regular_bytes( + path, "transcript", maximum=MAX_TRANSCRIPT_BYTES + ) + thread_id, _ = validate_structured_transcript( + transcript, capture_contract, arm, int(rep_text), name + ) + transcripts.append({"path": name, "sha256": digest_bytes(transcript)}) + threads.append({"path": name, "thread_id": thread_id}) + if len({entry["thread_id"] for entry in threads}) != len(threads): + raise MetadataError("each probe dispatch must have a distinct Codex thread_id") + + if reps != capture_contract["reps"]: + raise MetadataError( + f"capture reps {reps} do not match bound probe.json reps " + f"{capture_contract['reps']}" + ) + capture_inputs = capture_contract["capture_inputs"] + producer_request = validate_producer_request(capture_contract["producer_request"]) + expected_requested = { + "model": producer_request["model"], + "effort": producer_request["effort"], + } + supplied_requested = { + "model": require_text(requested_model, "requested model", nullable=True), + "effort": require_text(requested_effort, "requested effort", nullable=True), + } + if supplied_requested != expected_requested: + raise MetadataError( + "create requested producer does not match the pre-execution capture contract" + ) + validate_regular_file(harness, "probe harness") + validate_regular_file(preamble, "probe preamble") + validate_regular_file(dispatch_helper, "Codex dispatch helper") + helper = Path(__file__).resolve() + validate_regular_file(helper, "fixture metadata helper") + canonical_skill = capture_contract["canonical_skill"] + treatment_source = capture_contract["treatment_source"] + producer = {**producer_request, "threads": threads} + + return { + "schema": SCHEMA, + "probe": require_text(probe, "probe"), + "reps": reps, + "producer": producer, + "requested_producer": expected_requested, + "transcripts": transcripts, + "capture_contract": { + "path": CAPTURE_CONTRACT_NAME, + "sha256": digest_file(capture_path), + }, + "capture_inputs": capture_inputs, + "canonical_skill": canonical_skill, + "treatment_source": treatment_source, + "prompts": capture_contract["prompts"], + "schedule": capture_contract["schedule"], + "scoring": capture_contract["scoring"], + "capture_evaluator": { + "harness": { + "path": "scripts/probe-skill.sh", + "sha256": digest_file(harness), + }, + "preamble": { + "path": "scripts/lib/preamble.sh", + "sha256": digest_file(preamble), + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": digest_file(helper), + }, + "dispatch_helper": { + "path": "scripts/lib/codex-exec.sh", + "sha256": digest_file(dispatch_helper), + }, + }, + } + + +def write_manifest(fixture_dir: Path, payload: dict[str, Any]) -> dict[str, Any]: + """Create the stage manifest once without check/replace or rollback deletion.""" + manifest_path = fixture_dir / MANIFEST_NAME + manifest = dict(payload) + manifest["binding_sha256"] = digest_bytes(canonical_bytes(payload)) + encoded = ( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(manifest_path, flags, 0o644) + except FileExistsError as exc: + raise MetadataError( + f"refusing to replace existing immutable {MANIFEST_NAME}" + ) from exc + try: + identity = os.fstat(fd) + offset = 0 + while offset < len(encoded): + offset += os.write(fd, encoded[offset:]) + os.fsync(fd) + current = os.stat(manifest_path, follow_symlinks=False) + if not os.path.samestat(identity, current): + raise MetadataError("fixture manifest identity changed while writing") + finally: + os.close(fd) + return manifest + + +def rename_noreplace(source: Path, target: Path) -> None: + """Atomically rename one directory while refusing any existing target.""" + libc = ctypes.CDLL(None, use_errno=True) + source_bytes = os.fsencode(source) + target_bytes = os.fsencode(target) + result: int + if sys.platform == "darwin" and hasattr(libc, "renamex_np"): + renamex_np = libc.renamex_np + renamex_np.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint] + renamex_np.restype = ctypes.c_int + result = renamex_np(source_bytes, target_bytes, 0x00000004) # RENAME_EXCL + elif hasattr(libc, "renameat2"): + renameat2 = libc.renameat2 + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + getattr(os, "AT_FDCWD", -100), + source_bytes, + getattr(os, "AT_FDCWD", -100), + target_bytes, + 1, # RENAME_NOREPLACE + ) + else: + raise MetadataError( + "this platform lacks an atomic no-replace directory rename primitive" + ) + if result == 0: + return + error = ctypes.get_errno() + if error in {errno.EEXIST, errno.ENOTEMPTY}: + raise MetadataError( + f"refusing to replace existing immutable fixture set: {target}" + ) + raise MetadataError( + f"could not atomically publish immutable fixture set: {os.strerror(error)}" + ) + + +def publish_fixture_set( + stage_dir: Path, + target_dir: Path, + probe_dir: Path, + skills_dir: Path, + expected_probe: str, +) -> dict[str, Any]: + """Publish a hidden verified stage with one atomic no-replace rename.""" + validate_fixture_dir(stage_dir) + if not re.fullmatch( + r"fixtures(?:[_-][A-Za-z0-9][A-Za-z0-9._-]*)?", target_dir.name + ): + raise MetadataError(f"unsafe fixture set target name: {target_dir.name!r}") + if stage_dir.parent.is_symlink() or target_dir.parent.is_symlink(): + raise MetadataError("fixture set parent directory must not be a symlink") + try: + stage_parent = stage_dir.parent.resolve(strict=True) + target_parent = target_dir.parent.resolve(strict=True) + except OSError as exc: + raise MetadataError(f"fixture set parent directory not found: {exc}") from exc + if stage_parent != target_parent: + raise MetadataError("staged and published fixture sets must share one parent") + + directory_flags = ( + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + ) + parent_fd = os.open(stage_parent, directory_flags) + stage_fd = os.open(stage_dir.name, directory_flags, dir_fd=parent_fd) + try: + stage_identity = os.fstat(stage_fd) + manifest = validate_manifest( + stage_dir, + probe_dir, + skills_dir, + expected_probe, + require_current_inputs=True, + ) + if manifest["schema"] != SCHEMA: + raise MetadataError( + "new publication requires self-contained fixture metadata v3" + ) + verify_evaluator_files(manifest["capture_evaluator"], repository_root()) + manifest_after = validate_manifest( + stage_dir, + probe_dir, + skills_dir, + expected_probe, + require_current_inputs=True, + ) + if manifest_after["binding_sha256"] != manifest["binding_sha256"]: + raise MetadataError("staged fixture binding changed during publish") + verify_evaluator_files(manifest_after["capture_evaluator"], repository_root()) + stage_path_identity = os.stat( + stage_dir.name, dir_fd=parent_fd, follow_symlinks=False + ) + if not os.path.samestat(stage_identity, stage_path_identity): + raise MetadataError( + "staged fixture directory identity changed during publish" + ) + rename_noreplace(stage_dir, target_dir) + published_identity = os.stat( + target_dir.name, dir_fd=parent_fd, follow_symlinks=False + ) + if not os.path.samestat(stage_identity, published_identity): + raise MetadataError( + "published fixture directory identity is not the staged set" + ) + published = validate_manifest( + target_dir, + probe_dir, + skills_dir, + expected_probe, + require_current_inputs=True, + ) + verify_evaluator_files(published["capture_evaluator"], repository_root()) + if published["binding_sha256"] != manifest["binding_sha256"]: + raise MetadataError("published fixture binding differs from staged binding") + os.fsync(parent_fd) + finally: + os.close(stage_fd) + os.close(parent_fd) + + return {"binding_sha256": manifest.get("binding_sha256"), "target": target_dir.name} + + +def validate_hash_records( + records: Any, + expected_names: tuple[str, ...], + root: Path, + label: str, +) -> None: + if not isinstance(records, list) or len(records) != len(expected_names): + raise MetadataError(f"{label} must contain the exact declared file inventory") + seen: list[str] = [] + for index, entry in enumerate(records): + if not isinstance(entry, dict) or set(entry) != {"path", "sha256"}: + raise MetadataError( + f"{label}[{index}] must contain exactly path and sha256" + ) + name = entry["path"] + if name not in expected_names or name in seen: + raise MetadataError(f"unsafe or duplicate {label} path: {name!r}") + digest = entry["sha256"] + if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest): + raise MetadataError(f"invalid {label} digest for {name}") + path = root / name + validate_regular_file(path, label) + actual = digest_file(path) + if actual != digest: + raise MetadataError( + f"{label} digest mismatch for {name}: expected {digest}, got {actual}" + ) + seen.append(name) + if tuple(seen) != expected_names: + raise MetadataError(f"{label} inventory or ordering is invalid") + + +def validate_evaluator( + value: Any, field: str, *, require_dispatch: bool | None = True +) -> None: + legacy = {"harness", "metadata_helper"} + current = legacy | {"preamble", "dispatch_helper"} + allowed = {frozenset(current)} + if require_dispatch is False: + allowed = {frozenset(legacy)} + elif require_dispatch is None: + allowed.add(frozenset(legacy)) + if not isinstance(value, dict) or frozenset(value) not in allowed: + expected = "harness, preamble, metadata_helper, and dispatch_helper" + if require_dispatch is False: + expected = "harness and metadata_helper" + elif require_dispatch is None: + expected += " (or the legacy harness and metadata_helper pair)" + raise MetadataError(f"{field} must contain exactly {expected}") + expected_paths = { + "harness": "scripts/probe-skill.sh", + "preamble": "scripts/lib/preamble.sh", + "metadata_helper": "scripts/lib/probe-fixture-metadata.py", + "dispatch_helper": "scripts/lib/codex-exec.sh", + } + for key in value: + expected_path = expected_paths[key] + record = value[key] + if not isinstance(record, dict) or set(record) != {"path", "sha256"}: + raise MetadataError(f"{field}.{key} must contain exactly path and sha256") + if record["path"] != expected_path: + raise MetadataError(f"unexpected {field}.{key} path: {record['path']!r}") + if not isinstance(record["sha256"], str) or not SHA256_RE.fullmatch( + record["sha256"] + ): + raise MetadataError(f"invalid {field}.{key} digest") + + +def evaluator_identity( + harness: Path, preamble: Path, dispatch_helper: Path +) -> dict[str, Any]: + validate_regular_file(harness, "probe harness") + validate_regular_file(preamble, "probe preamble") + validate_regular_file(dispatch_helper, "Codex dispatch helper") + helper = Path(__file__).resolve() + validate_regular_file(helper, "fixture metadata helper") + return { + "harness": {"path": "scripts/probe-skill.sh", "sha256": digest_file(harness)}, + "preamble": { + "path": "scripts/lib/preamble.sh", + "sha256": digest_file(preamble), + }, + "metadata_helper": { + "path": "scripts/lib/probe-fixture-metadata.py", + "sha256": digest_file(helper), + }, + "dispatch_helper": { + "path": "scripts/lib/codex-exec.sh", + "sha256": digest_file(dispatch_helper), + }, + } + + +def repository_root() -> Path: + helper = Path(__file__).resolve() + root = helper.parents[2] + if not (root / "scripts" / "probe-skill.sh").is_file(): + raise MetadataError("could not resolve the probe evaluator repository root") + return root + + +def verify_evaluator_files(value: Any, repo_root: Path) -> dict[str, Any]: + validate_evaluator(value, "capture_evaluator", require_dispatch=True) + expected_paths = { + "harness": repo_root / "scripts" / "probe-skill.sh", + "preamble": repo_root / "scripts" / "lib" / "preamble.sh", + "metadata_helper": repo_root / "scripts" / "lib" / "probe-fixture-metadata.py", + "dispatch_helper": repo_root / "scripts" / "lib" / "codex-exec.sh", + } + actual: dict[str, Any] = {} + for key, path in expected_paths.items(): + validate_regular_file(path, f"repo-local evaluator {key}") + actual[key] = {"path": value[key]["path"], "sha256": digest_file(path)} + if actual != value: + raise MetadataError( + "capture evaluator hashes do not match the exact repo-local evaluator files" + ) + return actual + + +def safe_repo_path( + repo_root: Path, + relative: str, + *, + expected_prefix: tuple[str, ...] | None = None, + expect_directory: bool = False, +) -> Path: + """Resolve one normalized repo-relative path without following symlinks.""" + require_text(relative, "repository-relative path") + if "\\" in relative: + raise MetadataError(f"unsafe repository-relative path: {relative!r}") + pure = PurePosixPath(relative) + if ( + pure.is_absolute() + or not pure.parts + or any(part in {"", ".", ".."} for part in pure.parts) + or pure.as_posix() != relative + ): + raise MetadataError(f"unsafe repository-relative path: {relative!r}") + if ( + expected_prefix is not None + and pure.parts[: len(expected_prefix)] != expected_prefix + ): + prefix = "/".join(expected_prefix) + "/" + raise MetadataError(f"evidence path must stay under {prefix}: {relative!r}") + + try: + root = repo_root.resolve(strict=True) + except OSError as exc: + raise MetadataError(f"evidence root not found: {repo_root}") from exc + if not root.is_dir(): + raise MetadataError(f"evidence root is not a directory: {root}") + + candidate = root + for part in pure.parts: + candidate = candidate / part + if candidate.is_symlink(): + raise MetadataError( + f"evidence path must not traverse a symlink: {relative}" + ) + try: + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise MetadataError(f"evidence path not found: {relative}") from exc + try: + resolved.relative_to(root) + except ValueError as exc: + raise MetadataError( + f"evidence path escapes repository root: {relative}" + ) from exc + if expect_directory: + if not resolved.is_dir(): + raise MetadataError(f"evidence directory not found: {relative}") + elif not resolved.is_file(): + raise MetadataError(f"evidence file not found: {relative}") + return resolved + + +def validate_scorecard_stats(value: Any, field: str) -> dict[str, Any]: + stats = require_exact_object(value, {"present", "usable", "rate"}, field) + present = require_nonnegative_int(stats["present"], f"{field}.present") + usable = require_nonnegative_int(stats["usable"], f"{field}.usable") + if present > usable: + raise MetadataError(f"{field}.present cannot exceed {field}.usable") + rate = require_rate(stats["rate"], f"{field}.rate") + expected_rate = None if usable == 0 else round(present / usable, 4) + if rate != expected_rate: + raise MetadataError( + f"{field}.rate is inconsistent with counts: expected {expected_rate}, got {rate}" + ) + return {"present": present, "usable": usable, "rate": rate} + + +def extract_legacy_codex_response(transcript: bytes) -> bytes | None: + """Return the final response from a legacy human-formatted transcript.""" + lines = transcript.splitlines(keepends=True) + markers = [ + index for index, line in enumerate(lines) if line.rstrip(b"\r\n") == b"codex" + ] + if not markers: + return None + start = markers[-1] + 1 + end = None + for index in range(start, len(lines)): + stripped = lines[index].rstrip(b"\r\n") + if stripped == b"tokens used" or re.fullmatch( + rb"tokens used:[ \t]*[^\r\n]*", stripped + ): + end = index + break + if end is None or end <= start: + return None + response = b"".join(lines[start:end]) + return response if response.strip() else None + + +def parse_jsonl_events(data: bytes, label: str) -> list[dict[str, Any]]: + try: + text = data.decode("utf-8", errors="strict") + except UnicodeError as exc: + raise MetadataError(f"{label} is not UTF-8 JSONL") from exc + lines = text.splitlines() + if not lines or any(not line.strip() for line in lines): + raise MetadataError(f"{label} must contain non-blank JSONL events") + events: list[dict[str, Any]] = [] + for index, line in enumerate(lines, 1): + try: + event = json.loads(line, object_pairs_hook=no_duplicate_object) + except (json.JSONDecodeError, MetadataError) as exc: + raise MetadataError(f"{label} line {index} is invalid JSON: {exc}") from exc + if not isinstance(event, dict): + raise MetadataError(f"{label} line {index} must be a JSON object") + events.append(event) + return events + + +def scheduled_position(contract: dict[str, Any], arm: str, rep: int) -> int: + if arm not in {"control", "treatment"}: + raise MetadataError(f"unsupported probe arm: {arm!r}") + require_reps(rep) + matches = [ + entry + for entry in contract["schedule"] + if entry["arm"] == arm and entry["rep"] == rep + ] + if len(matches) != 1: + raise MetadataError(f"capture schedule has no unique {arm}-{rep} entry") + return matches[0]["position"] + + +def probe_input_event( + contract: dict[str, Any], arm: str, rep: int, prompt: bytes +) -> dict[str, Any]: + expected = decode_prompts(contract["prompts"])[arm] + if prompt != expected: + raise MetadataError(f"actual {arm}-{rep} prompt differs from capture contract") + return { + "type": PROBE_INPUT_EVENT, + "arm": arm, + "rep": rep, + "position": scheduled_position(contract, arm, rep), + "prompt": embedded_bytes_record(prompt, f"{arm}.prompt"), + } + + +def validate_codex_runtime_events( + events: list[dict[str, Any]], label: str +) -> tuple[str, bytes]: + if not events: + raise MetadataError(f"{label} has no Codex runtime events") + types = [event.get("type") for event in events] + if types[0] != "thread.started": + raise MetadataError(f"{label} must start with thread.started") + if types.count("thread.started") != 1 or types.count("turn.started") != 1: + raise MetadataError(f"{label} must contain one thread and one turn start") + if types.count("turn.completed") != 1 or types[-1] != "turn.completed": + raise MetadataError(f"{label} must end with exactly one turn.completed event") + if any(kind in {"turn.failed", "error"} for kind in types): + raise MetadataError(f"{label} contains a failed Codex turn") + thread_id = require_text(events[0].get("thread_id"), f"{label} thread_id") + assert isinstance(thread_id, str) + messages: list[str] = [] + for event in events: + if event.get("type") != "item.completed": + continue + item = event.get("item") + if not isinstance(item, dict) or item.get("type") != "agent_message": + continue + message = require_message_text(item.get("text"), f"{label} agent_message") + messages.append(message) + if not messages: + raise MetadataError(f"{label} has no completed agent_message") + return thread_id, messages[-1].encode("utf-8") + + +def validate_structured_transcript( + transcript: bytes, + contract: dict[str, Any], + arm: str, + rep: int, + label: str, +) -> tuple[str, bytes]: + events = parse_jsonl_events(transcript, label) + first = require_exact_object( + events[0], {"type", "arm", "rep", "position", "prompt"}, "probe input event" + ) + expected_prompt = decode_prompts(contract["prompts"])[arm] + expected_event = probe_input_event(contract, arm, rep, expected_prompt) + if first != expected_event: + raise MetadataError(f"{label} probe input event does not match bound {arm}-{rep} prompt") + if any(event.get("type") == PROBE_INPUT_EVENT for event in events[1:]): + raise MetadataError(f"{label} contains more than one probe input event") + return validate_codex_runtime_events(events[1:], label) + + +def assemble_transcript( + runtime_path: Path, + prompt_path: Path, + fixture_dir: Path, + expected_probe: str, + arm: str, + rep: int, +) -> bytes: + contract = load_capture_contract(fixture_dir, expected_probe) + prompt = read_regular_bytes(prompt_path, "actual dispatch prompt", maximum=MAX_INPUT_BYTES * 2 + 16) + input_event = probe_input_event(contract, arm, rep, prompt) + runtime = read_regular_bytes( + runtime_path, "Codex JSONL runtime stream", maximum=MAX_TRANSCRIPT_BYTES + ) + runtime_events = parse_jsonl_events(runtime, "Codex JSONL runtime stream") + validate_codex_runtime_events(runtime_events, "Codex JSONL runtime stream") + all_events = [input_event, *runtime_events] + return b"".join( + canonical_bytes(event) + b"\n" for event in all_events + ) + + +def run_bounded_discriminator(discriminator_bytes: bytes, response_bytes: bytes) -> int: + """Run the scorer on a prompt-free response envelope in private snapshots.""" + with tempfile.TemporaryDirectory(prefix="probe-score.") as directory: + root = Path(directory) + discriminator_path = root / "discriminator.sh" + response_path = root / "response.txt" + discriminator_path.write_bytes(discriminator_bytes) + # Keep the historical `codex` boundary for discriminators that already + # extracted the response themselves, but never expose the echoed user + # prompt. Response-native discriminators see the same response lines. + response_envelope = b"codex\n" + response_bytes + response_path.write_bytes(response_envelope) + discriminator_path.chmod(0o500) + response_path.chmod(0o400) + process = subprocess.Popen( + ["bash", str(discriminator_path), str(response_path)], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + try: + returncode = process.wait(timeout=DISCRIMINATOR_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as exc: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + raise MetadataError( + "discriminator timed out and its process group was terminated" + ) from exc + if ( + sorted(os.listdir(root)) != ["discriminator.sh", "response.txt"] + or read_regular_bytes(discriminator_path, "scoring discriminator") + != discriminator_bytes + or read_regular_bytes(response_path, "scoring response") + != response_envelope + ): + raise MetadataError("discriminator mutated an immutable scoring snapshot") + return returncode + + +def classify_bytes( + discriminator_bytes: bytes, + transcript_bytes: bytes, + *, + structured: tuple[dict[str, Any], str, int] | None = None, +) -> str: + if structured is None: + response = extract_legacy_codex_response(transcript_bytes) + else: + contract, arm, rep = structured + _, response = validate_structured_transcript( + transcript_bytes, contract, arm, rep, f"{arm}-{rep}.txt" + ) + if response is None: + return "DEGRADED" + try: + returncode = run_bounded_discriminator(discriminator_bytes, response) + except OSError as exc: + raise MetadataError(f"could not run discriminator: {exc}") from exc + if returncode == 0: + return "PRESENT" + if returncode == 1: + return "ABSENT" + return "DEGRADED" + + +def captured_input_bytes(manifest: dict[str, Any]) -> dict[str, bytes]: + probe = require_text(manifest.get("probe"), "captured probe") + source = require_text( + manifest.get("treatment_source"), "captured treatment_source" + ) + assert isinstance(probe, str) and isinstance(source, str) + decoded, _ = decode_capture_inputs( + manifest.get("capture_inputs"), probe, source + ) + return decoded + + +def verdict_for_rates(control: dict[str, Any], treatment: dict[str, Any]) -> str: + if control["usable"] == 0 or treatment["usable"] == 0: + return "UNMEASURED" + if treatment["rate"] > control["rate"]: + return "BEHAVIORAL" + if treatment["rate"] < control["rate"]: + return "REGRESSIVE" + return "INERT" + + +def recompute_score( + probe_dir: Path, fixture_dir: Path, manifest: dict[str, Any] +) -> dict[str, Any]: + reps = require_reps(manifest["reps"]) + if manifest["schema"] == SCHEMA: + discriminator_bytes = captured_input_bytes(manifest)["discriminator.sh"] + else: + discriminator_bytes = read_regular_bytes( + probe_dir / "discriminator.sh", + "discriminator", + maximum=MAX_INPUT_BYTES, + ) + transcript_snapshot = { + name: read_regular_bytes( + fixture_dir / name, "transcript", maximum=MAX_TRANSCRIPT_BYTES + ) + for name in expected_transcripts(reps) + } + per_rep: list[dict[str, Any]] = [] + counts = { + "control": {"present": 0, "usable": 0}, + "treatment": {"present": 0, "usable": 0}, + } + for rep in range(1, reps + 1): + entry: dict[str, Any] = {"rep": rep} + for arm in ("control", "treatment"): + name = f"{arm}-{rep}.txt" + structured = (manifest, arm, rep) if manifest["schema"] == SCHEMA else None + outcome = classify_bytes( + discriminator_bytes, + transcript_snapshot[name], + structured=structured, + ) + entry[arm] = outcome + if outcome != "DEGRADED": + counts[arm]["usable"] += 1 + if outcome == "PRESENT": + counts[arm]["present"] += 1 + per_rep.append(entry) + + scored: dict[str, Any] = {"per_rep": per_rep} + for arm in ("control", "treatment"): + present = counts[arm]["present"] + usable = counts[arm]["usable"] + scored[arm] = { + "present": present, + "usable": usable, + "rate": None if usable == 0 else round(present / usable, 4), + } + scored["verdict"] = verdict_for_rates(scored["control"], scored["treatment"]) + return scored + + +def read_open_fd(fd: int, path: Path) -> tuple[bytes, os.stat_result]: + try: + before = os.fstat(fd) + except OSError as exc: + raise MetadataError(f"capture transcript fd is not open: {fd}") from exc + if not stat.S_ISREG(before.st_mode): + raise MetadataError("capture transcript fd is not a regular file") + if before.st_size > MAX_TRANSCRIPT_BYTES: + raise MetadataError("capture transcript exceeds the safety limit") + try: + read_fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + except OSError as exc: + raise MetadataError("capture transcript path identity changed") from exc + try: + read_before = os.fstat(read_fd) + if not os.path.samestat(before, read_before): + raise MetadataError("capture transcript path does not name the owned sink") + chunks: list[bytes] = [] + total = 0 + while chunk := os.read(read_fd, 1024 * 1024): + total += len(chunk) + if total > MAX_TRANSCRIPT_BYTES: + raise MetadataError("capture transcript exceeds the safety limit") + chunks.append(chunk) + after = os.fstat(fd) + read_after = os.fstat(read_fd) + path_identity = os.stat(path, follow_symlinks=False) + if ( + not os.path.samestat(before, after) + or not os.path.samestat(before, read_after) + or not os.path.samestat(before, path_identity) + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or total != before.st_size + ): + raise MetadataError("capture transcript identity or bytes changed") + return b"".join(chunks), before + finally: + os.close(read_fd) + + +def classify_open_capture( + fd: int, + path: Path, + contract: dict[str, Any], + discriminator_bytes: bytes, + arm: str, + rep: int, +) -> dict[str, Any]: + transcript, identity = read_open_fd(fd, path) + thread_id, _ = validate_structured_transcript( + transcript, contract, arm, rep, path.name + ) + outcome = classify_bytes( + discriminator_bytes, + transcript, + structured=(contract, arm, rep), + ) + transcript_after, identity_after = read_open_fd(fd, path) + if not os.path.samestat(identity, identity_after) or transcript_after != transcript: + raise MetadataError("capture transcript changed during scoring") + return { + "outcome": outcome, + "sha256": digest_bytes(transcript), + "thread_id": thread_id, + "producer": contract["producer_request"], + } + + +def verify_scorecard( + repo_root: Path, + skills_dir: Path, + scorecard_relative: str, + ledger_skill: str, + ledger_probe: str, + ledger_verdict: str, +) -> dict[str, Any]: + for value, field in ( + (ledger_skill, "ledger skill"), + (ledger_probe, "ledger probe"), + (ledger_verdict, "ledger verdict"), + ): + require_text(value, field) + if not SAFE_ID_RE.fullmatch(ledger_skill) or not SAFE_ID_RE.fullmatch(ledger_probe): + raise MetadataError("ledger skill and probe must be safe identifiers") + if ledger_verdict not in CURRENT_VERDICTS: + raise MetadataError(f"ledger verdict is not a current result: {ledger_verdict}") + + try: + resolved_root = repo_root.resolve(strict=True) + resolved_skills = skills_dir.resolve(strict=True) + except OSError as exc: + raise MetadataError( + f"repository or canonical skills root not found: {exc}" + ) from exc + if skills_dir.is_symlink() or resolved_skills != resolved_root / "skills": + raise MetadataError( + "coverage canonical skills directory must be the repo-local skills/ tree" + ) + + if not scorecard_relative.endswith(".json"): + raise MetadataError("scorecard evidence path must end in .json") + scorecard_path = safe_repo_path( + repo_root, + scorecard_relative, + expected_prefix=("docs", "evals", "scorecards"), + ) + scorecard = load_json(scorecard_path) + scorecard_keys = { + "schema", + "probe", + "skill", + "mode", + "generated_at", + "reps", + "producer", + "requested_producer", + "fixture_set", + "treatment_source", + "evaluator", + "capture_evaluator", + "evaluator_matches_capture", + "honesty", + "schedule", + "scoring", + "control", + "treatment", + "verdict", + "per_rep", + } + require_exact_object(scorecard, scorecard_keys, "scorecard") + if scorecard["schema"] != SCORECARD_SCHEMA: + raise MetadataError(f"scorecard is not v3: {scorecard['schema']!r}") + if scorecard["skill"] != ledger_skill: + raise MetadataError( + f"scorecard/ledger skill mismatch: {scorecard['skill']!r} != {ledger_skill!r}" + ) + if scorecard["probe"] != ledger_probe: + raise MetadataError( + f"scorecard/ledger probe mismatch: {scorecard['probe']!r} != {ledger_probe!r}" + ) + if scorecard["verdict"] != ledger_verdict: + raise MetadataError( + f"scorecard/ledger verdict mismatch: {scorecard['verdict']!r} != {ledger_verdict!r}" + ) + if scorecard["mode"] not in {"live", "replay"}: + raise MetadataError(f"invalid scorecard mode: {scorecard['mode']!r}") + if scorecard["treatment_source"] not in TREATMENT_SOURCES: + raise MetadataError( + f"invalid scorecard treatment_source: {scorecard['treatment_source']!r}" + ) + require_text(scorecard["generated_at"], "scorecard generated_at") + require_text(scorecard["honesty"], "scorecard honesty") + reps = require_reps(scorecard["reps"]) + schedule = validate_schedule(scorecard["schedule"], reps) + scoring = require_exact_object( + scorecard["scoring"], + { + "response_extraction", + "transcript_format", + "discriminator_timeout_seconds", + }, + "scorecard scoring", + ) + if scoring != { + "response_extraction": RESPONSE_EXTRACTION, + "transcript_format": TRANSCRIPT_FORMAT, + "discriminator_timeout_seconds": DISCRIMINATOR_TIMEOUT_SECONDS, + }: + raise MetadataError("scorecard scoring contract is not the current contract") + + producer = require_exact_object( + scorecard["producer"], + {"adapter", "model", "effort", "identity", "threads"}, + "scorecard producer", + ) + producer_request = validate_producer_request( + {key: producer[key] for key in ("adapter", "model", "effort", "identity")} + ) + if not producer_request["identity"]["coverage_eligible"]: + raise MetadataError( + "tier coverage requires non-overrideable native Codex runtime evidence" + ) + requested = require_exact_object( + scorecard["requested_producer"], + {"model", "effort"}, + "scorecard requested_producer", + ) + require_text(requested["model"], "requested model", nullable=True) + require_text(requested["effort"], "requested effort", nullable=True) + + fixture = require_exact_object( + scorecard["fixture_set"], + {"name", "metadata", "binding_sha256", "schema"}, + "scorecard fixture_set", + ) + fixture_name = require_text(fixture["name"], "fixture set name") + if not re.fullmatch(r"fixtures(?:[_-][A-Za-z0-9][A-Za-z0-9._-]*)?", fixture_name): + raise MetadataError(f"unsafe fixture set name: {fixture_name!r}") + if fixture["metadata"] != MANIFEST_NAME: + raise MetadataError(f"scorecard fixture metadata must be {MANIFEST_NAME}") + if fixture["schema"] != SCHEMA: + raise MetadataError("tier coverage requires a self-contained v3 fixture set") + if not isinstance(fixture["binding_sha256"], str) or not SHA256_RE.fullmatch( + fixture["binding_sha256"] + ): + raise MetadataError("scorecard fixture binding must be a sha256 digest") + + validate_evaluator(scorecard["evaluator"], "scorecard evaluator") + verify_evaluator_files(scorecard["evaluator"], resolved_root) + validate_evaluator( + scorecard["capture_evaluator"], + "scorecard capture_evaluator", + require_dispatch=True, + ) + if not isinstance(scorecard["evaluator_matches_capture"], bool): + raise MetadataError("scorecard evaluator_matches_capture must be boolean") + if scorecard["evaluator_matches_capture"] != ( + scorecard["evaluator"] == scorecard["capture_evaluator"] + ): + raise MetadataError("scorecard evaluator_matches_capture is inconsistent") + control = validate_scorecard_stats(scorecard["control"], "scorecard control") + treatment = validate_scorecard_stats(scorecard["treatment"], "scorecard treatment") + + probe_relative = f"evals/skill-probes/{ledger_probe}" + probe_dir = safe_repo_path(repo_root, probe_relative, expect_directory=True) + fixture_relative = f"{probe_relative}/{fixture_name}" + fixture_dir = safe_repo_path(repo_root, fixture_relative, expect_directory=True) + manifest = validate_manifest( + fixture_dir, + probe_dir, + skills_dir, + ledger_probe, + require_current_inputs=True, + ) + if manifest["schema"] != SCHEMA: + raise MetadataError("tier coverage requires self-contained fixture metadata v3") + + if fixture["binding_sha256"] != manifest["binding_sha256"]: + raise MetadataError("scorecard/manifest fixture binding mismatch") + if reps != manifest["reps"]: + raise MetadataError("scorecard/manifest reps mismatch") + if producer != manifest["producer"]: + raise MetadataError("scorecard/manifest producer mismatch") + if requested != manifest["requested_producer"]: + raise MetadataError("scorecard/manifest requested producer mismatch") + if scorecard["capture_evaluator"] != manifest["capture_evaluator"]: + raise MetadataError("scorecard/manifest capture evaluator mismatch") + if scorecard["treatment_source"] != manifest["treatment_source"]: + raise MetadataError("scorecard/manifest treatment_source mismatch") + if schedule != manifest["schedule"]: + raise MetadataError("scorecard/manifest schedule mismatch") + if scoring != manifest["scoring"]: + raise MetadataError("scorecard/manifest scoring contract mismatch") + if manifest["treatment_source"] != "canonical-skill": + raise MetadataError( + "tier coverage requires treatment_source 'canonical-skill'; " + "injected-prelude evidence measures only the bound prelude" + ) + + probe_meta = load_json(probe_dir / "probe.json") + if probe_meta.get("id") != ledger_probe: + raise MetadataError("probe.json id does not match ledger probe") + if probe_meta.get("skill") != ledger_skill: + raise MetadataError("probe.json skill does not match ledger skill") + + recomputed = recompute_score(probe_dir, fixture_dir, manifest) + manifest_after = validate_manifest( + fixture_dir, + probe_dir, + skills_dir, + ledger_probe, + require_current_inputs=True, + ) + if manifest_after["binding_sha256"] != manifest["binding_sha256"]: + raise MetadataError("fixture binding changed during discriminator replay") + if scorecard["per_rep"] != recomputed["per_rep"]: + raise MetadataError( + "scorecard per_rep outcomes do not match discriminator replay" + ) + if control != recomputed["control"]: + raise MetadataError( + "scorecard control totals do not match discriminator replay" + ) + if treatment != recomputed["treatment"]: + raise MetadataError( + "scorecard treatment totals do not match discriminator replay" + ) + if scorecard["verdict"] != recomputed["verdict"]: + raise MetadataError( + f"scorecard verdict does not match discriminator replay: {recomputed['verdict']}" + ) + + return { + "binding_sha256": manifest["binding_sha256"], + "producer": manifest["producer"], + "probe": ledger_probe, + "reps": reps, + "skill": ledger_skill, + "verdict": ledger_verdict, + } + + +def validate_manifest( + fixture_dir: Path, + probe_dir: Path, + skills_dir: Path, + expected_probe: str, + *, + require_current_inputs: bool = False, +) -> dict[str, Any]: + manifest_path = fixture_dir / MANIFEST_NAME + if manifest_path.is_symlink() or not manifest_path.is_file(): + raise MetadataError( + f"verified replay requires immutable capture metadata at {manifest_path}" + ) + manifest = load_json(manifest_path) + common_keys = { + "schema", + "probe", + "reps", + "producer", + "requested_producer", + "transcripts", + "capture_evaluator", + "binding_sha256", + } + schema = manifest.get("schema") + if schema == SCHEMA: + expected_keys = common_keys | { + "capture_contract", + "capture_inputs", + "canonical_skill", + "treatment_source", + "prompts", + "schedule", + "scoring", + } + elif schema == LEGACY_CANONICAL_SCHEMA: + expected_keys = common_keys | { + "evaluation_inputs", + "canonical_skill", + "treatment_source", + } + elif schema == LEGACY_BOUND_SCHEMA: + expected_keys = common_keys | {"evaluation_inputs"} + else: + raise MetadataError(f"unsupported fixture metadata schema: {schema!r}") + if set(manifest) != expected_keys: + raise MetadataError("fixture metadata has unknown or missing top-level fields") + probe = require_text(manifest["probe"], "probe") + if probe != expected_probe: + raise MetadataError( + f"fixture metadata probe mismatch: expected {expected_probe}, got {probe}" + ) + reps = require_reps(manifest["reps"]) + + producer = manifest["producer"] + if schema == SCHEMA: + producer = require_exact_object( + producer, + {"adapter", "model", "effort", "identity", "threads"}, + "producer", + ) + producer_request = validate_producer_request( + {key: producer[key] for key in ("adapter", "model", "effort", "identity")} + ) + else: + producer = require_exact_object( + producer, {"adapter", "model", "effort"}, "producer" + ) + if producer["adapter"] != "codex": + raise MetadataError(f"unsupported producer adapter: {producer['adapter']!r}") + require_text(producer["model"], "producer model") + require_text(producer["effort"], "producer effort") + producer_request = None + + requested = manifest["requested_producer"] + if not isinstance(requested, dict) or set(requested) != {"model", "effort"}: + raise MetadataError("requested_producer must contain exactly model and effort") + require_text(requested["model"], "requested model", nullable=True) + require_text(requested["effort"], "requested effort", nullable=True) + if schema == SCHEMA and requested != { + "model": producer["model"], + "effort": producer["effort"], + }: + raise MetadataError("requested_producer disagrees with bound producer request") + + transcripts = manifest["transcripts"] + if not isinstance(transcripts, list): + raise MetadataError("transcripts must be an array") + expected = expected_transcripts(reps) + seen: list[str] = [] + paths: list[Path] = [] + for index, entry in enumerate(transcripts): + if not isinstance(entry, dict) or set(entry) != {"path", "sha256"}: + raise MetadataError( + f"transcripts[{index}] must contain exactly path and sha256" + ) + name = entry["path"] + if not isinstance(name, str) or not TRANSCRIPT_RE.fullmatch(name): + raise MetadataError(f"unsafe transcript path in metadata: {name!r}") + if not isinstance(entry["sha256"], str) or not SHA256_RE.fullmatch( + entry["sha256"] + ): + raise MetadataError(f"invalid transcript digest for {name}") + if name in seen: + raise MetadataError(f"duplicate transcript metadata: {name}") + seen.append(name) + path = fixture_dir / name + validate_transcript(path) + actual_digest = digest_file(path) + if actual_digest != entry["sha256"]: + raise MetadataError( + f"transcript digest mismatch for {name}: expected {entry['sha256']}, got {actual_digest}" + ) + paths.append(path) + if seen != expected: + raise MetadataError( + "fixture metadata transcript inventory or ordering is invalid" + ) + actual_names = sorted(os.listdir(fixture_dir)) + expected_names = sorted( + expected + + [MANIFEST_NAME] + + ([CAPTURE_CONTRACT_NAME] if schema == SCHEMA else []) + ) + if actual_names != expected_names: + raise MetadataError( + "fixture directory contains files outside the exact bound inventory" + ) + + if schema in {LEGACY_BOUND_SCHEMA, LEGACY_CANONICAL_SCHEMA}: + observed = observe_paths(paths) + if observed != producer: + raise MetadataError( + "producer metadata disagrees with observed transcript headers: " + f"metadata={producer['model']}/{producer['effort']} " + f"observed={observed['model']}/{observed['effort']}" + ) + validate_hash_records( + manifest["evaluation_inputs"], + LEGACY_EVALUATION_INPUT_NAMES, + probe_dir, + "evaluation_inputs", + ) + current_probe = load_json(probe_dir / "probe.json") + contract = validate_probe_metadata( + current_probe, + expected_probe, + require_treatment=schema == LEGACY_CANONICAL_SCHEMA, + ) + if contract["reps"] != reps: + raise MetadataError("fixture reps disagree with bound probe.json reps") + if schema == LEGACY_CANONICAL_SCHEMA: + validate_canonical_skill_record( + manifest["canonical_skill"], probe_dir, skills_dir, expected_probe + ) + treatment_source = declared_treatment_source(probe_dir, expected_probe) + if manifest["treatment_source"] != treatment_source: + raise MetadataError( + "fixture metadata treatment_source disagrees with bound probe.json: " + f"{manifest['treatment_source']!r} != {treatment_source!r}" + ) + elif schema == SCHEMA: + capture_contract_record = require_exact_object( + manifest["capture_contract"], + {"path", "sha256"}, + "capture_contract", + ) + if capture_contract_record["path"] != CAPTURE_CONTRACT_NAME: + raise MetadataError("capture_contract path is invalid") + capture_contract_digest = capture_contract_record["sha256"] + if not isinstance(capture_contract_digest, str) or not SHA256_RE.fullmatch( + capture_contract_digest + ): + raise MetadataError("capture_contract sha256 must be a sha256 digest") + if digest_file(fixture_dir / CAPTURE_CONTRACT_NAME) != capture_contract_digest: + raise MetadataError("capture contract file digest mismatch") + capture_contract = load_capture_contract(fixture_dir, expected_probe) + for field in ( + "reps", + "capture_inputs", + "canonical_skill", + "treatment_source", + "prompts", + "schedule", + "scoring", + ): + if manifest[field] != capture_contract[field]: + raise MetadataError( + f"fixture manifest {field} disagrees with capture contract" + ) + assert producer_request is not None + if producer_request != capture_contract["producer_request"]: + raise MetadataError( + "fixture producer identity disagrees with pre-execution capture contract" + ) + thread_records = producer["threads"] + if not isinstance(thread_records, list) or len(thread_records) != len(expected): + raise MetadataError("producer threads must cover the exact transcript inventory") + actual_threads = [] + for name, path in zip(expected, paths, strict=True): + match = TRANSCRIPT_RE.fullmatch(name) + assert match is not None + arm, rep_text = match.groups() + transcript = read_regular_bytes( + path, "transcript", maximum=MAX_TRANSCRIPT_BYTES + ) + thread_id, _ = validate_structured_transcript( + transcript, capture_contract, arm, int(rep_text), name + ) + actual_threads.append({"path": name, "thread_id": thread_id}) + if thread_records != actual_threads: + raise MetadataError("producer thread identities disagree with Codex JSON events") + if len({entry["thread_id"] for entry in actual_threads}) != len(actual_threads): + raise MetadataError("each probe dispatch must have a distinct Codex thread_id") + capture_inputs = captured_input_bytes(manifest) + captured_probe = parse_json_bytes( + capture_inputs["probe.json"], "captured probe.json" + ) + captured_contract = validate_probe_metadata( + captured_probe, expected_probe, require_treatment=True + ) + if captured_contract["reps"] != reps: + raise MetadataError("fixture reps disagree with captured probe.json reps") + if manifest["treatment_source"] != captured_contract["treatment_source"]: + raise MetadataError( + "fixture treatment_source disagrees with captured probe.json" + ) + embedded_skill, _ = validate_embedded_canonical_skill( + manifest["canonical_skill"], captured_probe, expected_probe + ) + validate_schedule(manifest["schedule"], reps) + scoring = require_exact_object( + manifest["scoring"], + { + "response_extraction", + "transcript_format", + "discriminator_timeout_seconds", + }, + "scoring", + ) + if scoring != { + "response_extraction": RESPONSE_EXTRACTION, + "transcript_format": TRANSCRIPT_FORMAT, + "discriminator_timeout_seconds": DISCRIMINATOR_TIMEOUT_SECONDS, + }: + raise MetadataError("fixture scoring contract is unsupported") + if require_current_inputs: + current_records = embedded_input_records( + probe_dir, manifest["treatment_source"] + ) + if current_records != manifest["capture_inputs"]: + raise MetadataError( + "current probe inputs differ from the self-contained capture" + ) + current_skill = build_canonical_skill_record( + probe_dir, skills_dir, expected_probe + ) + if any( + embedded_skill[field] != current_skill[field] + for field in ("name", "path", "sha256") + ): + raise MetadataError( + "current canonical skill differs from the self-contained capture" + ) + validate_evaluator( + manifest["capture_evaluator"], + "capture_evaluator", + require_dispatch=schema != LEGACY_BOUND_SCHEMA, + ) + + binding = manifest["binding_sha256"] + if not isinstance(binding, str) or not SHA256_RE.fullmatch(binding): + raise MetadataError("binding_sha256 must be a sha256 digest") + payload = {key: value for key, value in manifest.items() if key != "binding_sha256"} + actual_binding = digest_bytes(canonical_bytes(payload)) + if binding != actual_binding: + raise MetadataError( + f"fixture-set binding mismatch: expected {binding}, got {actual_binding}" + ) + return manifest + + +def summary(manifest: dict[str, Any]) -> dict[str, Any]: + return { + "schema": manifest["schema"], + "binding_sha256": manifest["binding_sha256"], + "producer": manifest["producer"], + "requested_producer": manifest["requested_producer"], + "capture_evaluator": manifest["capture_evaluator"], + "canonical_skill": manifest.get("canonical_skill"), + "reps": manifest["reps"], + "treatment_source": manifest.get("treatment_source", "injected-prelude"), + "schedule": manifest.get("schedule"), + "scoring": manifest.get("scoring"), + "transcripts": manifest["transcripts"], + } + + +def probe_contract_summary( + probe_dir: Path, skills_dir: Path, expected_probe: str +) -> dict[str, Any]: + probe_meta = load_json(probe_dir / "probe.json") + contract = validate_probe_metadata( + probe_meta, expected_probe, require_treatment=True + ) + treatment_source = contract["treatment_source"] + assert isinstance(treatment_source, str) + records = [] + for name in capture_input_names(treatment_source): + path = probe_dir / name + validate_regular_file(path, "evaluation input") + records.append({"path": name, "sha256": digest_file(path)}) + return { + "canonical_skill": build_canonical_skill_record( + probe_dir, skills_dir, expected_probe + ), + "evaluation_inputs": records, + "reps": contract["reps"], + "treatment_source": treatment_source, + "schedule": counterbalanced_schedule(contract["reps"]), + "scoring": { + "response_extraction": RESPONSE_EXTRACTION, + "transcript_format": TRANSCRIPT_FORMAT, + "discriminator_timeout_seconds": DISCRIMINATOR_TIMEOUT_SECONDS, + }, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + create = subparsers.add_parser("create") + create.add_argument("--fixture-dir", type=Path, required=True) + create.add_argument("--probe-dir", type=Path, required=True) + create.add_argument("--skills-dir", type=Path, required=True) + create.add_argument("--harness", type=Path, required=True) + create.add_argument("--preamble", type=Path, required=True) + create.add_argument("--dispatch-helper", type=Path, required=True) + create.add_argument("--probe", required=True) + create.add_argument("--reps", type=int, required=True) + create.add_argument("--requested-model") + create.add_argument("--requested-effort") + + verify = subparsers.add_parser("verify") + verify.add_argument("--fixture-dir", type=Path, required=True) + verify.add_argument("--probe-dir", type=Path, required=True) + verify.add_argument("--skills-dir", type=Path, required=True) + verify.add_argument("--probe", required=True) + + observe = subparsers.add_parser("observe") + observe.add_argument("--fixture-dir", type=Path, required=True) + + identity = subparsers.add_parser("identity") + identity.add_argument("--harness", type=Path, required=True) + identity.add_argument("--preamble", type=Path, required=True) + identity.add_argument("--dispatch-helper", type=Path, required=True) + + contract = subparsers.add_parser("probe-contract") + contract.add_argument("--probe-dir", type=Path, required=True) + contract.add_argument("--skills-dir", type=Path, required=True) + contract.add_argument("--probe", required=True) + + publish = subparsers.add_parser("publish") + publish.add_argument("--stage-dir", type=Path, required=True) + publish.add_argument("--target-dir", type=Path, required=True) + publish.add_argument("--probe-dir", type=Path, required=True) + publish.add_argument("--skills-dir", type=Path, required=True) + publish.add_argument("--probe", required=True) + + scorecard = subparsers.add_parser("verify-scorecard") + scorecard.add_argument("--repo-root", type=Path, required=True) + scorecard.add_argument("--skills-dir", type=Path, required=True) + scorecard.add_argument("--scorecard", required=True) + scorecard.add_argument("--ledger-skill", required=True) + scorecard.add_argument("--ledger-probe", required=True) + scorecard.add_argument("--ledger-verdict", required=True) + + score = subparsers.add_parser("score") + score.add_argument("--fixture-dir", type=Path, required=True) + score.add_argument("--probe-dir", type=Path, required=True) + score.add_argument("--skills-dir", type=Path, required=True) + score.add_argument("--probe", required=True) + + classify_open = subparsers.add_parser("classify-open") + classify_open.add_argument("--fd", type=int, required=True) + classify_open.add_argument("--path", type=Path, required=True) + classify_open.add_argument("--fixture-dir", type=Path, required=True) + classify_open.add_argument("--probe", required=True) + classify_open.add_argument("--arm", choices=("control", "treatment"), required=True) + classify_open.add_argument("--rep", type=int, required=True) + + assemble = subparsers.add_parser("assemble-transcript") + assemble.add_argument("--runtime-file", type=Path, required=True) + assemble.add_argument("--prompt-file", type=Path, required=True) + assemble.add_argument("--fixture-dir", type=Path, required=True) + assemble.add_argument("--probe", required=True) + assemble.add_argument("--arm", choices=("control", "treatment"), required=True) + assemble.add_argument("--rep", type=int, required=True) + + tiers = subparsers.add_parser("tier-skills") + tiers.add_argument("--skills-dir", type=Path, required=True) + + snapshot = subparsers.add_parser("snapshot") + snapshot.add_argument("--fixture-dir", type=Path, required=True) + snapshot.add_argument("--probe-dir", type=Path, required=True) + snapshot.add_argument("--skills-dir", type=Path, required=True) + snapshot.add_argument("--probe", required=True) + snapshot.add_argument("--requested-model") + snapshot.add_argument("--requested-effort") + snapshot.add_argument("--producer-override-bin") + + capture_file = subparsers.add_parser("capture-file") + capture_file.add_argument("--fixture-dir", type=Path, required=True) + capture_file.add_argument("--probe", required=True) + capture_file.add_argument("--name", required=True) + + write_output = subparsers.add_parser("write-output") + write_output.add_argument("--path", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + if args.command == "capture-file": + validate_fixture_dir(args.fixture_dir) + contract = load_capture_contract(args.fixture_dir, args.probe) + if args.name == "canonical-skill": + probe_meta = parse_json_bytes( + captured_input_bytes(contract)["probe.json"], + "captured probe.json", + ) + _, data = validate_embedded_canonical_skill( + contract["canonical_skill"], probe_meta, args.probe + ) + elif args.name in {"prompt-control", "prompt-treatment"}: + arm = args.name.removeprefix("prompt-") + data = decode_prompts(contract["prompts"])[arm] + elif args.name in capture_input_names(contract["treatment_source"]): + data = captured_input_bytes(contract)[args.name] + else: + raise MetadataError(f"unknown captured input: {args.name!r}") + sys.stdout.buffer.write(data) + return 0 + if args.command == "write-output": + data = sys.stdin.buffer.read(MAX_INPUT_BYTES + 1) + if len(data) > MAX_INPUT_BYTES: + raise MetadataError("scorecard output exceeds the safety limit") + parse_json_bytes(data, "scorecard output") + parent = args.path.parent + if parent.is_symlink() or not parent.is_dir(): + raise MetadataError("scorecard output parent must be a directory") + write_exclusive_bytes(args.path, data, "scorecard output") + result = {"path": str(args.path)} + print(json.dumps(result, separators=(",", ":"))) + return 0 + if args.command == "identity": + result = evaluator_identity( + args.harness, args.preamble, args.dispatch_helper + ) + elif args.command == "probe-contract": + result = probe_contract_summary(args.probe_dir, args.skills_dir, args.probe) + elif args.command == "publish": + result = publish_fixture_set( + args.stage_dir, + args.target_dir, + args.probe_dir, + args.skills_dir, + args.probe, + ) + elif args.command == "verify-scorecard": + result = verify_scorecard( + args.repo_root, + args.skills_dir, + args.scorecard, + args.ledger_skill, + args.ledger_probe, + args.ledger_verdict, + ) + elif args.command == "classify-open": + contract = load_capture_contract(args.fixture_dir, args.probe) + result = classify_open_capture( + args.fd, + args.path, + contract, + captured_input_bytes(contract)["discriminator.sh"], + args.arm, + args.rep, + ) + elif args.command == "assemble-transcript": + encoded = assemble_transcript( + args.runtime_file, + args.prompt_file, + args.fixture_dir, + args.probe, + args.arm, + args.rep, + ) + sys.stdout.buffer.write(encoded) + return 0 + elif args.command == "tier-skills": + result = {"skills": tier_skills(args.skills_dir)} + elif args.command == "snapshot": + validate_fixture_dir(args.fixture_dir) + contract = write_capture_contract( + args.fixture_dir, + args.probe_dir, + args.skills_dir, + args.probe, + args.requested_model, + args.requested_effort, + args.producer_override_bin, + ) + result = { + "binding_sha256": contract["binding_sha256"], + "probe": contract["probe"], + "reps": contract["reps"], + "schedule": contract["schedule"], + "scoring": contract["scoring"], + "treatment_source": contract["treatment_source"], + "canonical_skill": { + key: contract["canonical_skill"][key] + for key in ("name", "path", "sha256") + }, + } + elif args.command == "score": + validate_fixture_dir(args.fixture_dir) + manifest = validate_manifest( + args.fixture_dir, args.probe_dir, args.skills_dir, args.probe + ) + result = recompute_score(args.probe_dir, args.fixture_dir, manifest) + manifest_after = validate_manifest( + args.fixture_dir, args.probe_dir, args.skills_dir, args.probe + ) + if manifest_after["binding_sha256"] != manifest["binding_sha256"]: + raise MetadataError("fixture binding changed during scoring") + elif args.command in {"create", "verify", "observe"}: + validate_fixture_dir(args.fixture_dir) + if args.command == "create": + reps = require_reps(args.reps) + payload = build_payload( + args.fixture_dir, + args.probe_dir, + args.skills_dir, + args.harness, + args.preamble, + args.dispatch_helper, + args.probe, + reps, + args.requested_model, + args.requested_effort, + ) + manifest = write_manifest(args.fixture_dir, payload) + result = summary(manifest) + elif args.command == "verify": + result = summary( + validate_manifest( + args.fixture_dir, args.probe_dir, args.skills_dir, args.probe + ) + ) + else: + result = {"producer": observe_paths(transcript_paths(args.fixture_dir))} + else: + raise MetadataError(f"unsupported command: {args.command}") + except (MetadataError, OSError) as exc: + fail(str(exc)) + print(json.dumps(result, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe-skill.sh b/scripts/probe-skill.sh index 6decf3a68..f6590ef20 100755 --- a/scripts/probe-skill.sh +++ b/scripts/probe-skill.sh @@ -3,46 +3,61 @@ # # ============================ HONESTY HEADER ================================= # A probe measures BEHAVIOR-CHANGE, NOT quality-uplift. It answers exactly one -# question: when the skill is LOADED (treatment) versus NOT loaded (control), -# does the agent actually DO the thing differently — a tool call made, an +# question: when the declared treatment source is injected (treatment) versus +# omitted (control), does the agent actually DO the thing differently — a tool call made, an # artifact produced, a sequence followed? It NEVER scores whether the text # mentions the skill, and it NEVER claims the skill makes output better. A -# BEHAVIORAL verdict means "loading it changed what the agent did"; an INERT -# verdict means "it didn't" (the 2026-06-30 graphify result: a doc-instruction -# skill 0/2 treatment agents obeyed). Small N (default 2-3) is DIRECTIONAL, not -# statistical. Do not overclaim (ADR-0011 discipline). +# BEHAVIORAL verdict means the treatment increased the scored behavior; INERT +# means equal rates; REGRESSIVE means the treatment reduced it. The historical 2026-06-30 graphify report (0/2 +# treatment responses obeyed the guidance) predates immutable capture metadata +# and remains LEGACY-UNVERIFIED, not a current harness result. Small N (default +# 2-3) is DIRECTIONAL, not statistical. Do not overclaim (ADR-0011 discipline). # ============================================================================ # # A PROBE is a directory under evals/skill-probes// carrying: # probe.json metadata: id, skill, reps, behavior, discriminator # question.md the scenario question — IDENTICAL for both arms -# treatment-prelude.md the skill guidance injected ONLY in the treatment arm -# (the sole variable: control = question; treatment = -# prelude + question) -# discriminator.sh a DETERMINISTIC behavioral check over one transcript: +# treatment-prelude.md an optional distilled treatment injected ONLY when +# probe.json declares treatment_source=injected-prelude; +# this mode measures the bound prelude, not full-skill +# activation, and does not qualify as skill-tier coverage +# discriminator.sh a DETERMINISTIC check over a prompt-free response envelope: # exit 0 = behavior PRESENT, 1 = ABSENT, 2 = infra error # fixtures/ recorded transcripts control-.txt / treatment-.txt -# (used by --replay for deterministic calibration + a -# committed, reproducible evidence run) +# as one bound prompt event followed by native Codex JSONL +# fixtures/capture-contract.json +# pre-dispatch binding over exact prompt bytes, producer +# request/runtime executable identity, schedule, and scoring +# fixtures/fixture-set.json +# immutable capture metadata: exact transcripts and thread +# ids, evaluation-input and canonical SKILL.md inventories, +# evaluator identity, per-file SHA-256 digests, and one +# binding digest over the complete capture contract # # MODES: # live (default) dispatch a cross-family worker (codex exec — the sanctioned # headless path; NEVER claude -p, LAW 0) for each arm x rep, -# capture the transcript, run the discriminator. Writes the -# transcripts into fixtures/ so the run is reproducible. -# --replay skip dispatch; run the discriminator over the committed -# fixtures. Deterministic — this is what calibration and CI use. +# capture the transcript, run the discriminator, and publish +# a new immutable fixture set so its bound classification is +# replayable. Existing fixture sets are never overwritten. +# --replay skip dispatch; verify immutable capture metadata and then run +# the discriminator over the bound transcripts. Legacy fixture +# sets without metadata fail closed; they are not retroactively +# blessed as verified captures. # -# VERDICT: BEHAVIORAL iff treatment_rate > control_rate; INERT iff not; UNMEASURED -# iff no usable treatment reps (all degraded / missing). +# VERDICT: BEHAVIORAL iff treatment_rate > control_rate; REGRESSIVE iff lower; +# INERT iff equal; UNMEASURED iff either arm has no usable reps or a live capture +# is incomplete (a durable delta needs two measured arms and replayable evidence). # # Usage: # bash scripts/probe-skill.sh --probe rpi --replay +# bash scripts/probe-skill.sh --probe rpi --replay --fixtures fixtures-xhigh-2026-08-04 # bash scripts/probe-skill.sh --probe rpi --reps 2 --output out.json # bash scripts/probe-skill.sh --probe rpi --live --capture # bash scripts/probe-skill.sh --probe rpi --live --model gpt-5-mini # # Flags: --probe (required) · --replay | --live · --capture · --reps N · +# --fixtures · # --output · --timeout · --model (weaker producer, the # ratchet when a frontier producer aces both arms) · --effort # (low|medium|high|xhigh — sets codex model_reasoning_effort; the SECOND @@ -50,23 +65,32 @@ # default effort, lower the effort to surface headroom. 2026-08-04 wave-1 # finding: gpt-5.6-luna at xhigh aced 4/6 control arms). # -# Env overrides (test seams): SKILL_PROBES_DIR (default $REPO_ROOT/evals/skill-probes) +# Env overrides (test seams): SKILL_PROBES_DIR (default $REPO_ROOT/evals/skill-probes), +# SKILL_PROBE_SKILLS_DIR (default $REPO_ROOT/skills), PROBE_FIXTURE_SET +# (default fixtures) # # practices: [measurement-over-assertion, ab-testing] -# shellcheck disable=SC1007 +# shellcheck source=scripts/lib/preamble.sh disable=SC1007,SC1091 . "$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/preamble.sh" # shellcheck source=scripts/lib/codex-exec.sh disable=SC1091 . "$REPO_ROOT/scripts/lib/codex-exec.sh" PROBES_DIR="${SKILL_PROBES_DIR:-$REPO_ROOT/evals/skill-probes}" +SKILLS_DIR="${SKILL_PROBE_SKILLS_DIR:-$REPO_ROOT/skills}" PROBE="" REPLAY=0 CAPTURE=0 REPS="" +REPS_EXPLICIT=0 OUTPUT="" TIMEOUT="${PROBE_TIMEOUT:-240}" MODEL="${PROBE_MODEL:-}" EFFORT="${PROBE_EFFORT:-}" +MODEL_CONSTRAINT=0 +EFFORT_CONSTRAINT=0 +if [[ -n "$MODEL" ]]; then MODEL_CONSTRAINT=1; fi +if [[ -n "$EFFORT" ]]; then EFFORT_CONSTRAINT=1; fi +FIXTURE_SET="${PROBE_FIXTURE_SET:-fixtures}" usage() { grep '^#' "$0" | sed 's/^# \?//'; } @@ -76,151 +100,574 @@ while [[ $# -gt 0 ]]; do --replay) REPLAY=1; shift;; --live) REPLAY=0; shift;; --capture) CAPTURE=1; shift;; - --reps) REPS="${2:-}"; shift 2;; + --reps) REPS="${2:-}"; REPS_EXPLICIT=1; shift 2;; + --fixtures|--fixture-set) FIXTURE_SET="${2:-}"; shift 2;; --output) OUTPUT="${2:-}"; shift 2;; --timeout) TIMEOUT="${2:-}"; shift 2;; - --model) MODEL="${2:-}"; shift 2;; - --effort) EFFORT="${2:-}"; shift 2;; + --model) MODEL="${2:-}"; MODEL_CONSTRAINT=1; shift 2;; + --effort) EFFORT="${2:-}"; EFFORT_CONSTRAINT=1; shift 2;; -h|--help) usage; exit 0;; *) echo "Unknown flag: $1" >&2; exit 2;; esac done [[ -n "$PROBE" ]] || { echo "error: --probe required" >&2; exit 2; } +[[ "$PROBE" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \ + || { echo "error: unsafe probe id: $PROBE" >&2; exit 2; } +[[ "$FIXTURE_SET" =~ ^fixtures([_-][A-Za-z0-9][A-Za-z0-9._-]*)?$ ]] \ + || { echo "error: --fixtures must name a fixtures directory inside the probe: $FIXTURE_SET" >&2; exit 2; } PROBE_DIR="$PROBES_DIR/$PROBE" [[ -d "$PROBE_DIR" ]] || { echo "error: probe not found: $PROBE_DIR" >&2; exit 2; } DISC="$PROBE_DIR/discriminator.sh" QUESTION="$PROBE_DIR/question.md" -PRELUDE="$PROBE_DIR/treatment-prelude.md" META="$PROBE_DIR/probe.json" -for f in "$DISC" "$QUESTION" "$PRELUDE" "$META"; do - [[ -f "$f" ]] || { echo "error: probe file missing: $f" >&2; exit 2; } -done +if [[ $REPLAY -eq 0 ]]; then + for f in "$DISC" "$QUESTION" "$META"; do + [[ -f "$f" && ! -L "$f" ]] || { echo "error: probe file missing or unsafe: $f" >&2; exit 2; } + done +fi -# Read reps + skill from probe.json (python3, no jq dependency). +# Read probe metadata (python3, no jq dependency). json_get() { python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(d.get(sys.argv[2],""))' "$META" "$1"; } -[[ -n "$REPS" ]] || REPS="$(json_get reps)" -[[ -n "$REPS" ]] || REPS=2 -SKILL="$(json_get skill)" -FIXDIR="$PROBE_DIR/fixtures" -mkdir -p "$FIXDIR" +FIXDIR="$PROBE_DIR/$FIXTURE_SET" +FIXTURE_META_TOOL="$REPO_ROOT/scripts/lib/probe-fixture-metadata.py" +[[ -f "$FIXTURE_META_TOOL" ]] || { echo "error: fixture metadata helper missing: $FIXTURE_META_TOOL" >&2; exit 2; } +HARNESS_PATH="$REPO_ROOT/scripts/probe-skill.sh" +PREAMBLE_PATH="$REPO_ROOT/scripts/lib/preamble.sh" +DISPATCH_HELPER_PATH="$REPO_ROOT/scripts/lib/codex-exec.sh" -# --effort plumbs through the codex-exec lib's CODEX_EXEC_EXTRA_ARGS array +if [[ -n "$OUTPUT" ]]; then + OUTPUT_DIR="$(dirname "$OUTPUT")" + [[ -d "$OUTPUT_DIR" ]] || { echo "error: scorecard output directory does not exist: $OUTPUT_DIR" >&2; exit 2; } + [[ ! -e "$OUTPUT" && ! -L "$OUTPUT" ]] \ + || { echo "error: refusing to overwrite immutable scorecard output: $OUTPUT" >&2; exit 2; } +fi + +summary_get() { + local summary="$1" path="$2" + python3 -c ' +import json, sys +value = json.loads(sys.argv[1]) +for part in sys.argv[2].split("."): + if not isinstance(value, dict) or part not in value: + value = None + break + value = value[part] +print("" if value is None else value) +' "$summary" "$path" +} + +summary_json() { + local summary="$1" path="$2" + python3 -c ' +import json, sys +value = json.loads(sys.argv[1]) +for part in sys.argv[2].split("."): + value = value[part] +print(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) +' "$summary" "$path" +} + +SKILL="" +TREATMENT_SOURCE="" +if [[ $REPLAY -eq 0 ]]; then + if ! PROBE_CONTRACT="$(python3 "$FIXTURE_META_TOOL" probe-contract \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS_DIR" \ + --probe "$PROBE")"; then + echo "error: probe contract is incomplete or unsafe" >&2 + exit 2 + fi + SKILL="$(summary_get "$PROBE_CONTRACT" canonical_skill.name)" + CONTRACT_REPS="$(summary_get "$PROBE_CONTRACT" reps)" + TREATMENT_SOURCE="$(summary_get "$PROBE_CONTRACT" treatment_source)" +fi + +PRODUCER_MODEL="" +PRODUCER_EFFORT="" +PRODUCER_JSON="" +CAPTURE_REQUESTED_MODEL="$MODEL" +CAPTURE_REQUESTED_EFFORT="$EFFORT" +FIXTURE_BINDING="" +FIXTURE_SCHEMA="" +CAPTURE_EVALUATOR="" +CURRENT_EVALUATOR="$(python3 "$FIXTURE_META_TOOL" identity \ + --harness "$HARNESS_PATH" \ + --preamble "$PREAMBLE_PATH" \ + --dispatch-helper "$DISPATCH_HELPER_PATH")" + +if [[ $REPLAY -eq 1 ]]; then + [[ -d "$FIXDIR" && ! -L "$FIXDIR" ]] \ + || { echo "error: replay fixture set not found or unsafe: $FIXDIR" >&2; exit 2; } + if ! FIXTURE_METADATA="$(python3 "$FIXTURE_META_TOOL" verify --fixture-dir "$FIXDIR" --probe-dir "$PROBE_DIR" --skills-dir "$SKILLS_DIR" --probe "$PROBE")"; then + echo "error: replay refused: fixture metadata is missing or failed verification" >&2 + exit 2 + fi + MANIFEST_REPS="$(summary_get "$FIXTURE_METADATA" reps)" + if [[ $REPS_EXPLICIT -eq 1 && "$REPS" != "$MANIFEST_REPS" ]]; then + echo "error: --reps $REPS does not match fixture metadata reps $MANIFEST_REPS" >&2 + exit 2 + fi + REPS="$MANIFEST_REPS" + PRODUCER_MODEL="$(summary_get "$FIXTURE_METADATA" producer.model)" + PRODUCER_EFFORT="$(summary_get "$FIXTURE_METADATA" producer.effort)" + PRODUCER_JSON="$(summary_json "$FIXTURE_METADATA" producer)" + CAPTURE_REQUESTED_MODEL="$(summary_get "$FIXTURE_METADATA" requested_producer.model)" + CAPTURE_REQUESTED_EFFORT="$(summary_get "$FIXTURE_METADATA" requested_producer.effort)" + FIXTURE_BINDING="$(summary_get "$FIXTURE_METADATA" binding_sha256)" + FIXTURE_SCHEMA="$(summary_get "$FIXTURE_METADATA" schema)" + CAPTURE_EVALUATOR="$(python3 -c 'import json,sys; print(json.dumps(json.loads(sys.argv[1])["capture_evaluator"],sort_keys=True,separators=(",",":")))' "$FIXTURE_METADATA")" + TREATMENT_SOURCE="$(summary_get "$FIXTURE_METADATA" treatment_source)" + SKILL="$(summary_get "$FIXTURE_METADATA" canonical_skill.name)" + if [[ -z "$SKILL" && -f "$META" && ! -L "$META" ]]; then + SKILL="$(json_get skill)" + fi + [[ -n "$SKILL" ]] || { echo "error: verified fixture does not identify a skill" >&2; exit 2; } + if [[ $MODEL_CONSTRAINT -eq 1 && "$MODEL" != "$PRODUCER_MODEL" ]]; then + echo "error: replay --model $MODEL does not match bound fixture producer request $PRODUCER_MODEL" >&2 + exit 2 + fi + if [[ $EFFORT_CONSTRAINT -eq 1 && "$EFFORT" != "$PRODUCER_EFFORT" ]]; then + echo "error: replay --effort $EFFORT does not match bound fixture producer request $PRODUCER_EFFORT" >&2 + exit 2 + fi +else + [[ -n "$REPS" ]] || REPS="$CONTRACT_REPS" + if [[ "$REPS" != "$CONTRACT_REPS" ]]; then + echo "error: --reps $REPS does not match bound probe.json reps $CONTRACT_REPS" >&2 + exit 2 + fi + [[ ! -e "$FIXDIR" && ! -L "$FIXDIR" ]] || { + echo "error: refusing to overwrite immutable fixture set: $FIXDIR" >&2 + echo " choose a new --fixtures name for this live capture" >&2 + exit 2 + } +fi + +[[ "$REPS" =~ ^[1-9][0-9]*$ ]] || { echo "error: --reps must be a positive integer, got: $REPS" >&2; exit 2; } +[[ "$REPS" -le 20 ]] || { echo "error: --reps must not exceed 20, got: $REPS" >&2; exit 2; } +[[ "$TIMEOUT" =~ ^[0-9]+$ ]] || { echo "error: --timeout must be a non-negative integer, got: $TIMEOUT" >&2; exit 2; } + +# Structured JSONL is mandatory for new captures. --effort plumbs through the +# codex-exec lib's CODEX_EXEC_EXTRA_ARGS array # (arrays cannot cross a process boundary, so the flag lives here, in the same # shell that sources the lib). Applied to BOTH arms — the producer config must # stay symmetric or the delta is confounded. -if [[ -n "$EFFORT" ]]; then +if [[ $REPLAY -eq 0 ]]; then + # shellcheck disable=SC2034 # consumed by codex_exec_guarded in the sourced library + CODEX_EXEC_EXTRA_ARGS=(--json --ephemeral) +fi +if [[ $REPLAY -eq 0 && -n "$EFFORT" ]]; then # shellcheck disable=SC2034 # CODEX_EXEC_EXTRA_ARGS is consumed by codex_exec_guarded in the sourced codex-exec.sh case "$EFFORT" in - low|medium|high|xhigh) CODEX_EXEC_EXTRA_ARGS=(-c "model_reasoning_effort=\"$EFFORT\"");; + low|medium|high|xhigh) CODEX_EXEC_EXTRA_ARGS+=(-c "model_reasoning_effort=\"$EFFORT\"");; *) echo "error: --effort must be low|medium|high|xhigh, got: $EFFORT" >&2; exit 2;; esac fi -# run_discriminator TRANSCRIPT -> echoes PRESENT|ABSENT|DEGRADED -run_discriminator() { - local transcript="$1" rc=0 - [[ -s "$transcript" ]] || { echo "DEGRADED"; return; } - bash "$DISC" "$transcript" >/dev/null 2>&1 || rc=$? - case "$rc" in - 0) echo "PRESENT";; - 1) echo "ABSENT";; - *) echo "DEGRADED";; - esac -} - -# dispatch_live ARM REP TRANSCRIPT_OUT -> populate TRANSCRIPT_OUT via codex exec. -# ARM is control|treatment. Control prompt = question; treatment = prelude+question. +# dispatch_live ARM REP TRANSCRIPT_OUT -> bind the exact prompt event, capture +# native Codex JSONL, and populate TRANSCRIPT_OUT with one structured envelope. dispatch_live() { - local arm="$1" transcript="$2" prompt work - if [[ "$arm" == "treatment" ]]; then - prompt="$(cat "$PRELUDE")"$'\n\n---\n\n'"$(cat "$QUESTION")" - else - prompt="$(cat "$QUESTION")" + local arm="$1" rep="$2" transcript="$3" receipt_name="$4" result_name="$5" + local rc=0 receipt="" outcome="DEGRADED" had_noclobber=0 + local prompt_file="$LIVE_WORKSPACE/$arm-$rep.prompt" + local runtime_file="$LIVE_WORKSPACE/$arm-$rep.codex.jsonl" + local stderr_file="$LIVE_WORKSPACE/$arm-$rep.codex.stderr" + if ! python3 "$FIXTURE_META_TOOL" capture-file \ + --fixture-dir "$LIVE_STAGE" --probe "$PROBE" \ + --name "prompt-$arm" >"$prompt_file"; then + echo "probe-skill: could not materialize bound $arm-$rep prompt" >&2 + printf -v "$receipt_name" '%s' "" + printf -v "$result_name" '%s' "$outcome" + return 1 fi - work="$(mktemp -d "${TMPDIR:-/tmp}/probe-ws.XXXXXX")" + if [[ -o noclobber ]]; then + had_noclobber=1 + else + set -o noclobber + fi + if ! exec 9> "$transcript"; then + [[ $had_noclobber -eq 1 ]] || set +o noclobber + echo "probe-skill: $arm dispatch refused unsafe/existing transcript sink" >&2 + printf -v "$receipt_name" '%s' "" + printf -v "$result_name" '%s' "$outcome" + return 1 + fi + [[ $had_noclobber -eq 1 ]] || set +o noclobber # read-only sandbox: the probe only wants the agent's PLAN text, no mutation. # --model routes a WEAKER producer (e.g. gpt-5-mini) — the ratchet for # surfacing a skill's behavioral value when a frontier producer aces both arms # (the membrane-eval-too-easy lesson). Empty => the codex default (frontier). - CODEX_EXEC_PROMPT_ARG="$prompt" \ - CODEX_EXEC_DIR="$work" \ + REVIEWER=codex \ + REVIEWER_MARKER=turn.completed \ + CODEX_EXEC_PROMPT_FILE="$prompt_file" \ + CODEX_EXEC_DIR="$LIVE_WORKSPACE" \ CODEX_EXEC_SANDBOX=read-only \ CODEX_EXEC_SKIP_GIT_CHECK=1 \ CODEX_EXEC_TIMEOUT="$TIMEOUT" \ CODEX_EXEC_MODEL="$MODEL" \ - CODEX_EXEC_OUT_FILE="$transcript" \ + CODEX_EXEC_OUT_FILE="$runtime_file" \ + CODEX_EXEC_STDERR_FILE="$stderr_file" \ CODEX_EXEC_EXPECT_OUTPUT=1 \ - codex_exec_guarded >/dev/null 2>&1 || true - rm -rf "$work" + codex_exec_guarded >/dev/null || rc=$? + if [[ -s "$stderr_file" ]]; then + cat "$stderr_file" >&2 + [[ "$rc" -ne 0 ]] || rc=2 + fi + if [[ "$rc" -eq 0 ]]; then + python3 "$FIXTURE_META_TOOL" assemble-transcript \ + --runtime-file "$runtime_file" --prompt-file "$prompt_file" \ + --fixture-dir "$LIVE_STAGE" --probe "$PROBE" \ + --arm "$arm" --rep "$rep" >&9 || rc=$? + fi + if [[ "$rc" -eq 0 ]]; then + receipt="$(python3 "$FIXTURE_META_TOOL" classify-open \ + --fd 0 --path "$transcript" --fixture-dir "$LIVE_STAGE" \ + --probe "$PROBE" --arm "$arm" --rep "$rep" <&9)" || rc=$? + fi + exec 9>&- + if [[ "$rc" -ne 0 || -z "$receipt" ]]; then + echo "probe-skill: $arm dispatch degraded (rc=$rc); no transcript accepted" >&2 + printf -v "$receipt_name" '%s' "" + printf -v "$result_name" '%s' "$outcome" + return 1 + fi + outcome="$(summary_get "$receipt" outcome)" + printf -v "$receipt_name" '%s' "$receipt" + printf -v "$result_name" '%s' "$outcome" + return 0 } +LIVE_STAGE="" +LIVE_ALL_DISPATCH_OK=1 + +publish_fixture_set() { + local stage="$1" target="$2" + python3 "$FIXTURE_META_TOOL" publish \ + --stage-dir "$stage" \ + --target-dir "$target" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS_DIR" \ + --probe "$PROBE" >/dev/null || return 1 + LIVE_STAGE="" + return 0 +} + +if [[ $REPLAY -eq 0 ]]; then + LIVE_STAGE="$(mktemp -d "$PROBE_DIR/.${FIXTURE_SET}.capture.XXXXXX")" + chmod 0700 "$LIVE_STAGE" + SNAPSHOT_ARGS=( + snapshot + --fixture-dir "$LIVE_STAGE" + --probe-dir "$PROBE_DIR" + --skills-dir "$SKILLS_DIR" + --probe "$PROBE" + ) + if [[ -n "$MODEL" ]]; then SNAPSHOT_ARGS+=(--requested-model "$MODEL"); fi + if [[ -n "$EFFORT" ]]; then SNAPSHOT_ARGS+=(--requested-effort "$EFFORT"); fi + if [[ -n "${CODEX_EXEC_BIN:-}" ]]; then + SNAPSHOT_ARGS+=(--producer-override-bin "$CODEX_EXEC_BIN") + fi + if ! python3 "$FIXTURE_META_TOOL" "${SNAPSHOT_ARGS[@]}" >/dev/null; then + echo "error: live capture inputs changed while creating the pre-dispatch snapshot" >&2 + exit 2 + fi + if ! CURRENT_CONTRACT="$(python3 "$FIXTURE_META_TOOL" probe-contract \ + --probe-dir "$PROBE_DIR" --skills-dir "$SKILLS_DIR" \ + --probe "$PROBE")" || [[ "$CURRENT_CONTRACT" != "$PROBE_CONTRACT" ]]; then + echo "error: live capture inputs changed while creating the pre-dispatch snapshot" >&2 + exit 2 + fi + LIVE_WORKSPACE="$(mktemp -d "${TMPDIR:-/tmp}/probe-ws.XXXXXX")" + chmod 0700 "$LIVE_WORKSPACE" +fi + C_PRESENT=0; C_USABLE=0; T_PRESENT=0; T_USABLE=0 PER_REP_JSON="" +declare -A LIVE_RESULTS=() +declare -A LIVE_DIGESTS=() -for ((n=1; n<=REPS; n++)); do - c_tx="$FIXDIR/control-$n.txt" - t_tx="$FIXDIR/treatment-$n.txt" - if [[ $REPLAY -eq 0 ]]; then - dispatch_live control "$c_tx" - dispatch_live treatment "$t_tx" +if [[ $REPLAY -eq 0 ]]; then + while IFS=$'\t' read -r arm rep; do + key="$arm-$rep" + transcript="$LIVE_STAGE/$key.txt" + dispatch_receipt="" + dispatch_result="DEGRADED" + if dispatch_live "$arm" "$rep" "$transcript" dispatch_receipt dispatch_result; then + model="$(summary_get "$dispatch_receipt" producer.model)" + effort="$(summary_get "$dispatch_receipt" producer.effort)" + PRODUCER_MODEL="$model" + PRODUCER_EFFORT="$effort" + LIVE_DIGESTS["$key"]="$(summary_get "$dispatch_receipt" sha256)" + else + LIVE_ALL_DISPATCH_OK=0 + fi + LIVE_RESULTS["$key"]="$dispatch_result" + done < <(python3 -c ' +import json, sys +for entry in json.loads(sys.argv[1])["schedule"]: + print(f"{entry['"'"'arm'"'"']}\t{entry['"'"'rep'"'"']}") +' "$PROBE_CONTRACT") + + for ((n=1; n<=REPS; n++)); do + c_res="${LIVE_RESULTS[control-$n]:-DEGRADED}" + t_res="${LIVE_RESULTS[treatment-$n]:-DEGRADED}" + if [[ "$c_res" != "DEGRADED" ]]; then C_USABLE=$((C_USABLE + 1)); fi + if [[ "$c_res" == "PRESENT" ]]; then C_PRESENT=$((C_PRESENT + 1)); fi + if [[ "$t_res" != "DEGRADED" ]]; then T_USABLE=$((T_USABLE + 1)); fi + if [[ "$t_res" == "PRESENT" ]]; then T_PRESENT=$((T_PRESENT + 1)); fi + entry="$(printf '{"rep":%d,"control":"%s","treatment":"%s"}' "$n" "$c_res" "$t_res")" + PER_REP_JSON="${PER_REP_JSON:+$PER_REP_JSON,}$entry" + done +else + if ! SCORE_SUMMARY="$(python3 "$FIXTURE_META_TOOL" score \ + --fixture-dir "$FIXDIR" --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS_DIR" --probe "$PROBE")"; then + echo "error: replay scoring failed; no scorecard will be emitted" >&2 + exit 2 fi - c_res="$(run_discriminator "$c_tx")" - t_res="$(run_discriminator "$t_tx")" + C_PRESENT="$(summary_get "$SCORE_SUMMARY" control.present)" + C_USABLE="$(summary_get "$SCORE_SUMMARY" control.usable)" + T_PRESENT="$(summary_get "$SCORE_SUMMARY" treatment.present)" + T_USABLE="$(summary_get "$SCORE_SUMMARY" treatment.usable)" + PER_REP_JSON="$(python3 -c ' +import json, sys +print(",".join(json.dumps(item,separators=(",",":")) for item in json.loads(sys.argv[1])["per_rep"])) +' "$SCORE_SUMMARY")" +fi - [[ "$c_res" != "DEGRADED" ]] && C_USABLE=$((C_USABLE + 1)) - [[ "$c_res" == "PRESENT" ]] && C_PRESENT=$((C_PRESENT + 1)) - [[ "$t_res" != "DEGRADED" ]] && T_USABLE=$((T_USABLE + 1)) - [[ "$t_res" == "PRESENT" ]] && T_PRESENT=$((T_PRESENT + 1)) - - entry="$(printf '{"rep":%d,"control":"%s","treatment":"%s"}' "$n" "$c_res" "$t_res")" - PER_REP_JSON="${PER_REP_JSON:+$PER_REP_JSON,}$entry" -done +if [[ $REPLAY -eq 0 && "$LIVE_ALL_DISPATCH_OK" -eq 1 ]]; then + if ! CURRENT_CONTRACT="$(python3 "$FIXTURE_META_TOOL" probe-contract \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS_DIR" \ + --probe "$PROBE")" || [[ "$CURRENT_CONTRACT" != "$PROBE_CONTRACT" ]]; then + echo "error: live capture inputs changed during dispatch; fixture set not published" >&2 + exit 2 + fi + CREATE_ARGS=( + create + --fixture-dir "$LIVE_STAGE" + --probe-dir "$PROBE_DIR" + --skills-dir "$SKILLS_DIR" + --harness "$HARNESS_PATH" + --preamble "$PREAMBLE_PATH" + --dispatch-helper "$DISPATCH_HELPER_PATH" + --probe "$PROBE" + --reps "$REPS" + ) + if [[ -n "$MODEL" ]]; then CREATE_ARGS+=(--requested-model "$MODEL"); fi + if [[ -n "$EFFORT" ]]; then CREATE_ARGS+=(--requested-effort "$EFFORT"); fi + if ! FIXTURE_METADATA="$(python3 "$FIXTURE_META_TOOL" "${CREATE_ARGS[@]}")"; then + echo "error: live capture refused: structured transcripts or bound producer identity failed verification" >&2 + exit 2 + fi + FIXTURE_BINDING="$(summary_get "$FIXTURE_METADATA" binding_sha256)" + FIXTURE_SCHEMA="$(summary_get "$FIXTURE_METADATA" schema)" + CAPTURE_EVALUATOR="$(python3 -c 'import json,sys; print(json.dumps(json.loads(sys.argv[1])["capture_evaluator"],sort_keys=True,separators=(",",":")))' "$FIXTURE_METADATA")" + if [[ "$CAPTURE_EVALUATOR" != "$CURRENT_EVALUATOR" ]]; then + echo "error: capture evaluator changed during live dispatch; fixture set not published" >&2 + exit 2 + fi + TREATMENT_SOURCE="$(summary_get "$FIXTURE_METADATA" treatment_source)" + PRODUCER_MODEL="$(summary_get "$FIXTURE_METADATA" producer.model)" + PRODUCER_EFFORT="$(summary_get "$FIXTURE_METADATA" producer.effort)" + PRODUCER_JSON="$(summary_json "$FIXTURE_METADATA" producer)" + for ((n=1; n<=REPS; n++)); do + for arm in control treatment; do + key="$arm-$n" + bound_digest="$(python3 -c ' +import json, sys +records=json.loads(sys.argv[1])["transcripts"] +print(next(item["sha256"] for item in records if item["path"] == sys.argv[2] + ".txt")) +' "$FIXTURE_METADATA" "$key")" + if [[ "$bound_digest" != "${LIVE_DIGESTS[$key]:-}" ]]; then + echo "error: live transcript identity changed after scoring: $key" >&2 + exit 2 + fi + done + done + if ! python3 "$FIXTURE_META_TOOL" verify \ + --fixture-dir "$LIVE_STAGE" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS_DIR" \ + --probe "$PROBE" >/dev/null; then + echo "error: staged fixture set failed post-capture verification" >&2 + exit 2 + fi + if ! CURRENT_CONTRACT="$(python3 "$FIXTURE_META_TOOL" probe-contract \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS_DIR" \ + --probe "$PROBE")" || [[ "$CURRENT_CONTRACT" != "$PROBE_CONTRACT" ]]; then + echo "error: live capture inputs changed before publish; fixture set not published" >&2 + exit 2 + fi + if ! SCORE_SUMMARY="$(python3 "$FIXTURE_META_TOOL" score \ + --fixture-dir "$LIVE_STAGE" --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS_DIR" --probe "$PROBE")"; then + echo "error: staged fixture set failed bound response-only scoring" >&2 + exit 2 + fi + if ! python3 -c ' +import json, sys +raise SystemExit(0 if json.loads("[" + sys.argv[1] + "]") == json.loads(sys.argv[2])["per_rep"] else 1) +' "$PER_REP_JSON" "$SCORE_SUMMARY"; then + echo "error: bound scoring disagrees with live transcript receipts" >&2 + exit 2 + fi + C_PRESENT="$(summary_get "$SCORE_SUMMARY" control.present)" + C_USABLE="$(summary_get "$SCORE_SUMMARY" control.usable)" + T_PRESENT="$(summary_get "$SCORE_SUMMARY" treatment.present)" + T_USABLE="$(summary_get "$SCORE_SUMMARY" treatment.usable)" + if ! publish_fixture_set "$LIVE_STAGE" "$FIXDIR"; then + echo "error: failed to publish fixture set atomically: $FIXDIR" >&2 + exit 1 + fi + if ! PUBLISHED_METADATA="$(python3 "$FIXTURE_META_TOOL" verify \ + --fixture-dir "$FIXDIR" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS_DIR" \ + --probe "$PROBE")" || \ + [[ "$(summary_get "$PUBLISHED_METADATA" binding_sha256)" != "$FIXTURE_BINDING" ]]; then + echo "error: published fixture target failed exact binding verification: $FIXDIR" >&2 + exit 1 + fi +elif [[ $REPLAY -eq 0 ]]; then + echo "probe-skill: incomplete live run; fixture set not published" >&2 +fi # --- verdict ------------------------------------------------------------------ rate() { local n="$1" d="$2"; [[ "$d" -eq 0 ]] && { echo "null"; return; }; python3 -c "print(round($n/$d,4))"; } C_RATE="$(rate "$C_PRESENT" "$C_USABLE")" T_RATE="$(rate "$T_PRESENT" "$T_USABLE")" -if [[ "$T_USABLE" -eq 0 ]]; then +if [[ "$C_USABLE" -eq 0 || "$T_USABLE" -eq 0 || ( $REPLAY -eq 0 && "$LIVE_ALL_DISPATCH_OK" -ne 1 ) ]]; then VERDICT="UNMEASURED" else - # BEHAVIORAL iff treatment did the thing strictly more than control. + # Direction is part of the result; a lower treatment rate is not a null. cmp_res="$(python3 -c " tu=$T_USABLE; cu=$C_USABLE tr=$T_PRESENT/tu -cr=($C_PRESENT/cu) if cu>0 else 0.0 -print('BEHAVIORAL' if tr>cr else 'INERT')")" +cr=$C_PRESENT/cu +print('BEHAVIORAL' if tr>cr else 'REGRESSIVE' if tr&2; exit 1; } +print(json.dumps(scorecard, ensure_ascii=False, indent=2, sort_keys=False)) +PY +)" || { echo "error: failed to serialize scorecard JSON" >&2; exit 1; } if [[ -n "$OUTPUT" ]]; then - printf '%s\n' "$SCORECARD" > "$OUTPUT" + if ! printf '%s\n' "$SCORECARD" | python3 "$FIXTURE_META_TOOL" \ + write-output --path "$OUTPUT" >/dev/null; then + echo "error: refusing to overwrite immutable scorecard output: $OUTPUT" >&2 + exit 2 + fi echo "scorecard written: $OUTPUT" >&2 else printf '%s\n' "$SCORECARD" @@ -228,5 +675,7 @@ fi # In --capture we keep the fixtures (default for live). In pure --replay we never # wrote them. Nothing else to do. -[[ $CAPTURE -eq 1 ]] && echo "fixtures captured under: $FIXDIR" >&2 +if [[ $CAPTURE -eq 1 && -n "$FIXTURE_BINDING" ]]; then + echo "fixtures captured under: $FIXDIR" >&2 +fi exit 0 diff --git a/scripts/prune-agents.sh b/scripts/prune-agents.sh index 44355c334..44088439c 100755 --- a/scripts/prune-agents.sh +++ b/scripts/prune-agents.sh @@ -1,276 +1,34 @@ #!/usr/bin/env bash -# prune-agents.sh — Enforce .agents/ retention policies +# prune-agents.sh — compatibility wrapper over `ao session prune-agents` # # Usage: -# ./scripts/prune-agents.sh # Dry run (default) — show what would be deleted -# ./scripts/prune-agents.sh --execute # Actually delete files -# ./scripts/prune-agents.sh --quiet # Suppress per-file output (summary only) -# ./scripts/prune-agents.sh --execute --quiet # Auto-prune with minimal output +# ./scripts/prune-agents.sh # Dry run (default) +# ./scripts/prune-agents.sh --execute # Apply retention deletions +# ./scripts/prune-agents.sh --quiet # Summary only # -# Policies defined in .agents/README.md ## Pruning section. -# Never touches: learnings/, patterns/, plans/, research/, retros/ (knowledge assets) +# Retention policy and every filesystem mutation live in the Go CLI. This +# wrapper only resolves the checkout root and matching ao binary, then forwards +# arguments. AGENTOPS_AO_BIN is the explicit binary seam used by CI and tests. set -euo pipefail -# Anchor to repo root to avoid pruning wrong .agents/ when cwd differs +# Anchor to this checkout (or the fixture override) before invoking ao, so a +# caller's current directory cannot select another repository's .agents tree. # shellcheck disable=SC1007,SC1091 . "$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/repo-root.sh" -REPO_ROOT="$(resolve_repo_root)" -AGENTS_DIR="${REPO_ROOT}/.agents" -DRY_RUN=true -TOTAL_FILES=0 -TOTAL_BYTES=0 +repo_root="$(resolve_repo_root)" -QUIET=false -for arg in "$@"; do - case "$arg" in - --execute) DRY_RUN=false ;; - --quiet) QUIET=true ;; - esac -done - -if [[ "$QUIET" == false ]]; then - if [[ "$DRY_RUN" == true ]]; then - echo "=== DRY RUN — no files will be deleted (pass --execute to delete) ===" - else - echo "=== EXECUTE MODE — files will be deleted ===" - fi - echo "" +ao_bin="${AGENTOPS_AO_BIN:-}" +if [[ -z "$ao_bin" && -x "$repo_root/cli/bin/ao" ]]; then + ao_bin="$repo_root/cli/bin/ao" +fi +if [[ -z "$ao_bin" ]]; then + ao_bin="$(command -v ao 2>/dev/null || true)" +fi +if [[ -z "$ao_bin" || ! -x "$ao_bin" ]]; then + echo "prune-agents: ao binary not found; build cli/bin/ao or set AGENTOPS_AO_BIN" >&2 + exit 1 fi -# Helper: list files to prune, sorted oldest first -prune_keep_newest() { - local dir="$1" - local keep="$2" - local label="$3" - - if [[ ! -d "$dir" ]]; then - return - fi - - local count - count=$(find "$dir" -maxdepth 1 -type f 2>/dev/null | wc -l | tr -d ' ') - - if [[ "$count" -le "$keep" ]]; then - [[ "$QUIET" == false ]] && echo "[$label] $count files — within limit ($keep). Nothing to prune." - return - fi - - local to_delete=$((count - keep)) - [[ "$QUIET" == false ]] && echo "[$label] $count files — keeping newest $keep, pruning $to_delete" - - # List oldest files first (by modification time) - find "$dir" -maxdepth 1 -type f -print0 2>/dev/null \ - | xargs -0 ls -t 2>/dev/null \ - | tail -n "$to_delete" \ - | while read -r f; do - local size - size=$(stat -f%z "$f" 2>/dev/null || stat --format=%s "$f" 2>/dev/null || echo 0) - TOTAL_BYTES=$((TOTAL_BYTES + size)) - TOTAL_FILES=$((TOTAL_FILES + 1)) - if [[ "$DRY_RUN" == true ]]; then - [[ "$QUIET" == false ]] && echo " would delete: $f ($(numfmt_size "$size"))" - else - rm -f "$f" - [[ "$QUIET" == false ]] && echo " deleted: $f ($(numfmt_size "$size"))" - fi - done -} - -prune_older_than() { - local dir="$1" - local days="$2" - local pattern="$3" - local label="$4" - - if [[ ! -d "$dir" ]]; then - return - fi - - local found - found=$(find "$dir" -maxdepth 1 -name "$pattern" -type f -mtime +"$days" 2>/dev/null | wc -l | tr -d ' ') - - if [[ "$found" -eq 0 ]]; then - [[ "$QUIET" == false ]] && echo "[$label] No files older than ${days}d matching '$pattern'. Nothing to prune." - return - fi - - [[ "$QUIET" == false ]] && echo "[$label] $found files older than ${days}d" - - find "$dir" -maxdepth 1 -name "$pattern" -type f -mtime +"$days" -print0 2>/dev/null \ - | while IFS= read -r -d '' f; do - local size - size=$(stat -f%z "$f" 2>/dev/null || stat --format=%s "$f" 2>/dev/null || echo 0) - TOTAL_BYTES=$((TOTAL_BYTES + size)) - TOTAL_FILES=$((TOTAL_FILES + 1)) - if [[ "$DRY_RUN" == true ]]; then - [[ "$QUIET" == false ]] && echo " would delete: $f ($(numfmt_size "$size"))" - else - rm -f "$f" - [[ "$QUIET" == false ]] && echo " deleted: $f ($(numfmt_size "$size"))" - fi - done -} - -numfmt_size() { - local bytes="$1" - if [[ "$bytes" -ge 1073741824 ]]; then - echo "$(( bytes / 1073741824 ))GB" - elif [[ "$bytes" -ge 1048576 ]]; then - echo "$(( bytes / 1048576 ))MB" - elif [[ "$bytes" -ge 1024 ]]; then - echo "$(( bytes / 1024 ))KB" - else - echo "${bytes}B" - fi -} - -# --- Policy: council/ — keep last 30 --- -prune_keep_newest "$AGENTS_DIR/council" 30 "council" -[[ "$QUIET" == false ]] && echo "" - -# --- tooling/ and security/ no longer live in .agents/ (moved to $TMPDIR) --- -# Clean up any legacy directories left from older versions -for legacy_dir in "$AGENTS_DIR/tooling" "$AGENTS_DIR/security"; do - if [[ -d "$legacy_dir" ]]; then - legacy_count=$(find "$legacy_dir" -type f 2>/dev/null | wc -l | tr -d ' ') - if [[ "$legacy_count" -gt 0 ]]; then - [[ "$QUIET" == false ]] && echo "[legacy] $legacy_dir has $legacy_count files (scanner output moved to \$TMPDIR)" - if [[ "$DRY_RUN" == true ]]; then - [[ "$QUIET" == false ]] && echo " would delete: $legacy_dir/ ($legacy_count files)" - else - rm -rf "$legacy_dir" - mkdir -p "$legacy_dir" - [[ "$QUIET" == false ]] && echo " deleted: $legacy_dir/ ($legacy_count files)" - fi - fi - fi -done -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: knowledge/pending/ — older than 14 days --- -prune_older_than "$AGENTS_DIR/knowledge/pending" 14 "*.md" "knowledge/pending" -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: rpi/ phase summaries — older than 30 days --- -prune_older_than "$AGENTS_DIR/rpi" 30 "phase-*-summary-*" "rpi/phase-summaries" -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: ao/sessions/ — keep last 50 --- -prune_keep_newest "$AGENTS_DIR/ao/sessions" 50 "ao/sessions" -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: handoff/ — keep last 10 --- -prune_keep_newest "$AGENTS_DIR/handoff" 10 "handoff" -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: opencode-tests/ — logs older than 7 days --- -prune_older_than "$AGENTS_DIR/opencode-tests" 7 "*.log" "opencode-tests" -prune_older_than "$AGENTS_DIR/opencode-tests" 7 "*.txt" "opencode-tests/summaries" -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: ao/subagent-outputs/ — keep last 50 --- -prune_keep_newest "$AGENTS_DIR/ao/subagent-outputs" 50 "ao/subagent-outputs" -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: releases/local-ci/ — keep last 3 runs --- -# Local CI validation runs dump ~2GB each (scanner output, SBOMs, etc.) -if [[ -d "$AGENTS_DIR/releases/local-ci" ]]; then - ci_runs=$(find "$AGENTS_DIR/releases/local-ci" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l | tr -d ' ') - keep_ci=3 - if [[ "$ci_runs" -gt "$keep_ci" ]]; then - to_delete_ci=$((ci_runs - keep_ci)) - [[ "$QUIET" == false ]] && echo "[releases/local-ci] $ci_runs runs — keeping newest $keep_ci, pruning $to_delete_ci" - find "$AGENTS_DIR/releases/local-ci" -maxdepth 1 -mindepth 1 -type d -print0 2>/dev/null \ - | xargs -0 ls -dt 2>/dev/null \ - | tail -n "$to_delete_ci" \ - | while read -r d; do - local_size=$(du -sk "$d" 2>/dev/null | cut -f1 || echo 0) - TOTAL_FILES=$((TOTAL_FILES + 1)) - if [[ "$DRY_RUN" == true ]]; then - [[ "$QUIET" == false ]] && echo " would delete: $d (~${local_size}KB)" - else - rm -rf "$d" - [[ "$QUIET" == false ]] && echo " deleted: $d (~${local_size}KB)" - fi - done - else - [[ "$QUIET" == false ]] && echo "[releases/local-ci] $ci_runs runs — within limit ($keep_ci). Nothing to prune." - fi -fi -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: vibe/ vibecheck/ — keep last 20 --- -prune_keep_newest "$AGENTS_DIR/vibe" 20 "vibe" -prune_keep_newest "$AGENTS_DIR/vibecheck" 20 "vibecheck" -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: brainstorm/ — keep last 10 --- -prune_keep_newest "$AGENTS_DIR/brainstorm" 10 "brainstorm" -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: compaction-snapshots/ — older than 7 days --- -prune_older_than "$AGENTS_DIR/compaction-snapshots" 7 "*.md" "compaction-snapshots" -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: swarm/ — keep last 10 --- -prune_keep_newest "$AGENTS_DIR/swarm" 10 "swarm" -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: status dashboards — keep last 5 --- -if [[ -d "$AGENTS_DIR" ]]; then - dashboard_count=$(find "$AGENTS_DIR" -maxdepth 1 -name "status-dashboard*" -type f 2>/dev/null | wc -l | tr -d ' ') - if [[ "$dashboard_count" -gt 5 ]]; then - to_delete_dash=$((dashboard_count - 5)) - [[ "$QUIET" == false ]] && echo "[status-dashboards] $dashboard_count files — keeping newest 5, pruning $to_delete_dash" - find "$AGENTS_DIR" -maxdepth 1 -name "status-dashboard*" -type f -print0 2>/dev/null \ - | xargs -0 ls -t 2>/dev/null \ - | tail -n "$to_delete_dash" \ - | while read -r f; do - if [[ "$DRY_RUN" == true ]]; then - [[ "$QUIET" == false ]] && echo " would delete: $f" - else - rm -f "$f" - [[ "$QUIET" == false ]] && echo " deleted: $f" - fi - TOTAL_FILES=$((TOTAL_FILES + 1)) - done - fi -fi -[[ "$QUIET" == false ]] && echo "" - -# --- Policy: archived-worktrees/ — older than 7 days --- -if [[ -d "$AGENTS_DIR/archived-worktrees" ]]; then - old_wt=$(find "$AGENTS_DIR/archived-worktrees" -maxdepth 1 -mindepth 1 -type d -mtime +7 2>/dev/null | wc -l | tr -d ' ') - if [[ "$old_wt" -gt 0 ]]; then - [[ "$QUIET" == false ]] && echo "[archived-worktrees] $old_wt directories older than 7d" - find "$AGENTS_DIR/archived-worktrees" -maxdepth 1 -mindepth 1 -type d -mtime +7 -print0 2>/dev/null \ - | while IFS= read -r -d '' d; do - if [[ "$DRY_RUN" == true ]]; then - [[ "$QUIET" == false ]] && echo " would delete: $d" - else - rm -rf "$d" - [[ "$QUIET" == false ]] && echo " deleted: $d" - fi - TOTAL_FILES=$((TOTAL_FILES + 1)) - done - else - [[ "$QUIET" == false ]] && echo "[archived-worktrees] No directories older than 7d. Nothing to prune." - fi -fi -[[ "$QUIET" == false ]] && echo "" - -# --- Summary --- -echo "========================================" -if [[ "$DRY_RUN" == true ]]; then - echo "DRY RUN COMPLETE" - echo "Files that would be deleted: $TOTAL_FILES" -else - echo "PRUNE COMPLETE" - echo "Files deleted: $TOTAL_FILES" -fi -if [[ "$QUIET" == false ]]; then - echo "" - echo "Protected directories (never pruned):" - echo " learnings/ patterns/ plans/ research/ retros/" -fi +cd "$repo_root" +exec "$ao_bin" session prune-agents "$@" diff --git a/skills-codex/.agentops-manifest.json b/skills-codex/.agentops-manifest.json index 4b63739b6..e29b2ccf5 100644 --- a/skills-codex/.agentops-manifest.json +++ b/skills-codex/.agentops-manifest.json @@ -357,8 +357,8 @@ { "name": "agent-mail", "source_skill": "skills/agent-mail", - "source_hash": "ffa68ba868e41b84bafb514842d2e870b551dccb86dcf6579549ba9e9897ecbf", - "generated_hash": "b83599604595f8914ee7845458711c61ce4f04b4d6d185fe61aedbf0aae8f568" + "source_hash": "f21a53e9da7bfc6047763d1a17552de25eb8d48f1a4aab6c4adb66f5f81118a5", + "generated_hash": "86fedf4d643ca4b375305c846cbbb6d8883116bf74b6990b3e63975592e598f2" }, { "name": "agent-native", @@ -405,8 +405,8 @@ { "name": "codebase-recon", "source_skill": "skills/codebase-recon", - "source_hash": "65da8420795f88d4d3f20da8e93585f8af34f619ce1d8b523fa8d4136deb0376", - "generated_hash": "eccd4073aace12f96c7ec62a7f01c21a821ef7d27658b60958abb8dea524bde1" + "source_hash": "2ee84ce6be213b02db860d9206e0a6307a53c5718c75311a2e69830b89ebd327", + "generated_hash": "26238e641ee39ee94d39dc3ddd21726d1db2c7e26e689c2092f217f3abda108e" }, { "name": "codex-exec", @@ -465,8 +465,8 @@ { "name": "handoff", "source_skill": "skills/handoff", - "source_hash": "11f7039b9e9533556fcec30c9dd130bc8889c6dd58b8c70dc0e72e314e927302", - "generated_hash": "e0aac3ade529c89b17ed8b95d49e16c40ab259d0ed278efde882d82e87e1faa3" + "source_hash": "8bf6b07f4db91ac2a7f13b27ee5bee1cd960b6bca7f2b4716ccf904de04561ad", + "generated_hash": "49d4e2099d491b52c7d7440f3d1d70f2ae9cad9850df484f1ffe6605efbb3098" }, { "name": "idea-genie", @@ -477,7 +477,7 @@ { "name": "implement", "source_skill": "skills/implement", - "source_hash": "5aa7e7b5b988a8f192d64ef7347ea7ca715f4979f325ed0507556ac0ff5ee82d", + "source_hash": "2a01b04555e02e0ccae7d670da6b6db67f26e6123d65103b68eb513f10d5026f", "generated_hash": "8b7f0c9f95ce32043dc9bf287313987027f557f2dbfcec6fbc67e852442d239e" }, { @@ -513,7 +513,7 @@ { "name": "plan", "source_skill": "skills/plan", - "source_hash": "16026fda4df366feaa3c99980f6a49d78d331e8563f8b0a7a4c0f45d0a60338b", + "source_hash": "6133824e806dd44f5c3fcfec39ed46251d951802d573e16fb02ffac9074e23aa", "generated_hash": "4144d7daa581c0352c5cb4231363bd19e99dd4a53dfe1a8daa9baee1a72d8d41" }, { @@ -561,8 +561,8 @@ { "name": "reverse-engineer", "source_skill": "skills/reverse-engineer", - "source_hash": "6ce0e7470b88ba775d15bc6a9ccf5aca3937a7238e4a333f2877b0bf35381769", - "generated_hash": "af78e0faf6f44e02ceb7503c9c691e5265f7c740b5d0ba627327b0c6f4722581" + "source_hash": "c4c1d351deda610d6a848b31aca64559d603d065b719a1a91cda3504d7ede98a", + "generated_hash": "e2330646793a9d980ebed4a2d10438ebef90ae790265520a10595192f3e564e8" }, { "name": "rpi", @@ -603,14 +603,14 @@ { "name": "skill-builder", "source_skill": "skills/skill-builder", - "source_hash": "775e73f3ebe87960a245790645cb2ba9083f18e0c8d0ee60569e6b747529b830", - "generated_hash": "9ab3978eca5991203690b4c5ae36c75b12e2d860002d8ccff5e4ceb5cdfdc6c5" + "source_hash": "e7383dd770bef7595bf2e6db5dff94ef0ba74eff77e5c4db9ad750084eedbb4c", + "generated_hash": "b401b04fc6bbf86ccbdc67e882a7d6831925bb4768a2eaf61de359a83b976ae1" }, { "name": "standards", "source_skill": "skills/standards", - "source_hash": "02caa3fe0a55bfbe46c068323cf91d8651e508fe07c000d011ae0f3fb5078d44", - "generated_hash": "72d77ad41f4fbf98f0cd43dc26abf3b4a5cae88c995c860f74ae209150cfbeb1" + "source_hash": "d493ba253d8eaafcd19c3e62b98328a535f3a8d9720bd17a144fa77b1c369268", + "generated_hash": "bec8e7c8b82b76ea771c6264783e4561e7d9eec61f52a7415e196f018cc8de2c" }, { "name": "status", @@ -639,7 +639,7 @@ { "name": "using-flywheel", "source_skill": "skills/using-flywheel", - "source_hash": "73efa5c718907ace8722a180162e24d08fb7721a42cd601a234ac83e9c523424", + "source_hash": "876ddaec0998ca32f917b49101ffaed1248ac1c676deb985914efaea342cbae5", "generated_hash": "ccc09a1240f753a136cf1e337d60ebd03c7dc705224720a1421dbedc7452d797" }, { diff --git a/skills-codex/agent-mail/.agentops-generated.json b/skills-codex/agent-mail/.agentops-generated.json index 0cc4a86f1..86e547eab 100644 --- a/skills-codex/agent-mail/.agentops-generated.json +++ b/skills-codex/agent-mail/.agentops-generated.json @@ -2,6 +2,6 @@ "generator": "codex-sync", "source_skill": "skills/agent-mail", "layout": "modular", - "source_hash": "ffa68ba868e41b84bafb514842d2e870b551dccb86dcf6579549ba9e9897ecbf", - "generated_hash": "b83599604595f8914ee7845458711c61ce4f04b4d6d185fe61aedbf0aae8f568" + "source_hash": "f21a53e9da7bfc6047763d1a17552de25eb8d48f1a4aab6c4adb66f5f81118a5", + "generated_hash": "86fedf4d643ca4b375305c846cbbb6d8883116bf74b6990b3e63975592e598f2" } diff --git a/skills-codex/agent-mail/SKILL.md b/skills-codex/agent-mail/SKILL.md index 946cb7367..e69fafc20 100644 --- a/skills-codex/agent-mail/SKILL.md +++ b/skills-codex/agent-mail/SKILL.md @@ -29,7 +29,9 @@ changes are the caller's call. create work ownership or affect Plan, Candidate, or verdict semantics. - Mail silence proves nothing about work status. - A message or acknowledgement is evidence that communication occurred, not - evidence that a change is correct or complete. The adapter cannot select AgentOps semantics, issue a binding verdict, or turn factory completion into delivery or validation proof. + evidence that a change is correct or complete. The adapter cannot select + AgentOps semantics, issue a binding verdict, or turn factory completion into + delivery or validation proof. - Release a reservation, including any `force_release`, only on the caller's explicit request for that exact reservation. Force-release has no autonomous trigger; a conflict is reported, not force-cleared. @@ -54,20 +56,35 @@ Two disjoint surfaces; do not reach the second from the first: ## Surfaces +Choose exactly one mailbox owner and access mode for each storage root. When an +HTTP/MCP daemon owns the root, use its MCP tools; do not point the direct `am` +CLI at the same database. Use the CLI fallback only with a root not owned by a +running Agent Mail runtime. A busy mailbox activity lock or a bounded read +timeout is a degraded adapter result, not permission to restart the service, +repair the database, or silently switch roots. + Use the MCP tools when they are present. Otherwise use the self-describing `am` -CLI. Discover current syntax with `am mail --help`, -`am file_reservations --help`, and related group help; do not infer commands -from remembered aliases. +CLI. Pin the intended storage root explicitly, and discover current syntax with +`am mail --help`, `am file_reservations --help`, and related group help; do not +infer commands from remembered aliases. If a direct macOS read rejects a +symlinked snapshot directory such as `/var`, use a caller-scoped, non-symlinked +temporary directory for that isolated invocation or report the adapter +degraded; never weaken the traversal check. ## One-shot use 1. Confirm that multiple explicitly coordinated writers share the repository. -2. Register the caller-supplied identity against the same absolute project path. -3. Reserve only the supplied paths, with a bounded TTL. -4. Report conflicts without waiting, narrowing scope, or changing the plan. -5. Send the supplied message once and record its id. -6. Read or acknowledge only the requested thread. -7. Release only reservations the caller explicitly asks to release. +2. Freeze one storage root and either MCP/server mode or direct-CLI mode; never + mix both against the same live database. +3. Register the caller-supplied identity against the same absolute project path. +4. Reserve only the supplied paths, with a bounded TTL. +5. Report conflicts without waiting, narrowing scope, or changing the plan. +6. Send the supplied message once and record its id. +7. Read or acknowledge only the requested thread. +8. Before the caller advances a declared transition, verify every + acknowledgement-required message in that transition has the intended + recipient acknowledgement. Later traffic is not an implicit acknowledgement. +9. Release only reservations the caller explicitly asks to release. ## Output @@ -82,6 +99,12 @@ Terminal outcomes are explicit, never silent: hand-written coordination or treat the absence as "no conflicts". - **Reservation conflict** — report the conflicting reservation as-is; do not narrow, widen, renew, or force-release it. +- **Mailbox ownership conflict** — a daemon and direct CLI contend for one + storage root: report the lock owner/mode and stop; do not restart, repair, or + bypass the lock as a coordination side effect. +- **Required acknowledgement pending** — report the exact message and intended + recipient and stop the dependent transition. Do not infer acknowledgement + from a later reply or repair it after validation. - **Timeout / degraded surface** — report the operation as timed out or degraded with what was and was not observed; a timeout is evidence, not "done". - **Cleanup** — reservations released this session are listed by id; any left diff --git a/skills-codex/agent-mail/references/RECOVERY.md b/skills-codex/agent-mail/references/RECOVERY.md index abe310ecf..de8bedf76 100644 --- a/skills-codex/agent-mail/references/RECOVERY.md +++ b/skills-codex/agent-mail/references/RECOVERY.md @@ -230,6 +230,8 @@ am acks remind /abs/path/project GreenCastle --min-age-minutes 30 |---------|-----------|-----| | Stale reservations accumulating | Agent crashed without releasing | `doctor repair --yes` | | FTS search returns wrong results | Index out of sync | `doctor repair --yes` | -| "database is locked" | Concurrent access issue | Restart server, retry | +| "database is locked" | Another runtime may own or be actively using the selected storage root | Identify the owner and use that root's frozen access mode; report degraded if it remains busy, and do not restart the server as a coordination side effect | +| "mailbox activity lock is busy" | A daemon or another direct runtime owns the same storage root | Use the running daemon through MCP, or a separately authorized isolated CLI root; do not restart or repair as a coordination side effect | +| "refusing to traverse symlinked snapshot directory /var" on macOS | Direct-read snapshot temporary path resolves through macOS's `/var` symlink | For an isolated invocation, set `TMPDIR` to a non-symlinked caller-scoped temporary root; do not disable traversal protection | | Corrupted git archive | Interrupted write | Restore from backup | | Server won't start | Port conflict | `config set-port 9000` | diff --git a/skills-codex/codebase-recon/.agentops-generated.json b/skills-codex/codebase-recon/.agentops-generated.json index ca40993c4..a8f0ec1fc 100644 --- a/skills-codex/codebase-recon/.agentops-generated.json +++ b/skills-codex/codebase-recon/.agentops-generated.json @@ -2,6 +2,6 @@ "generator": "codex-sync", "source_skill": "skills/codebase-recon", "layout": "modular", - "source_hash": "65da8420795f88d4d3f20da8e93585f8af34f619ce1d8b523fa8d4136deb0376", - "generated_hash": "eccd4073aace12f96c7ec62a7f01c21a821ef7d27658b60958abb8dea524bde1" + "source_hash": "2ee84ce6be213b02db860d9206e0a6307a53c5718c75311a2e69830b89ebd327", + "generated_hash": "26238e641ee39ee94d39dc3ddd21726d1db2c7e26e689c2092f217f3abda108e" } diff --git a/skills-codex/codebase-recon/SKILL.md b/skills-codex/codebase-recon/SKILL.md index aef36f303..f0b4683ff 100644 --- a/skills-codex/codebase-recon/SKILL.md +++ b/skills-codex/codebase-recon/SKILL.md @@ -45,7 +45,10 @@ leads with. Pattern packaging beyond evidence pointers belongs in ## Workflow 1. Record the current commit and the repository's local source-of-truth - precedence. Search for a prior recon pack before starting. + precedence. Search for validated prior manifests before starting with + `skills/codebase-recon/scripts/validate-output.sh --repo-root --discover-priors`. + Successful empty output means no prior pack exists at either documented + default. 2. If no prior pack exists, use `baseline` mode. If one exists, verify its still-valid claims against the current commit and use `delta` mode. Preserve valid evidence by reference and describe only changed paths and synthesis. @@ -87,12 +90,15 @@ The durable output doc earns its keep only if a future reader can re-verify a claim without redoing the recon. Every `fact` cites file:line; every `inference` cites the file:line facts it rests on. A claim that cannot be cited is downgraded to `unknown` before the report ships — never shipped -uncited at its original confidence. The manifest validator accepts a bare file -path (it requires the path resolve to an existing regular file, so a bare -directory is rejected as a coverage gap), but does not require the line number; -hold the companion report to the stricter floor: a path without a line is a -pointer to homework, not a citation, and counts as a coverage gap in the -report's own terms. +uncited at its original confidence. The manifest validator checks citations +against the exact Git commit declared by that manifest. They must be safe +repository-relative regular-file paths; artifact-local and external paths are +rejected because this schema has no digest field for those bytes. A supplied +line number must exist in the committed blob. The validator also resolves each +representative flow path at that commit. It does not require every citation to +carry a line number; hold the companion report to the stricter floor: a path +without a line is a pointer to homework, not a citation, and counts as a +coverage gap in the report's own terms. When reconstructing a repository other than the one that ships this skill, pass `--repo-root ` to the validator so evidence resolves against the target @@ -105,17 +111,47 @@ tree rather than the skill's own checkout. `codebase-recon.md` in the same directory. - **Format:** `codebase-recon.v1` JSON manifest plus an evidence-cited Markdown report covering the same commit, mode, flows, claims, and scope boundaries. + The manifest's `report` object names `codebase-recon.md` and binds its + lowercase SHA-256. The report carries one + `` marker plus `manifest_commit`, + `manifest_mode`, `flows_sha256`, `claims_sha256`, and `coverage_sha256` + markers computed from canonical compact sorted JSON for those sections. - **Validation command:** `skills/codebase-recon/scripts/validate-output.sh ` - validates the machine-readable manifest; the cited Markdown report remains - its human-readable companion. + snapshots and validates both artifacts, then rechecks their identities and + the repository HEAD/index/worktree before returning. - **Downstream handoff:** pass both validated artifact paths to the requesting research, planning, review, or documentation workflow; the consumer owns any decision or code-change plan. -Baseline manifests carry at least one complete entry-to-test flow. Delta -manifests name an existing prior recon, prove `baseline_verified: true`, and -describe at least one changed path. Every manifest lists both inspected and -uninspected scope. +### Earlier default compatibility + +Packs already stored under `.agents/recon//` remain in place. The +validator's `--discover-priors` mode enumerates validated +`codebase-recon.json` manifests under both that legacy root and the current +scratch root. Record the selected manifest's exact path in `prior_recon`; delta +validation re-validates the cited manifest and its prior chain instead of +accepting a path merely because it exists. New packs use the current default +unless the caller supplies a different path. Never move, copy, or delete an +earlier pack merely to make its directory match the new state tier, because +that would obscure the identity a delta cites. Downstream consumers use the +exact returned artifact paths rather than scanning only one default root. An +earlier pack without a digest-bound companion report remains untouched but is +not returned as validated prior evidence under the current contract. + +Baseline manifests carry at least one complete entry-to-test flow. A manifest +being handed off must name the target repository's current `HEAD` by its full +object-format OID; abbreviations and hex-looking refs are rejected. Historical +manifests cited as priors must likewise carry full immutable commit OIDs that +resolve in that repository. +Delta manifests name an existing prior recon, set `baseline_verified: true`, +and list exactly the paths in Git's prior-commit-to-current-commit diff. The +validator derives those facts rather than trusting the boolean or path list. +It also refuses dirty tracked, staged, or untracked source state outside +`.agents/`, because those bytes are not bound by the declared commit. Every +manifest lists both inspected and uninspected scope. Manifests and companions +must be real regular files, are read from one snapshot, and are rechecked along +with HEAD and source status after validation so a mid-run swap cannot earn a +green result for different bytes. The validator is the machine boundary: @@ -123,17 +159,24 @@ The validator is the machine boundary: skills/codebase-recon/scripts/validate-output.sh ``` -Evidence entries are existing file paths, optionally followed by a line number. -Delta manifests require an existing prior pack, `baseline_verified: true`, and -at least one described change. +Evidence entries are repository-relative files at the manifest's commit, +optionally followed by a line number. +Delta manifests require a valid prior `codebase-recon.json` chain, an ancestor +commit, `baseline_verified: true`, and an exact changed-path match to the Git +diff ending at current `HEAD`. Enumerate validated manifests at both documented +defaults with: + +```bash +skills/codebase-recon/scripts/validate-output.sh --repo-root --discover-priors +``` Executable behavior: [references/codebase-recon.feature](references/codebase-recon.feature). ## Quality -- Every fact and inference resolves to existing evidence; unknowns remain - visibly typed and never masquerade as established behavior. +- Every fact and inference resolves to evidence in the manifest's exact commit; + unknowns remain visibly typed and never masquerade as established behavior. - Representative flows reach entry, domain, integration, and test surfaces, while inspected and uninspected scope stay explicit. - The named validator passes before the JSON manifest and companion report are diff --git a/skills-codex/codebase-recon/references/codebase-recon.feature b/skills-codex/codebase-recon/references/codebase-recon.feature index 1408c6f99..32326b5fa 100644 --- a/skills-codex/codebase-recon/references/codebase-recon.feature +++ b/skills-codex/codebase-recon/references/codebase-recon.feature @@ -4,12 +4,29 @@ Feature: Evidence-bounded repository reconstruction Scenario: A baseline explains representative repository flows Given repository precedence and the current commit are known When entry, domain, integration, and test paths are traced - Then material claims are typed and cited + Then material claims are typed and cited against that exact commit And inspected and uninspected scope are explicit @covered-by:tests/scripts/agentops-native-skills.bats::delta Scenario: A later run preserves a verified baseline Given an earlier recon pack exists When the repository is reconstructed again - Then the earlier baseline is checked against the current commit - And the new artifact records a delta instead of replacing valid evidence + Then the cited prior manifest chain passes the recon validator + And the earlier commit is an ancestor of the current repository HEAD + And the new artifact's changed paths equal the Git diff between those commits + And dirty source bytes outside the declared commits are rejected + + @covered-by:tests/scripts/agentops-native-skills.bats::prior-discovery + Scenario: Current and earlier default packs are discoverable + Given validated prior manifests under .agents/scratch/codebase-recon and .agents/recon + When prior discovery runs + Then both manifests are returned at their existing paths + And an invalid manifest is never accepted as a delta's prior pack + + @covered-by:tests/scripts/agentops-native-skills.bats::companion + Scenario: The manifest and human report are one stable evidence pack + Given codebase-recon.json binds codebase-recon.md by SHA-256 + And the report binds the manifest commit, mode, flows, claims, and coverage + When validation runs over immutable snapshots of both files + Then a missing mismatched or symlinked companion is rejected + And a manifest, report, HEAD, index, or worktree change before return is rejected diff --git a/skills-codex/codebase-recon/scripts/validate-output.sh b/skills-codex/codebase-recon/scripts/validate-output.sh index 9fe608433..2d9d83b1d 100755 --- a/skills-codex/codebase-recon/scripts/validate-output.sh +++ b/skills-codex/codebase-recon/scripts/validate-output.sh @@ -9,7 +9,7 @@ set -euo pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" usage() { - echo "usage: $0 [--repo-root ] " >&2 + echo "usage: $0 [--repo-root ] [--discover-priors | ]" >&2 } # Evidence paths in a recon manifest are relative to the repository being @@ -18,6 +18,7 @@ usage() { # to the skill's own checkout for the in-repo self-test case. repo_root="" artifact="" +discover_priors=0 while [[ $# -gt 0 ]]; do case "$1" in --repo-root) @@ -26,6 +27,7 @@ while [[ $# -gt 0 ]]; do repo_root="$1" ;; --repo-root=*) repo_root="${1#--repo-root=}" ;; + --discover-priors) discover_priors=1 ;; -h|--help) usage; exit 0 ;; -*) echo "unknown flag: $1" >&2; usage; exit 2 ;; *) @@ -39,7 +41,12 @@ while [[ $# -gt 0 ]]; do shift done -if [[ -z "$artifact" || ! -f "$artifact" ]]; then +if [[ "$discover_priors" == "1" && -n "$artifact" ]]; then + echo "--discover-priors does not accept an artifact" >&2 + usage + exit 2 +fi +if [[ "$discover_priors" != "1" && ( -z "$artifact" || ! -f "$artifact" || -L "$artifact" ) ]]; then usage exit 2 fi @@ -52,59 +59,116 @@ if [[ ! -d "$repo_root" ]]; then exit 2 fi repo_root="$(cd "$repo_root" && pwd -P)" -artifact_dir="$(cd "$(dirname "$artifact")" && pwd -P)" -jq -e ' - def text: type == "string" and length > 0; - .schema_version == "codebase-recon.v1" - and (.mode == "baseline" or .mode == "delta") - and (.commit | text) - and (.flows - | type == "array" - and all(.[]; - (.entry | text) - and (.domain | text) - and (.integration | text) - and (.tests | text))) - and (.claims - | type == "array" - and all(.[]; - (.kind == "fact" or .kind == "inference" or .kind == "unknown") - and (.text | text) - and (.confidence == "high" or .confidence == "medium" or .confidence == "low") - and (.evidence | type == "array" and all(.[]; text)) - and (if .kind == "unknown" then true else (.evidence | length > 0) end))) - and (.coverage | type == "object") - and (.coverage.inspected | type == "array" and length > 0 and all(.[]; text)) - and (.coverage.uninspected | type == "array" and length > 0 and all(.[]; text)) - and ( - if .mode == "baseline" then - (.flows | length > 0) - and ((has("prior_recon") | not) or .prior_recon == "" or .prior_recon == null) - else - (.prior_recon | text) - and .baseline_verified == true - and (.delta - | type == "array" and length > 0 - and all(.[]; (.path | text) and (.change | text))) - end - ) -' "$artifact" >/dev/null || { - echo "invalid codebase-recon.v1 artifact: $artifact" >&2 - exit 1 +snapshot_root="$(mktemp -d "${TMPDIR:-/tmp}/codebase-recon-validate.XXXXXX")" +cleanup() { + rm -rf -- "$snapshot_root" +} +trap cleanup EXIT HUP INT TERM + +declare -a watched_sources=() +declare -a watched_identities=() +declare -a watched_hashes=() +snapshot_counter=0 + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi } -# resolve_evidence CANDIDATE MUST_BE_FILE -# MUST_BE_FILE=1 → claim evidence: the contract is "existing file paths, -# optionally followed by a line number", so the resolved path must be a -# regular file. A directory is a coverage gap, not a citation, and no longer -# passes silently (the old `-e` accepted directories). -# MUST_BE_FILE=0 → prior-recon pack reference: any existing path resolves. -resolve_evidence() { - local candidate="$1" must_file="$2" - if [[ "$candidate" =~ ^(.+):[0-9]+$ ]]; then - candidate="${BASH_REMATCH[1]}" +sha256_stream() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + else + shasum -a 256 | awk '{print $1}' fi +} + +file_identity() { + if stat -f '%d:%i:%z:%m' "$1" >/dev/null 2>&1; then + stat -f '%d:%i:%z:%m' "$1" + else + stat -c '%d:%i:%s:%Y' "$1" + fi +} + +# Snapshot each manifest/report exactly once. cp -P copies a raced-in symlink as +# a symlink rather than following it; the destination type check then fails. +watch_regular_file() { + local source="$1" label="$2" before after source_hash snapshot_hash snapshot + [[ -f "$source" && ! -L "$source" ]] || { + echo "$label must be a real regular file: $source" >&2 + return 1 + } + before="$(file_identity "$source")" || return 1 + snapshot_counter=$((snapshot_counter + 1)) + snapshot="$snapshot_root/$snapshot_counter" + cp -P -- "$source" "$snapshot" + [[ -f "$snapshot" && ! -L "$snapshot" ]] || { + echo "$label changed shape while being snapshotted: $source" >&2 + return 1 + } + after="$(file_identity "$source")" || return 1 + [[ "$before" == "$after" ]] || { + echo "$label changed identity while being snapshotted: $source" >&2 + return 1 + } + source_hash="$(sha256_file "$source")" + snapshot_hash="$(sha256_file "$snapshot")" + [[ "$source_hash" == "$snapshot_hash" ]] || { + echo "$label changed bytes while being snapshotted: $source" >&2 + return 1 + } + watched_sources+=("$source") + watched_identities+=("$before") + watched_hashes+=("$snapshot_hash") + WATCHED_SNAPSHOT="$snapshot" +} + +recheck_watched_files() { + local i source + for ((i = 0; i < ${#watched_sources[@]}; i++)); do + source="${watched_sources[$i]}" + [[ -f "$source" && ! -L "$source" ]] || { + echo "validated artifact changed shape during validation: $source" >&2 + return 1 + } + [[ "$(file_identity "$source")" == "${watched_identities[$i]}" ]] || { + echo "validated artifact changed identity during validation: $source" >&2 + return 1 + } + [[ "$(sha256_file "$source")" == "${watched_hashes[$i]}" ]] || { + echo "validated artifact changed bytes during validation: $source" >&2 + return 1 + } + done +} + +repo_head_initial="$(git -C "$repo_root" rev-parse --verify 'HEAD^{commit}' 2>/dev/null || true)" +repo_status_initial="$(git -C "$repo_root" status --porcelain=v1 --untracked-files=all -- . ':(exclude).agents' 2>/dev/null || true)" + +recheck_repo_state() { + local current_head current_status + current_head="$(git -C "$repo_root" rev-parse --verify 'HEAD^{commit}' 2>/dev/null || true)" + current_status="$(git -C "$repo_root" status --porcelain=v1 --untracked-files=all -- . ':(exclude).agents' 2>/dev/null || true)" + [[ -n "$repo_head_initial" && "$current_head" == "$repo_head_initial" ]] || { + echo "target repository HEAD changed during validation" >&2 + return 1 + } + [[ "$current_status" == "$repo_status_initial" && -z "$current_status" ]] || { + echo "target repository index or worktree changed during validation" >&2 + return 1 + } +} + +# Resolve the prior manifest's exact path. Unlike evidence citations, a prior +# reference has no :LINE syntax: silently stripping such a suffix would accept +# a different path than the manifest declared. +resolve_prior_manifest() { + local candidate="$1" artifact_dir="$2" local -a roots=() if [[ "$candidate" = /* ]]; then roots=("$candidate") @@ -113,28 +177,358 @@ resolve_evidence() { fi local p for p in "${roots[@]}"; do - if [[ "$must_file" == "1" ]]; then - [[ -f "$p" ]] && return 0 - else - [[ -e "$p" ]] && return 0 - fi + [[ -f "$p" ]] && { printf '%s\n' "$p"; return 0; } done return 1 } -while IFS= read -r evidence; do - if ! resolve_evidence "$evidence" 1; then - echo "missing or non-file claim evidence: $evidence" >&2 - exit 1 +# resolve_manifest_commit ARTIFACT +# +# A manifest's commit is evidence only when it resolves to an immutable commit +# in the target repository. Symbolic names such as HEAD are deliberately +# rejected because their meaning changes after the artifact is written. +resolve_manifest_commit() { + local manifest="$1" declared declared_normalized resolved resolved_normalized object_format oid_length + declared="$(jq -r '.commit // empty' "$manifest")" + if ! object_format="$(git -C "$repo_root" rev-parse --show-object-format=storage 2>/dev/null)"; then + object_format="$(git -C "$repo_root" rev-parse --show-object-format 2>/dev/null)" || { + echo "could not determine target repository object format" >&2 + return 1 + } fi -done < <(jq -r '.claims[] | select(.kind == "fact" or .kind == "inference") | .evidence[]' "$artifact") + object_format="${object_format%%$'\n'*}" + case "$object_format" in + sha1) oid_length=40 ;; + sha256) oid_length=64 ;; + *) echo "unsupported target repository object format: $object_format" >&2; return 1 ;; + esac + if [[ ! "$declared" =~ ^[0-9a-fA-F]{$oid_length}$ ]]; then + echo "manifest commit is not a full $object_format object id: $declared" >&2 + return 1 + fi + if ! resolved="$(git -C "$repo_root" rev-parse --verify "${declared}^{commit}" 2>/dev/null)"; then + echo "manifest commit does not resolve in target repository: $declared" >&2 + return 1 + fi + declared_normalized="$(printf '%s' "$declared" | tr '[:upper:]' '[:lower:]')" + resolved_normalized="$(printf '%s' "$resolved" | tr '[:upper:]' '[:lower:]')" + if [[ "$resolved_normalized" != "$declared_normalized" ]]; then + echo "manifest commit resolved through a mutable or abbreviated name: $declared" >&2 + return 1 + fi + printf '%s\n' "$resolved_normalized" +} -if [[ "$(jq -r '.mode' "$artifact")" == "delta" ]]; then - prior="$(jq -r '.prior_recon' "$artifact")" - if ! resolve_evidence "$prior" 0; then - echo "missing prior recon pack: $prior" >&2 - exit 1 +# resolve_repo_path_at_commit CITATION COMMIT MUST_BE_FILE +# +# Fact/inference evidence belongs to the repository commit named by the +# manifest, never to whichever bytes happen to be in the current worktree or +# beside the artifact. A trailing :LINE is checked against that committed blob. +resolve_repo_path_at_commit() { + local citation="$1" commit="$2" must_file="$3" candidate="$1" line="" line_number="" + local tree_entry mode object_type object_id line_count + if [[ "$candidate" =~ ^(.+):([0-9]+)$ ]]; then + candidate="${BASH_REMATCH[1]}" + line="${BASH_REMATCH[2]}" fi + while [[ "$candidate" == ./* ]]; do candidate="${candidate#./}"; done + if [[ -z "$candidate" || "$candidate" == "." || "$candidate" = /* || "$candidate" == */ || "$candidate" == ".." || "$candidate" == ../* || "$candidate" == */../* || "$candidate" == */.. ]]; then + echo "evidence citation is not a safe repository-relative path: $citation" >&2 + return 1 + fi + if ! tree_entry="$(git -C "$repo_root" ls-tree "$commit" -- ":(literal)$candidate")" || [[ -z "$tree_entry" ]]; then + echo "evidence path is absent from manifest commit: $citation" >&2 + return 1 + fi + read -r mode object_type object_id _ <<<"$tree_entry" + if [[ "$must_file" == "1" && ( "$mode" != 100* || "$object_type" != "blob" ) ]]; then + echo "evidence citation is not a regular file in manifest commit: $citation" >&2 + return 1 + fi + if [[ -n "$line" ]]; then + line_number=$((10#$line)) + if [[ "$must_file" != "1" || "$line_number" -lt 1 ]]; then + echo "invalid evidence line citation: $citation" >&2 + return 1 + fi + if ! line_count="$(git -C "$repo_root" cat-file blob "$object_id" | awk 'END { print NR }')"; then + echo "could not read evidence blob from manifest commit: $citation" >&2 + return 1 + fi + if (( line_number > line_count )); then + echo "evidence line is outside committed blob: $citation" >&2 + return 1 + fi + fi + printf '%s\n' "$candidate" +} + +require_clean_source_tree() { + local source_status + if ! source_status="$(git -C "$repo_root" status --porcelain=v1 --untracked-files=all -- . ':(exclude).agents' 2>&1)"; then + echo "could not inspect target repository worktree: $source_status" >&2 + return 1 + fi + if [[ -n "$source_status" ]]; then + echo "target repository has source changes not bound by the manifest commit:" >&2 + printf '%s\n' "$source_status" >&2 + return 1 + fi +} + +require_report_marker() { + local report="$1" key="$2" expected="$3" count + count="$(grep -Fxc "$key: $expected" "$report" || true)" + if [[ "$count" != "1" ]]; then + echo "companion report must contain exactly one '$key: $expected' marker" >&2 + return 1 + fi +} + +validate_companion_report() { + local manifest="$1" manifest_dir="$2" report_rel report_source report_snapshot declared_sha actual_sha + local commit mode flows_sha claims_sha coverage_sha + report_rel="$(jq -r '.report.path // empty' "$manifest")" + declared_sha="$(jq -r '.report.sha256 // empty' "$manifest")" + if [[ "$report_rel" != "codebase-recon.md" || ! "$declared_sha" =~ ^[0-9a-f]{64}$ ]]; then + echo "manifest must bind companion report codebase-recon.md by lowercase SHA-256" >&2 + return 1 + fi + report_source="$manifest_dir/$report_rel" + if ! watch_regular_file "$report_source" "companion codebase-recon report"; then + return 1 + fi + report_snapshot="$WATCHED_SNAPSHOT" + actual_sha="$(sha256_file "$report_snapshot")" + if [[ "$actual_sha" != "$declared_sha" ]]; then + echo "companion report digest does not match manifest: $report_source" >&2 + return 1 + fi + + commit="$(jq -r '.commit' "$manifest")" + mode="$(jq -r '.mode' "$manifest")" + flows_sha="$(jq -cS '.flows' "$manifest" | sha256_stream)" + claims_sha="$(jq -cS '.claims' "$manifest" | sha256_stream)" + coverage_sha="$(jq -cS '.coverage' "$manifest" | sha256_stream)" + grep -Fqx '' "$report_snapshot" || { + echo "companion report lacks codebase-recon-report.v1 identity marker" >&2 + return 1 + } + require_report_marker "$report_snapshot" manifest_commit "$commit" || return 1 + require_report_marker "$report_snapshot" manifest_mode "$mode" || return 1 + require_report_marker "$report_snapshot" flows_sha256 "$flows_sha" || return 1 + require_report_marker "$report_snapshot" claims_sha256 "$claims_sha" || return 1 + require_report_marker "$report_snapshot" coverage_sha256 "$coverage_sha" || return 1 +} + +# validate_artifact ARTIFACT DEPTH STACK REQUIRE_CURRENT_HEAD +# +# Delta manifests form a provenance chain. Validate every cited manifest in +# that chain, with a bounded depth and cycle check, before accepting the leaf. +# STACK is a newline-delimited list of normalized artifact paths. +# REQUIRE_CURRENT_HEAD=1 is used for the artifact the caller is validating; +# recursively cited/discovered historical manifests need only resolve in the +# repository because their commit is expected to predate HEAD. +validate_artifact() { + local current_input="$1" depth="$2" stack="$3" require_current_head="$4" + if [[ ! -f "$current_input" || -L "$current_input" ]]; then + echo "missing codebase-recon.v1 artifact: $current_input" >&2 + return 1 + fi + + if (( depth > 32 )); then + echo "prior recon chain exceeds 32 manifests: $current_input" >&2 + return 1 + fi + + local current_dir current_source current + current_dir="$(cd "$(dirname "$current_input")" && pwd -P)" + current_source="$current_dir/$(basename "$current_input")" + case $'\n'"$stack"$'\n' in + *$'\n'"$current_source"$'\n'*) + echo "cyclic prior recon chain: $current_source" >&2 + return 1 + ;; + esac + + local next_stack + if [[ -n "$stack" ]]; then + next_stack="$stack"$'\n'"$current_source" + else + next_stack="$current_source" + fi + + if ! watch_regular_file "$current_source" "codebase-recon manifest"; then + return 1 + fi + current="$WATCHED_SNAPSHOT" + + jq -e ' + def text: type == "string" and length > 0; + def path_text: text and (test("[\u0000-\u001f\u007f]") | not); + .schema_version == "codebase-recon.v1" + and (.mode == "baseline" or .mode == "delta") + and (.commit | text) + and (.flows + | type == "array" + and all(.[]; + (.entry | path_text) + and (.domain | path_text) + and (.integration | path_text) + and (.tests | path_text))) + and (.claims + | type == "array" + and all(.[]; + (.kind == "fact" or .kind == "inference" or .kind == "unknown") + and (.text | text) + and (.confidence == "high" or .confidence == "medium" or .confidence == "low") + and (.evidence | type == "array" and all(.[]; path_text)) + and (if .kind == "unknown" then true else (.evidence | length > 0) end))) + and (.coverage | type == "object") + and (.coverage.inspected | type == "array" and length > 0 and all(.[]; text)) + and (.coverage.uninspected | type == "array" and length > 0 and all(.[]; text)) + and (.report | type == "object") + and (.report.path == "codebase-recon.md") + and (.report.sha256 | type == "string" and test("^[0-9a-f]{64}$")) + and ( + if .mode == "baseline" then + (.flows | length > 0) + and ((has("prior_recon") | not) or .prior_recon == "" or .prior_recon == null) + else + (.prior_recon | path_text) + and .baseline_verified == true + and (.delta + | type == "array" and length > 0 + and all(.[]; (.path | path_text) and (.change | text))) + end + ) + ' "$current" >/dev/null || { + echo "invalid codebase-recon.v1 artifact: $current_source" >&2 + return 1 + } + if ! validate_companion_report "$current" "$current_dir"; then + echo "invalid companion report for: $current_source" >&2 + return 1 + fi + + if ! git -C "$repo_root" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "target is not a git repository: $repo_root" >&2 + return 1 + fi + if ! require_clean_source_tree; then + return 1 + fi + + local current_commit + if ! current_commit="$(resolve_manifest_commit "$current")"; then + return 1 + fi + if [[ "$require_current_head" == "1" ]]; then + local target_head + if ! target_head="$(git -C "$repo_root" rev-parse --verify 'HEAD^{commit}' 2>/dev/null)"; then + echo "target repository has no current commit: $repo_root" >&2 + return 1 + fi + if [[ "$current_commit" != "$target_head" ]]; then + echo "manifest commit is not the target repository's current commit: $(jq -r '.commit' "$current")" >&2 + return 1 + fi + fi + + local evidence + while IFS= read -r evidence; do + if ! resolve_repo_path_at_commit "$evidence" "$current_commit" 1 >/dev/null; then + echo "invalid or unbound claim evidence: $evidence" >&2 + return 1 + fi + done < <(jq -r '.claims[] | select(.kind == "fact" or .kind == "inference") | .evidence[]' "$current") + + local flow_path + while IFS= read -r flow_path; do + if ! resolve_repo_path_at_commit "$flow_path" "$current_commit" 1 >/dev/null; then + echo "invalid or unbound flow file: $flow_path" >&2 + return 1 + fi + done < <(jq -r '.flows[] | .entry, .tests' "$current") + while IFS= read -r flow_path; do + if ! resolve_repo_path_at_commit "$flow_path" "$current_commit" 0 >/dev/null; then + echo "invalid or unbound flow path: $flow_path" >&2 + return 1 + fi + done < <(jq -r '.flows[] | .domain, .integration' "$current") + + if [[ "$(jq -r '.mode' "$current")" == "delta" ]]; then + local prior prior_path prior_commit + prior="$(jq -r '.prior_recon' "$current")" + if ! prior_path="$(resolve_prior_manifest "$prior" "$current_dir")"; then + echo "missing or non-file prior recon pack: $prior" >&2 + return 1 + fi + if ! validate_artifact "$prior_path" "$((depth + 1))" "$next_stack" 0; then + echo "invalid prior recon pack: $prior" >&2 + return 1 + fi + prior_commit="$VALIDATED_COMMIT" + if ! git -C "$repo_root" merge-base --is-ancestor "$prior_commit" "$current_commit"; then + echo "prior recon commit is not an ancestor of manifest commit: $prior" >&2 + return 1 + fi + + local declared_delta actual_delta declared_count unique_count + declared_delta="$(jq -r '.delta[].path' "$current" | LC_ALL=C sort -u)" + declared_count="$(jq -r '.delta | length' "$current")" + unique_count="$(printf '%s\n' "$declared_delta" | sed '/^$/d' | wc -l | tr -d ' ')" + if [[ "$declared_count" != "$unique_count" ]]; then + echo "delta contains duplicate changed paths: $current" >&2 + return 1 + fi + if ! actual_delta="$(git -C "$repo_root" diff --name-only --diff-filter=ACDMRTUXB "$prior_commit" "$current_commit" -- | LC_ALL=C sort -u)"; then + echo "could not derive repository delta for $current" >&2 + return 1 + fi + if [[ "$declared_delta" != "$actual_delta" ]]; then + echo "declared delta paths do not match git diff ${prior_commit}..${current_commit}" >&2 + return 1 + fi + fi + VALIDATED_COMMIT="$current_commit" +} + +discover_valid_priors() { + local -a candidates=() + shopt -s nullglob + candidates+=("$repo_root"/.agents/scratch/codebase-recon/*/codebase-recon.json) + candidates+=("$repo_root"/.agents/recon/*/codebase-recon.json) + shopt -u nullglob + + if [[ "${#candidates[@]}" -eq 0 ]]; then + return 0 + fi + + local candidate found=0 + while IFS= read -r candidate; do + if validate_artifact "$candidate" 0 "" 0 >/dev/null 2>&1; then + printf '%s\n' "$candidate" + found=1 + else + echo "ignoring invalid prior recon pack: $candidate" >&2 + fi + done < <(printf '%s\n' "${candidates[@]}" | LC_ALL=C sort) + + if [[ "$found" == "0" ]]; then + echo "no validated prior recon packs found under current or earlier default roots" >&2 + return 1 + fi +} + +if [[ "$discover_priors" == "1" ]]; then + discover_valid_priors + recheck_watched_files + recheck_repo_state + exit $? fi +validate_artifact "$artifact" 0 "" 1 +recheck_watched_files +recheck_repo_state echo "valid codebase-recon.v1: $artifact" diff --git a/skills-codex/handoff/.agentops-generated.json b/skills-codex/handoff/.agentops-generated.json index 857b7705c..8e79b787e 100644 --- a/skills-codex/handoff/.agentops-generated.json +++ b/skills-codex/handoff/.agentops-generated.json @@ -2,6 +2,6 @@ "generator": "codex-sync", "source_skill": "skills/handoff", "layout": "modular", - "source_hash": "11f7039b9e9533556fcec30c9dd130bc8889c6dd58b8c70dc0e72e314e927302", - "generated_hash": "e0aac3ade529c89b17ed8b95d49e16c40ab259d0ed278efde882d82e87e1faa3" + "source_hash": "8bf6b07f4db91ac2a7f13b27ee5bee1cd960b6bca7f2b4716ccf904de04561ad", + "generated_hash": "49d4e2099d491b52c7d7440f3d1d70f2ae9cad9850df484f1ffe6605efbb3098" } diff --git a/skills-codex/handoff/SKILL.md b/skills-codex/handoff/SKILL.md index d4cf1b3ff..30e751421 100644 --- a/skills-codex/handoff/SKILL.md +++ b/skills-codex/handoff/SKILL.md @@ -37,4 +37,15 @@ boundary for JSON artifacts under `.agents/ao/handoff/`. The skill may write Markdown when that better serves a human, but the content semantics remain identical. +### Earlier default compatibility + +JSON artifacts already stored under `.agents/handoff/` remain read-only +evidence. `ao session handoff` writes new JSON to `.agents/ao/handoff/`, while +`ao session rehydrate` searches both directories and selects the newest +lexical handoff id; if an identical filename exists in both, the canonical +`.agents/ao/handoff/` copy wins. No command moves or deletes the legacy files. +Human-authored Markdown consumers receive the exact path, so they do not need +to scan either default. This owning skill contract is the compatibility +authority; no separate migration artifact is required. + Return the artifact path and stop. diff --git a/skills-codex/implement/.agentops-generated.json b/skills-codex/implement/.agentops-generated.json index abb1c1aa8..d42426b18 100644 --- a/skills-codex/implement/.agentops-generated.json +++ b/skills-codex/implement/.agentops-generated.json @@ -2,6 +2,6 @@ "generator": "codex-sync", "source_skill": "skills/implement", "layout": "modular", - "source_hash": "5aa7e7b5b988a8f192d64ef7347ea7ca715f4979f325ed0507556ac0ff5ee82d", + "source_hash": "2a01b04555e02e0ccae7d670da6b6db67f26e6123d65103b68eb513f10d5026f", "generated_hash": "8b7f0c9f95ce32043dc9bf287313987027f557f2dbfcec6fbc67e852442d239e" } diff --git a/skills-codex/plan/.agentops-generated.json b/skills-codex/plan/.agentops-generated.json index 741fae8b5..7dec3506e 100644 --- a/skills-codex/plan/.agentops-generated.json +++ b/skills-codex/plan/.agentops-generated.json @@ -2,6 +2,6 @@ "generator": "codex-sync", "source_skill": "skills/plan", "layout": "modular", - "source_hash": "16026fda4df366feaa3c99980f6a49d78d331e8563f8b0a7a4c0f45d0a60338b", + "source_hash": "6133824e806dd44f5c3fcfec39ed46251d951802d573e16fb02ffac9074e23aa", "generated_hash": "4144d7daa581c0352c5cb4231363bd19e99dd4a53dfe1a8daa9baee1a72d8d41" } diff --git a/skills-codex/reverse-engineer/.agentops-generated.json b/skills-codex/reverse-engineer/.agentops-generated.json index 3cd07eb25..067841d27 100644 --- a/skills-codex/reverse-engineer/.agentops-generated.json +++ b/skills-codex/reverse-engineer/.agentops-generated.json @@ -2,6 +2,6 @@ "generator": "codex-sync", "source_skill": "skills/reverse-engineer", "layout": "modular", - "source_hash": "6ce0e7470b88ba775d15bc6a9ccf5aca3937a7238e4a333f2877b0bf35381769", - "generated_hash": "af78e0faf6f44e02ceb7503c9c691e5265f7c740b5d0ba627327b0c6f4722581" + "source_hash": "c4c1d351deda610d6a848b31aca64559d603d065b719a1a91cda3504d7ede98a", + "generated_hash": "e2330646793a9d980ebed4a2d10438ebef90ae790265520a10595192f3e564e8" } diff --git a/skills-codex/reverse-engineer/SKILL.md b/skills-codex/reverse-engineer/SKILL.md index 250ffd2b7..0bbc79cbd 100644 --- a/skills-codex/reverse-engineer/SKILL.md +++ b/skills-codex/reverse-engineer/SKILL.md @@ -32,6 +32,13 @@ Binary mode requires `--authorized` (see Invocation Contract + Self-Test). Use t Map each capability the teardown found onto **our** surfaces. This is the part that turns research into a decision. Emit `.agents/scratch/reverse-engineer//steal-map.md` with a table; every row cites the teardown evidence **and** the matching surface in our repo. +The mechanical script intentionally stops after validating Phase 1. It cannot +truthfully decide whether our live tree has, lacks, or should adopt a capability. +The caller authors `steal-map.md` from the generated registry plus a fresh read +of our repository, then runs the complete-output validator below. A missing or +malformed map is therefore an incomplete skill result, not a script success +silently relabelled as a decision. + | Their capability | Our surface today | Verdict | |---|---|---| | `` | `` | **have** / **gap** / **steal** / **park** / **reject** | @@ -63,11 +70,11 @@ neither strategy grants readiness or continuation authority. ## Invocation Contract -Required: `product_name`. Common flags: `--mode=repo|binary|both`, `--upstream-repo`, `--upstream-ref` (pins the clone to a specific commit/tag/branch; the resolved SHA is recorded in `clone-metadata.json` on any clone), `--output-dir` (default `.agents/scratch/reverse-engineer//`), `--security-audit`, `--materialize-archives` (authorized-only opt-in; embedded-archive extraction is off/index-only by default), `--authorized` (mandatory for binary mode — refuses without it). Full list: `python3 skills/reverse-engineer/scripts/reverse_engineer.py --help`. +Required: `product_name`. Common flags: `--mode=repo|binary|both`, `--upstream-repo`, `--upstream-ref` (requires the selected checkout to be at that exact commit and records its resolved SHA in `clone-metadata.json`), `--local-clone-dir` (selects that exact tree, including a non-Git tree; it never falls back to the caller's checkout), `--output-dir` (default `.agents/scratch/reverse-engineer//`), `--security-audit`, `--materialize-archives` (authorized-only opt-in; embedded-archive extraction is off/index-only by default), `--authorized` (mandatory for binary mode — refuses without it). Full list: `python3 skills/reverse-engineer/scripts/reverse_engineer.py --help`. ## Output Specification -Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry.yaml`, `feature-catalog.md`, `spec-architecture.md`, `spec-code-map.md`, `spec-clone-vs-use.md`, `spec-clone-mvp.md`, plus `spec-cli-surface.md` only when a CLI is detected and `clone-metadata.json` only when the script performs a clone (i.e., `--upstream-repo` is supplied and the target is not already checked out); `--upstream-ref` pins which commit, it is not what triggers the file. Security mode adds `output_dir/security/`: `threat-model.md`, `attack-surface.md`, `dataflow.md`, `crypto-review.md`, `authn-authz.md`, `findings.md`, `reproducibility.md`, `validate-security-audit.sh`. Phase-2: `steal-map.md`. +Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry.yaml`, `feature-catalog.md`, `spec-architecture.md`, `spec-code-map.md`, `spec-clone-vs-use.md`, `spec-clone-mvp.md`, plus `spec-cli-surface.md` only when a CLI is detected. `clone-metadata.json` is written whenever an upstream repo/ref is selected and binds the exact analyzed commit, including an already-present checkout. Security mode adds `output_dir/security/`: `threat-model.md`, `attack-surface.md`, `dataflow.md`, `crypto-review.md`, `authn-authz.md`, `findings.md`, `reproducibility.md`, `validate-security-audit.sh`. Phase-2 adds the caller-authored `steal-map.md`. - **Artifact directory:** the exact `--output-dir`, defaulting to `$REPO/.agents/scratch/reverse-engineer//`. @@ -75,53 +82,40 @@ Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry. files live only in the `security/` child directory. - **Serialization/schema format:** registry is YAML, clone metadata is one JSON object, and inventories/specs/steal-map are nonempty Markdown files. -- **Validator command:** with `$output_dir`, `$security_audit`, `$sbom`, and - `$upstream_ref_set` (each flag `0|1`) set: +- **Validator command:** Phase 1 runs this automatically with + `--phase teardown`. After authoring `steal-map.md`, validate the complete + skill output with `$output_dir`, `$security_audit`, `$sbom`, and + `$upstream_ref_set` (each numeric flag `0|1`): ```bash - set -euo pipefail - required=(feature-inventory.md feature-registry.yaml feature-catalog.md spec-architecture.md spec-code-map.md spec-clone-vs-use.md spec-clone-mvp.md analysis-root-path.txt validate-feature-registry.py steal-map.md) - for name in "${required[@]}"; do - test -f "$output_dir/$name" - test ! -L "$output_dir/$name" - test -s "$output_dir/$name" - done - test -f "$output_dir/docs-features.txt" - test ! -L "$output_dir/docs-features.txt" - test ! -L "$output_dir/spec-cli-surface.md" - if [[ -e "$output_dir/spec-cli-surface.md" ]]; then - test -f "$output_dir/spec-cli-surface.md" - test -s "$output_dir/spec-cli-surface.md" - fi - python3 "$output_dir/validate-feature-registry.py" - if [[ "$upstream_ref_set" == 1 ]]; then - test -f "$output_dir/clone-metadata.json" - test ! -L "$output_dir/clone-metadata.json" - jq -e 'type == "object"' "$output_dir/clone-metadata.json" >/dev/null - else - [[ "$upstream_ref_set" == 0 ]] - fi - grep -Fqx '| Their capability | Our surface today | Verdict |' "$output_dir/steal-map.md" - if [[ "$security_audit" == 1 ]]; then - test -x "$output_dir/security/validate-security-audit.sh" - if [[ "$sbom" == 1 ]]; then - "$output_dir/security/validate-security-audit.sh" "$output_dir" --sbom - else - [[ "$sbom" == 0 ]] - "$output_dir/security/validate-security-audit.sh" "$output_dir" --no-sbom - fi - else - [[ "$security_audit" == 0 ]] - [[ "$sbom" == 0 ]] - fi + bash skills/reverse-engineer/scripts/validate-output.sh \ + --output-dir "$output_dir" --phase complete \ + --security-audit "$security_audit" --sbom "$sbom" \ + --upstream-ref-set "$upstream_ref_set" ``` - **Downstream handoff:** give the validated `steal-map.md` to Plan for one-way-door candidates; ordinary `have`, `park`, and `reject` decisions remain evidence-backed terminal rows. +### Earlier default compatibility + +Existing teardowns under `.agents/research//` remain in place and +usable. The script accepts that directory when it is passed explicitly with +`--output-dir`; that flag is caller authorization to write the teardown at the +exact selected path. It does not relocate or duplicate existing artifacts. An +invocation that omits the flag writes only to the current scratch default and +never creates output under the earlier root. +Consumers must retain the exact selected `output_dir` with their evidence +references instead of rediscovering outputs by globbing one root. This owning +skill contract is the compatibility authority; no separate migration receipt +is required. + ## Reproducibility + fixtures -`--upstream-ref` pins the clone (fetch `FETCH_HEAD`, record SHA) so contracts can be committed as golden fixtures and diffed across runs. Regression test: `bash skills/reverse-engineer/scripts/repo_fixture_test.sh`. To update a fixture when contracts legitimately change, re-run with the new pinned ref, copy the contract files into `fixtures//`, and commit. +`--upstream-ref` binds the selected checkout to one full commit: a new clone is +checked out detached at the fetched ref, while an existing checkout must already +match or the run refuses before analysis. `clone-metadata.json` records that +resolved commit. Regression test: `bash skills/reverse-engineer/scripts/repo_fixture_test.sh`. To update a fixture when contracts legitimately change, re-run with the new pinned ref, copy the contract files into `fixtures//`, and commit. ## Self-Test (acceptance) @@ -129,13 +123,17 @@ Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry. bash skills/reverse-engineer/scripts/self_test.sh ``` -Must show: feature inventory generated, registry generated, registry validator exits 0; in security mode `validate-security-audit.sh` exits 0 and the secret scan passes. +Must show: feature inventory and registry generated; the exact Phase-1 validator +passes; the complete validator rejects a missing and malformed steal-map and +accepts a valid caller-authored fixture; existing-checkout ref mismatch and +output symlinks fail closed; in security mode `validate-security-audit.sh` +exits 0 only after the scaffold is completed and the secret scan passes. ## Examples ### Reverse-engineer an OSS CLI (repo mode) → steal-map -Run the skill for `cc-sdd` with `--mode=repo --upstream-repo="https://github.com/gotalab/cc-sdd.git" --upstream-ref=v1.0.0`. It clones the pinned source, scans the surface, writes inventory/registry/specs, and maps each feature onto our surfaces (`have`, `gap`, `steal`, `park`, or `reject`) in `steal-map.md`. Supply selected steals to Plan. +Run Phase 1 for `cc-sdd` with `--mode=repo --upstream-repo="https://github.com/gotalab/cc-sdd.git" --upstream-ref=v1.0.0`. It clones the pinned source, scans the surface, writes inventory/registry/specs, and validates the teardown. Then inspect our live surfaces, author each `have`/`gap`/`steal`/`park`/`reject` row in `steal-map.md`, and run the complete-output validator. Supply selected steals to Plan. ### Binary analysis with security audit @@ -148,6 +146,7 @@ Run the skill for `ao` with `--authorized --mode=binary --binary-path="$(command | Refuses binary analysis | Missing `--authorized` | Add `--authorized` (explicit written authorization required). | | No `clone-metadata.json` | `--upstream-repo` not passed | Pass `--upstream-repo` (and optionally `--upstream-ref`). | | Fixture diff fails | Upstream changed / stale golden | Re-run pinned, refresh `fixtures/`, commit. | +| Existing teardown is under `.agents/research/` | It used the earlier default | Pass that exact directory with `--output-dir`; new runs otherwise use the scratch default. | | `spec-cli-surface.md` missing | No Node/Python/Go CLI detected | Surface is documented in `spec-code-map.md` instead. | | Steal-map is all "steal" | Skipped the park/reject rules | Substrate we delegate is **park**; doctrine conflicts are **reject** — not everything novel is worth adopting. | diff --git a/skills-codex/reverse-engineer/references/reverse-engineer.feature b/skills-codex/reverse-engineer/references/reverse-engineer.feature index 8975730e6..047116caf 100644 --- a/skills-codex/reverse-engineer/references/reverse-engineer.feature +++ b/skills-codex/reverse-engineer/references/reverse-engineer.feature @@ -23,3 +23,27 @@ Feature: Reverse-engineer reconstructs specs from an existing system Scenario: Output is a reusable spec set When reconstruction completes Then it emits a feature catalog, code map, and specs as durable artifacts + + Scenario: A steal-map is a separate checked decision + Given a validated mechanical teardown + When the caller compares its registry with the live destination repository + Then the caller authors steal-map.md with evidence-backed verdict rows + And the complete-output validator rejects a missing or malformed steal-map + + Scenario: An explicit analysis root cannot drift + Given --local-clone-dir selects a particular tree + When the selected tree is non-Git + Then that exact tree is analyzed instead of the caller's current checkout + When --upstream-ref also selects a Git commit + Then a mismatched existing checkout is refused before outputs are trusted + + Scenario: Managed output paths do not follow links + Given an output parent or managed artifact is a symbolic link + When reverse engineering starts + Then it refuses before writing through that link + + Scenario: An earlier-default output directory remains explicit and usable + Given an existing teardown under .agents/research + When that exact directory is supplied with --output-dir + Then the teardown writes and validates in that directory + And it does not move existing artifacts into the current scratch default diff --git a/skills-codex/reverse-engineer/scripts/reverse_engineer.py b/skills-codex/reverse-engineer/scripts/reverse_engineer.py index e183b4333..8338cd8d3 100755 --- a/skills-codex/reverse-engineer/scripts/reverse_engineer.py +++ b/skills-codex/reverse-engineer/scripts/reverse_engineer.py @@ -5,8 +5,10 @@ import argparse import datetime as _dt import hashlib import json +import os import re import shutil +import stat import subprocess import sys from pathlib import Path @@ -43,13 +45,81 @@ def _die(msg: str, code: int = 2) -> None: raise SystemExit(code) -def _run(cmd: list[str], *, cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess: +def _run( + cmd: list[str], *, cwd: Path | None = None, check: bool = True +) -> subprocess.CompletedProcess: return subprocess.run(cmd, cwd=str(cwd) if cwd else None, check=check) +def _lexical_absolute(path: Path) -> Path: + """Return an absolute normalized path without following filesystem links.""" + + return Path(os.path.abspath(os.fspath(path.expanduser()))) + + +def _ensure_real_directory(path: Path) -> tuple[int, int]: + """Create/traverse *path* one component at a time without following links. + + The returned device/inode pair lets the caller detect replacement of the + selected output root after setup. Every existing component must be a real + directory; a symlink or special file is a hard error. + """ + + absolute = _lexical_absolute(path) + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + nofollow = getattr(os, "O_NOFOLLOW", 0) + current_fd = os.open(absolute.anchor, flags) + try: + for part in absolute.parts[1:]: + try: + os.mkdir(part, mode=0o755, dir_fd=current_fd) + except FileExistsError: + pass + try: + next_fd = os.open(part, flags | nofollow, dir_fd=current_fd) + except OSError as exc: + _die(f"directory component is not a real directory: {absolute}: {exc}") + os.close(current_fd) + current_fd = next_fd + info = os.fstat(current_fd) + return info.st_dev, info.st_ino + finally: + os.close(current_fd) + + +def _assert_directory_identity( + path: Path, identity: tuple[int, int], label: str +) -> None: + try: + info = os.lstat(path) + except OSError as exc: + _die(f"{label} disappeared during the run: {path}: {exc}") + if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode): + _die(f"{label} is no longer a real directory: {path}") + if (info.st_dev, info.st_ino) != identity: + _die(f"{label} was replaced during the run: {path}") + + +def _assert_no_symlinks(root: Path) -> None: + """Reject pre-existing or concurrently introduced links below *root*.""" + + if not root.exists(): + return + root_info = os.lstat(root) + if stat.S_ISLNK(root_info.st_mode) or not stat.S_ISDIR(root_info.st_mode): + _die(f"output root must be a real directory: {root}") + for directory, dirnames, filenames in os.walk(root, followlinks=False): + base = Path(directory) + for name in [*dirnames, *filenames]: + child = base / name + info = os.lstat(child) + if stat.S_ISLNK(info.st_mode): + _die(f"refusing symlink inside managed output tree: {child}") + + def _ensure_dirs(paths: list[Path]) -> None: for p in paths: - p.mkdir(parents=True, exist_ok=True) + _ensure_real_directory(p) def _today_ymd() -> str: @@ -181,7 +251,9 @@ def _extract_ts_string_const(src: Path, const_name: str) -> str | None: return None -def _extract_agents_from_registry_ts(registry_ts: Path) -> tuple[list[str], list[str]] | None: +def _extract_agents_from_registry_ts( + registry_ts: Path, +) -> tuple[list[str], list[str]] | None: """ Best-effort parser for agent keys + alias flags from a TS registry. Intended to resolve help text interpolations like `${agentKeys.join('|')}`. @@ -227,7 +299,9 @@ def _extract_agents_from_registry_ts(registry_ts: Path) -> tuple[list[str], list return agent_keys, sorted(alias_flags) -def _find_node_cli_package(repo_root: Path, product_slug: str, product_name: str) -> dict[str, object] | None: +def _find_node_cli_package( + repo_root: Path, product_slug: str, product_name: str +) -> dict[str, object] | None: # Detect Node CLI packages by locating a package.json with a "bin" field and matching name/bin key. product_name_lc = product_name.strip().lower() candidates: list[tuple[int, Path, dict[str, object]]] = [] @@ -299,7 +373,12 @@ def _find_node_cli_package(repo_root: Path, product_slug: str, product_name: str def _find_python_cli(repo_root: Path) -> dict[str, object] | None: """Detect Python CLI packages via pyproject.toml or setup.cfg entry_points.""" - result: dict[str, object] = {"language": "python", "bin": {}, "framework": None, "entry_module": None} + result: dict[str, object] = { + "language": "python", + "bin": {}, + "framework": None, + "entry_module": None, + } # Try pyproject.toml first (modern standard). for pyproject in sorted(repo_root.rglob("pyproject.toml")): @@ -307,7 +386,7 @@ def _find_python_cli(repo_root: Path) -> dict[str, object] | None: continue text = _read_text(pyproject) # [project.scripts] section (PEP 621). - m = re.search(r'\[project\.scripts\]\s*\n((?:[^\[].+\n)*)', text) + m = re.search(r"\[project\.scripts\]\s*\n((?:[^\[].+\n)*)", text) if m: for line in m.group(1).strip().splitlines(): parts = line.split("=", 1) @@ -316,9 +395,11 @@ def _find_python_cli(repo_root: Path) -> dict[str, object] | None: entry = parts[1].strip().strip('"').strip("'") result["bin"][name] = entry # type: ignore[index] if not result["entry_module"]: - result["entry_module"] = entry.split(":")[0] if ":" in entry else entry + result["entry_module"] = ( + entry.split(":")[0] if ":" in entry else entry + ) # [tool.poetry.scripts] section. - m2 = re.search(r'\[tool\.poetry\.scripts\]\s*\n((?:[^\[].+\n)*)', text) + m2 = re.search(r"\[tool\.poetry\.scripts\]\s*\n((?:[^\[].+\n)*)", text) if m2: for line in m2.group(1).strip().splitlines(): parts = line.split("=", 1) @@ -335,7 +416,10 @@ def _find_python_cli(repo_root: Path) -> dict[str, object] | None: if _should_skip_repo_scan_path(setup_cfg, repo_root): continue text = _read_text(setup_cfg) - m = re.search(r'\[options\.entry_points\]\s*\nconsole_scripts\s*=\s*\n((?:\s+.+\n)*)', text) + m = re.search( + r"\[options\.entry_points\]\s*\nconsole_scripts\s*=\s*\n((?:\s+.+\n)*)", + text, + ) if m: for line in m.group(1).strip().splitlines(): parts = line.strip().split("=", 1) @@ -370,7 +454,12 @@ def _find_python_cli(repo_root: Path) -> dict[str, object] | None: def _find_go_cli(repo_root: Path) -> dict[str, object] | None: """Detect Go CLI packages via go.mod + main.go + flag/cobra usage.""" - result: dict[str, object] = {"language": "go", "bin": {}, "framework": None, "module": None} + result: dict[str, object] = { + "language": "go", + "bin": {}, + "framework": None, + "module": None, + } # Find go.mod for module name. go_mod = repo_root / "go.mod" @@ -383,7 +472,7 @@ def _find_go_cli(repo_root: Path) -> dict[str, object] | None: break if go_mod.exists(): text = _read_text(go_mod) - m = re.search(r'^module\s+(.+)$', text, re.MULTILINE) + m = re.search(r"^module\s+(.+)$", text, re.MULTILINE) if m: result["module"] = m.group(1).strip() @@ -416,7 +505,10 @@ def _find_go_cli(repo_root: Path) -> dict[str, object] | None: # Detect CLI framework (cobra vs stdlib flag). scanned = 0 for go_file in sorted(repo_root.rglob("*.go")): - if _should_skip_repo_scan_path(go_file, repo_root) or "testdata" in go_file.parts: + if ( + _should_skip_repo_scan_path(go_file, repo_root) + or "testdata" in go_file.parts + ): continue scanned += 1 if scanned > 200: @@ -491,11 +583,20 @@ def _enrich_registry_with_binary_evidence( for raw_line in text.splitlines(): stripped = raw_line.strip() if stripped.startswith("docs_features_prefix:"): - reg["docs_features_prefix"] = stripped.split(":", 1)[1].strip().strip("'\"") + reg["docs_features_prefix"] = ( + stripped.split(":", 1)[1].strip().strip("'\"") + ) elif stripped.startswith("docs_features:"): reg.setdefault("docs_features", []) - elif raw_line.startswith(" - ") and "docs_features" in reg and "groups" not in text.split(raw_line)[0].rsplit("docs_features:", 1)[-1]: - reg.setdefault("docs_features", []).append(stripped[2:].strip().strip("'\"")) + elif ( + raw_line.startswith(" - ") + and "docs_features" in reg + and "groups" + not in text.split(raw_line)[0].rsplit("docs_features:", 1)[-1] + ): + reg.setdefault("docs_features", []).append( + stripped[2:].strip().strip("'\"") + ) # Parse groups using the same logic as the validator cur = None in_groups = False @@ -509,7 +610,11 @@ def _enrich_registry_with_binary_evidence( continue if not in_groups: continue - if line.startswith(" ") and not line.startswith(" ") and line.endswith(":"): + if ( + line.startswith(" ") + and not line.startswith(" ") + and line.endswith(":") + ): name = line.strip()[:-1] cur = {"impl": None, "anchors": [], "notes": ""} reg["groups"][name] = cur @@ -598,19 +703,29 @@ def _write_binary_cli_surface_spec( lines.append(f"# CLI Surface Spec: {product_name}") lines.append("") lines.append(f"- Date: {date}") - lines.append("- Source: binary --help output" if help_tree.exists() else "- Source: binary string extraction") + lines.append( + "- Source: binary --help output" + if help_tree.exists() + else "- Source: binary string extraction" + ) lines.append("") cmd_count = 0 if commands_file.exists(): - cmds = [c.strip() for c in commands_file.read_text(encoding="utf-8").splitlines() if c.strip()] + cmds = [ + c.strip() + for c in commands_file.read_text(encoding="utf-8").splitlines() + if c.strip() + ] cmd_count = len(cmds) if help_tree.exists(): tree_text = help_tree.read_text(encoding="utf-8") lines.append("## Command Count") lines.append("") - lines.append(f"- **{cmd_count} commands** discovered via recursive `--help` execution") + lines.append( + f"- **{cmd_count} commands** discovered via recursive `--help` execution" + ) lines.append("") # Extract top-level commands and subcommands @@ -623,7 +738,9 @@ def _write_binary_cli_surface_spec( for top in top_level: subs = [c for c in cmds if c.startswith(top + " ") and c != top] sub_names = [c.split(maxsplit=1)[1] if " " in c else "" for c in subs] - sub_str = ", ".join(f"`{s}`" for s in sub_names if s) if sub_names else "—" + sub_str = ( + ", ".join(f"`{s}`" for s in sub_names if s) if sub_names else "—" + ) lines.append(f"| `{top}` | {sub_str} |") lines.append("") @@ -642,7 +759,11 @@ def _write_binary_cli_surface_spec( elif strings_file.exists(): # Fallback: extract command-like patterns from strings raw = strings_file.read_text(encoding="utf-8", errors="replace") - usage_lines = [line.strip() for line in raw.splitlines() if "usage" in line.lower() or "Usage" in line] + usage_lines = [ + line.strip() + for line in raw.splitlines() + if "usage" in line.lower() or "Usage" in line + ] lines.append("## CLI Surface (from binary strings, best-effort)") lines.append("") if usage_lines: @@ -715,18 +836,26 @@ def _write_cli_surface_spec( lines.append("") lines.append("## Notes For 1:1 Fidelity") lines.append("") - lines.append("- Run ` --help` to capture the full CLI contract as a golden test fixture.") + lines.append( + "- Run ` --help` to capture the full CLI contract as a golden test fixture." + ) if lang == "Python": - lines.append("- For Click/Typer apps, consider ` --help` per subcommand for full coverage.") + lines.append( + "- For Click/Typer apps, consider ` --help` per subcommand for full coverage." + ) elif lang == "Go": - lines.append("- For Cobra apps, consider ` help ` for full coverage.") + lines.append( + "- For Cobra apps, consider ` help ` for full coverage." + ) out.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") return True out = output_dir / "spec-cli-surface.md" pkg_dir = Path(str(node_cli["package_dir"])) - pkg_json_rel = Path(str(node_cli["package_json"])).relative_to(analysis_root).as_posix() + pkg_json_rel = ( + Path(str(node_cli["package_json"])).relative_to(analysis_root).as_posix() + ) src_index = pkg_dir / "src" / "index.ts" src_cli = pkg_dir / "src" / "cli.ts" src_store = pkg_dir / "src" / "cli" / "store.ts" @@ -741,7 +870,9 @@ def _write_cli_surface_spec( if extracted: agent_keys, alias_flags = extracted if agent_keys: - help_text = help_text.replace("${agentKeys.join('|')}", "|".join(agent_keys)) + help_text = help_text.replace( + "${agentKeys.join('|')}", "|".join(agent_keys) + ) alias_line = "" if alias_flags: alias_line = f" {' | '.join(alias_flags)} Agent alias flags\n" @@ -754,7 +885,14 @@ def _write_cli_surface_spec( pat = re.compile(r"\bprocess\.env\.([A-Z][A-Z0-9_]*)\b") found = set() for p in sorted(src_root.rglob("*")): - if not p.is_file() or p.suffix.lower() not in (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"): + if not p.is_file() or p.suffix.lower() not in ( + ".ts", + ".tsx", + ".js", + ".jsx", + ".mjs", + ".cjs", + ): continue for m in pat.finditer(_read_text(p)): found.add(m.group(1)) @@ -789,7 +927,9 @@ def _write_cli_surface_spec( lines.append("") lines.append("### Source Entry (Heuristic)") lines.append("") - lines.append(f"- `{src_cli.relative_to(analysis_root).as_posix()}` (node shebang entry; typically calls `runCli`)") + lines.append( + f"- `{src_cli.relative_to(analysis_root).as_posix()}` (node shebang entry; typically calls `runCli`)" + ) lines.append("") lines.append("## Usage / Help (Code-Proven Where Possible)") @@ -800,7 +940,9 @@ def _write_cli_surface_spec( lines.append("```") lines.append("") lines.append("Evidence:") - lines.append(f"- `{src_index.relative_to(analysis_root).as_posix()}` (`helpText`)") + lines.append( + f"- `{src_index.relative_to(analysis_root).as_posix()}` (`helpText`)" + ) else: lines.append("- _Help text not extracted (pattern not found)._") lines.append("Evidence:") @@ -816,7 +958,9 @@ def _write_cli_surface_spec( wrote_any = True if env_vars: lines.append(f"- Environment variables: `{', '.join(env_vars)}`") - lines.append(f" Evidence: scan of `{src_root.relative_to(analysis_root).as_posix()}` for `process.env.`.") + lines.append( + f" Evidence: scan of `{src_root.relative_to(analysis_root).as_posix()}` for `process.env.`." + ) wrote_any = True if not wrote_any: lines.append("- _No config/env surface extracted._") @@ -824,8 +968,12 @@ def _write_cli_surface_spec( lines.append("") lines.append("## Notes For 1:1 Fidelity") lines.append("") - lines.append("- Treat `--help` output as the CLI contract; include it as a golden test fixture for regressions.") - lines.append("- If the repo does not ship built artifacts (ex: `dist/`), building may be required to execute the CLI directly.") + lines.append( + "- Treat `--help` output as the CLI contract; include it as a golden test fixture for regressions." + ) + lines.append( + "- If the repo does not ship built artifacts (ex: `dist/`), building may be required to execute the CLI directly." + ) out.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") return True @@ -918,7 +1066,11 @@ def _write_artifact_surface_spec( from_dir = source.get("fromDir") if not isinstance(from_dir, str): continue - from_dir_res = _render_placeholders(from_dir, placeholder_vars) if placeholder_vars else from_dir + from_dir_res = ( + _render_placeholders(from_dir, placeholder_vars) + if placeholder_vars + else from_dir + ) abs_from = pkg_dir / from_dir_res if abs_from.exists() and abs_from.is_dir(): for fp in sorted(abs_from.rglob("*")): @@ -938,7 +1090,11 @@ def _write_artifact_surface_spec( from_file = source.get("from") if not isinstance(from_file, str): continue - from_file_res = _render_placeholders(from_file, placeholder_vars) if placeholder_vars else from_file + from_file_res = ( + _render_placeholders(from_file, placeholder_vars) + if placeholder_vars + else from_file + ) abs_from = pkg_dir / from_file_res if abs_from.exists() and abs_from.is_file(): resolved_sources.append( @@ -976,7 +1132,9 @@ def _write_artifact_surface_spec( lines.append(f"- Date: {date}") lines.append(f"- Analysis root: `{analysis_root}`") lines.append(f"- Node package: `{pkg_dir.relative_to(analysis_root).as_posix()}`") - lines.append(f"- Manifests dir: `{manifests_dir.relative_to(analysis_root).as_posix()}`") + lines.append( + f"- Manifests dir: `{manifests_dir.relative_to(analysis_root).as_posix()}`" + ) lines.append(f"- Machine registry: `{out_json.relative_to(output_dir).as_posix()}`") lines.append("") lines.append("## Manifest Inventory (Code-Proven)") @@ -998,7 +1156,9 @@ def _write_artifact_surface_spec( lines.append("## Template Source File Inventory (Hashed)") lines.append("") lines.append(f"- Files hashed: `{len(resolved_sources)}`") - lines.append("- Use `artifact-registry.json` as the source of truth for 1:1 template content equivalence.") + lines.append( + "- Use `artifact-registry.json` as the source of truth for 1:1 template content equivalence." + ) out_md.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") @@ -1033,11 +1193,30 @@ def _collect_env_vars_with_evidence( var_files: dict[str, set[str]] = {} patterns: list[tuple[re.Pattern[str], set[str]]] = [ - (re.compile(r"\bprocess\.env\.([A-Z][A-Z0-9_]+)\b"), {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"}), - (re.compile(r"""os\.environ(?:\.get)?\s*\(\s*['"]([A-Z][A-Z0-9_]+)['"]\s*\)"""), {".py"}), - (re.compile(r"""\bos\.getenv\s*\(\s*['"]([A-Z][A-Z0-9_]+)['"]\s*\)"""), {".py"}), - (re.compile(r"""\bos\.(?:Getenv|LookupEnv)\s*\(\s*"([A-Z][A-Z0-9_]+)"\s*\)"""), {".go"}), - (re.compile(r'\$\{?([A-Z][A-Z0-9_]{2,})\}?'), {".sh", ".bash", ".env", ".envrc"}), + ( + re.compile(r"\bprocess\.env\.([A-Z][A-Z0-9_]+)\b"), + {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"}, + ), + ( + re.compile( + r"""os\.environ(?:\.get)?\s*\(\s*['"]([A-Z][A-Z0-9_]+)['"]\s*\)""" + ), + {".py"}, + ), + ( + re.compile(r"""\bos\.getenv\s*\(\s*['"]([A-Z][A-Z0-9_]+)['"]\s*\)"""), + {".py"}, + ), + ( + re.compile( + r"""\bos\.(?:Getenv|LookupEnv)\s*\(\s*"([A-Z][A-Z0-9_]+)"\s*\)""" + ), + {".go"}, + ), + ( + re.compile(r"\$\{?([A-Z][A-Z0-9_]{2,})\}?"), + {".sh", ".bash", ".env", ".envrc"}, + ), ] scanned = 0 @@ -1045,7 +1224,14 @@ def _collect_env_vars_with_evidence( if not p.is_file(): continue # Skip irrelevant dirs - skip_dirs = {"node_modules", ".git", ".venv", "vendor", "testdata", "__pycache__"} + skip_dirs = { + "node_modules", + ".git", + ".venv", + "vendor", + "testdata", + "__pycache__", + } if any(part in skip_dirs for part in p.parts): continue suffix = p.suffix.lower() @@ -1067,10 +1253,12 @@ def _collect_env_vars_with_evidence( result: list[dict[str, object]] = [] for var_name in sorted(var_files.keys()): - result.append({ - "name": var_name, - "files": sorted(var_files[var_name]), - }) + result.append( + { + "name": var_name, + "files": sorted(var_files[var_name]), + } + ) return result @@ -1174,7 +1362,11 @@ def _write_repo_contract_json( node_cli = _find_node_cli_package(analysis_root, product_slug, product_name) python_cli_info = _find_python_cli(analysis_root) if node_cli is None else None - go_cli_info = _find_go_cli(analysis_root) if node_cli is None and python_cli_info is None else None + go_cli_info = ( + _find_go_cli(analysis_root) + if node_cli is None and python_cli_info is None + else None + ) if node_cli: pkg_dir = Path(str(node_cli["package_dir"])) @@ -1185,7 +1377,9 @@ def _write_repo_contract_json( for k, v in raw_bin.items(): bin_map[k] = v cli_surface["language"] = "node" - cli_surface["package_json"] = Path(str(node_cli["package_json"])).relative_to(analysis_root).as_posix() + cli_surface["package_json"] = ( + Path(str(node_cli["package_json"])).relative_to(analysis_root).as_posix() + ) cli_surface["package_dir"] = pkg_dir.relative_to(analysis_root).as_posix() cli_surface["package_name"] = str(node_cli.get("name") or "") cli_surface["bin"] = {k: bin_map[k] for k in sorted(bin_map)} @@ -1199,35 +1393,53 @@ def _write_repo_contract_json( if extracted: agent_keys, alias_flags = extracted if agent_keys: - help_text = help_text.replace("${agentKeys.join('|')}", "|".join(agent_keys)) + help_text = help_text.replace( + "${agentKeys.join('|')}", "|".join(agent_keys) + ) alias_line = "" if alias_flags: alias_line = f" {' | '.join(alias_flags)} Agent alias flags\n" help_text = help_text.replace("${agentAliasLine}", alias_line) if help_text is not None: cli_surface["help_text"] = help_text - cli_surface["help_text_source"] = src_index.relative_to(analysis_root).as_posix() if src_index.exists() else None + cli_surface["help_text_source"] = ( + src_index.relative_to(analysis_root).as_posix() + if src_index.exists() + else None + ) # Config file from store.ts src_store = pkg_dir / "src" / "cli" / "store.ts" config_file = _extract_ts_string_const(src_store, "CONFIG_FILE") if config_file: cli_surface["config_file"] = config_file - cli_surface["config_file_source"] = src_store.relative_to(analysis_root).as_posix() if src_store.exists() else None + cli_surface["config_file_source"] = ( + src_store.relative_to(analysis_root).as_posix() + if src_store.exists() + else None + ) elif python_cli_info: raw_bin_py = python_cli_info.get("bin") or {} cli_surface["language"] = "python" cli_surface["framework"] = python_cli_info.get("framework") cli_surface["entry_module"] = python_cli_info.get("entry_module") - cli_surface["bin"] = {k: str(raw_bin_py[k]) for k in sorted(raw_bin_py)} if isinstance(raw_bin_py, dict) else {} + cli_surface["bin"] = ( + {k: str(raw_bin_py[k]) for k in sorted(raw_bin_py)} + if isinstance(raw_bin_py, dict) + else {} + ) elif go_cli_info: raw_bin_go = go_cli_info.get("bin") or {} cli_surface["language"] = "go" cli_surface["framework"] = go_cli_info.get("framework") cli_surface["module"] = go_cli_info.get("module") - cli_surface["bin"] = {k: str(raw_bin_go[k]) for k in sorted(raw_bin_go)} if isinstance(raw_bin_go, dict) else {} + cli_surface["bin"] = ( + {k: str(raw_bin_go[k]) for k in sorted(raw_bin_go)} + if isinstance(raw_bin_go, dict) + else {} + ) contract["cli"] = cli_surface @@ -1253,15 +1465,21 @@ def _write_repo_contract_json( # Template files: keep path, sha256 (no absolute paths; already relative in artifact-registry) template_hashes: list[dict[str, object]] = [] for tf in template_files_raw: - template_hashes.append({ - "file": tf.get("file"), - "manifest": tf.get("manifest"), - "sha256": tf.get("sha256"), - "source_type": tf.get("source_type"), - }) + template_hashes.append( + { + "file": tf.get("file"), + "manifest": tf.get("manifest"), + "sha256": tf.get("sha256"), + "source_type": tf.get("source_type"), + } + ) - contract["manifests"] = sorted(manifests_clean, key=lambda x: str(x.get("path", ""))) - contract["template_files"] = sorted(template_hashes, key=lambda x: str(x.get("file", ""))) + contract["manifests"] = sorted( + manifests_clean, key=lambda x: str(x.get("path", "")) + ) + contract["template_files"] = sorted( + template_hashes, key=lambda x: str(x.get("file", "")) + ) except Exception: pass @@ -1293,7 +1511,11 @@ def _write_comparison_report( binary_cmds: list[str] = [] commands_file = tmp_dir / "binary" / "cli-commands.txt" if commands_file.exists(): - binary_cmds = [c.strip() for c in commands_file.read_text(encoding="utf-8").splitlines() if c.strip()] + binary_cmds = [ + c.strip() + for c in commands_file.read_text(encoding="utf-8").splitlines() + if c.strip() + ] repo_cmds: list[str] = [] repo_cli_spec = output_dir / "spec-cli-surface.md" @@ -1326,7 +1548,11 @@ def _write_comparison_report( if not in_groups: continue # Group entries are 2-space indented, end with ':' - if line.startswith(" ") and not line.startswith(" ") and line.rstrip().endswith(":"): + if ( + line.startswith(" ") + and not line.startswith(" ") + and line.rstrip().endswith(":") + ): # Determine source from notes field binary_groups += 1 @@ -1344,7 +1570,9 @@ def _write_comparison_report( # --- Coverage percentage --- if repo_cmds: coverage_pct = round(len(binary_set & repo_set) / len(repo_set) * 100) - coverage_line = f"Binary analysis found {coverage_pct}% of repo-discovered commands." + coverage_line = ( + f"Binary analysis found {coverage_pct}% of repo-discovered commands." + ) elif binary_cmds: coverage_line = f"Binary analysis found {len(binary_cmds)} commands; repo analysis found none (no CLI detected in repo)." else: @@ -1403,7 +1631,9 @@ def _write_comparison_report( def _write_wrapper_validate_feature_registry(output_dir: Path) -> None: - skill_validate_path = (SKILL_DIR / "scripts" / "validate_feature_registry.py").resolve() + skill_validate_path = ( + SKILL_DIR / "scripts" / "validate_feature_registry.py" + ).resolve() wrapper = output_dir / "validate-feature-registry.py" wrapper.write_text( f"""#!/usr/bin/env python3 @@ -1467,6 +1697,160 @@ def _copy_security_validators(output_dir: Path) -> None: dst.chmod(0o755) +def _git_text(repo: Path, *args: str) -> str: + return subprocess.check_output( + ["git", "-C", str(repo), *args], text=True, stderr=subprocess.STDOUT + ).strip() + + +def _is_git_checkout(path: Path) -> bool: + try: + return _git_text(path, "rev-parse", "--is-inside-work-tree") == "true" + except (OSError, subprocess.CalledProcessError): + return False + + +def _write_source_metadata( + output_dir: Path, + *, + upstream_repo: str | None, + upstream_ref: str | None, + resolved_commit: str, + source_kind: str, +) -> None: + payload = { + "upstream_repo": upstream_repo, + "upstream_ref": upstream_ref, + "resolved_commit": resolved_commit, + "source_kind": source_kind, + "clone_date": _today_ymd(), + } + (output_dir / "clone-metadata.json").write_text( + json.dumps(payload, indent=2) + "\n", encoding="utf-8" + ) + + +def _prepare_repo_analysis( + *, + local_clone_dir: Path, + output_dir: Path, + explicit_local_dir: bool, + upstream_repo: str | None, + upstream_ref: str | None, +) -> Path: + """Select one unambiguous repo analysis root and bind its requested ref.""" + + exists_before = local_clone_dir.exists() and any(local_clone_dir.iterdir()) + if upstream_repo and not exists_before: + clone_cmd = ["git", "clone"] + if not upstream_ref: + clone_cmd.append("--depth=1") + clone_cmd.extend([upstream_repo, str(local_clone_dir)]) + _run(clone_cmd, check=True) + if upstream_ref: + _run( + [ + "git", + "-C", + str(local_clone_dir), + "fetch", + "--depth=1", + "origin", + upstream_ref, + ], + check=True, + ) + _run( + [ + "git", + "-C", + str(local_clone_dir), + "checkout", + "--detach", + "FETCH_HEAD", + ], + check=True, + ) + + if explicit_local_dir: + analysis_root = local_clone_dir + elif upstream_repo: + analysis_root = local_clone_dir + else: + try: + top = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + except (OSError, subprocess.CalledProcessError): + top = "" + analysis_root = _lexical_absolute(Path(top)) if top else local_clone_dir + + if upstream_repo and not _is_git_checkout(analysis_root): + _die(f"--upstream-repo did not produce a Git checkout: {analysis_root}") + if upstream_repo and exists_before: + try: + origin = _git_text(analysis_root, "config", "--get", "remote.origin.url") + except subprocess.CalledProcessError: + _die( + "existing checkout has no origin URL to verify against --upstream-repo" + ) + if origin != upstream_repo: + _die( + "existing checkout origin does not match --upstream-repo " + f"(origin={origin!r}, requested={upstream_repo!r})" + ) + + if upstream_ref: + if not _is_git_checkout(analysis_root): + _die( + "--upstream-ref requires the selected analysis root to be a Git checkout" + ) + try: + requested = _git_text( + analysis_root, "rev-parse", "--verify", f"{upstream_ref}^{{commit}}" + ) + except subprocess.CalledProcessError: + if not upstream_repo: + _die( + f"requested ref is not present in the selected checkout: {upstream_ref}" + ) + _run( + [ + "git", + "-C", + str(analysis_root), + "fetch", + "--depth=1", + "origin", + upstream_ref, + ], + check=True, + ) + requested = _git_text( + analysis_root, "rev-parse", "--verify", "FETCH_HEAD^{commit}" + ) + current = _git_text(analysis_root, "rev-parse", "--verify", "HEAD^{commit}") + if current != requested: + _die( + "selected checkout is not at --upstream-ref; refusing to analyze the " + f"wrong commit (HEAD={current}, requested={requested})" + ) + + if _is_git_checkout(analysis_root) and (upstream_repo or upstream_ref): + resolved = _git_text(analysis_root, "rev-parse", "--verify", "HEAD^{commit}") + _write_source_metadata( + output_dir, + upstream_repo=upstream_repo, + upstream_ref=upstream_ref, + resolved_commit=resolved, + source_kind="existing-checkout" if exists_before else "clone", + ) + + return analysis_root + + def main() -> int: ap = argparse.ArgumentParser(prog="reverse_engineer.py") ap.add_argument("product_name") @@ -1483,9 +1867,22 @@ def main() -> int: help="Docs slug prefix, e.g. docs/features/. Use 'auto' to detect from repo/sitemap (default).", ) ap.add_argument("--upstream-repo", default=None) - ap.add_argument("--upstream-ref", default=None, help="Pin clone to a specific commit, tag, or branch. Records resolved SHA in clone-metadata.json.") + ap.add_argument( + "--upstream-ref", + default=None, + help="Pin clone to a specific commit, tag, or branch. Records resolved SHA in clone-metadata.json.", + ) ap.add_argument("--local-clone-dir", default=None) - ap.add_argument("--output-dir", default=None) + ap.add_argument( + "--output-dir", + default=None, + help=( + "Artifact directory. Defaults to " + ".agents/scratch/reverse-engineer//. The earlier " + ".agents/research// path remains accepted when supplied " + "explicitly; existing artifacts are never moved automatically." + ), + ) ap.add_argument("--mode", default="repo", choices=["repo", "binary", "both"]) ap.add_argument("--binary-path", default=None) @@ -1506,70 +1903,59 @@ def main() -> int: args = ap.parse_args() product_slug = _slugify(args.product_name) - local_clone_dir = Path(args.local_clone_dir or f".tmp/{product_slug}").resolve() - output_dir = Path(args.output_dir or f".agents/scratch/reverse-engineer/{product_slug}/").resolve() + explicit_local_dir = args.local_clone_dir is not None + local_clone_dir = _lexical_absolute( + Path(args.local_clone_dir or f".tmp/{product_slug}") + ) + output_dir = _lexical_absolute( + Path(args.output_dir or f".agents/scratch/reverse-engineer/{product_slug}/") + ) analysis_root = local_clone_dir - tmp_dir = (REPO_ROOT / ".tmp" / f"reverse-engineer-{product_slug}").resolve() - _ensure_dirs([local_clone_dir, output_dir, tmp_dir]) + tmp_dir = _lexical_absolute(REPO_ROOT / ".tmp" / f"reverse-engineer-{product_slug}") + _ensure_real_directory(local_clone_dir) + output_identity = _ensure_real_directory(output_dir) + _ensure_real_directory(tmp_dir) + _assert_no_symlinks(output_dir) docs_features_txt = output_dir / "docs-features.txt" effective_docs_prefix = args.docs_features_prefix - # Acquire code (repo mode): shallow clone if requested. - # NOTE: this must happen before docs inventory, otherwise docs/features extraction runs against an empty dir. if args.mode in ("repo", "both"): - if args.upstream_repo and not (local_clone_dir / ".git").exists(): - clone_cmd = ["git", "clone"] - if not args.upstream_ref: - clone_cmd.append("--depth=1") - clone_cmd.extend([args.upstream_repo, str(local_clone_dir)]) - _run(clone_cmd, check=True) - if args.upstream_ref: - _run(["git", "-C", str(local_clone_dir), "fetch", "--depth=1", "origin", args.upstream_ref], check=True) - _run(["git", "-C", str(local_clone_dir), "checkout", "FETCH_HEAD"], check=True) - # Record clone metadata for reproducibility. - resolved_sha = subprocess.check_output( - ["git", "-C", str(local_clone_dir), "rev-parse", "HEAD"], text=True, - ).strip() - clone_meta = { - "upstream_repo": args.upstream_repo, - "upstream_ref": args.upstream_ref, - "resolved_commit": resolved_sha, - "clone_date": _today_ymd(), - } - (output_dir / "clone-metadata.json").write_text( - json.dumps(clone_meta, indent=2) + "\n", encoding="utf-8", - ) - analysis_root = local_clone_dir - - # Determine an analysis root for repo mode. - # Priority: - # 1) local_clone_dir if it looks like a git checkout already - # 2) git toplevel of the current working directory (if inside a repo) - # 3) local_clone_dir (created) - if args.mode in ("repo", "both"): - if (local_clone_dir / ".git").exists(): - analysis_root = local_clone_dir - else: - try: - top = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() - if top: - analysis_root = Path(top).resolve() - except Exception: - analysis_root = local_clone_dir + # Acquire/select the repo before inventory. An explicit local path is + # always the selected root, including when it is intentionally non-Git; + # never replace it with the caller's current checkout. + analysis_root = _prepare_repo_analysis( + local_clone_dir=local_clone_dir, + output_dir=output_dir, + explicit_local_dir=explicit_local_dir, + upstream_repo=args.upstream_repo, + upstream_ref=args.upstream_ref, + ) # 1) Mechanical docs inventory (NO heavy crawling). if args.docs_sitemap_url: sitemap_xml = tmp_dir / f"{product_slug}-sitemap.xml" - _run([sys.executable, str(SKILL_DIR / "scripts" / "fetch_url.py"), args.docs_sitemap_url, str(sitemap_xml)]) + _run( + [ + sys.executable, + str(SKILL_DIR / "scripts" / "fetch_url.py"), + args.docs_sitemap_url, + str(sitemap_xml), + ] + ) paths_txt = tmp_dir / f"{product_slug}-sitemap-paths.txt" - sitemap_paths = subprocess.check_output([str(SKILL_DIR / "scripts" / "extract_sitemap_paths.sh"), str(sitemap_xml)], text=True) + sitemap_paths = subprocess.check_output( + [str(SKILL_DIR / "scripts" / "extract_sitemap_paths.sh"), str(sitemap_xml)], + text=True, + ) paths_txt.write_text(sitemap_paths, encoding="utf-8") if args.docs_features_prefix in ("", "auto"): - effective_docs_prefix = _detect_docs_prefix_from_paths(sitemap_paths.splitlines()) + effective_docs_prefix = _detect_docs_prefix_from_paths( + sitemap_paths.splitlines() + ) docs_features = subprocess.check_output( [ @@ -1587,7 +1973,10 @@ def main() -> int: effective_docs_prefix = _detect_docs_prefix_for_repo(analysis_root) # Backward-compatibility fallback for explicit old default. elif args.docs_features_prefix == "docs/features/": - if not (analysis_root / "docs" / "features").exists() and (analysis_root / "docs").exists(): + if ( + not (analysis_root / "docs" / "features").exists() + and (analysis_root / "docs").exists() + ): effective_docs_prefix = "docs/" prefix_dir = effective_docs_prefix.strip("/").rstrip("/") @@ -1602,7 +1991,9 @@ def main() -> int: rel = p.relative_to(analysis_root).as_posix() # Normalize to slug without extension to match sitemap-style slugs. slugs.append(rel[: -len(p.suffix)]) - docs_features_txt.write_text("\n".join(slugs) + ("\n" if slugs else ""), encoding="utf-8") + docs_features_txt.write_text( + "\n".join(slugs) + ("\n" if slugs else ""), encoding="utf-8" + ) else: docs_features_txt.write_text("", encoding="utf-8") @@ -1634,7 +2025,12 @@ def main() -> int: capture_script = SKILL_DIR / "scripts" / "binary" / "capture_cli_help.sh" if capture_script.exists(): _run( - ["bash", str(capture_script), str(binary_path), str(tmp_dir / "binary")], + [ + "bash", + str(capture_script), + str(binary_path), + str(tmp_dir / "binary"), + ], check=False, # best-effort ) # Copy results to output dir if they exist @@ -1671,7 +2067,12 @@ def main() -> int: _run( [ sys.executable, - str(SKILL_DIR / "scripts" / "binary" / "extract_embedded_archives.py"), + str( + SKILL_DIR + / "scripts" + / "binary" + / "extract_embedded_archives.py" + ), "--binary", str(binary_path), "--out-dir", @@ -1798,13 +2199,17 @@ def main() -> int: # 6c) Comparison report (binary vs repo) when both sources are available. if args.mode == "both": - _write_comparison_report(output_dir, tmp_dir, product_name=args.product_name, date=_today_ymd()) + _write_comparison_report( + output_dir, tmp_dir, product_name=args.product_name, date=_today_ymd() + ) # 7) Validation gate: produce a self-contained validator in the output dir and run it once. _write_wrapper_validate_feature_registry(output_dir) # Store analysis root pointer for validators (repo clone dir or a placeholder). (output_dir / "analysis-root").mkdir(exist_ok=True) - (output_dir / "analysis-root-path.txt").write_text(str(analysis_root), encoding="utf-8") + (output_dir / "analysis-root-path.txt").write_text( + str(analysis_root), encoding="utf-8" + ) # Keep docs-features alongside outputs for deterministic validation. # (Already written as output_dir/docs-features.txt) _run( @@ -1816,7 +2221,11 @@ def main() -> int: "--docs-features", str(docs_features_txt), "--local-clone-dir", - str(analysis_root if analysis_root.exists() else output_dir / "analysis-root"), + str( + analysis_root + if analysis_root.exists() + else output_dir / "analysis-root" + ), ], check=True, ) @@ -1834,12 +2243,19 @@ def main() -> int: "findings.md.tmpl", "reproducibility.md.tmpl", ]: - _render_template(TEMPLATES_DIR / "security" / name, sec_dir / name.replace(".tmpl", ""), vars) + _render_template( + TEMPLATES_DIR / "security" / name, + sec_dir / name.replace(".tmpl", ""), + vars, + ) _copy_security_validators(output_dir) if args.sbom: - _run([str(sec_dir / "generate-sbom.sh"), str(analysis_root), str(sec_dir)], check=False) + _run( + [str(sec_dir / "generate-sbom.sh"), str(analysis_root), str(sec_dir)], + check=False, + ) # Scaffold-time safety check: scan the generated output for leaked # secrets. The full certifying gate (validate-security-audit.sh) is NOT @@ -1859,8 +2275,37 @@ def main() -> int: vibe_path = reports_dir / f"{_today_ymd()}-vibe-{product_slug}.md" post_path = reports_dir / f"{_today_ymd()}-postmortem-{product_slug}.md" - _render_template(TEMPLATES_DIR / "vibe-report.md.tmpl", vibe_path, {**vars, "OUTPUT_DIR": str(output_dir)}) - _render_template(TEMPLATES_DIR / "postmortem.md.tmpl", post_path, {**vars, "OUTPUT_DIR": str(output_dir)}) + _render_template( + TEMPLATES_DIR / "vibe-report.md.tmpl", + vibe_path, + {**vars, "OUTPUT_DIR": str(output_dir)}, + ) + _render_template( + TEMPLATES_DIR / "postmortem.md.tmpl", + post_path, + {**vars, "OUTPUT_DIR": str(output_dir)}, + ) + + # Phase 1 deliberately stops at a validated teardown. The evidence-backed + # steal-map is a caller-authored Phase-2 judgment over this output and the + # live destination repository; the script must not manufacture that choice. + _assert_directory_identity(output_dir, output_identity, "output directory") + _assert_no_symlinks(output_dir) + _run( + [ + "bash", + str(SKILL_DIR / "scripts" / "validate-output.sh"), + "--output-dir", + str(output_dir), + "--phase", + "teardown", + "--upstream-ref-set", + "1" if args.upstream_ref else "0", + ], + check=True, + ) + _assert_directory_identity(output_dir, output_identity, "output directory") + _assert_no_symlinks(output_dir) return 0 diff --git a/skills-codex/reverse-engineer/scripts/self_test.sh b/skills-codex/reverse-engineer/scripts/self_test.sh index 92f626427..2613690a0 100755 --- a/skills-codex/reverse-engineer/scripts/self_test.sh +++ b/skills-codex/reverse-engineer/scripts/self_test.sh @@ -19,6 +19,13 @@ SITEMAP="$TMP/sitemap.xml" rm -rf "$TMP" mkdir -p "$SRC" "$OUT1" "$OUT2" +HELP="$(python3 "$SKILL/scripts/reverse_engineer.py" --help)" +grep -Fq '.agents/scratch/reverse-engineer//' <<<"$HELP" +grep -Fq '.agents/research// path remains' <<<"$HELP" +grep -Fq 'are never moved automatically.' <<<"$HELP" +grep -Fq -- "- '.agents/scratch/reverse-engineer/*/'" "$SKILL/SKILL.md" +echo "OK: output-path migration contract is visible in --help" + python3 - "$SRC" <<'PY' import sys, zipfile from pathlib import Path @@ -75,6 +82,33 @@ python3 "$SKILL/scripts/reverse_engineer.py" demo \ python3 "$OUT1/validate-feature-registry.py" +VALIDATE_OUTPUT="$SKILL/scripts/validate-output.sh" +"$VALIDATE_OUTPUT" --output-dir "$OUT1" --phase teardown \ + --security-audit 0 --sbom 0 --upstream-ref-set 0 +if "$VALIDATE_OUTPUT" --output-dir "$OUT1" --phase complete \ + --security-audit 0 --sbom 0 --upstream-ref-set 0 >/dev/null 2>&1; then + echo "FAIL: complete validator accepted a missing steal-map.md" >&2 + exit 1 +fi +cat >"$OUT1/steal-map.md" <<'EOF' +# Steal map: demo + +| Their capability | Our surface today | Verdict | +|---|---|---| +| Embedded archive inventory (`feature-registry.yaml`) | `skills/reverse-engineer/` | **have** | +EOF +"$VALIDATE_OUTPUT" --output-dir "$OUT1" --phase complete \ + --security-audit 0 --sbom 0 --upstream-ref-set 0 +cp "$OUT1/steal-map.md" "$OUT1/steal-map.valid" +printf '# malformed map\n' >"$OUT1/steal-map.md" +if "$VALIDATE_OUTPUT" --output-dir "$OUT1" --phase complete \ + --security-audit 0 --sbom 0 --upstream-ref-set 0 >/dev/null 2>&1; then + echo "FAIL: complete validator accepted a malformed steal-map.md" >&2 + exit 1 +fi +mv "$OUT1/steal-map.valid" "$OUT1/steal-map.md" +echo "OK: exact output validator distinguishes teardown from complete decision output" + # --- Binary mode capability assertions --- echo "--- binary mode capability checks ---" @@ -193,6 +227,83 @@ if [ ! -f "$OUT_REF/clone-metadata.json" ]; then fi echo "OK: clone-metadata.json created with --upstream-ref" +echo "--- existing-checkout ref mismatch test ---" +WRONG_REPO="$TMP/local-wrong-ref" +WRONG_OUT="$TMP/out-wrong-ref" +mkdir -p "$WRONG_REPO" +git -C "$WRONG_REPO" init -q +git -C "$WRONG_REPO" config user.name reverse-self-test +git -C "$WRONG_REPO" config user.email reverse-self-test@example.invalid +printf 'one\n' >"$WRONG_REPO/unique.txt" +git -C "$WRONG_REPO" add unique.txt +git -C "$WRONG_REPO" commit -qm one +first_commit="$(git -C "$WRONG_REPO" rev-parse HEAD)" +printf 'two\n' >"$WRONG_REPO/unique.txt" +git -C "$WRONG_REPO" commit -qam two +second_commit="$(git -C "$WRONG_REPO" rev-parse HEAD)" +git -C "$WRONG_REPO" checkout -q --detach "$first_commit" +if python3 "$SKILL/scripts/reverse_engineer.py" wrong-ref \ + --mode=repo --local-clone-dir="$WRONG_REPO" \ + --upstream-ref="$second_commit" --output-dir="$WRONG_OUT" >/dev/null 2>&1; then + echo "FAIL: existing checkout at the wrong commit was analyzed" >&2 + exit 1 +fi +if [ -e "$WRONG_OUT/feature-registry.yaml" ]; then + echo "FAIL: ref mismatch wrote trusted teardown artifacts" >&2 + exit 1 +fi +echo "OK: existing checkout must match the requested ref" + +echo "--- explicit non-Git root test ---" +EXPLICIT_TREE="$TMP/explicit-nongit" +EXPLICIT_OUT="$TMP/out-explicit-nongit" +mkdir -p "$EXPLICIT_TREE" +printf 'only-in-explicit-tree\n' >"$EXPLICIT_TREE/unique-source.txt" +python3 "$SKILL/scripts/reverse_engineer.py" explicit-nongit \ + --mode=repo --local-clone-dir="$EXPLICIT_TREE" --output-dir="$EXPLICIT_OUT" +if ! grep -Fqx "$EXPLICIT_TREE" "$EXPLICIT_OUT/analysis-root-path.txt"; then + echo "FAIL: explicit non-Git tree was replaced by the caller checkout" >&2 + exit 1 +fi +echo "OK: explicit non-Git analysis root wins" + +echo "--- output symlink refusal tests ---" +SYMLINK_CASE="$TMP/symlink-case" +SYMLINK_OUTSIDE="$TMP/symlink-outside" +mkdir -p "$SYMLINK_CASE/.agents" "$SYMLINK_OUTSIDE" "$SYMLINK_CASE/local" +printf 'outside sentinel\n' >"$SYMLINK_OUTSIDE/sentinel" +ln -s "$SYMLINK_OUTSIDE" "$SYMLINK_CASE/.agents/scratch" +if ( + cd "$SYMLINK_CASE" + python3 "$SKILL/scripts/reverse_engineer.py" escaped \ + --mode=repo --local-clone-dir="$SYMLINK_CASE/local" >/dev/null 2>&1 +); then + echo "FAIL: default output followed a symlinked scratch parent" >&2 + exit 1 +fi +if ! grep -Fqx 'outside sentinel' "$SYMLINK_OUTSIDE/sentinel" \ + || [ -e "$SYMLINK_OUTSIDE/reverse-engineer" ]; then + echo "FAIL: symlinked parent allowed an outside write" >&2 + exit 1 +fi + +MANAGED_OUT="$TMP/out-managed-link" +MANAGED_OUTSIDE="$TMP/managed-outside.yaml" +mkdir -p "$MANAGED_OUT" +printf 'outside registry\n' >"$MANAGED_OUTSIDE" +ln -s "$MANAGED_OUTSIDE" "$MANAGED_OUT/feature-registry.yaml" +if python3 "$SKILL/scripts/reverse_engineer.py" managed-link \ + --mode=repo --local-clone-dir="$EXPLICIT_TREE" \ + --output-dir="$MANAGED_OUT" >/dev/null 2>&1; then + echo "FAIL: managed artifact symlink was followed" >&2 + exit 1 +fi +if ! grep -Fqx 'outside registry' "$MANAGED_OUTSIDE"; then + echo "FAIL: managed artifact symlink changed the outside target" >&2 + exit 1 +fi +echo "OK: output parent and managed-file symlinks fail closed" + # --- Multi-language CLI graceful degradation test --- echo "--- multi-language CLI degradation test ---" @@ -218,6 +329,54 @@ if ! grep -q "no CLI surface detected" "$OUT_NONCLI/spec-code-map.md" 2>/dev/nul fi echo "OK: multi-language CLI graceful degradation works" +echo "--- default output-path parity test ---" +DEFAULT_OUT="$TMP/.agents/scratch/reverse-engineer/default-demo" +( + cd "$TMP" + python3 "$SKILL/scripts/reverse_engineer.py" default-demo \ + --mode=repo \ + --local-clone-dir="$TMP/local-noncli" \ + --docs-sitemap-url="file://$SITEMAP" +) +if [ ! -s "$DEFAULT_OUT/feature-registry.yaml" ] \ + || [ ! -s "$DEFAULT_OUT/contracts/repo-contract.json" ] \ + || [ ! -s "$DEFAULT_OUT/reports/$(date +%F)-vibe-default-demo.md" ] \ + || [ ! -s "$DEFAULT_OUT/docs-features.txt" ] \ + || [ ! -s "$DEFAULT_OUT/validate-feature-registry.py" ]; then + echo "FAIL: executable default did not emit the declared product output directory" >&2 + exit 1 +fi +echo "OK: frontmatter output directory matches the executable default" + +echo "--- earlier output-path compatibility test ---" +LEGACY_OUT="$TMP/.agents/research/legacy-demo" +LEGACY_EXPECTED="$TMP/legacy-sentinel.expected" +LEGACY_DEFAULT="$TMP/.agents/scratch/reverse-engineer/legacy-demo" +mkdir -p "$LEGACY_OUT" +printf 'caller-owned sentinel\n\n' > "$LEGACY_OUT/caller-sentinel.txt" +cp "$LEGACY_OUT/caller-sentinel.txt" "$LEGACY_EXPECTED" +( + cd "$TMP" + python3 "$SKILL/scripts/reverse_engineer.py" legacy-demo \ + --mode=repo \ + --local-clone-dir="$TMP/local-noncli" \ + --output-dir="$LEGACY_OUT" \ + --docs-sitemap-url="file://$SITEMAP" +) +if [ ! -s "$LEGACY_OUT/feature-registry.yaml" ]; then + echo "FAIL: explicit earlier-default output directory was not honored" >&2 + exit 1 +fi +if ! cmp -s "$LEGACY_EXPECTED" "$LEGACY_OUT/caller-sentinel.txt"; then + echo "FAIL: explicit earlier-default invocation changed a pre-existing artifact" >&2 + exit 1 +fi +if [ -e "$LEGACY_DEFAULT" ]; then + echo "FAIL: explicit earlier-default invocation also wrote to the scratch default" >&2 + exit 1 +fi +echo "OK: explicit earlier-default output directory remains supported" + echo "--- generated-tree hygiene regression test ---" HYGIENE_REPO="$TMP/local-hygiene" HYGIENE_OUT="$TMP/out-hygiene" diff --git a/skills-codex/reverse-engineer/scripts/validate-output.sh b/skills-codex/reverse-engineer/scripts/validate-output.sh new file mode 100755 index 000000000..d785b382d --- /dev/null +++ b/skills-codex/reverse-engineer/scripts/validate-output.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: validate-output.sh --output-dir DIR [--phase teardown|complete] + [--security-audit 0|1] [--sbom 0|1] + [--upstream-ref-set 0|1] +EOF + exit 2 +} + +output_dir="" +phase="complete" +security_audit=0 +sbom=0 +upstream_ref_set=0 +while (($#)); do + case "$1" in + --output-dir) (($# >= 2)) || usage; output_dir=$2; shift 2 ;; + --phase) (($# >= 2)) || usage; phase=$2; shift 2 ;; + --security-audit) (($# >= 2)) || usage; security_audit=$2; shift 2 ;; + --sbom) (($# >= 2)) || usage; sbom=$2; shift 2 ;; + --upstream-ref-set) (($# >= 2)) || usage; upstream_ref_set=$2; shift 2 ;; + -h|--help) usage ;; + *) usage ;; + esac +done + +[[ -n "$output_dir" ]] || usage +[[ "$phase" == teardown || "$phase" == complete ]] || usage +[[ "$security_audit" =~ ^[01]$ ]] || usage +[[ "$sbom" =~ ^[01]$ ]] || usage +[[ "$upstream_ref_set" =~ ^[01]$ ]] || usage +[[ -d "$output_dir" && ! -L "$output_dir" ]] || { + echo "error: output directory must be a real directory: $output_dir" >&2 + exit 1 +} + +required=( + feature-inventory.md + feature-registry.yaml + feature-catalog.md + spec-architecture.md + spec-code-map.md + spec-clone-vs-use.md + spec-clone-mvp.md + analysis-root-path.txt + validate-feature-registry.py +) +for name in "${required[@]}"; do + path="$output_dir/$name" + [[ -f "$path" && ! -L "$path" && -s "$path" ]] || { + echo "error: required regular nonempty artifact missing: $path" >&2 + exit 1 + } +done + +[[ -f "$output_dir/docs-features.txt" && ! -L "$output_dir/docs-features.txt" ]] || { + echo "error: docs-features.txt must be a regular file" >&2 + exit 1 +} +if [[ -e "$output_dir/spec-cli-surface.md" || -L "$output_dir/spec-cli-surface.md" ]]; then + [[ -f "$output_dir/spec-cli-surface.md" && ! -L "$output_dir/spec-cli-surface.md" && -s "$output_dir/spec-cli-surface.md" ]] || { + echo "error: spec-cli-surface.md must be a regular nonempty file when present" >&2 + exit 1 + } +fi + +python3 "$output_dir/validate-feature-registry.py" + +if [[ "$upstream_ref_set" == 1 ]]; then + metadata="$output_dir/clone-metadata.json" + [[ -f "$metadata" && ! -L "$metadata" && -s "$metadata" ]] || { + echo "error: --upstream-ref requires clone-metadata.json" >&2 + exit 1 + } + python3 - "$metadata" <<'PY' +import json, pathlib, re, sys +path = pathlib.Path(sys.argv[1]) +data = json.loads(path.read_text(encoding="utf-8")) +if not isinstance(data, dict): + raise SystemExit("clone metadata must be an object") +commit = data.get("resolved_commit") +if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-fA-F]{40,64}", commit): + raise SystemExit("clone metadata lacks a full resolved commit OID") +if not data.get("upstream_ref"): + raise SystemExit("clone metadata lacks upstream_ref") +PY +fi + +if [[ "$phase" == complete ]]; then + steal_map="$output_dir/steal-map.md" + [[ -f "$steal_map" && ! -L "$steal_map" && -s "$steal_map" ]] || { + echo "error: complete output requires a regular nonempty steal-map.md" >&2 + exit 1 + } + grep -Fqx '| Their capability | Our surface today | Verdict |' "$steal_map" || { + echo "error: steal-map.md lacks the required table header" >&2 + exit 1 + } + awk -F'|' ' + BEGIN { found = 0 } + /^\|/ { + capability=$2; ours=$3; verdict=$4 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", capability) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", ours) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", verdict) + gsub(/\*\*/, "", verdict) + if (capability != "" && capability != "Their capability" && capability !~ /^-+$/ && + ours != "" && verdict ~ /^(have|gap|steal|park|reject)$/) found = 1 + } + END { exit found ? 0 : 1 } + ' "$steal_map" || { + echo "error: steal-map.md needs at least one nonempty row with a valid verdict" >&2 + exit 1 + } +fi + +if [[ "$security_audit" == 1 ]]; then + gate="$output_dir/security/validate-security-audit.sh" + [[ -x "$gate" && ! -L "$gate" ]] || { + echo "error: security validator is missing or unsafe" >&2 + exit 1 + } + if [[ "$sbom" == 1 ]]; then + "$gate" "$output_dir" --sbom + else + "$gate" "$output_dir" --no-sbom + fi +else + [[ "$sbom" == 0 ]] || { + echo "error: --sbom requires --security-audit 1" >&2 + exit 1 + } +fi + +echo "PASS: reverse-engineer $phase output is structurally valid" diff --git a/skills-codex/skill-builder/.agentops-generated.json b/skills-codex/skill-builder/.agentops-generated.json index 6f762f89e..bd65c3914 100644 --- a/skills-codex/skill-builder/.agentops-generated.json +++ b/skills-codex/skill-builder/.agentops-generated.json @@ -2,6 +2,6 @@ "generator": "codex-sync", "source_skill": "skills/skill-builder", "layout": "modular", - "source_hash": "775e73f3ebe87960a245790645cb2ba9083f18e0c8d0ee60569e6b747529b830", - "generated_hash": "9ab3978eca5991203690b4c5ae36c75b12e2d860002d8ccff5e4ceb5cdfdc6c5" + "source_hash": "e7383dd770bef7595bf2e6db5dff94ef0ba74eff77e5c4db9ad750084eedbb4c", + "generated_hash": "b401b04fc6bbf86ccbdc67e882a7d6831925bb4768a2eaf61de359a83b976ae1" } diff --git a/skills-codex/skill-builder/references/skill-auditor.feature b/skills-codex/skill-builder/references/skill-auditor.feature index 49777d995..69203ea24 100644 --- a/skills-codex/skill-builder/references/skill-auditor.feature +++ b/skills-codex/skill-builder/references/skill-auditor.feature @@ -2,8 +2,8 @@ # skill template audit (BC1 Corpus / Skill Catalog). # The audit checks an existing SKILL.md against the unified template: Pass 1 gates # through heal.sh --strict, Pass 2 runs additional structural checks, then it emits a -# density report and a productization score. Hexagon: supporting; consumes: a SKILL.md + -# the template; produces: audit-report.json. (soc-qk4b) +# density report and a static package-readiness score. Hexagon: supporting; consumes: a +# SKILL.md + the template; produces: audit-report.json. (soc-qk4b) Feature: Skill-auditor scores a skill against the unified template As a catalog maintainer @@ -21,6 +21,7 @@ Feature: Skill-auditor scores a skill against the unified template When Pass 1 completes Then Pass 2 runs the additional template-conformance checks - Scenario: A density report and productization score are emitted + Scenario: A density report and static package-readiness score are emitted When both passes complete - Then it emits an advisory density report and a productization score in audit-report.json + Then it emits an advisory density report and a static package-readiness score in audit-report.json + And the score says that safety and effectiveness were not evaluated diff --git a/skills-codex/skill-builder/references/skill-template.md b/skills-codex/skill-builder/references/skill-template.md index 3f8c146cd..e42a9c87d 100644 --- a/skills-codex/skill-builder/references/skill-template.md +++ b/skills-codex/skill-builder/references/skill-template.md @@ -158,7 +158,7 @@ Each NEW Pass-2 check maps to AgentOps' design principles in PRODUCT.md, so the | `quality-rubric` | Operational Principle #3 (context quality determines output quality) | | `references-modularization` | Finding `f-2026-05-01-025` (SKILL.md churn budget — every Skill() invocation reloads 5-15KB) | | `trigger-clarity` | Operational Principle #1 (agents are ephemeral) — invocation criteria must be in artifact | -| `description-has-triggers` (renamed from `description-multiline`) | Pillar #6 (knowledge flywheel) — searchability requires structured description. Three valid forms preserve AgentOps' single-line convention. | +| `description-has-triggers` (renamed from `description-multiline`) | Product surfaces — structured invocation criteria make the right capability discoverable without loading every skill body. Three valid forms preserve AgentOps' single-line convention. | --- diff --git a/skills-codex/skill-builder/schemas/audit-report.json b/skills-codex/skill-builder/schemas/audit-report.json index 7164c762c..d17999bef 100644 --- a/skills-codex/skill-builder/schemas/audit-report.json +++ b/skills-codex/skill-builder/schemas/audit-report.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema#", "title": "Skill Audit Report", - "description": "Output contract for the skill-builder deep audit. Three passes: Pass 1 wraps heal.sh structural checks; Pass 2 adds 8 NEW content-discipline checks beyond heal.sh; Pass 3 folds the 10-category Skill Quality Rubric (docs/reference/skill-quality-rubric.md) in as an advisory 0-30 productization score.", + "description": "Output contract for the skill-builder deep audit. Pass 1 wraps heal.sh structural checks; Pass 2 adds 8 content-discipline checks beyond heal.sh; Pass 3 reports an advisory 0-30 static package-readiness score that evaluates neither safety nor behavioral effectiveness; later passes add advisory craft and authoring signals.", "type": "object", "required": ["target", "profile_id", "verdict", "pass1", "pass2"], "properties": { @@ -155,13 +155,28 @@ "additionalProperties": false }, "rubric": { - "description": "Advisory-only Pass-3 Skill Quality Rubric score (docs/reference/skill-quality-rubric.md). Folded in from score_agentops_skill.py --audit-block. Never affects the aggregate verdict. Emitted as null when python3 or the scorer is unavailable (fail-open).", + "description": "Advisory-only Pass-3 static package-readiness score (docs/reference/skill-quality-rubric.md). Folded in from score_agentops_skill.py --audit-block. It evaluates neither the safety gate nor behavioral effectiveness and never affects the aggregate verdict. Emitted as null when python3 or the scorer is unavailable (fail-open).", "oneOf": [ {"type": "null"}, { "type": "object", - "required": ["total_score", "max_score", "rating", "advisory", "categories"], + "required": ["scope", "safety_gate_evaluated", "effectiveness_evaluated", "total_score", "max_score", "rating", "advisory", "categories"], "properties": { + "scope": { + "type": "string", + "const": "static-package-readiness", + "description": "The score covers visible package properties only." + }, + "safety_gate_evaluated": { + "type": "boolean", + "const": false, + "description": "Always false: boundary-word heuristics are not a full-bundle safety review." + }, + "effectiveness_evaluated": { + "type": "boolean", + "const": false, + "description": "Always false: structural scoring contains no baseline-versus-treatment behavioral evaluation." + }, "total_score": { "type": "integer", "minimum": 0, @@ -175,12 +190,12 @@ "rating": { "type": "string", "enum": ["C", "B", "A", "S"], - "description": "Rating band: C (0-10), B (11-20), A (21-26), S (27-30)." + "description": "Static readiness band: C (0-10), B (11-20), A (21-26), S (27-30)." }, "advisory": { "type": "boolean", "const": true, - "description": "Always true. The rubric score is report-only and never gates the verdict." + "description": "Always true. The static readiness score is report-only and never gates the verdict." }, "categories": { "type": "array", diff --git a/skills-codex/skill-builder/scripts/audit.sh b/skills-codex/skill-builder/scripts/audit.sh index 7560fd81c..035575d5e 100755 --- a/skills-codex/skill-builder/scripts/audit.sh +++ b/skills-codex/skill-builder/scripts/audit.sh @@ -314,14 +314,14 @@ else DENSITY_STATUS="warn" fi -# --- Pass 3: rubric scoring (advisory) ----------------------------------- +# --- Pass 3: static package-readiness scoring (advisory) ----------------- # Folds the 10-category Skill Quality Rubric (docs/reference/skill-quality-rubric.md) # into the report via score_agentops_skill.py --audit-block. Each category gets a # deterministic 0-3 score plus an explainable reason; total is 0-30 with a C/B/A/S -# rating band. Advisory-only: it never changes the PASS/WARN/FAIL verdict — the -# rubric measures market-facing maturity, not template conformance (which Pass 1+2 -# already gate). Reason: a low rubric score on a structurally-clean skill is a -# productization backlog signal, not a ship blocker. +# readiness band. Advisory-only: it never changes the PASS/WARN/FAIL verdict and +# explicitly evaluates neither the safety gate nor behavioral effectiveness. +# Reason: a low score on a structurally clean skill is a triage signal, while a +# high score still cannot prove that the skill is safe or improves outcomes. RUBRIC_JSON="null" RUBRIC_SUMMARY="" RUBRIC_SCORE="n/a" @@ -331,7 +331,7 @@ if [[ -f "$SCORE_PY" ]] && command -v python3 >/dev/null 2>&1; then RUBRIC_JSON="$rubric_out" RUBRIC_SCORE="$(printf '%s' "$rubric_out" | awk -F': ' '/"total_score"/{gsub(/[, ]/,"",$2); print $2; exit}')" RUBRIC_RATING="$(printf '%s' "$rubric_out" | awk -F'"' '/"rating"/{print $4; exit}')" - RUBRIC_SUMMARY=" Rubric: ${RUBRIC_SCORE}/30 (${RUBRIC_RATING}) [advisory]." + RUBRIC_SUMMARY=" Static readiness: ${RUBRIC_SCORE}/30 (${RUBRIC_RATING}) [advisory; safety/effectiveness not evaluated]." fi fi @@ -482,7 +482,7 @@ fi printf " [%-4s] %s\n" "${CHECK_STATUS[$id]}" "$id" done echo "Density advisory: $density_present_count/6 fields present ($DENSITY_STATUS)" - echo "Pass 3 rubric (advisory): ${RUBRIC_SCORE}/30 (${RUBRIC_RATING})" + echo "Pass 3 static readiness (advisory): ${RUBRIC_SCORE}/30 (${RUBRIC_RATING}); safety/effectiveness not evaluated" if [[ -n "$CRAFT_LINES" ]]; then echo "$CRAFT_LINES" fi diff --git a/skills-codex/skill-builder/scripts/score_agentops_skill.py b/skills-codex/skill-builder/scripts/score_agentops_skill.py index ad703c260..af2d0301b 100755 --- a/skills-codex/skill-builder/scripts/score_agentops_skill.py +++ b/skills-codex/skill-builder/scripts/score_agentops_skill.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Score an AgentOps skill against the local product-grade skill rubric.""" +"""Score static package readiness for an AgentOps skill.""" from __future__ import annotations @@ -248,6 +248,9 @@ def score_skill(path: Path) -> dict: return { "skill": str(path), "name": path.name, + "scope": "static-package-readiness", + "safety_gate_evaluated": False, + "effectiveness_evaluated": False, "total_score": total, "max_score": 30, "rating": rating, @@ -273,13 +276,16 @@ def score_skill(path: Path) -> dict: def audit_block(report: dict) -> dict: - """Compact rubric object for embedding in the skill-builder deep audit's audit-report.json (Pass 3). + """Compact static-readiness object for the deep audit report (Pass 3). Mirrors the rubric schema block: per-category 0-3 score plus an explainable - reason, the 0-30 total, max, and the C/B/A/S rating band. Deterministic — - derived only from the skill directory contents. + reason, the 0-30 total, max, and the C/B/A/S readiness band. It is derived + only from directory contents and cannot evaluate safety or effectiveness. """ return { + "scope": report["scope"], + "safety_gate_evaluated": report["safety_gate_evaluated"], + "effectiveness_evaluated": report["effectiveness_evaluated"], "total_score": report["total_score"], "max_score": report["max_score"], "rating": report["rating"], @@ -290,9 +296,11 @@ def audit_block(report: dict) -> dict: def markdown_report(report: dict) -> str: lines = [ - f"# Skill Quality Score: {report['name']}", + f"# Static Skill Package Readiness: {report['name']}", "", - f"Score: {report['total_score']}/{report['max_score']} ({report['rating']})", + f"Static score: {report['total_score']}/{report['max_score']} ({report['rating']})", + "", + "This score does not evaluate the safety gate or behavioral effectiveness.", "", "## Category Scores", "", diff --git a/skills-codex/standards/.agentops-generated.json b/skills-codex/standards/.agentops-generated.json index 86fe47fd0..d54ac4d52 100644 --- a/skills-codex/standards/.agentops-generated.json +++ b/skills-codex/standards/.agentops-generated.json @@ -2,6 +2,6 @@ "generator": "codex-sync", "source_skill": "skills/standards", "layout": "modular", - "source_hash": "02caa3fe0a55bfbe46c068323cf91d8651e508fe07c000d011ae0f3fb5078d44", - "generated_hash": "72d77ad41f4fbf98f0cd43dc26abf3b4a5cae88c995c860f74ae209150cfbeb1" + "source_hash": "d493ba253d8eaafcd19c3e62b98328a535f3a8d9720bd17a144fa77b1c369268", + "generated_hash": "bec8e7c8b82b76ea771c6264783e4561e7d9eec61f52a7415e196f018cc8de2c" } diff --git a/skills-codex/standards/references/test-pyramid.md b/skills-codex/standards/references/test-pyramid.md index 2346cd8a0..04e00501e 100644 --- a/skills-codex/standards/references/test-pyramid.md +++ b/skills-codex/standards/references/test-pyramid.md @@ -33,7 +33,7 @@ Add test levels only when each one covers a distinct risk. Do not require L2 by default, duplicate the same assertion at every level, or treat test count as evidence quality. -## Operating-loop use +## RPI traversal use - **Plan** names the active behavior, edge scenario, required evidence, and first acceptance check. diff --git a/skills-codex/using-flywheel/.agentops-generated.json b/skills-codex/using-flywheel/.agentops-generated.json index 844511d04..d8be3720c 100644 --- a/skills-codex/using-flywheel/.agentops-generated.json +++ b/skills-codex/using-flywheel/.agentops-generated.json @@ -2,6 +2,6 @@ "generator": "codex-sync", "source_skill": "skills/using-flywheel", "layout": "modular", - "source_hash": "73efa5c718907ace8722a180162e24d08fb7721a42cd601a234ac83e9c523424", + "source_hash": "876ddaec0998ca32f917b49101ffaed1248ac1c676deb985914efaea342cbae5", "generated_hash": "ccc09a1240f753a136cf1e337d60ebd03c7dc705224720a1421dbedc7452d797" } diff --git a/skills/agent-mail/SKILL.md b/skills/agent-mail/SKILL.md index 6c2b9381e..a7971520c 100644 --- a/skills/agent-mail/SKILL.md +++ b/skills/agent-mail/SKILL.md @@ -51,7 +51,9 @@ changes are the caller's call. create work ownership or affect Plan, Candidate, or verdict semantics. - Mail silence proves nothing about work status. - A message or acknowledgement is evidence that communication occurred, not - evidence that a change is correct or complete. The adapter cannot select AgentOps semantics, issue a binding verdict, or turn factory completion into delivery or validation proof. + evidence that a change is correct or complete. The adapter cannot select + AgentOps semantics, issue a binding verdict, or turn factory completion into + delivery or validation proof. - Release a reservation, including any `force_release`, only on the caller's explicit request for that exact reservation. Force-release has no autonomous trigger; a conflict is reported, not force-cleared. @@ -76,20 +78,35 @@ Two disjoint surfaces; do not reach the second from the first: ## Surfaces +Choose exactly one mailbox owner and access mode for each storage root. When an +HTTP/MCP daemon owns the root, use its MCP tools; do not point the direct `am` +CLI at the same database. Use the CLI fallback only with a root not owned by a +running Agent Mail runtime. A busy mailbox activity lock or a bounded read +timeout is a degraded adapter result, not permission to restart the service, +repair the database, or silently switch roots. + Use the MCP tools when they are present. Otherwise use the self-describing `am` -CLI. Discover current syntax with `am mail --help`, -`am file_reservations --help`, and related group help; do not infer commands -from remembered aliases. +CLI. Pin the intended storage root explicitly, and discover current syntax with +`am mail --help`, `am file_reservations --help`, and related group help; do not +infer commands from remembered aliases. If a direct macOS read rejects a +symlinked snapshot directory such as `/var`, use a caller-scoped, non-symlinked +temporary directory for that isolated invocation or report the adapter +degraded; never weaken the traversal check. ## One-shot use 1. Confirm that multiple explicitly coordinated writers share the repository. -2. Register the caller-supplied identity against the same absolute project path. -3. Reserve only the supplied paths, with a bounded TTL. -4. Report conflicts without waiting, narrowing scope, or changing the plan. -5. Send the supplied message once and record its id. -6. Read or acknowledge only the requested thread. -7. Release only reservations the caller explicitly asks to release. +2. Freeze one storage root and either MCP/server mode or direct-CLI mode; never + mix both against the same live database. +3. Register the caller-supplied identity against the same absolute project path. +4. Reserve only the supplied paths, with a bounded TTL. +5. Report conflicts without waiting, narrowing scope, or changing the plan. +6. Send the supplied message once and record its id. +7. Read or acknowledge only the requested thread. +8. Before the caller advances a declared transition, verify every + acknowledgement-required message in that transition has the intended + recipient acknowledgement. Later traffic is not an implicit acknowledgement. +9. Release only reservations the caller explicitly asks to release. ## Output @@ -104,6 +121,12 @@ Terminal outcomes are explicit, never silent: hand-written coordination or treat the absence as "no conflicts". - **Reservation conflict** — report the conflicting reservation as-is; do not narrow, widen, renew, or force-release it. +- **Mailbox ownership conflict** — a daemon and direct CLI contend for one + storage root: report the lock owner/mode and stop; do not restart, repair, or + bypass the lock as a coordination side effect. +- **Required acknowledgement pending** — report the exact message and intended + recipient and stop the dependent transition. Do not infer acknowledgement + from a later reply or repair it after validation. - **Timeout / degraded surface** — report the operation as timed out or degraded with what was and was not observed; a timeout is evidence, not "done". - **Cleanup** — reservations released this session are listed by id; any left diff --git a/skills/agent-mail/references/RECOVERY.md b/skills/agent-mail/references/RECOVERY.md index abe310ecf..de8bedf76 100644 --- a/skills/agent-mail/references/RECOVERY.md +++ b/skills/agent-mail/references/RECOVERY.md @@ -230,6 +230,8 @@ am acks remind /abs/path/project GreenCastle --min-age-minutes 30 |---------|-----------|-----| | Stale reservations accumulating | Agent crashed without releasing | `doctor repair --yes` | | FTS search returns wrong results | Index out of sync | `doctor repair --yes` | -| "database is locked" | Concurrent access issue | Restart server, retry | +| "database is locked" | Another runtime may own or be actively using the selected storage root | Identify the owner and use that root's frozen access mode; report degraded if it remains busy, and do not restart the server as a coordination side effect | +| "mailbox activity lock is busy" | A daemon or another direct runtime owns the same storage root | Use the running daemon through MCP, or a separately authorized isolated CLI root; do not restart or repair as a coordination side effect | +| "refusing to traverse symlinked snapshot directory /var" on macOS | Direct-read snapshot temporary path resolves through macOS's `/var` symlink | For an isolated invocation, set `TMPDIR` to a non-symlinked caller-scoped temporary root; do not disable traversal protection | | Corrupted git archive | Interrupted write | Restore from backup | | Server won't start | Port conflict | `config set-port 9000` | diff --git a/skills/catalog.json b/skills/catalog.json index 5985a75c8..06db747d2 100644 --- a/skills/catalog.json +++ b/skills/catalog.json @@ -656,7 +656,7 @@ "code-complete" ], "produces": [ - "caller-selected handoff path or .agents/ao/handoff/*.md" + "caller-selected handoff path or .agents/ao/handoff/*" ], "references_count": 1, "tier": "session", @@ -1200,7 +1200,7 @@ "adr" ], "produces": [ - ".agents/scratch/reverse-engineer/*.md" + ".agents/scratch/reverse-engineer/*/" ], "references_count": 2, "tier": "execution", diff --git a/skills/codebase-recon/SKILL.md b/skills/codebase-recon/SKILL.md index 19e29d2cc..fecd7ced6 100644 --- a/skills/codebase-recon/SKILL.md +++ b/skills/codebase-recon/SKILL.md @@ -82,7 +82,10 @@ leads with. Pattern packaging beyond evidence pointers belongs in ## Workflow 1. Record the current commit and the repository's local source-of-truth - precedence. Search for a prior recon pack before starting. + precedence. Search for validated prior manifests before starting with + `skills/codebase-recon/scripts/validate-output.sh --repo-root --discover-priors`. + Successful empty output means no prior pack exists at either documented + default. 2. If no prior pack exists, use `baseline` mode. If one exists, verify its still-valid claims against the current commit and use `delta` mode. Preserve valid evidence by reference and describe only changed paths and synthesis. @@ -124,12 +127,15 @@ The durable output doc earns its keep only if a future reader can re-verify a claim without redoing the recon. Every `fact` cites file:line; every `inference` cites the file:line facts it rests on. A claim that cannot be cited is downgraded to `unknown` before the report ships — never shipped -uncited at its original confidence. The manifest validator accepts a bare file -path (it requires the path resolve to an existing regular file, so a bare -directory is rejected as a coverage gap), but does not require the line number; -hold the companion report to the stricter floor: a path without a line is a -pointer to homework, not a citation, and counts as a coverage gap in the -report's own terms. +uncited at its original confidence. The manifest validator checks citations +against the exact Git commit declared by that manifest. They must be safe +repository-relative regular-file paths; artifact-local and external paths are +rejected because this schema has no digest field for those bytes. A supplied +line number must exist in the committed blob. The validator also resolves each +representative flow path at that commit. It does not require every citation to +carry a line number; hold the companion report to the stricter floor: a path +without a line is a pointer to homework, not a citation, and counts as a +coverage gap in the report's own terms. When reconstructing a repository other than the one that ships this skill, pass `--repo-root ` to the validator so evidence resolves against the target @@ -142,17 +148,47 @@ tree rather than the skill's own checkout. `codebase-recon.md` in the same directory. - **Format:** `codebase-recon.v1` JSON manifest plus an evidence-cited Markdown report covering the same commit, mode, flows, claims, and scope boundaries. + The manifest's `report` object names `codebase-recon.md` and binds its + lowercase SHA-256. The report carries one + `` marker plus `manifest_commit`, + `manifest_mode`, `flows_sha256`, `claims_sha256`, and `coverage_sha256` + markers computed from canonical compact sorted JSON for those sections. - **Validation command:** `skills/codebase-recon/scripts/validate-output.sh ` - validates the machine-readable manifest; the cited Markdown report remains - its human-readable companion. + snapshots and validates both artifacts, then rechecks their identities and + the repository HEAD/index/worktree before returning. - **Downstream handoff:** pass both validated artifact paths to the requesting research, planning, review, or documentation workflow; the consumer owns any decision or code-change plan. -Baseline manifests carry at least one complete entry-to-test flow. Delta -manifests name an existing prior recon, prove `baseline_verified: true`, and -describe at least one changed path. Every manifest lists both inspected and -uninspected scope. +### Earlier default compatibility + +Packs already stored under `.agents/recon//` remain in place. The +validator's `--discover-priors` mode enumerates validated +`codebase-recon.json` manifests under both that legacy root and the current +scratch root. Record the selected manifest's exact path in `prior_recon`; delta +validation re-validates the cited manifest and its prior chain instead of +accepting a path merely because it exists. New packs use the current default +unless the caller supplies a different path. Never move, copy, or delete an +earlier pack merely to make its directory match the new state tier, because +that would obscure the identity a delta cites. Downstream consumers use the +exact returned artifact paths rather than scanning only one default root. An +earlier pack without a digest-bound companion report remains untouched but is +not returned as validated prior evidence under the current contract. + +Baseline manifests carry at least one complete entry-to-test flow. A manifest +being handed off must name the target repository's current `HEAD` by its full +object-format OID; abbreviations and hex-looking refs are rejected. Historical +manifests cited as priors must likewise carry full immutable commit OIDs that +resolve in that repository. +Delta manifests name an existing prior recon, set `baseline_verified: true`, +and list exactly the paths in Git's prior-commit-to-current-commit diff. The +validator derives those facts rather than trusting the boolean or path list. +It also refuses dirty tracked, staged, or untracked source state outside +`.agents/`, because those bytes are not bound by the declared commit. Every +manifest lists both inspected and uninspected scope. Manifests and companions +must be real regular files, are read from one snapshot, and are rechecked along +with HEAD and source status after validation so a mid-run swap cannot earn a +green result for different bytes. The validator is the machine boundary: @@ -160,17 +196,24 @@ The validator is the machine boundary: skills/codebase-recon/scripts/validate-output.sh ``` -Evidence entries are existing file paths, optionally followed by a line number. -Delta manifests require an existing prior pack, `baseline_verified: true`, and -at least one described change. +Evidence entries are repository-relative files at the manifest's commit, +optionally followed by a line number. +Delta manifests require a valid prior `codebase-recon.json` chain, an ancestor +commit, `baseline_verified: true`, and an exact changed-path match to the Git +diff ending at current `HEAD`. Enumerate validated manifests at both documented +defaults with: + +```bash +skills/codebase-recon/scripts/validate-output.sh --repo-root --discover-priors +``` Executable behavior: [references/codebase-recon.feature](references/codebase-recon.feature). ## Quality -- Every fact and inference resolves to existing evidence; unknowns remain - visibly typed and never masquerade as established behavior. +- Every fact and inference resolves to evidence in the manifest's exact commit; + unknowns remain visibly typed and never masquerade as established behavior. - Representative flows reach entry, domain, integration, and test surfaces, while inspected and uninspected scope stay explicit. - The named validator passes before the JSON manifest and companion report are diff --git a/skills/codebase-recon/references/codebase-recon.feature b/skills/codebase-recon/references/codebase-recon.feature index 1408c6f99..32326b5fa 100644 --- a/skills/codebase-recon/references/codebase-recon.feature +++ b/skills/codebase-recon/references/codebase-recon.feature @@ -4,12 +4,29 @@ Feature: Evidence-bounded repository reconstruction Scenario: A baseline explains representative repository flows Given repository precedence and the current commit are known When entry, domain, integration, and test paths are traced - Then material claims are typed and cited + Then material claims are typed and cited against that exact commit And inspected and uninspected scope are explicit @covered-by:tests/scripts/agentops-native-skills.bats::delta Scenario: A later run preserves a verified baseline Given an earlier recon pack exists When the repository is reconstructed again - Then the earlier baseline is checked against the current commit - And the new artifact records a delta instead of replacing valid evidence + Then the cited prior manifest chain passes the recon validator + And the earlier commit is an ancestor of the current repository HEAD + And the new artifact's changed paths equal the Git diff between those commits + And dirty source bytes outside the declared commits are rejected + + @covered-by:tests/scripts/agentops-native-skills.bats::prior-discovery + Scenario: Current and earlier default packs are discoverable + Given validated prior manifests under .agents/scratch/codebase-recon and .agents/recon + When prior discovery runs + Then both manifests are returned at their existing paths + And an invalid manifest is never accepted as a delta's prior pack + + @covered-by:tests/scripts/agentops-native-skills.bats::companion + Scenario: The manifest and human report are one stable evidence pack + Given codebase-recon.json binds codebase-recon.md by SHA-256 + And the report binds the manifest commit, mode, flows, claims, and coverage + When validation runs over immutable snapshots of both files + Then a missing mismatched or symlinked companion is rejected + And a manifest, report, HEAD, index, or worktree change before return is rejected diff --git a/skills/codebase-recon/scripts/validate-output.sh b/skills/codebase-recon/scripts/validate-output.sh index 9fe608433..2d9d83b1d 100755 --- a/skills/codebase-recon/scripts/validate-output.sh +++ b/skills/codebase-recon/scripts/validate-output.sh @@ -9,7 +9,7 @@ set -euo pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" usage() { - echo "usage: $0 [--repo-root ] " >&2 + echo "usage: $0 [--repo-root ] [--discover-priors | ]" >&2 } # Evidence paths in a recon manifest are relative to the repository being @@ -18,6 +18,7 @@ usage() { # to the skill's own checkout for the in-repo self-test case. repo_root="" artifact="" +discover_priors=0 while [[ $# -gt 0 ]]; do case "$1" in --repo-root) @@ -26,6 +27,7 @@ while [[ $# -gt 0 ]]; do repo_root="$1" ;; --repo-root=*) repo_root="${1#--repo-root=}" ;; + --discover-priors) discover_priors=1 ;; -h|--help) usage; exit 0 ;; -*) echo "unknown flag: $1" >&2; usage; exit 2 ;; *) @@ -39,7 +41,12 @@ while [[ $# -gt 0 ]]; do shift done -if [[ -z "$artifact" || ! -f "$artifact" ]]; then +if [[ "$discover_priors" == "1" && -n "$artifact" ]]; then + echo "--discover-priors does not accept an artifact" >&2 + usage + exit 2 +fi +if [[ "$discover_priors" != "1" && ( -z "$artifact" || ! -f "$artifact" || -L "$artifact" ) ]]; then usage exit 2 fi @@ -52,59 +59,116 @@ if [[ ! -d "$repo_root" ]]; then exit 2 fi repo_root="$(cd "$repo_root" && pwd -P)" -artifact_dir="$(cd "$(dirname "$artifact")" && pwd -P)" -jq -e ' - def text: type == "string" and length > 0; - .schema_version == "codebase-recon.v1" - and (.mode == "baseline" or .mode == "delta") - and (.commit | text) - and (.flows - | type == "array" - and all(.[]; - (.entry | text) - and (.domain | text) - and (.integration | text) - and (.tests | text))) - and (.claims - | type == "array" - and all(.[]; - (.kind == "fact" or .kind == "inference" or .kind == "unknown") - and (.text | text) - and (.confidence == "high" or .confidence == "medium" or .confidence == "low") - and (.evidence | type == "array" and all(.[]; text)) - and (if .kind == "unknown" then true else (.evidence | length > 0) end))) - and (.coverage | type == "object") - and (.coverage.inspected | type == "array" and length > 0 and all(.[]; text)) - and (.coverage.uninspected | type == "array" and length > 0 and all(.[]; text)) - and ( - if .mode == "baseline" then - (.flows | length > 0) - and ((has("prior_recon") | not) or .prior_recon == "" or .prior_recon == null) - else - (.prior_recon | text) - and .baseline_verified == true - and (.delta - | type == "array" and length > 0 - and all(.[]; (.path | text) and (.change | text))) - end - ) -' "$artifact" >/dev/null || { - echo "invalid codebase-recon.v1 artifact: $artifact" >&2 - exit 1 +snapshot_root="$(mktemp -d "${TMPDIR:-/tmp}/codebase-recon-validate.XXXXXX")" +cleanup() { + rm -rf -- "$snapshot_root" +} +trap cleanup EXIT HUP INT TERM + +declare -a watched_sources=() +declare -a watched_identities=() +declare -a watched_hashes=() +snapshot_counter=0 + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi } -# resolve_evidence CANDIDATE MUST_BE_FILE -# MUST_BE_FILE=1 → claim evidence: the contract is "existing file paths, -# optionally followed by a line number", so the resolved path must be a -# regular file. A directory is a coverage gap, not a citation, and no longer -# passes silently (the old `-e` accepted directories). -# MUST_BE_FILE=0 → prior-recon pack reference: any existing path resolves. -resolve_evidence() { - local candidate="$1" must_file="$2" - if [[ "$candidate" =~ ^(.+):[0-9]+$ ]]; then - candidate="${BASH_REMATCH[1]}" +sha256_stream() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + else + shasum -a 256 | awk '{print $1}' fi +} + +file_identity() { + if stat -f '%d:%i:%z:%m' "$1" >/dev/null 2>&1; then + stat -f '%d:%i:%z:%m' "$1" + else + stat -c '%d:%i:%s:%Y' "$1" + fi +} + +# Snapshot each manifest/report exactly once. cp -P copies a raced-in symlink as +# a symlink rather than following it; the destination type check then fails. +watch_regular_file() { + local source="$1" label="$2" before after source_hash snapshot_hash snapshot + [[ -f "$source" && ! -L "$source" ]] || { + echo "$label must be a real regular file: $source" >&2 + return 1 + } + before="$(file_identity "$source")" || return 1 + snapshot_counter=$((snapshot_counter + 1)) + snapshot="$snapshot_root/$snapshot_counter" + cp -P -- "$source" "$snapshot" + [[ -f "$snapshot" && ! -L "$snapshot" ]] || { + echo "$label changed shape while being snapshotted: $source" >&2 + return 1 + } + after="$(file_identity "$source")" || return 1 + [[ "$before" == "$after" ]] || { + echo "$label changed identity while being snapshotted: $source" >&2 + return 1 + } + source_hash="$(sha256_file "$source")" + snapshot_hash="$(sha256_file "$snapshot")" + [[ "$source_hash" == "$snapshot_hash" ]] || { + echo "$label changed bytes while being snapshotted: $source" >&2 + return 1 + } + watched_sources+=("$source") + watched_identities+=("$before") + watched_hashes+=("$snapshot_hash") + WATCHED_SNAPSHOT="$snapshot" +} + +recheck_watched_files() { + local i source + for ((i = 0; i < ${#watched_sources[@]}; i++)); do + source="${watched_sources[$i]}" + [[ -f "$source" && ! -L "$source" ]] || { + echo "validated artifact changed shape during validation: $source" >&2 + return 1 + } + [[ "$(file_identity "$source")" == "${watched_identities[$i]}" ]] || { + echo "validated artifact changed identity during validation: $source" >&2 + return 1 + } + [[ "$(sha256_file "$source")" == "${watched_hashes[$i]}" ]] || { + echo "validated artifact changed bytes during validation: $source" >&2 + return 1 + } + done +} + +repo_head_initial="$(git -C "$repo_root" rev-parse --verify 'HEAD^{commit}' 2>/dev/null || true)" +repo_status_initial="$(git -C "$repo_root" status --porcelain=v1 --untracked-files=all -- . ':(exclude).agents' 2>/dev/null || true)" + +recheck_repo_state() { + local current_head current_status + current_head="$(git -C "$repo_root" rev-parse --verify 'HEAD^{commit}' 2>/dev/null || true)" + current_status="$(git -C "$repo_root" status --porcelain=v1 --untracked-files=all -- . ':(exclude).agents' 2>/dev/null || true)" + [[ -n "$repo_head_initial" && "$current_head" == "$repo_head_initial" ]] || { + echo "target repository HEAD changed during validation" >&2 + return 1 + } + [[ "$current_status" == "$repo_status_initial" && -z "$current_status" ]] || { + echo "target repository index or worktree changed during validation" >&2 + return 1 + } +} + +# Resolve the prior manifest's exact path. Unlike evidence citations, a prior +# reference has no :LINE syntax: silently stripping such a suffix would accept +# a different path than the manifest declared. +resolve_prior_manifest() { + local candidate="$1" artifact_dir="$2" local -a roots=() if [[ "$candidate" = /* ]]; then roots=("$candidate") @@ -113,28 +177,358 @@ resolve_evidence() { fi local p for p in "${roots[@]}"; do - if [[ "$must_file" == "1" ]]; then - [[ -f "$p" ]] && return 0 - else - [[ -e "$p" ]] && return 0 - fi + [[ -f "$p" ]] && { printf '%s\n' "$p"; return 0; } done return 1 } -while IFS= read -r evidence; do - if ! resolve_evidence "$evidence" 1; then - echo "missing or non-file claim evidence: $evidence" >&2 - exit 1 +# resolve_manifest_commit ARTIFACT +# +# A manifest's commit is evidence only when it resolves to an immutable commit +# in the target repository. Symbolic names such as HEAD are deliberately +# rejected because their meaning changes after the artifact is written. +resolve_manifest_commit() { + local manifest="$1" declared declared_normalized resolved resolved_normalized object_format oid_length + declared="$(jq -r '.commit // empty' "$manifest")" + if ! object_format="$(git -C "$repo_root" rev-parse --show-object-format=storage 2>/dev/null)"; then + object_format="$(git -C "$repo_root" rev-parse --show-object-format 2>/dev/null)" || { + echo "could not determine target repository object format" >&2 + return 1 + } fi -done < <(jq -r '.claims[] | select(.kind == "fact" or .kind == "inference") | .evidence[]' "$artifact") + object_format="${object_format%%$'\n'*}" + case "$object_format" in + sha1) oid_length=40 ;; + sha256) oid_length=64 ;; + *) echo "unsupported target repository object format: $object_format" >&2; return 1 ;; + esac + if [[ ! "$declared" =~ ^[0-9a-fA-F]{$oid_length}$ ]]; then + echo "manifest commit is not a full $object_format object id: $declared" >&2 + return 1 + fi + if ! resolved="$(git -C "$repo_root" rev-parse --verify "${declared}^{commit}" 2>/dev/null)"; then + echo "manifest commit does not resolve in target repository: $declared" >&2 + return 1 + fi + declared_normalized="$(printf '%s' "$declared" | tr '[:upper:]' '[:lower:]')" + resolved_normalized="$(printf '%s' "$resolved" | tr '[:upper:]' '[:lower:]')" + if [[ "$resolved_normalized" != "$declared_normalized" ]]; then + echo "manifest commit resolved through a mutable or abbreviated name: $declared" >&2 + return 1 + fi + printf '%s\n' "$resolved_normalized" +} -if [[ "$(jq -r '.mode' "$artifact")" == "delta" ]]; then - prior="$(jq -r '.prior_recon' "$artifact")" - if ! resolve_evidence "$prior" 0; then - echo "missing prior recon pack: $prior" >&2 - exit 1 +# resolve_repo_path_at_commit CITATION COMMIT MUST_BE_FILE +# +# Fact/inference evidence belongs to the repository commit named by the +# manifest, never to whichever bytes happen to be in the current worktree or +# beside the artifact. A trailing :LINE is checked against that committed blob. +resolve_repo_path_at_commit() { + local citation="$1" commit="$2" must_file="$3" candidate="$1" line="" line_number="" + local tree_entry mode object_type object_id line_count + if [[ "$candidate" =~ ^(.+):([0-9]+)$ ]]; then + candidate="${BASH_REMATCH[1]}" + line="${BASH_REMATCH[2]}" fi + while [[ "$candidate" == ./* ]]; do candidate="${candidate#./}"; done + if [[ -z "$candidate" || "$candidate" == "." || "$candidate" = /* || "$candidate" == */ || "$candidate" == ".." || "$candidate" == ../* || "$candidate" == */../* || "$candidate" == */.. ]]; then + echo "evidence citation is not a safe repository-relative path: $citation" >&2 + return 1 + fi + if ! tree_entry="$(git -C "$repo_root" ls-tree "$commit" -- ":(literal)$candidate")" || [[ -z "$tree_entry" ]]; then + echo "evidence path is absent from manifest commit: $citation" >&2 + return 1 + fi + read -r mode object_type object_id _ <<<"$tree_entry" + if [[ "$must_file" == "1" && ( "$mode" != 100* || "$object_type" != "blob" ) ]]; then + echo "evidence citation is not a regular file in manifest commit: $citation" >&2 + return 1 + fi + if [[ -n "$line" ]]; then + line_number=$((10#$line)) + if [[ "$must_file" != "1" || "$line_number" -lt 1 ]]; then + echo "invalid evidence line citation: $citation" >&2 + return 1 + fi + if ! line_count="$(git -C "$repo_root" cat-file blob "$object_id" | awk 'END { print NR }')"; then + echo "could not read evidence blob from manifest commit: $citation" >&2 + return 1 + fi + if (( line_number > line_count )); then + echo "evidence line is outside committed blob: $citation" >&2 + return 1 + fi + fi + printf '%s\n' "$candidate" +} + +require_clean_source_tree() { + local source_status + if ! source_status="$(git -C "$repo_root" status --porcelain=v1 --untracked-files=all -- . ':(exclude).agents' 2>&1)"; then + echo "could not inspect target repository worktree: $source_status" >&2 + return 1 + fi + if [[ -n "$source_status" ]]; then + echo "target repository has source changes not bound by the manifest commit:" >&2 + printf '%s\n' "$source_status" >&2 + return 1 + fi +} + +require_report_marker() { + local report="$1" key="$2" expected="$3" count + count="$(grep -Fxc "$key: $expected" "$report" || true)" + if [[ "$count" != "1" ]]; then + echo "companion report must contain exactly one '$key: $expected' marker" >&2 + return 1 + fi +} + +validate_companion_report() { + local manifest="$1" manifest_dir="$2" report_rel report_source report_snapshot declared_sha actual_sha + local commit mode flows_sha claims_sha coverage_sha + report_rel="$(jq -r '.report.path // empty' "$manifest")" + declared_sha="$(jq -r '.report.sha256 // empty' "$manifest")" + if [[ "$report_rel" != "codebase-recon.md" || ! "$declared_sha" =~ ^[0-9a-f]{64}$ ]]; then + echo "manifest must bind companion report codebase-recon.md by lowercase SHA-256" >&2 + return 1 + fi + report_source="$manifest_dir/$report_rel" + if ! watch_regular_file "$report_source" "companion codebase-recon report"; then + return 1 + fi + report_snapshot="$WATCHED_SNAPSHOT" + actual_sha="$(sha256_file "$report_snapshot")" + if [[ "$actual_sha" != "$declared_sha" ]]; then + echo "companion report digest does not match manifest: $report_source" >&2 + return 1 + fi + + commit="$(jq -r '.commit' "$manifest")" + mode="$(jq -r '.mode' "$manifest")" + flows_sha="$(jq -cS '.flows' "$manifest" | sha256_stream)" + claims_sha="$(jq -cS '.claims' "$manifest" | sha256_stream)" + coverage_sha="$(jq -cS '.coverage' "$manifest" | sha256_stream)" + grep -Fqx '' "$report_snapshot" || { + echo "companion report lacks codebase-recon-report.v1 identity marker" >&2 + return 1 + } + require_report_marker "$report_snapshot" manifest_commit "$commit" || return 1 + require_report_marker "$report_snapshot" manifest_mode "$mode" || return 1 + require_report_marker "$report_snapshot" flows_sha256 "$flows_sha" || return 1 + require_report_marker "$report_snapshot" claims_sha256 "$claims_sha" || return 1 + require_report_marker "$report_snapshot" coverage_sha256 "$coverage_sha" || return 1 +} + +# validate_artifact ARTIFACT DEPTH STACK REQUIRE_CURRENT_HEAD +# +# Delta manifests form a provenance chain. Validate every cited manifest in +# that chain, with a bounded depth and cycle check, before accepting the leaf. +# STACK is a newline-delimited list of normalized artifact paths. +# REQUIRE_CURRENT_HEAD=1 is used for the artifact the caller is validating; +# recursively cited/discovered historical manifests need only resolve in the +# repository because their commit is expected to predate HEAD. +validate_artifact() { + local current_input="$1" depth="$2" stack="$3" require_current_head="$4" + if [[ ! -f "$current_input" || -L "$current_input" ]]; then + echo "missing codebase-recon.v1 artifact: $current_input" >&2 + return 1 + fi + + if (( depth > 32 )); then + echo "prior recon chain exceeds 32 manifests: $current_input" >&2 + return 1 + fi + + local current_dir current_source current + current_dir="$(cd "$(dirname "$current_input")" && pwd -P)" + current_source="$current_dir/$(basename "$current_input")" + case $'\n'"$stack"$'\n' in + *$'\n'"$current_source"$'\n'*) + echo "cyclic prior recon chain: $current_source" >&2 + return 1 + ;; + esac + + local next_stack + if [[ -n "$stack" ]]; then + next_stack="$stack"$'\n'"$current_source" + else + next_stack="$current_source" + fi + + if ! watch_regular_file "$current_source" "codebase-recon manifest"; then + return 1 + fi + current="$WATCHED_SNAPSHOT" + + jq -e ' + def text: type == "string" and length > 0; + def path_text: text and (test("[\u0000-\u001f\u007f]") | not); + .schema_version == "codebase-recon.v1" + and (.mode == "baseline" or .mode == "delta") + and (.commit | text) + and (.flows + | type == "array" + and all(.[]; + (.entry | path_text) + and (.domain | path_text) + and (.integration | path_text) + and (.tests | path_text))) + and (.claims + | type == "array" + and all(.[]; + (.kind == "fact" or .kind == "inference" or .kind == "unknown") + and (.text | text) + and (.confidence == "high" or .confidence == "medium" or .confidence == "low") + and (.evidence | type == "array" and all(.[]; path_text)) + and (if .kind == "unknown" then true else (.evidence | length > 0) end))) + and (.coverage | type == "object") + and (.coverage.inspected | type == "array" and length > 0 and all(.[]; text)) + and (.coverage.uninspected | type == "array" and length > 0 and all(.[]; text)) + and (.report | type == "object") + and (.report.path == "codebase-recon.md") + and (.report.sha256 | type == "string" and test("^[0-9a-f]{64}$")) + and ( + if .mode == "baseline" then + (.flows | length > 0) + and ((has("prior_recon") | not) or .prior_recon == "" or .prior_recon == null) + else + (.prior_recon | path_text) + and .baseline_verified == true + and (.delta + | type == "array" and length > 0 + and all(.[]; (.path | path_text) and (.change | text))) + end + ) + ' "$current" >/dev/null || { + echo "invalid codebase-recon.v1 artifact: $current_source" >&2 + return 1 + } + if ! validate_companion_report "$current" "$current_dir"; then + echo "invalid companion report for: $current_source" >&2 + return 1 + fi + + if ! git -C "$repo_root" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "target is not a git repository: $repo_root" >&2 + return 1 + fi + if ! require_clean_source_tree; then + return 1 + fi + + local current_commit + if ! current_commit="$(resolve_manifest_commit "$current")"; then + return 1 + fi + if [[ "$require_current_head" == "1" ]]; then + local target_head + if ! target_head="$(git -C "$repo_root" rev-parse --verify 'HEAD^{commit}' 2>/dev/null)"; then + echo "target repository has no current commit: $repo_root" >&2 + return 1 + fi + if [[ "$current_commit" != "$target_head" ]]; then + echo "manifest commit is not the target repository's current commit: $(jq -r '.commit' "$current")" >&2 + return 1 + fi + fi + + local evidence + while IFS= read -r evidence; do + if ! resolve_repo_path_at_commit "$evidence" "$current_commit" 1 >/dev/null; then + echo "invalid or unbound claim evidence: $evidence" >&2 + return 1 + fi + done < <(jq -r '.claims[] | select(.kind == "fact" or .kind == "inference") | .evidence[]' "$current") + + local flow_path + while IFS= read -r flow_path; do + if ! resolve_repo_path_at_commit "$flow_path" "$current_commit" 1 >/dev/null; then + echo "invalid or unbound flow file: $flow_path" >&2 + return 1 + fi + done < <(jq -r '.flows[] | .entry, .tests' "$current") + while IFS= read -r flow_path; do + if ! resolve_repo_path_at_commit "$flow_path" "$current_commit" 0 >/dev/null; then + echo "invalid or unbound flow path: $flow_path" >&2 + return 1 + fi + done < <(jq -r '.flows[] | .domain, .integration' "$current") + + if [[ "$(jq -r '.mode' "$current")" == "delta" ]]; then + local prior prior_path prior_commit + prior="$(jq -r '.prior_recon' "$current")" + if ! prior_path="$(resolve_prior_manifest "$prior" "$current_dir")"; then + echo "missing or non-file prior recon pack: $prior" >&2 + return 1 + fi + if ! validate_artifact "$prior_path" "$((depth + 1))" "$next_stack" 0; then + echo "invalid prior recon pack: $prior" >&2 + return 1 + fi + prior_commit="$VALIDATED_COMMIT" + if ! git -C "$repo_root" merge-base --is-ancestor "$prior_commit" "$current_commit"; then + echo "prior recon commit is not an ancestor of manifest commit: $prior" >&2 + return 1 + fi + + local declared_delta actual_delta declared_count unique_count + declared_delta="$(jq -r '.delta[].path' "$current" | LC_ALL=C sort -u)" + declared_count="$(jq -r '.delta | length' "$current")" + unique_count="$(printf '%s\n' "$declared_delta" | sed '/^$/d' | wc -l | tr -d ' ')" + if [[ "$declared_count" != "$unique_count" ]]; then + echo "delta contains duplicate changed paths: $current" >&2 + return 1 + fi + if ! actual_delta="$(git -C "$repo_root" diff --name-only --diff-filter=ACDMRTUXB "$prior_commit" "$current_commit" -- | LC_ALL=C sort -u)"; then + echo "could not derive repository delta for $current" >&2 + return 1 + fi + if [[ "$declared_delta" != "$actual_delta" ]]; then + echo "declared delta paths do not match git diff ${prior_commit}..${current_commit}" >&2 + return 1 + fi + fi + VALIDATED_COMMIT="$current_commit" +} + +discover_valid_priors() { + local -a candidates=() + shopt -s nullglob + candidates+=("$repo_root"/.agents/scratch/codebase-recon/*/codebase-recon.json) + candidates+=("$repo_root"/.agents/recon/*/codebase-recon.json) + shopt -u nullglob + + if [[ "${#candidates[@]}" -eq 0 ]]; then + return 0 + fi + + local candidate found=0 + while IFS= read -r candidate; do + if validate_artifact "$candidate" 0 "" 0 >/dev/null 2>&1; then + printf '%s\n' "$candidate" + found=1 + else + echo "ignoring invalid prior recon pack: $candidate" >&2 + fi + done < <(printf '%s\n' "${candidates[@]}" | LC_ALL=C sort) + + if [[ "$found" == "0" ]]; then + echo "no validated prior recon packs found under current or earlier default roots" >&2 + return 1 + fi +} + +if [[ "$discover_priors" == "1" ]]; then + discover_valid_priors + recheck_watched_files + recheck_repo_state + exit $? fi +validate_artifact "$artifact" 0 "" 1 +recheck_watched_files +recheck_repo_state echo "valid codebase-recon.v1: $artifact" diff --git a/skills/handoff/SKILL.md b/skills/handoff/SKILL.md index 9e9670e67..b2493c17b 100644 --- a/skills/handoff/SKILL.md +++ b/skills/handoff/SKILL.md @@ -4,7 +4,7 @@ description: 'Write compact caller-authored session evidence without choosing co practices: [adr, wiki-knowledge-surface, code-complete] hexagonal_role: supporting consumes: [] -produces: [caller-selected handoff path or .agents/ao/handoff/*.md] +produces: [caller-selected handoff path or .agents/ao/handoff/*] context_rel: [] skill_api_version: 1 context: @@ -57,4 +57,15 @@ boundary for JSON artifacts under `.agents/ao/handoff/`. The skill may write Markdown when that better serves a human, but the content semantics remain identical. +### Earlier default compatibility + +JSON artifacts already stored under `.agents/handoff/` remain read-only +evidence. `ao session handoff` writes new JSON to `.agents/ao/handoff/`, while +`ao session rehydrate` searches both directories and selects the newest +lexical handoff id; if an identical filename exists in both, the canonical +`.agents/ao/handoff/` copy wins. No command moves or deletes the legacy files. +Human-authored Markdown consumers receive the exact path, so they do not need +to scan either default. This owning skill contract is the compatibility +authority; no separate migration artifact is required. + Return the artifact path and stop. diff --git a/skills/implement/SKILL.md b/skills/implement/SKILL.md index 51037912d..c429e1abd 100644 --- a/skills/implement/SKILL.md +++ b/skills/implement/SKILL.md @@ -9,6 +9,7 @@ hexagonal_role: driving-adapter consumes: [] produces: - subject-manifest.v1 +output_contract: 'subject-manifest.v1 digest, author context ID, and exact acceptance-check receipts returned through the response or runtime channel' context_rel: - kind: customer-of with: plan diff --git a/skills/plan/SKILL.md b/skills/plan/SKILL.md index 359e97bbb..22f6d45a6 100644 --- a/skills/plan/SKILL.md +++ b/skills/plan/SKILL.md @@ -8,6 +8,7 @@ practices: hexagonal_role: domain consumes: [] produces: [] +output_contract: 'in-place caller intent update or concise proposed amendment; never an AgentOps planning artifact' context_rel: [] skill_api_version: 1 user-invocable: true diff --git a/skills/reverse-engineer/SKILL.md b/skills/reverse-engineer/SKILL.md index ff56b4129..f839a3d91 100644 --- a/skills/reverse-engineer/SKILL.md +++ b/skills/reverse-engineer/SKILL.md @@ -8,7 +8,7 @@ practices: hexagonal_role: supporting consumes: [] produces: -- .agents/scratch/reverse-engineer/*.md +- '.agents/scratch/reverse-engineer/*/' context_rel: [] skill_api_version: 1 context: @@ -27,7 +27,7 @@ metadata: disposition: keep_specialist tier: execution internal: false -output_contract: feature inventory, feature-registry.yaml, spec set, steal-map.md +output_contract: validated phase-1 teardown directory, followed by a caller-authored and validated phase-2 steal-map.md --- # Reverse Engineer @@ -59,6 +59,13 @@ Binary mode requires `--authorized` (see Invocation Contract + Self-Test). Use t Map each capability the teardown found onto **our** surfaces. This is the part that turns research into a decision. Emit `.agents/scratch/reverse-engineer//steal-map.md` with a table; every row cites the teardown evidence **and** the matching surface in our repo. +The mechanical script intentionally stops after validating Phase 1. It cannot +truthfully decide whether our live tree has, lacks, or should adopt a capability. +The caller authors `steal-map.md` from the generated registry plus a fresh read +of our repository, then runs the complete-output validator below. A missing or +malformed map is therefore an incomplete skill result, not a script success +silently relabelled as a decision. + | Their capability | Our surface today | Verdict | |---|---|---| | `` | `` | **have** / **gap** / **steal** / **park** / **reject** | @@ -90,11 +97,11 @@ neither strategy grants readiness or continuation authority. ## Invocation Contract -Required: `product_name`. Common flags: `--mode=repo|binary|both`, `--upstream-repo`, `--upstream-ref` (pins the clone to a specific commit/tag/branch; the resolved SHA is recorded in `clone-metadata.json` on any clone), `--output-dir` (default `.agents/scratch/reverse-engineer//`), `--security-audit`, `--materialize-archives` (authorized-only opt-in; embedded-archive extraction is off/index-only by default), `--authorized` (mandatory for binary mode — refuses without it). Full list: `python3 skills/reverse-engineer/scripts/reverse_engineer.py --help`. +Required: `product_name`. Common flags: `--mode=repo|binary|both`, `--upstream-repo`, `--upstream-ref` (requires the selected checkout to be at that exact commit and records its resolved SHA in `clone-metadata.json`), `--local-clone-dir` (selects that exact tree, including a non-Git tree; it never falls back to the caller's checkout), `--output-dir` (default `.agents/scratch/reverse-engineer//`), `--security-audit`, `--materialize-archives` (authorized-only opt-in; embedded-archive extraction is off/index-only by default), `--authorized` (mandatory for binary mode — refuses without it). Full list: `python3 skills/reverse-engineer/scripts/reverse_engineer.py --help`. ## Output Specification -Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry.yaml`, `feature-catalog.md`, `spec-architecture.md`, `spec-code-map.md`, `spec-clone-vs-use.md`, `spec-clone-mvp.md`, plus `spec-cli-surface.md` only when a CLI is detected and `clone-metadata.json` only when the script performs a clone (i.e., `--upstream-repo` is supplied and the target is not already checked out); `--upstream-ref` pins which commit, it is not what triggers the file. Security mode adds `output_dir/security/`: `threat-model.md`, `attack-surface.md`, `dataflow.md`, `crypto-review.md`, `authn-authz.md`, `findings.md`, `reproducibility.md`, `validate-security-audit.sh`. Phase-2: `steal-map.md`. +Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry.yaml`, `feature-catalog.md`, `spec-architecture.md`, `spec-code-map.md`, `spec-clone-vs-use.md`, `spec-clone-mvp.md`, plus `spec-cli-surface.md` only when a CLI is detected. `clone-metadata.json` is written whenever an upstream repo/ref is selected and binds the exact analyzed commit, including an already-present checkout. Security mode adds `output_dir/security/`: `threat-model.md`, `attack-surface.md`, `dataflow.md`, `crypto-review.md`, `authn-authz.md`, `findings.md`, `reproducibility.md`, `validate-security-audit.sh`. Phase-2 adds the caller-authored `steal-map.md`. - **Artifact directory:** the exact `--output-dir`, defaulting to `$REPO/.agents/scratch/reverse-engineer//`. @@ -102,53 +109,40 @@ Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry. files live only in the `security/` child directory. - **Serialization/schema format:** registry is YAML, clone metadata is one JSON object, and inventories/specs/steal-map are nonempty Markdown files. -- **Validator command:** with `$output_dir`, `$security_audit`, `$sbom`, and - `$upstream_ref_set` (each flag `0|1`) set: +- **Validator command:** Phase 1 runs this automatically with + `--phase teardown`. After authoring `steal-map.md`, validate the complete + skill output with `$output_dir`, `$security_audit`, `$sbom`, and + `$upstream_ref_set` (each numeric flag `0|1`): ```bash - set -euo pipefail - required=(feature-inventory.md feature-registry.yaml feature-catalog.md spec-architecture.md spec-code-map.md spec-clone-vs-use.md spec-clone-mvp.md analysis-root-path.txt validate-feature-registry.py steal-map.md) - for name in "${required[@]}"; do - test -f "$output_dir/$name" - test ! -L "$output_dir/$name" - test -s "$output_dir/$name" - done - test -f "$output_dir/docs-features.txt" - test ! -L "$output_dir/docs-features.txt" - test ! -L "$output_dir/spec-cli-surface.md" - if [[ -e "$output_dir/spec-cli-surface.md" ]]; then - test -f "$output_dir/spec-cli-surface.md" - test -s "$output_dir/spec-cli-surface.md" - fi - python3 "$output_dir/validate-feature-registry.py" - if [[ "$upstream_ref_set" == 1 ]]; then - test -f "$output_dir/clone-metadata.json" - test ! -L "$output_dir/clone-metadata.json" - jq -e 'type == "object"' "$output_dir/clone-metadata.json" >/dev/null - else - [[ "$upstream_ref_set" == 0 ]] - fi - grep -Fqx '| Their capability | Our surface today | Verdict |' "$output_dir/steal-map.md" - if [[ "$security_audit" == 1 ]]; then - test -x "$output_dir/security/validate-security-audit.sh" - if [[ "$sbom" == 1 ]]; then - "$output_dir/security/validate-security-audit.sh" "$output_dir" --sbom - else - [[ "$sbom" == 0 ]] - "$output_dir/security/validate-security-audit.sh" "$output_dir" --no-sbom - fi - else - [[ "$security_audit" == 0 ]] - [[ "$sbom" == 0 ]] - fi + bash skills/reverse-engineer/scripts/validate-output.sh \ + --output-dir "$output_dir" --phase complete \ + --security-audit "$security_audit" --sbom "$sbom" \ + --upstream-ref-set "$upstream_ref_set" ``` - **Downstream handoff:** give the validated `steal-map.md` to Plan for one-way-door candidates; ordinary `have`, `park`, and `reject` decisions remain evidence-backed terminal rows. +### Earlier default compatibility + +Existing teardowns under `.agents/research//` remain in place and +usable. The script accepts that directory when it is passed explicitly with +`--output-dir`; that flag is caller authorization to write the teardown at the +exact selected path. It does not relocate or duplicate existing artifacts. An +invocation that omits the flag writes only to the current scratch default and +never creates output under the earlier root. +Consumers must retain the exact selected `output_dir` with their evidence +references instead of rediscovering outputs by globbing one root. This owning +skill contract is the compatibility authority; no separate migration receipt +is required. + ## Reproducibility + fixtures -`--upstream-ref` pins the clone (fetch `FETCH_HEAD`, record SHA) so contracts can be committed as golden fixtures and diffed across runs. Regression test: `bash skills/reverse-engineer/scripts/repo_fixture_test.sh`. To update a fixture when contracts legitimately change, re-run with the new pinned ref, copy the contract files into `fixtures//`, and commit. +`--upstream-ref` binds the selected checkout to one full commit: a new clone is +checked out detached at the fetched ref, while an existing checkout must already +match or the run refuses before analysis. `clone-metadata.json` records that +resolved commit. Regression test: `bash skills/reverse-engineer/scripts/repo_fixture_test.sh`. To update a fixture when contracts legitimately change, re-run with the new pinned ref, copy the contract files into `fixtures//`, and commit. ## Self-Test (acceptance) @@ -156,13 +150,17 @@ Phase-1 teardown under `output_dir/`: `feature-inventory.md`, `feature-registry. bash skills/reverse-engineer/scripts/self_test.sh ``` -Must show: feature inventory generated, registry generated, registry validator exits 0; in security mode `validate-security-audit.sh` exits 0 and the secret scan passes. +Must show: feature inventory and registry generated; the exact Phase-1 validator +passes; the complete validator rejects a missing and malformed steal-map and +accepts a valid caller-authored fixture; existing-checkout ref mismatch and +output symlinks fail closed; in security mode `validate-security-audit.sh` +exits 0 only after the scaffold is completed and the secret scan passes. ## Examples ### Reverse-engineer an OSS CLI (repo mode) → steal-map -Run the skill for `cc-sdd` with `--mode=repo --upstream-repo="https://github.com/gotalab/cc-sdd.git" --upstream-ref=v1.0.0`. It clones the pinned source, scans the surface, writes inventory/registry/specs, and maps each feature onto our surfaces (`have`, `gap`, `steal`, `park`, or `reject`) in `steal-map.md`. Supply selected steals to Plan. +Run Phase 1 for `cc-sdd` with `--mode=repo --upstream-repo="https://github.com/gotalab/cc-sdd.git" --upstream-ref=v1.0.0`. It clones the pinned source, scans the surface, writes inventory/registry/specs, and validates the teardown. Then inspect our live surfaces, author each `have`/`gap`/`steal`/`park`/`reject` row in `steal-map.md`, and run the complete-output validator. Supply selected steals to Plan. ### Binary analysis with security audit @@ -175,6 +173,7 @@ Run the skill for `ao` with `--authorized --mode=binary --binary-path="$(command | Refuses binary analysis | Missing `--authorized` | Add `--authorized` (explicit written authorization required). | | No `clone-metadata.json` | `--upstream-repo` not passed | Pass `--upstream-repo` (and optionally `--upstream-ref`). | | Fixture diff fails | Upstream changed / stale golden | Re-run pinned, refresh `fixtures/`, commit. | +| Existing teardown is under `.agents/research/` | It used the earlier default | Pass that exact directory with `--output-dir`; new runs otherwise use the scratch default. | | `spec-cli-surface.md` missing | No Node/Python/Go CLI detected | Surface is documented in `spec-code-map.md` instead. | | Steal-map is all "steal" | Skipped the park/reject rules | Substrate we delegate is **park**; doctrine conflicts are **reject** — not everything novel is worth adopting. | diff --git a/skills/reverse-engineer/references/reverse-engineer.feature b/skills/reverse-engineer/references/reverse-engineer.feature index 8975730e6..047116caf 100644 --- a/skills/reverse-engineer/references/reverse-engineer.feature +++ b/skills/reverse-engineer/references/reverse-engineer.feature @@ -23,3 +23,27 @@ Feature: Reverse-engineer reconstructs specs from an existing system Scenario: Output is a reusable spec set When reconstruction completes Then it emits a feature catalog, code map, and specs as durable artifacts + + Scenario: A steal-map is a separate checked decision + Given a validated mechanical teardown + When the caller compares its registry with the live destination repository + Then the caller authors steal-map.md with evidence-backed verdict rows + And the complete-output validator rejects a missing or malformed steal-map + + Scenario: An explicit analysis root cannot drift + Given --local-clone-dir selects a particular tree + When the selected tree is non-Git + Then that exact tree is analyzed instead of the caller's current checkout + When --upstream-ref also selects a Git commit + Then a mismatched existing checkout is refused before outputs are trusted + + Scenario: Managed output paths do not follow links + Given an output parent or managed artifact is a symbolic link + When reverse engineering starts + Then it refuses before writing through that link + + Scenario: An earlier-default output directory remains explicit and usable + Given an existing teardown under .agents/research + When that exact directory is supplied with --output-dir + Then the teardown writes and validates in that directory + And it does not move existing artifacts into the current scratch default diff --git a/skills/reverse-engineer/scripts/reverse_engineer.py b/skills/reverse-engineer/scripts/reverse_engineer.py index e183b4333..8338cd8d3 100755 --- a/skills/reverse-engineer/scripts/reverse_engineer.py +++ b/skills/reverse-engineer/scripts/reverse_engineer.py @@ -5,8 +5,10 @@ import argparse import datetime as _dt import hashlib import json +import os import re import shutil +import stat import subprocess import sys from pathlib import Path @@ -43,13 +45,81 @@ def _die(msg: str, code: int = 2) -> None: raise SystemExit(code) -def _run(cmd: list[str], *, cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess: +def _run( + cmd: list[str], *, cwd: Path | None = None, check: bool = True +) -> subprocess.CompletedProcess: return subprocess.run(cmd, cwd=str(cwd) if cwd else None, check=check) +def _lexical_absolute(path: Path) -> Path: + """Return an absolute normalized path without following filesystem links.""" + + return Path(os.path.abspath(os.fspath(path.expanduser()))) + + +def _ensure_real_directory(path: Path) -> tuple[int, int]: + """Create/traverse *path* one component at a time without following links. + + The returned device/inode pair lets the caller detect replacement of the + selected output root after setup. Every existing component must be a real + directory; a symlink or special file is a hard error. + """ + + absolute = _lexical_absolute(path) + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + nofollow = getattr(os, "O_NOFOLLOW", 0) + current_fd = os.open(absolute.anchor, flags) + try: + for part in absolute.parts[1:]: + try: + os.mkdir(part, mode=0o755, dir_fd=current_fd) + except FileExistsError: + pass + try: + next_fd = os.open(part, flags | nofollow, dir_fd=current_fd) + except OSError as exc: + _die(f"directory component is not a real directory: {absolute}: {exc}") + os.close(current_fd) + current_fd = next_fd + info = os.fstat(current_fd) + return info.st_dev, info.st_ino + finally: + os.close(current_fd) + + +def _assert_directory_identity( + path: Path, identity: tuple[int, int], label: str +) -> None: + try: + info = os.lstat(path) + except OSError as exc: + _die(f"{label} disappeared during the run: {path}: {exc}") + if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode): + _die(f"{label} is no longer a real directory: {path}") + if (info.st_dev, info.st_ino) != identity: + _die(f"{label} was replaced during the run: {path}") + + +def _assert_no_symlinks(root: Path) -> None: + """Reject pre-existing or concurrently introduced links below *root*.""" + + if not root.exists(): + return + root_info = os.lstat(root) + if stat.S_ISLNK(root_info.st_mode) or not stat.S_ISDIR(root_info.st_mode): + _die(f"output root must be a real directory: {root}") + for directory, dirnames, filenames in os.walk(root, followlinks=False): + base = Path(directory) + for name in [*dirnames, *filenames]: + child = base / name + info = os.lstat(child) + if stat.S_ISLNK(info.st_mode): + _die(f"refusing symlink inside managed output tree: {child}") + + def _ensure_dirs(paths: list[Path]) -> None: for p in paths: - p.mkdir(parents=True, exist_ok=True) + _ensure_real_directory(p) def _today_ymd() -> str: @@ -181,7 +251,9 @@ def _extract_ts_string_const(src: Path, const_name: str) -> str | None: return None -def _extract_agents_from_registry_ts(registry_ts: Path) -> tuple[list[str], list[str]] | None: +def _extract_agents_from_registry_ts( + registry_ts: Path, +) -> tuple[list[str], list[str]] | None: """ Best-effort parser for agent keys + alias flags from a TS registry. Intended to resolve help text interpolations like `${agentKeys.join('|')}`. @@ -227,7 +299,9 @@ def _extract_agents_from_registry_ts(registry_ts: Path) -> tuple[list[str], list return agent_keys, sorted(alias_flags) -def _find_node_cli_package(repo_root: Path, product_slug: str, product_name: str) -> dict[str, object] | None: +def _find_node_cli_package( + repo_root: Path, product_slug: str, product_name: str +) -> dict[str, object] | None: # Detect Node CLI packages by locating a package.json with a "bin" field and matching name/bin key. product_name_lc = product_name.strip().lower() candidates: list[tuple[int, Path, dict[str, object]]] = [] @@ -299,7 +373,12 @@ def _find_node_cli_package(repo_root: Path, product_slug: str, product_name: str def _find_python_cli(repo_root: Path) -> dict[str, object] | None: """Detect Python CLI packages via pyproject.toml or setup.cfg entry_points.""" - result: dict[str, object] = {"language": "python", "bin": {}, "framework": None, "entry_module": None} + result: dict[str, object] = { + "language": "python", + "bin": {}, + "framework": None, + "entry_module": None, + } # Try pyproject.toml first (modern standard). for pyproject in sorted(repo_root.rglob("pyproject.toml")): @@ -307,7 +386,7 @@ def _find_python_cli(repo_root: Path) -> dict[str, object] | None: continue text = _read_text(pyproject) # [project.scripts] section (PEP 621). - m = re.search(r'\[project\.scripts\]\s*\n((?:[^\[].+\n)*)', text) + m = re.search(r"\[project\.scripts\]\s*\n((?:[^\[].+\n)*)", text) if m: for line in m.group(1).strip().splitlines(): parts = line.split("=", 1) @@ -316,9 +395,11 @@ def _find_python_cli(repo_root: Path) -> dict[str, object] | None: entry = parts[1].strip().strip('"').strip("'") result["bin"][name] = entry # type: ignore[index] if not result["entry_module"]: - result["entry_module"] = entry.split(":")[0] if ":" in entry else entry + result["entry_module"] = ( + entry.split(":")[0] if ":" in entry else entry + ) # [tool.poetry.scripts] section. - m2 = re.search(r'\[tool\.poetry\.scripts\]\s*\n((?:[^\[].+\n)*)', text) + m2 = re.search(r"\[tool\.poetry\.scripts\]\s*\n((?:[^\[].+\n)*)", text) if m2: for line in m2.group(1).strip().splitlines(): parts = line.split("=", 1) @@ -335,7 +416,10 @@ def _find_python_cli(repo_root: Path) -> dict[str, object] | None: if _should_skip_repo_scan_path(setup_cfg, repo_root): continue text = _read_text(setup_cfg) - m = re.search(r'\[options\.entry_points\]\s*\nconsole_scripts\s*=\s*\n((?:\s+.+\n)*)', text) + m = re.search( + r"\[options\.entry_points\]\s*\nconsole_scripts\s*=\s*\n((?:\s+.+\n)*)", + text, + ) if m: for line in m.group(1).strip().splitlines(): parts = line.strip().split("=", 1) @@ -370,7 +454,12 @@ def _find_python_cli(repo_root: Path) -> dict[str, object] | None: def _find_go_cli(repo_root: Path) -> dict[str, object] | None: """Detect Go CLI packages via go.mod + main.go + flag/cobra usage.""" - result: dict[str, object] = {"language": "go", "bin": {}, "framework": None, "module": None} + result: dict[str, object] = { + "language": "go", + "bin": {}, + "framework": None, + "module": None, + } # Find go.mod for module name. go_mod = repo_root / "go.mod" @@ -383,7 +472,7 @@ def _find_go_cli(repo_root: Path) -> dict[str, object] | None: break if go_mod.exists(): text = _read_text(go_mod) - m = re.search(r'^module\s+(.+)$', text, re.MULTILINE) + m = re.search(r"^module\s+(.+)$", text, re.MULTILINE) if m: result["module"] = m.group(1).strip() @@ -416,7 +505,10 @@ def _find_go_cli(repo_root: Path) -> dict[str, object] | None: # Detect CLI framework (cobra vs stdlib flag). scanned = 0 for go_file in sorted(repo_root.rglob("*.go")): - if _should_skip_repo_scan_path(go_file, repo_root) or "testdata" in go_file.parts: + if ( + _should_skip_repo_scan_path(go_file, repo_root) + or "testdata" in go_file.parts + ): continue scanned += 1 if scanned > 200: @@ -491,11 +583,20 @@ def _enrich_registry_with_binary_evidence( for raw_line in text.splitlines(): stripped = raw_line.strip() if stripped.startswith("docs_features_prefix:"): - reg["docs_features_prefix"] = stripped.split(":", 1)[1].strip().strip("'\"") + reg["docs_features_prefix"] = ( + stripped.split(":", 1)[1].strip().strip("'\"") + ) elif stripped.startswith("docs_features:"): reg.setdefault("docs_features", []) - elif raw_line.startswith(" - ") and "docs_features" in reg and "groups" not in text.split(raw_line)[0].rsplit("docs_features:", 1)[-1]: - reg.setdefault("docs_features", []).append(stripped[2:].strip().strip("'\"")) + elif ( + raw_line.startswith(" - ") + and "docs_features" in reg + and "groups" + not in text.split(raw_line)[0].rsplit("docs_features:", 1)[-1] + ): + reg.setdefault("docs_features", []).append( + stripped[2:].strip().strip("'\"") + ) # Parse groups using the same logic as the validator cur = None in_groups = False @@ -509,7 +610,11 @@ def _enrich_registry_with_binary_evidence( continue if not in_groups: continue - if line.startswith(" ") and not line.startswith(" ") and line.endswith(":"): + if ( + line.startswith(" ") + and not line.startswith(" ") + and line.endswith(":") + ): name = line.strip()[:-1] cur = {"impl": None, "anchors": [], "notes": ""} reg["groups"][name] = cur @@ -598,19 +703,29 @@ def _write_binary_cli_surface_spec( lines.append(f"# CLI Surface Spec: {product_name}") lines.append("") lines.append(f"- Date: {date}") - lines.append("- Source: binary --help output" if help_tree.exists() else "- Source: binary string extraction") + lines.append( + "- Source: binary --help output" + if help_tree.exists() + else "- Source: binary string extraction" + ) lines.append("") cmd_count = 0 if commands_file.exists(): - cmds = [c.strip() for c in commands_file.read_text(encoding="utf-8").splitlines() if c.strip()] + cmds = [ + c.strip() + for c in commands_file.read_text(encoding="utf-8").splitlines() + if c.strip() + ] cmd_count = len(cmds) if help_tree.exists(): tree_text = help_tree.read_text(encoding="utf-8") lines.append("## Command Count") lines.append("") - lines.append(f"- **{cmd_count} commands** discovered via recursive `--help` execution") + lines.append( + f"- **{cmd_count} commands** discovered via recursive `--help` execution" + ) lines.append("") # Extract top-level commands and subcommands @@ -623,7 +738,9 @@ def _write_binary_cli_surface_spec( for top in top_level: subs = [c for c in cmds if c.startswith(top + " ") and c != top] sub_names = [c.split(maxsplit=1)[1] if " " in c else "" for c in subs] - sub_str = ", ".join(f"`{s}`" for s in sub_names if s) if sub_names else "—" + sub_str = ( + ", ".join(f"`{s}`" for s in sub_names if s) if sub_names else "—" + ) lines.append(f"| `{top}` | {sub_str} |") lines.append("") @@ -642,7 +759,11 @@ def _write_binary_cli_surface_spec( elif strings_file.exists(): # Fallback: extract command-like patterns from strings raw = strings_file.read_text(encoding="utf-8", errors="replace") - usage_lines = [line.strip() for line in raw.splitlines() if "usage" in line.lower() or "Usage" in line] + usage_lines = [ + line.strip() + for line in raw.splitlines() + if "usage" in line.lower() or "Usage" in line + ] lines.append("## CLI Surface (from binary strings, best-effort)") lines.append("") if usage_lines: @@ -715,18 +836,26 @@ def _write_cli_surface_spec( lines.append("") lines.append("## Notes For 1:1 Fidelity") lines.append("") - lines.append("- Run ` --help` to capture the full CLI contract as a golden test fixture.") + lines.append( + "- Run ` --help` to capture the full CLI contract as a golden test fixture." + ) if lang == "Python": - lines.append("- For Click/Typer apps, consider ` --help` per subcommand for full coverage.") + lines.append( + "- For Click/Typer apps, consider ` --help` per subcommand for full coverage." + ) elif lang == "Go": - lines.append("- For Cobra apps, consider ` help ` for full coverage.") + lines.append( + "- For Cobra apps, consider ` help ` for full coverage." + ) out.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") return True out = output_dir / "spec-cli-surface.md" pkg_dir = Path(str(node_cli["package_dir"])) - pkg_json_rel = Path(str(node_cli["package_json"])).relative_to(analysis_root).as_posix() + pkg_json_rel = ( + Path(str(node_cli["package_json"])).relative_to(analysis_root).as_posix() + ) src_index = pkg_dir / "src" / "index.ts" src_cli = pkg_dir / "src" / "cli.ts" src_store = pkg_dir / "src" / "cli" / "store.ts" @@ -741,7 +870,9 @@ def _write_cli_surface_spec( if extracted: agent_keys, alias_flags = extracted if agent_keys: - help_text = help_text.replace("${agentKeys.join('|')}", "|".join(agent_keys)) + help_text = help_text.replace( + "${agentKeys.join('|')}", "|".join(agent_keys) + ) alias_line = "" if alias_flags: alias_line = f" {' | '.join(alias_flags)} Agent alias flags\n" @@ -754,7 +885,14 @@ def _write_cli_surface_spec( pat = re.compile(r"\bprocess\.env\.([A-Z][A-Z0-9_]*)\b") found = set() for p in sorted(src_root.rglob("*")): - if not p.is_file() or p.suffix.lower() not in (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"): + if not p.is_file() or p.suffix.lower() not in ( + ".ts", + ".tsx", + ".js", + ".jsx", + ".mjs", + ".cjs", + ): continue for m in pat.finditer(_read_text(p)): found.add(m.group(1)) @@ -789,7 +927,9 @@ def _write_cli_surface_spec( lines.append("") lines.append("### Source Entry (Heuristic)") lines.append("") - lines.append(f"- `{src_cli.relative_to(analysis_root).as_posix()}` (node shebang entry; typically calls `runCli`)") + lines.append( + f"- `{src_cli.relative_to(analysis_root).as_posix()}` (node shebang entry; typically calls `runCli`)" + ) lines.append("") lines.append("## Usage / Help (Code-Proven Where Possible)") @@ -800,7 +940,9 @@ def _write_cli_surface_spec( lines.append("```") lines.append("") lines.append("Evidence:") - lines.append(f"- `{src_index.relative_to(analysis_root).as_posix()}` (`helpText`)") + lines.append( + f"- `{src_index.relative_to(analysis_root).as_posix()}` (`helpText`)" + ) else: lines.append("- _Help text not extracted (pattern not found)._") lines.append("Evidence:") @@ -816,7 +958,9 @@ def _write_cli_surface_spec( wrote_any = True if env_vars: lines.append(f"- Environment variables: `{', '.join(env_vars)}`") - lines.append(f" Evidence: scan of `{src_root.relative_to(analysis_root).as_posix()}` for `process.env.`.") + lines.append( + f" Evidence: scan of `{src_root.relative_to(analysis_root).as_posix()}` for `process.env.`." + ) wrote_any = True if not wrote_any: lines.append("- _No config/env surface extracted._") @@ -824,8 +968,12 @@ def _write_cli_surface_spec( lines.append("") lines.append("## Notes For 1:1 Fidelity") lines.append("") - lines.append("- Treat `--help` output as the CLI contract; include it as a golden test fixture for regressions.") - lines.append("- If the repo does not ship built artifacts (ex: `dist/`), building may be required to execute the CLI directly.") + lines.append( + "- Treat `--help` output as the CLI contract; include it as a golden test fixture for regressions." + ) + lines.append( + "- If the repo does not ship built artifacts (ex: `dist/`), building may be required to execute the CLI directly." + ) out.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") return True @@ -918,7 +1066,11 @@ def _write_artifact_surface_spec( from_dir = source.get("fromDir") if not isinstance(from_dir, str): continue - from_dir_res = _render_placeholders(from_dir, placeholder_vars) if placeholder_vars else from_dir + from_dir_res = ( + _render_placeholders(from_dir, placeholder_vars) + if placeholder_vars + else from_dir + ) abs_from = pkg_dir / from_dir_res if abs_from.exists() and abs_from.is_dir(): for fp in sorted(abs_from.rglob("*")): @@ -938,7 +1090,11 @@ def _write_artifact_surface_spec( from_file = source.get("from") if not isinstance(from_file, str): continue - from_file_res = _render_placeholders(from_file, placeholder_vars) if placeholder_vars else from_file + from_file_res = ( + _render_placeholders(from_file, placeholder_vars) + if placeholder_vars + else from_file + ) abs_from = pkg_dir / from_file_res if abs_from.exists() and abs_from.is_file(): resolved_sources.append( @@ -976,7 +1132,9 @@ def _write_artifact_surface_spec( lines.append(f"- Date: {date}") lines.append(f"- Analysis root: `{analysis_root}`") lines.append(f"- Node package: `{pkg_dir.relative_to(analysis_root).as_posix()}`") - lines.append(f"- Manifests dir: `{manifests_dir.relative_to(analysis_root).as_posix()}`") + lines.append( + f"- Manifests dir: `{manifests_dir.relative_to(analysis_root).as_posix()}`" + ) lines.append(f"- Machine registry: `{out_json.relative_to(output_dir).as_posix()}`") lines.append("") lines.append("## Manifest Inventory (Code-Proven)") @@ -998,7 +1156,9 @@ def _write_artifact_surface_spec( lines.append("## Template Source File Inventory (Hashed)") lines.append("") lines.append(f"- Files hashed: `{len(resolved_sources)}`") - lines.append("- Use `artifact-registry.json` as the source of truth for 1:1 template content equivalence.") + lines.append( + "- Use `artifact-registry.json` as the source of truth for 1:1 template content equivalence." + ) out_md.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") @@ -1033,11 +1193,30 @@ def _collect_env_vars_with_evidence( var_files: dict[str, set[str]] = {} patterns: list[tuple[re.Pattern[str], set[str]]] = [ - (re.compile(r"\bprocess\.env\.([A-Z][A-Z0-9_]+)\b"), {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"}), - (re.compile(r"""os\.environ(?:\.get)?\s*\(\s*['"]([A-Z][A-Z0-9_]+)['"]\s*\)"""), {".py"}), - (re.compile(r"""\bos\.getenv\s*\(\s*['"]([A-Z][A-Z0-9_]+)['"]\s*\)"""), {".py"}), - (re.compile(r"""\bos\.(?:Getenv|LookupEnv)\s*\(\s*"([A-Z][A-Z0-9_]+)"\s*\)"""), {".go"}), - (re.compile(r'\$\{?([A-Z][A-Z0-9_]{2,})\}?'), {".sh", ".bash", ".env", ".envrc"}), + ( + re.compile(r"\bprocess\.env\.([A-Z][A-Z0-9_]+)\b"), + {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"}, + ), + ( + re.compile( + r"""os\.environ(?:\.get)?\s*\(\s*['"]([A-Z][A-Z0-9_]+)['"]\s*\)""" + ), + {".py"}, + ), + ( + re.compile(r"""\bos\.getenv\s*\(\s*['"]([A-Z][A-Z0-9_]+)['"]\s*\)"""), + {".py"}, + ), + ( + re.compile( + r"""\bos\.(?:Getenv|LookupEnv)\s*\(\s*"([A-Z][A-Z0-9_]+)"\s*\)""" + ), + {".go"}, + ), + ( + re.compile(r"\$\{?([A-Z][A-Z0-9_]{2,})\}?"), + {".sh", ".bash", ".env", ".envrc"}, + ), ] scanned = 0 @@ -1045,7 +1224,14 @@ def _collect_env_vars_with_evidence( if not p.is_file(): continue # Skip irrelevant dirs - skip_dirs = {"node_modules", ".git", ".venv", "vendor", "testdata", "__pycache__"} + skip_dirs = { + "node_modules", + ".git", + ".venv", + "vendor", + "testdata", + "__pycache__", + } if any(part in skip_dirs for part in p.parts): continue suffix = p.suffix.lower() @@ -1067,10 +1253,12 @@ def _collect_env_vars_with_evidence( result: list[dict[str, object]] = [] for var_name in sorted(var_files.keys()): - result.append({ - "name": var_name, - "files": sorted(var_files[var_name]), - }) + result.append( + { + "name": var_name, + "files": sorted(var_files[var_name]), + } + ) return result @@ -1174,7 +1362,11 @@ def _write_repo_contract_json( node_cli = _find_node_cli_package(analysis_root, product_slug, product_name) python_cli_info = _find_python_cli(analysis_root) if node_cli is None else None - go_cli_info = _find_go_cli(analysis_root) if node_cli is None and python_cli_info is None else None + go_cli_info = ( + _find_go_cli(analysis_root) + if node_cli is None and python_cli_info is None + else None + ) if node_cli: pkg_dir = Path(str(node_cli["package_dir"])) @@ -1185,7 +1377,9 @@ def _write_repo_contract_json( for k, v in raw_bin.items(): bin_map[k] = v cli_surface["language"] = "node" - cli_surface["package_json"] = Path(str(node_cli["package_json"])).relative_to(analysis_root).as_posix() + cli_surface["package_json"] = ( + Path(str(node_cli["package_json"])).relative_to(analysis_root).as_posix() + ) cli_surface["package_dir"] = pkg_dir.relative_to(analysis_root).as_posix() cli_surface["package_name"] = str(node_cli.get("name") or "") cli_surface["bin"] = {k: bin_map[k] for k in sorted(bin_map)} @@ -1199,35 +1393,53 @@ def _write_repo_contract_json( if extracted: agent_keys, alias_flags = extracted if agent_keys: - help_text = help_text.replace("${agentKeys.join('|')}", "|".join(agent_keys)) + help_text = help_text.replace( + "${agentKeys.join('|')}", "|".join(agent_keys) + ) alias_line = "" if alias_flags: alias_line = f" {' | '.join(alias_flags)} Agent alias flags\n" help_text = help_text.replace("${agentAliasLine}", alias_line) if help_text is not None: cli_surface["help_text"] = help_text - cli_surface["help_text_source"] = src_index.relative_to(analysis_root).as_posix() if src_index.exists() else None + cli_surface["help_text_source"] = ( + src_index.relative_to(analysis_root).as_posix() + if src_index.exists() + else None + ) # Config file from store.ts src_store = pkg_dir / "src" / "cli" / "store.ts" config_file = _extract_ts_string_const(src_store, "CONFIG_FILE") if config_file: cli_surface["config_file"] = config_file - cli_surface["config_file_source"] = src_store.relative_to(analysis_root).as_posix() if src_store.exists() else None + cli_surface["config_file_source"] = ( + src_store.relative_to(analysis_root).as_posix() + if src_store.exists() + else None + ) elif python_cli_info: raw_bin_py = python_cli_info.get("bin") or {} cli_surface["language"] = "python" cli_surface["framework"] = python_cli_info.get("framework") cli_surface["entry_module"] = python_cli_info.get("entry_module") - cli_surface["bin"] = {k: str(raw_bin_py[k]) for k in sorted(raw_bin_py)} if isinstance(raw_bin_py, dict) else {} + cli_surface["bin"] = ( + {k: str(raw_bin_py[k]) for k in sorted(raw_bin_py)} + if isinstance(raw_bin_py, dict) + else {} + ) elif go_cli_info: raw_bin_go = go_cli_info.get("bin") or {} cli_surface["language"] = "go" cli_surface["framework"] = go_cli_info.get("framework") cli_surface["module"] = go_cli_info.get("module") - cli_surface["bin"] = {k: str(raw_bin_go[k]) for k in sorted(raw_bin_go)} if isinstance(raw_bin_go, dict) else {} + cli_surface["bin"] = ( + {k: str(raw_bin_go[k]) for k in sorted(raw_bin_go)} + if isinstance(raw_bin_go, dict) + else {} + ) contract["cli"] = cli_surface @@ -1253,15 +1465,21 @@ def _write_repo_contract_json( # Template files: keep path, sha256 (no absolute paths; already relative in artifact-registry) template_hashes: list[dict[str, object]] = [] for tf in template_files_raw: - template_hashes.append({ - "file": tf.get("file"), - "manifest": tf.get("manifest"), - "sha256": tf.get("sha256"), - "source_type": tf.get("source_type"), - }) + template_hashes.append( + { + "file": tf.get("file"), + "manifest": tf.get("manifest"), + "sha256": tf.get("sha256"), + "source_type": tf.get("source_type"), + } + ) - contract["manifests"] = sorted(manifests_clean, key=lambda x: str(x.get("path", ""))) - contract["template_files"] = sorted(template_hashes, key=lambda x: str(x.get("file", ""))) + contract["manifests"] = sorted( + manifests_clean, key=lambda x: str(x.get("path", "")) + ) + contract["template_files"] = sorted( + template_hashes, key=lambda x: str(x.get("file", "")) + ) except Exception: pass @@ -1293,7 +1511,11 @@ def _write_comparison_report( binary_cmds: list[str] = [] commands_file = tmp_dir / "binary" / "cli-commands.txt" if commands_file.exists(): - binary_cmds = [c.strip() for c in commands_file.read_text(encoding="utf-8").splitlines() if c.strip()] + binary_cmds = [ + c.strip() + for c in commands_file.read_text(encoding="utf-8").splitlines() + if c.strip() + ] repo_cmds: list[str] = [] repo_cli_spec = output_dir / "spec-cli-surface.md" @@ -1326,7 +1548,11 @@ def _write_comparison_report( if not in_groups: continue # Group entries are 2-space indented, end with ':' - if line.startswith(" ") and not line.startswith(" ") and line.rstrip().endswith(":"): + if ( + line.startswith(" ") + and not line.startswith(" ") + and line.rstrip().endswith(":") + ): # Determine source from notes field binary_groups += 1 @@ -1344,7 +1570,9 @@ def _write_comparison_report( # --- Coverage percentage --- if repo_cmds: coverage_pct = round(len(binary_set & repo_set) / len(repo_set) * 100) - coverage_line = f"Binary analysis found {coverage_pct}% of repo-discovered commands." + coverage_line = ( + f"Binary analysis found {coverage_pct}% of repo-discovered commands." + ) elif binary_cmds: coverage_line = f"Binary analysis found {len(binary_cmds)} commands; repo analysis found none (no CLI detected in repo)." else: @@ -1403,7 +1631,9 @@ def _write_comparison_report( def _write_wrapper_validate_feature_registry(output_dir: Path) -> None: - skill_validate_path = (SKILL_DIR / "scripts" / "validate_feature_registry.py").resolve() + skill_validate_path = ( + SKILL_DIR / "scripts" / "validate_feature_registry.py" + ).resolve() wrapper = output_dir / "validate-feature-registry.py" wrapper.write_text( f"""#!/usr/bin/env python3 @@ -1467,6 +1697,160 @@ def _copy_security_validators(output_dir: Path) -> None: dst.chmod(0o755) +def _git_text(repo: Path, *args: str) -> str: + return subprocess.check_output( + ["git", "-C", str(repo), *args], text=True, stderr=subprocess.STDOUT + ).strip() + + +def _is_git_checkout(path: Path) -> bool: + try: + return _git_text(path, "rev-parse", "--is-inside-work-tree") == "true" + except (OSError, subprocess.CalledProcessError): + return False + + +def _write_source_metadata( + output_dir: Path, + *, + upstream_repo: str | None, + upstream_ref: str | None, + resolved_commit: str, + source_kind: str, +) -> None: + payload = { + "upstream_repo": upstream_repo, + "upstream_ref": upstream_ref, + "resolved_commit": resolved_commit, + "source_kind": source_kind, + "clone_date": _today_ymd(), + } + (output_dir / "clone-metadata.json").write_text( + json.dumps(payload, indent=2) + "\n", encoding="utf-8" + ) + + +def _prepare_repo_analysis( + *, + local_clone_dir: Path, + output_dir: Path, + explicit_local_dir: bool, + upstream_repo: str | None, + upstream_ref: str | None, +) -> Path: + """Select one unambiguous repo analysis root and bind its requested ref.""" + + exists_before = local_clone_dir.exists() and any(local_clone_dir.iterdir()) + if upstream_repo and not exists_before: + clone_cmd = ["git", "clone"] + if not upstream_ref: + clone_cmd.append("--depth=1") + clone_cmd.extend([upstream_repo, str(local_clone_dir)]) + _run(clone_cmd, check=True) + if upstream_ref: + _run( + [ + "git", + "-C", + str(local_clone_dir), + "fetch", + "--depth=1", + "origin", + upstream_ref, + ], + check=True, + ) + _run( + [ + "git", + "-C", + str(local_clone_dir), + "checkout", + "--detach", + "FETCH_HEAD", + ], + check=True, + ) + + if explicit_local_dir: + analysis_root = local_clone_dir + elif upstream_repo: + analysis_root = local_clone_dir + else: + try: + top = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + except (OSError, subprocess.CalledProcessError): + top = "" + analysis_root = _lexical_absolute(Path(top)) if top else local_clone_dir + + if upstream_repo and not _is_git_checkout(analysis_root): + _die(f"--upstream-repo did not produce a Git checkout: {analysis_root}") + if upstream_repo and exists_before: + try: + origin = _git_text(analysis_root, "config", "--get", "remote.origin.url") + except subprocess.CalledProcessError: + _die( + "existing checkout has no origin URL to verify against --upstream-repo" + ) + if origin != upstream_repo: + _die( + "existing checkout origin does not match --upstream-repo " + f"(origin={origin!r}, requested={upstream_repo!r})" + ) + + if upstream_ref: + if not _is_git_checkout(analysis_root): + _die( + "--upstream-ref requires the selected analysis root to be a Git checkout" + ) + try: + requested = _git_text( + analysis_root, "rev-parse", "--verify", f"{upstream_ref}^{{commit}}" + ) + except subprocess.CalledProcessError: + if not upstream_repo: + _die( + f"requested ref is not present in the selected checkout: {upstream_ref}" + ) + _run( + [ + "git", + "-C", + str(analysis_root), + "fetch", + "--depth=1", + "origin", + upstream_ref, + ], + check=True, + ) + requested = _git_text( + analysis_root, "rev-parse", "--verify", "FETCH_HEAD^{commit}" + ) + current = _git_text(analysis_root, "rev-parse", "--verify", "HEAD^{commit}") + if current != requested: + _die( + "selected checkout is not at --upstream-ref; refusing to analyze the " + f"wrong commit (HEAD={current}, requested={requested})" + ) + + if _is_git_checkout(analysis_root) and (upstream_repo or upstream_ref): + resolved = _git_text(analysis_root, "rev-parse", "--verify", "HEAD^{commit}") + _write_source_metadata( + output_dir, + upstream_repo=upstream_repo, + upstream_ref=upstream_ref, + resolved_commit=resolved, + source_kind="existing-checkout" if exists_before else "clone", + ) + + return analysis_root + + def main() -> int: ap = argparse.ArgumentParser(prog="reverse_engineer.py") ap.add_argument("product_name") @@ -1483,9 +1867,22 @@ def main() -> int: help="Docs slug prefix, e.g. docs/features/. Use 'auto' to detect from repo/sitemap (default).", ) ap.add_argument("--upstream-repo", default=None) - ap.add_argument("--upstream-ref", default=None, help="Pin clone to a specific commit, tag, or branch. Records resolved SHA in clone-metadata.json.") + ap.add_argument( + "--upstream-ref", + default=None, + help="Pin clone to a specific commit, tag, or branch. Records resolved SHA in clone-metadata.json.", + ) ap.add_argument("--local-clone-dir", default=None) - ap.add_argument("--output-dir", default=None) + ap.add_argument( + "--output-dir", + default=None, + help=( + "Artifact directory. Defaults to " + ".agents/scratch/reverse-engineer//. The earlier " + ".agents/research// path remains accepted when supplied " + "explicitly; existing artifacts are never moved automatically." + ), + ) ap.add_argument("--mode", default="repo", choices=["repo", "binary", "both"]) ap.add_argument("--binary-path", default=None) @@ -1506,70 +1903,59 @@ def main() -> int: args = ap.parse_args() product_slug = _slugify(args.product_name) - local_clone_dir = Path(args.local_clone_dir or f".tmp/{product_slug}").resolve() - output_dir = Path(args.output_dir or f".agents/scratch/reverse-engineer/{product_slug}/").resolve() + explicit_local_dir = args.local_clone_dir is not None + local_clone_dir = _lexical_absolute( + Path(args.local_clone_dir or f".tmp/{product_slug}") + ) + output_dir = _lexical_absolute( + Path(args.output_dir or f".agents/scratch/reverse-engineer/{product_slug}/") + ) analysis_root = local_clone_dir - tmp_dir = (REPO_ROOT / ".tmp" / f"reverse-engineer-{product_slug}").resolve() - _ensure_dirs([local_clone_dir, output_dir, tmp_dir]) + tmp_dir = _lexical_absolute(REPO_ROOT / ".tmp" / f"reverse-engineer-{product_slug}") + _ensure_real_directory(local_clone_dir) + output_identity = _ensure_real_directory(output_dir) + _ensure_real_directory(tmp_dir) + _assert_no_symlinks(output_dir) docs_features_txt = output_dir / "docs-features.txt" effective_docs_prefix = args.docs_features_prefix - # Acquire code (repo mode): shallow clone if requested. - # NOTE: this must happen before docs inventory, otherwise docs/features extraction runs against an empty dir. if args.mode in ("repo", "both"): - if args.upstream_repo and not (local_clone_dir / ".git").exists(): - clone_cmd = ["git", "clone"] - if not args.upstream_ref: - clone_cmd.append("--depth=1") - clone_cmd.extend([args.upstream_repo, str(local_clone_dir)]) - _run(clone_cmd, check=True) - if args.upstream_ref: - _run(["git", "-C", str(local_clone_dir), "fetch", "--depth=1", "origin", args.upstream_ref], check=True) - _run(["git", "-C", str(local_clone_dir), "checkout", "FETCH_HEAD"], check=True) - # Record clone metadata for reproducibility. - resolved_sha = subprocess.check_output( - ["git", "-C", str(local_clone_dir), "rev-parse", "HEAD"], text=True, - ).strip() - clone_meta = { - "upstream_repo": args.upstream_repo, - "upstream_ref": args.upstream_ref, - "resolved_commit": resolved_sha, - "clone_date": _today_ymd(), - } - (output_dir / "clone-metadata.json").write_text( - json.dumps(clone_meta, indent=2) + "\n", encoding="utf-8", - ) - analysis_root = local_clone_dir - - # Determine an analysis root for repo mode. - # Priority: - # 1) local_clone_dir if it looks like a git checkout already - # 2) git toplevel of the current working directory (if inside a repo) - # 3) local_clone_dir (created) - if args.mode in ("repo", "both"): - if (local_clone_dir / ".git").exists(): - analysis_root = local_clone_dir - else: - try: - top = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() - if top: - analysis_root = Path(top).resolve() - except Exception: - analysis_root = local_clone_dir + # Acquire/select the repo before inventory. An explicit local path is + # always the selected root, including when it is intentionally non-Git; + # never replace it with the caller's current checkout. + analysis_root = _prepare_repo_analysis( + local_clone_dir=local_clone_dir, + output_dir=output_dir, + explicit_local_dir=explicit_local_dir, + upstream_repo=args.upstream_repo, + upstream_ref=args.upstream_ref, + ) # 1) Mechanical docs inventory (NO heavy crawling). if args.docs_sitemap_url: sitemap_xml = tmp_dir / f"{product_slug}-sitemap.xml" - _run([sys.executable, str(SKILL_DIR / "scripts" / "fetch_url.py"), args.docs_sitemap_url, str(sitemap_xml)]) + _run( + [ + sys.executable, + str(SKILL_DIR / "scripts" / "fetch_url.py"), + args.docs_sitemap_url, + str(sitemap_xml), + ] + ) paths_txt = tmp_dir / f"{product_slug}-sitemap-paths.txt" - sitemap_paths = subprocess.check_output([str(SKILL_DIR / "scripts" / "extract_sitemap_paths.sh"), str(sitemap_xml)], text=True) + sitemap_paths = subprocess.check_output( + [str(SKILL_DIR / "scripts" / "extract_sitemap_paths.sh"), str(sitemap_xml)], + text=True, + ) paths_txt.write_text(sitemap_paths, encoding="utf-8") if args.docs_features_prefix in ("", "auto"): - effective_docs_prefix = _detect_docs_prefix_from_paths(sitemap_paths.splitlines()) + effective_docs_prefix = _detect_docs_prefix_from_paths( + sitemap_paths.splitlines() + ) docs_features = subprocess.check_output( [ @@ -1587,7 +1973,10 @@ def main() -> int: effective_docs_prefix = _detect_docs_prefix_for_repo(analysis_root) # Backward-compatibility fallback for explicit old default. elif args.docs_features_prefix == "docs/features/": - if not (analysis_root / "docs" / "features").exists() and (analysis_root / "docs").exists(): + if ( + not (analysis_root / "docs" / "features").exists() + and (analysis_root / "docs").exists() + ): effective_docs_prefix = "docs/" prefix_dir = effective_docs_prefix.strip("/").rstrip("/") @@ -1602,7 +1991,9 @@ def main() -> int: rel = p.relative_to(analysis_root).as_posix() # Normalize to slug without extension to match sitemap-style slugs. slugs.append(rel[: -len(p.suffix)]) - docs_features_txt.write_text("\n".join(slugs) + ("\n" if slugs else ""), encoding="utf-8") + docs_features_txt.write_text( + "\n".join(slugs) + ("\n" if slugs else ""), encoding="utf-8" + ) else: docs_features_txt.write_text("", encoding="utf-8") @@ -1634,7 +2025,12 @@ def main() -> int: capture_script = SKILL_DIR / "scripts" / "binary" / "capture_cli_help.sh" if capture_script.exists(): _run( - ["bash", str(capture_script), str(binary_path), str(tmp_dir / "binary")], + [ + "bash", + str(capture_script), + str(binary_path), + str(tmp_dir / "binary"), + ], check=False, # best-effort ) # Copy results to output dir if they exist @@ -1671,7 +2067,12 @@ def main() -> int: _run( [ sys.executable, - str(SKILL_DIR / "scripts" / "binary" / "extract_embedded_archives.py"), + str( + SKILL_DIR + / "scripts" + / "binary" + / "extract_embedded_archives.py" + ), "--binary", str(binary_path), "--out-dir", @@ -1798,13 +2199,17 @@ def main() -> int: # 6c) Comparison report (binary vs repo) when both sources are available. if args.mode == "both": - _write_comparison_report(output_dir, tmp_dir, product_name=args.product_name, date=_today_ymd()) + _write_comparison_report( + output_dir, tmp_dir, product_name=args.product_name, date=_today_ymd() + ) # 7) Validation gate: produce a self-contained validator in the output dir and run it once. _write_wrapper_validate_feature_registry(output_dir) # Store analysis root pointer for validators (repo clone dir or a placeholder). (output_dir / "analysis-root").mkdir(exist_ok=True) - (output_dir / "analysis-root-path.txt").write_text(str(analysis_root), encoding="utf-8") + (output_dir / "analysis-root-path.txt").write_text( + str(analysis_root), encoding="utf-8" + ) # Keep docs-features alongside outputs for deterministic validation. # (Already written as output_dir/docs-features.txt) _run( @@ -1816,7 +2221,11 @@ def main() -> int: "--docs-features", str(docs_features_txt), "--local-clone-dir", - str(analysis_root if analysis_root.exists() else output_dir / "analysis-root"), + str( + analysis_root + if analysis_root.exists() + else output_dir / "analysis-root" + ), ], check=True, ) @@ -1834,12 +2243,19 @@ def main() -> int: "findings.md.tmpl", "reproducibility.md.tmpl", ]: - _render_template(TEMPLATES_DIR / "security" / name, sec_dir / name.replace(".tmpl", ""), vars) + _render_template( + TEMPLATES_DIR / "security" / name, + sec_dir / name.replace(".tmpl", ""), + vars, + ) _copy_security_validators(output_dir) if args.sbom: - _run([str(sec_dir / "generate-sbom.sh"), str(analysis_root), str(sec_dir)], check=False) + _run( + [str(sec_dir / "generate-sbom.sh"), str(analysis_root), str(sec_dir)], + check=False, + ) # Scaffold-time safety check: scan the generated output for leaked # secrets. The full certifying gate (validate-security-audit.sh) is NOT @@ -1859,8 +2275,37 @@ def main() -> int: vibe_path = reports_dir / f"{_today_ymd()}-vibe-{product_slug}.md" post_path = reports_dir / f"{_today_ymd()}-postmortem-{product_slug}.md" - _render_template(TEMPLATES_DIR / "vibe-report.md.tmpl", vibe_path, {**vars, "OUTPUT_DIR": str(output_dir)}) - _render_template(TEMPLATES_DIR / "postmortem.md.tmpl", post_path, {**vars, "OUTPUT_DIR": str(output_dir)}) + _render_template( + TEMPLATES_DIR / "vibe-report.md.tmpl", + vibe_path, + {**vars, "OUTPUT_DIR": str(output_dir)}, + ) + _render_template( + TEMPLATES_DIR / "postmortem.md.tmpl", + post_path, + {**vars, "OUTPUT_DIR": str(output_dir)}, + ) + + # Phase 1 deliberately stops at a validated teardown. The evidence-backed + # steal-map is a caller-authored Phase-2 judgment over this output and the + # live destination repository; the script must not manufacture that choice. + _assert_directory_identity(output_dir, output_identity, "output directory") + _assert_no_symlinks(output_dir) + _run( + [ + "bash", + str(SKILL_DIR / "scripts" / "validate-output.sh"), + "--output-dir", + str(output_dir), + "--phase", + "teardown", + "--upstream-ref-set", + "1" if args.upstream_ref else "0", + ], + check=True, + ) + _assert_directory_identity(output_dir, output_identity, "output directory") + _assert_no_symlinks(output_dir) return 0 diff --git a/skills/reverse-engineer/scripts/self_test.sh b/skills/reverse-engineer/scripts/self_test.sh index 92f626427..2613690a0 100755 --- a/skills/reverse-engineer/scripts/self_test.sh +++ b/skills/reverse-engineer/scripts/self_test.sh @@ -19,6 +19,13 @@ SITEMAP="$TMP/sitemap.xml" rm -rf "$TMP" mkdir -p "$SRC" "$OUT1" "$OUT2" +HELP="$(python3 "$SKILL/scripts/reverse_engineer.py" --help)" +grep -Fq '.agents/scratch/reverse-engineer//' <<<"$HELP" +grep -Fq '.agents/research// path remains' <<<"$HELP" +grep -Fq 'are never moved automatically.' <<<"$HELP" +grep -Fq -- "- '.agents/scratch/reverse-engineer/*/'" "$SKILL/SKILL.md" +echo "OK: output-path migration contract is visible in --help" + python3 - "$SRC" <<'PY' import sys, zipfile from pathlib import Path @@ -75,6 +82,33 @@ python3 "$SKILL/scripts/reverse_engineer.py" demo \ python3 "$OUT1/validate-feature-registry.py" +VALIDATE_OUTPUT="$SKILL/scripts/validate-output.sh" +"$VALIDATE_OUTPUT" --output-dir "$OUT1" --phase teardown \ + --security-audit 0 --sbom 0 --upstream-ref-set 0 +if "$VALIDATE_OUTPUT" --output-dir "$OUT1" --phase complete \ + --security-audit 0 --sbom 0 --upstream-ref-set 0 >/dev/null 2>&1; then + echo "FAIL: complete validator accepted a missing steal-map.md" >&2 + exit 1 +fi +cat >"$OUT1/steal-map.md" <<'EOF' +# Steal map: demo + +| Their capability | Our surface today | Verdict | +|---|---|---| +| Embedded archive inventory (`feature-registry.yaml`) | `skills/reverse-engineer/` | **have** | +EOF +"$VALIDATE_OUTPUT" --output-dir "$OUT1" --phase complete \ + --security-audit 0 --sbom 0 --upstream-ref-set 0 +cp "$OUT1/steal-map.md" "$OUT1/steal-map.valid" +printf '# malformed map\n' >"$OUT1/steal-map.md" +if "$VALIDATE_OUTPUT" --output-dir "$OUT1" --phase complete \ + --security-audit 0 --sbom 0 --upstream-ref-set 0 >/dev/null 2>&1; then + echo "FAIL: complete validator accepted a malformed steal-map.md" >&2 + exit 1 +fi +mv "$OUT1/steal-map.valid" "$OUT1/steal-map.md" +echo "OK: exact output validator distinguishes teardown from complete decision output" + # --- Binary mode capability assertions --- echo "--- binary mode capability checks ---" @@ -193,6 +227,83 @@ if [ ! -f "$OUT_REF/clone-metadata.json" ]; then fi echo "OK: clone-metadata.json created with --upstream-ref" +echo "--- existing-checkout ref mismatch test ---" +WRONG_REPO="$TMP/local-wrong-ref" +WRONG_OUT="$TMP/out-wrong-ref" +mkdir -p "$WRONG_REPO" +git -C "$WRONG_REPO" init -q +git -C "$WRONG_REPO" config user.name reverse-self-test +git -C "$WRONG_REPO" config user.email reverse-self-test@example.invalid +printf 'one\n' >"$WRONG_REPO/unique.txt" +git -C "$WRONG_REPO" add unique.txt +git -C "$WRONG_REPO" commit -qm one +first_commit="$(git -C "$WRONG_REPO" rev-parse HEAD)" +printf 'two\n' >"$WRONG_REPO/unique.txt" +git -C "$WRONG_REPO" commit -qam two +second_commit="$(git -C "$WRONG_REPO" rev-parse HEAD)" +git -C "$WRONG_REPO" checkout -q --detach "$first_commit" +if python3 "$SKILL/scripts/reverse_engineer.py" wrong-ref \ + --mode=repo --local-clone-dir="$WRONG_REPO" \ + --upstream-ref="$second_commit" --output-dir="$WRONG_OUT" >/dev/null 2>&1; then + echo "FAIL: existing checkout at the wrong commit was analyzed" >&2 + exit 1 +fi +if [ -e "$WRONG_OUT/feature-registry.yaml" ]; then + echo "FAIL: ref mismatch wrote trusted teardown artifacts" >&2 + exit 1 +fi +echo "OK: existing checkout must match the requested ref" + +echo "--- explicit non-Git root test ---" +EXPLICIT_TREE="$TMP/explicit-nongit" +EXPLICIT_OUT="$TMP/out-explicit-nongit" +mkdir -p "$EXPLICIT_TREE" +printf 'only-in-explicit-tree\n' >"$EXPLICIT_TREE/unique-source.txt" +python3 "$SKILL/scripts/reverse_engineer.py" explicit-nongit \ + --mode=repo --local-clone-dir="$EXPLICIT_TREE" --output-dir="$EXPLICIT_OUT" +if ! grep -Fqx "$EXPLICIT_TREE" "$EXPLICIT_OUT/analysis-root-path.txt"; then + echo "FAIL: explicit non-Git tree was replaced by the caller checkout" >&2 + exit 1 +fi +echo "OK: explicit non-Git analysis root wins" + +echo "--- output symlink refusal tests ---" +SYMLINK_CASE="$TMP/symlink-case" +SYMLINK_OUTSIDE="$TMP/symlink-outside" +mkdir -p "$SYMLINK_CASE/.agents" "$SYMLINK_OUTSIDE" "$SYMLINK_CASE/local" +printf 'outside sentinel\n' >"$SYMLINK_OUTSIDE/sentinel" +ln -s "$SYMLINK_OUTSIDE" "$SYMLINK_CASE/.agents/scratch" +if ( + cd "$SYMLINK_CASE" + python3 "$SKILL/scripts/reverse_engineer.py" escaped \ + --mode=repo --local-clone-dir="$SYMLINK_CASE/local" >/dev/null 2>&1 +); then + echo "FAIL: default output followed a symlinked scratch parent" >&2 + exit 1 +fi +if ! grep -Fqx 'outside sentinel' "$SYMLINK_OUTSIDE/sentinel" \ + || [ -e "$SYMLINK_OUTSIDE/reverse-engineer" ]; then + echo "FAIL: symlinked parent allowed an outside write" >&2 + exit 1 +fi + +MANAGED_OUT="$TMP/out-managed-link" +MANAGED_OUTSIDE="$TMP/managed-outside.yaml" +mkdir -p "$MANAGED_OUT" +printf 'outside registry\n' >"$MANAGED_OUTSIDE" +ln -s "$MANAGED_OUTSIDE" "$MANAGED_OUT/feature-registry.yaml" +if python3 "$SKILL/scripts/reverse_engineer.py" managed-link \ + --mode=repo --local-clone-dir="$EXPLICIT_TREE" \ + --output-dir="$MANAGED_OUT" >/dev/null 2>&1; then + echo "FAIL: managed artifact symlink was followed" >&2 + exit 1 +fi +if ! grep -Fqx 'outside registry' "$MANAGED_OUTSIDE"; then + echo "FAIL: managed artifact symlink changed the outside target" >&2 + exit 1 +fi +echo "OK: output parent and managed-file symlinks fail closed" + # --- Multi-language CLI graceful degradation test --- echo "--- multi-language CLI degradation test ---" @@ -218,6 +329,54 @@ if ! grep -q "no CLI surface detected" "$OUT_NONCLI/spec-code-map.md" 2>/dev/nul fi echo "OK: multi-language CLI graceful degradation works" +echo "--- default output-path parity test ---" +DEFAULT_OUT="$TMP/.agents/scratch/reverse-engineer/default-demo" +( + cd "$TMP" + python3 "$SKILL/scripts/reverse_engineer.py" default-demo \ + --mode=repo \ + --local-clone-dir="$TMP/local-noncli" \ + --docs-sitemap-url="file://$SITEMAP" +) +if [ ! -s "$DEFAULT_OUT/feature-registry.yaml" ] \ + || [ ! -s "$DEFAULT_OUT/contracts/repo-contract.json" ] \ + || [ ! -s "$DEFAULT_OUT/reports/$(date +%F)-vibe-default-demo.md" ] \ + || [ ! -s "$DEFAULT_OUT/docs-features.txt" ] \ + || [ ! -s "$DEFAULT_OUT/validate-feature-registry.py" ]; then + echo "FAIL: executable default did not emit the declared product output directory" >&2 + exit 1 +fi +echo "OK: frontmatter output directory matches the executable default" + +echo "--- earlier output-path compatibility test ---" +LEGACY_OUT="$TMP/.agents/research/legacy-demo" +LEGACY_EXPECTED="$TMP/legacy-sentinel.expected" +LEGACY_DEFAULT="$TMP/.agents/scratch/reverse-engineer/legacy-demo" +mkdir -p "$LEGACY_OUT" +printf 'caller-owned sentinel\n\n' > "$LEGACY_OUT/caller-sentinel.txt" +cp "$LEGACY_OUT/caller-sentinel.txt" "$LEGACY_EXPECTED" +( + cd "$TMP" + python3 "$SKILL/scripts/reverse_engineer.py" legacy-demo \ + --mode=repo \ + --local-clone-dir="$TMP/local-noncli" \ + --output-dir="$LEGACY_OUT" \ + --docs-sitemap-url="file://$SITEMAP" +) +if [ ! -s "$LEGACY_OUT/feature-registry.yaml" ]; then + echo "FAIL: explicit earlier-default output directory was not honored" >&2 + exit 1 +fi +if ! cmp -s "$LEGACY_EXPECTED" "$LEGACY_OUT/caller-sentinel.txt"; then + echo "FAIL: explicit earlier-default invocation changed a pre-existing artifact" >&2 + exit 1 +fi +if [ -e "$LEGACY_DEFAULT" ]; then + echo "FAIL: explicit earlier-default invocation also wrote to the scratch default" >&2 + exit 1 +fi +echo "OK: explicit earlier-default output directory remains supported" + echo "--- generated-tree hygiene regression test ---" HYGIENE_REPO="$TMP/local-hygiene" HYGIENE_OUT="$TMP/out-hygiene" diff --git a/skills/reverse-engineer/scripts/validate-output.sh b/skills/reverse-engineer/scripts/validate-output.sh new file mode 100755 index 000000000..d785b382d --- /dev/null +++ b/skills/reverse-engineer/scripts/validate-output.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: validate-output.sh --output-dir DIR [--phase teardown|complete] + [--security-audit 0|1] [--sbom 0|1] + [--upstream-ref-set 0|1] +EOF + exit 2 +} + +output_dir="" +phase="complete" +security_audit=0 +sbom=0 +upstream_ref_set=0 +while (($#)); do + case "$1" in + --output-dir) (($# >= 2)) || usage; output_dir=$2; shift 2 ;; + --phase) (($# >= 2)) || usage; phase=$2; shift 2 ;; + --security-audit) (($# >= 2)) || usage; security_audit=$2; shift 2 ;; + --sbom) (($# >= 2)) || usage; sbom=$2; shift 2 ;; + --upstream-ref-set) (($# >= 2)) || usage; upstream_ref_set=$2; shift 2 ;; + -h|--help) usage ;; + *) usage ;; + esac +done + +[[ -n "$output_dir" ]] || usage +[[ "$phase" == teardown || "$phase" == complete ]] || usage +[[ "$security_audit" =~ ^[01]$ ]] || usage +[[ "$sbom" =~ ^[01]$ ]] || usage +[[ "$upstream_ref_set" =~ ^[01]$ ]] || usage +[[ -d "$output_dir" && ! -L "$output_dir" ]] || { + echo "error: output directory must be a real directory: $output_dir" >&2 + exit 1 +} + +required=( + feature-inventory.md + feature-registry.yaml + feature-catalog.md + spec-architecture.md + spec-code-map.md + spec-clone-vs-use.md + spec-clone-mvp.md + analysis-root-path.txt + validate-feature-registry.py +) +for name in "${required[@]}"; do + path="$output_dir/$name" + [[ -f "$path" && ! -L "$path" && -s "$path" ]] || { + echo "error: required regular nonempty artifact missing: $path" >&2 + exit 1 + } +done + +[[ -f "$output_dir/docs-features.txt" && ! -L "$output_dir/docs-features.txt" ]] || { + echo "error: docs-features.txt must be a regular file" >&2 + exit 1 +} +if [[ -e "$output_dir/spec-cli-surface.md" || -L "$output_dir/spec-cli-surface.md" ]]; then + [[ -f "$output_dir/spec-cli-surface.md" && ! -L "$output_dir/spec-cli-surface.md" && -s "$output_dir/spec-cli-surface.md" ]] || { + echo "error: spec-cli-surface.md must be a regular nonempty file when present" >&2 + exit 1 + } +fi + +python3 "$output_dir/validate-feature-registry.py" + +if [[ "$upstream_ref_set" == 1 ]]; then + metadata="$output_dir/clone-metadata.json" + [[ -f "$metadata" && ! -L "$metadata" && -s "$metadata" ]] || { + echo "error: --upstream-ref requires clone-metadata.json" >&2 + exit 1 + } + python3 - "$metadata" <<'PY' +import json, pathlib, re, sys +path = pathlib.Path(sys.argv[1]) +data = json.loads(path.read_text(encoding="utf-8")) +if not isinstance(data, dict): + raise SystemExit("clone metadata must be an object") +commit = data.get("resolved_commit") +if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-fA-F]{40,64}", commit): + raise SystemExit("clone metadata lacks a full resolved commit OID") +if not data.get("upstream_ref"): + raise SystemExit("clone metadata lacks upstream_ref") +PY +fi + +if [[ "$phase" == complete ]]; then + steal_map="$output_dir/steal-map.md" + [[ -f "$steal_map" && ! -L "$steal_map" && -s "$steal_map" ]] || { + echo "error: complete output requires a regular nonempty steal-map.md" >&2 + exit 1 + } + grep -Fqx '| Their capability | Our surface today | Verdict |' "$steal_map" || { + echo "error: steal-map.md lacks the required table header" >&2 + exit 1 + } + awk -F'|' ' + BEGIN { found = 0 } + /^\|/ { + capability=$2; ours=$3; verdict=$4 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", capability) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", ours) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", verdict) + gsub(/\*\*/, "", verdict) + if (capability != "" && capability != "Their capability" && capability !~ /^-+$/ && + ours != "" && verdict ~ /^(have|gap|steal|park|reject)$/) found = 1 + } + END { exit found ? 0 : 1 } + ' "$steal_map" || { + echo "error: steal-map.md needs at least one nonempty row with a valid verdict" >&2 + exit 1 + } +fi + +if [[ "$security_audit" == 1 ]]; then + gate="$output_dir/security/validate-security-audit.sh" + [[ -x "$gate" && ! -L "$gate" ]] || { + echo "error: security validator is missing or unsafe" >&2 + exit 1 + } + if [[ "$sbom" == 1 ]]; then + "$gate" "$output_dir" --sbom + else + "$gate" "$output_dir" --no-sbom + fi +else + [[ "$sbom" == 0 ]] || { + echo "error: --sbom requires --security-audit 1" >&2 + exit 1 + } +fi + +echo "PASS: reverse-engineer $phase output is structurally valid" diff --git a/skills/skill-builder/references/skill-auditor.feature b/skills/skill-builder/references/skill-auditor.feature index 49777d995..69203ea24 100644 --- a/skills/skill-builder/references/skill-auditor.feature +++ b/skills/skill-builder/references/skill-auditor.feature @@ -2,8 +2,8 @@ # skill template audit (BC1 Corpus / Skill Catalog). # The audit checks an existing SKILL.md against the unified template: Pass 1 gates # through heal.sh --strict, Pass 2 runs additional structural checks, then it emits a -# density report and a productization score. Hexagon: supporting; consumes: a SKILL.md + -# the template; produces: audit-report.json. (soc-qk4b) +# density report and a static package-readiness score. Hexagon: supporting; consumes: a +# SKILL.md + the template; produces: audit-report.json. (soc-qk4b) Feature: Skill-auditor scores a skill against the unified template As a catalog maintainer @@ -21,6 +21,7 @@ Feature: Skill-auditor scores a skill against the unified template When Pass 1 completes Then Pass 2 runs the additional template-conformance checks - Scenario: A density report and productization score are emitted + Scenario: A density report and static package-readiness score are emitted When both passes complete - Then it emits an advisory density report and a productization score in audit-report.json + Then it emits an advisory density report and a static package-readiness score in audit-report.json + And the score says that safety and effectiveness were not evaluated diff --git a/skills/skill-builder/references/skill-template.md b/skills/skill-builder/references/skill-template.md index 3f8c146cd..e42a9c87d 100644 --- a/skills/skill-builder/references/skill-template.md +++ b/skills/skill-builder/references/skill-template.md @@ -158,7 +158,7 @@ Each NEW Pass-2 check maps to AgentOps' design principles in PRODUCT.md, so the | `quality-rubric` | Operational Principle #3 (context quality determines output quality) | | `references-modularization` | Finding `f-2026-05-01-025` (SKILL.md churn budget — every Skill() invocation reloads 5-15KB) | | `trigger-clarity` | Operational Principle #1 (agents are ephemeral) — invocation criteria must be in artifact | -| `description-has-triggers` (renamed from `description-multiline`) | Pillar #6 (knowledge flywheel) — searchability requires structured description. Three valid forms preserve AgentOps' single-line convention. | +| `description-has-triggers` (renamed from `description-multiline`) | Product surfaces — structured invocation criteria make the right capability discoverable without loading every skill body. Three valid forms preserve AgentOps' single-line convention. | --- diff --git a/skills/skill-builder/schemas/audit-report.json b/skills/skill-builder/schemas/audit-report.json index 7164c762c..d17999bef 100644 --- a/skills/skill-builder/schemas/audit-report.json +++ b/skills/skill-builder/schemas/audit-report.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema#", "title": "Skill Audit Report", - "description": "Output contract for the skill-builder deep audit. Three passes: Pass 1 wraps heal.sh structural checks; Pass 2 adds 8 NEW content-discipline checks beyond heal.sh; Pass 3 folds the 10-category Skill Quality Rubric (docs/reference/skill-quality-rubric.md) in as an advisory 0-30 productization score.", + "description": "Output contract for the skill-builder deep audit. Pass 1 wraps heal.sh structural checks; Pass 2 adds 8 content-discipline checks beyond heal.sh; Pass 3 reports an advisory 0-30 static package-readiness score that evaluates neither safety nor behavioral effectiveness; later passes add advisory craft and authoring signals.", "type": "object", "required": ["target", "profile_id", "verdict", "pass1", "pass2"], "properties": { @@ -155,13 +155,28 @@ "additionalProperties": false }, "rubric": { - "description": "Advisory-only Pass-3 Skill Quality Rubric score (docs/reference/skill-quality-rubric.md). Folded in from score_agentops_skill.py --audit-block. Never affects the aggregate verdict. Emitted as null when python3 or the scorer is unavailable (fail-open).", + "description": "Advisory-only Pass-3 static package-readiness score (docs/reference/skill-quality-rubric.md). Folded in from score_agentops_skill.py --audit-block. It evaluates neither the safety gate nor behavioral effectiveness and never affects the aggregate verdict. Emitted as null when python3 or the scorer is unavailable (fail-open).", "oneOf": [ {"type": "null"}, { "type": "object", - "required": ["total_score", "max_score", "rating", "advisory", "categories"], + "required": ["scope", "safety_gate_evaluated", "effectiveness_evaluated", "total_score", "max_score", "rating", "advisory", "categories"], "properties": { + "scope": { + "type": "string", + "const": "static-package-readiness", + "description": "The score covers visible package properties only." + }, + "safety_gate_evaluated": { + "type": "boolean", + "const": false, + "description": "Always false: boundary-word heuristics are not a full-bundle safety review." + }, + "effectiveness_evaluated": { + "type": "boolean", + "const": false, + "description": "Always false: structural scoring contains no baseline-versus-treatment behavioral evaluation." + }, "total_score": { "type": "integer", "minimum": 0, @@ -175,12 +190,12 @@ "rating": { "type": "string", "enum": ["C", "B", "A", "S"], - "description": "Rating band: C (0-10), B (11-20), A (21-26), S (27-30)." + "description": "Static readiness band: C (0-10), B (11-20), A (21-26), S (27-30)." }, "advisory": { "type": "boolean", "const": true, - "description": "Always true. The rubric score is report-only and never gates the verdict." + "description": "Always true. The static readiness score is report-only and never gates the verdict." }, "categories": { "type": "array", diff --git a/skills/skill-builder/scripts/audit.sh b/skills/skill-builder/scripts/audit.sh index 7560fd81c..035575d5e 100755 --- a/skills/skill-builder/scripts/audit.sh +++ b/skills/skill-builder/scripts/audit.sh @@ -314,14 +314,14 @@ else DENSITY_STATUS="warn" fi -# --- Pass 3: rubric scoring (advisory) ----------------------------------- +# --- Pass 3: static package-readiness scoring (advisory) ----------------- # Folds the 10-category Skill Quality Rubric (docs/reference/skill-quality-rubric.md) # into the report via score_agentops_skill.py --audit-block. Each category gets a # deterministic 0-3 score plus an explainable reason; total is 0-30 with a C/B/A/S -# rating band. Advisory-only: it never changes the PASS/WARN/FAIL verdict — the -# rubric measures market-facing maturity, not template conformance (which Pass 1+2 -# already gate). Reason: a low rubric score on a structurally-clean skill is a -# productization backlog signal, not a ship blocker. +# readiness band. Advisory-only: it never changes the PASS/WARN/FAIL verdict and +# explicitly evaluates neither the safety gate nor behavioral effectiveness. +# Reason: a low score on a structurally clean skill is a triage signal, while a +# high score still cannot prove that the skill is safe or improves outcomes. RUBRIC_JSON="null" RUBRIC_SUMMARY="" RUBRIC_SCORE="n/a" @@ -331,7 +331,7 @@ if [[ -f "$SCORE_PY" ]] && command -v python3 >/dev/null 2>&1; then RUBRIC_JSON="$rubric_out" RUBRIC_SCORE="$(printf '%s' "$rubric_out" | awk -F': ' '/"total_score"/{gsub(/[, ]/,"",$2); print $2; exit}')" RUBRIC_RATING="$(printf '%s' "$rubric_out" | awk -F'"' '/"rating"/{print $4; exit}')" - RUBRIC_SUMMARY=" Rubric: ${RUBRIC_SCORE}/30 (${RUBRIC_RATING}) [advisory]." + RUBRIC_SUMMARY=" Static readiness: ${RUBRIC_SCORE}/30 (${RUBRIC_RATING}) [advisory; safety/effectiveness not evaluated]." fi fi @@ -482,7 +482,7 @@ fi printf " [%-4s] %s\n" "${CHECK_STATUS[$id]}" "$id" done echo "Density advisory: $density_present_count/6 fields present ($DENSITY_STATUS)" - echo "Pass 3 rubric (advisory): ${RUBRIC_SCORE}/30 (${RUBRIC_RATING})" + echo "Pass 3 static readiness (advisory): ${RUBRIC_SCORE}/30 (${RUBRIC_RATING}); safety/effectiveness not evaluated" if [[ -n "$CRAFT_LINES" ]]; then echo "$CRAFT_LINES" fi diff --git a/skills/skill-builder/scripts/score_agentops_skill.py b/skills/skill-builder/scripts/score_agentops_skill.py index ad703c260..af2d0301b 100755 --- a/skills/skill-builder/scripts/score_agentops_skill.py +++ b/skills/skill-builder/scripts/score_agentops_skill.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Score an AgentOps skill against the local product-grade skill rubric.""" +"""Score static package readiness for an AgentOps skill.""" from __future__ import annotations @@ -248,6 +248,9 @@ def score_skill(path: Path) -> dict: return { "skill": str(path), "name": path.name, + "scope": "static-package-readiness", + "safety_gate_evaluated": False, + "effectiveness_evaluated": False, "total_score": total, "max_score": 30, "rating": rating, @@ -273,13 +276,16 @@ def score_skill(path: Path) -> dict: def audit_block(report: dict) -> dict: - """Compact rubric object for embedding in the skill-builder deep audit's audit-report.json (Pass 3). + """Compact static-readiness object for the deep audit report (Pass 3). Mirrors the rubric schema block: per-category 0-3 score plus an explainable - reason, the 0-30 total, max, and the C/B/A/S rating band. Deterministic — - derived only from the skill directory contents. + reason, the 0-30 total, max, and the C/B/A/S readiness band. It is derived + only from directory contents and cannot evaluate safety or effectiveness. """ return { + "scope": report["scope"], + "safety_gate_evaluated": report["safety_gate_evaluated"], + "effectiveness_evaluated": report["effectiveness_evaluated"], "total_score": report["total_score"], "max_score": report["max_score"], "rating": report["rating"], @@ -290,9 +296,11 @@ def audit_block(report: dict) -> dict: def markdown_report(report: dict) -> str: lines = [ - f"# Skill Quality Score: {report['name']}", + f"# Static Skill Package Readiness: {report['name']}", "", - f"Score: {report['total_score']}/{report['max_score']} ({report['rating']})", + f"Static score: {report['total_score']}/{report['max_score']} ({report['rating']})", + "", + "This score does not evaluate the safety gate or behavioral effectiveness.", "", "## Category Scores", "", diff --git a/skills/standards/references/test-pyramid.md b/skills/standards/references/test-pyramid.md index 2346cd8a0..04e00501e 100644 --- a/skills/standards/references/test-pyramid.md +++ b/skills/standards/references/test-pyramid.md @@ -33,7 +33,7 @@ Add test levels only when each one covers a distinct risk. Do not require L2 by default, duplicate the same assertion at every level, or treat test count as evidence quality. -## Operating-loop use +## RPI traversal use - **Plan** names the active behavior, edge scenario, required evidence, and first acceptance check. diff --git a/skills/using-flywheel/SKILL.md b/skills/using-flywheel/SKILL.md index e2fcc9a5c..bffb9c2c1 100644 --- a/skills/using-flywheel/SKILL.md +++ b/skills/using-flywheel/SKILL.md @@ -6,6 +6,7 @@ skill_api_version: 1 hexagonal_role: driving-adapter consumes: [explicit-packets] produces: [flywheel-runtime-evidence] +output_contract: 'runtime evidence pointers for processed beads, candidate commits or worktrees, and invoked AgentOps skills; never an AgentOps verdict' context_rel: - kind: partnership with: using-gc diff --git a/tests/scripts/agentops-native-skills.bats b/tests/scripts/agentops-native-skills.bats index 6ad446b58..5c827957d 100644 --- a/tests/scripts/agentops-native-skills.bats +++ b/tests/scripts/agentops-native-skills.bats @@ -28,6 +28,156 @@ assert_rejects() { [ "$status" -ne 0 ] } +write_valid_recon_baseline() { + local path="$1" commit="${2:-deadbeef}" + write_recon_pack "$path" "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"$commit\",\"flows\":[{\"entry\":\"cli/main.go\",\"domain\":\"internal/domain\",\"integration\":\"internal/adapters\",\"tests\":\"internal/domain/x_test.go\"}],\"claims\":[],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}" +} + +sha256_stream() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + else + shasum -a 256 | awk '{print $1}' + fi +} + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +write_recon_pack() { + local path="$1" payload="$2" dir raw report commit mode flows_sha claims_sha coverage_sha report_sha + dir="$(dirname "$path")" + mkdir -p "$dir" + raw="$dir/.manifest.raw.json" + report="$dir/codebase-recon.md" + printf '%s\n' "$payload" > "$raw" + commit="$(jq -r '.commit // empty' "$raw")" + mode="$(jq -r '.mode // empty' "$raw")" + flows_sha="$(jq -cS '.flows' "$raw" | sha256_stream)" + claims_sha="$(jq -cS '.claims' "$raw" | sha256_stream)" + coverage_sha="$(jq -cS '.coverage' "$raw" | sha256_stream)" + cat > "$report" < +manifest_commit: $commit +manifest_mode: $mode +flows_sha256: $flows_sha +claims_sha256: $claims_sha +coverage_sha256: $coverage_sha + +# Codebase recon fixture +EOF + report_sha="$(sha256_file "$report")" + jq --arg report_sha "$report_sha" \ + '. + {report:{path:"codebase-recon.md",sha256:$report_sha}}' \ + "$raw" > "$path" + rm -f "$raw" +} + +recon_file() { + CASE=$((CASE + 1)) + local dir="$BATS_TEST_TMPDIR/recon-case-$CASE" path="$BATS_TEST_TMPDIR/recon-case-$CASE/codebase-recon.json" + mkdir -p "$dir" + write_recon_pack "$path" "$1" + printf '%s\n' "$path" +} + +init_recon_repo() { + local target="$1" + mkdir -p "$target/cli" "$target/internal/domain" "$target/internal/adapters" + git -C "$target" init -q + git -C "$target" config user.name fixture + git -C "$target" config user.email fixture@example.invalid + printf 'package main\n' > "$target/cli/main.go" + printf 'package domain\n' > "$target/internal/domain/model.go" + printf 'package domain\n' > "$target/internal/domain/x_test.go" + printf 'package adapters\n' > "$target/internal/adapters/adapter.go" + printf 'entry -> domain -> test\n' > "$target/evidence.txt" + git -C "$target" add cli/main.go internal evidence.txt + git -C "$target" commit -qm baseline + git -C "$target" rev-parse HEAD +} + +write_slow_recon_baseline() { + local path="$1" commit="$2" payload + payload="$(jq -cn --arg commit "$commit" ' + { + schema_version: "codebase-recon.v1", + mode: "baseline", + commit: $commit, + flows: [{ + entry: "cli/main.go", + domain: "internal/domain", + integration: "internal/adapters", + tests: "internal/domain/x_test.go" + }], + claims: [range(0; 150) | { + kind: "fact", + text: ("race witness " + tostring), + confidence: "high", + evidence: ["evidence.txt:1"] + }], + coverage: {inspected: ["cli"], uninspected: ["images"]} + } + ')" + write_recon_pack "$path" "$payload" +} + +run_recon_race() { + local validator="$1" target="$2" pack="$3" mutation="$4" + local sync_root="$BATS_TEST_TMPDIR/race-$mutation" output_file pid snapshot="" file_count=0 ready=0 i + mkdir -p "$sync_root" + output_file="$sync_root/output" + + TMPDIR="$sync_root" "$validator" --repo-root "$target" "$pack" >"$output_file" 2>&1 & + pid=$! + for ((i = 0; i < 1000; i++)); do + snapshot="$(find "$sync_root" -mindepth 1 -maxdepth 1 -type d -name 'codebase-recon-validate.*' -print -quit)" + if [[ -n "$snapshot" ]]; then + file_count="$(find "$snapshot" -mindepth 1 -maxdepth 1 -type f | wc -l | tr -d ' ')" + if [[ "$file_count" -ge 2 ]]; then + ready=1 + break + fi + fi + kill -0 "$pid" 2>/dev/null || break + sleep 0.005 + done + + if [[ "$ready" != "1" ]]; then + if wait "$pid"; then + RACE_STATUS=0 + else + RACE_STATUS=$? + fi + RACE_OUTPUT="$(cat "$output_file")" + return 1 + fi + + case "$mutation" in + manifest) printf ' ' >> "$pack" ;; + report) printf '\nlate report mutation\n' >> "$(dirname "$pack")/codebase-recon.md" ;; + worktree) printf '// late worktree mutation\n' >> "$target/cli/main.go" ;; + index) + printf '// late index mutation\n' >> "$target/cli/main.go" + git -C "$target" add cli/main.go + ;; + head) git -C "$target" commit --allow-empty -qm 'race commit' ;; + *) return 2 ;; + esac + + if wait "$pid"; then + RACE_STATUS=0 + else + RACE_STATUS=$? + fi + RACE_OUTPUT="$(cat "$output_file")" +} + # B1.1 @test "idea-genie produces an evidence-grounded idea-portfolio artifact" { v="$REPO_ROOT/skills/idea-genie/scripts/validate-output.sh" @@ -60,17 +210,241 @@ assert_rejects() { # B3.1 @test "codebase-recon validates evidence-bounded fact inference unknown claims" { v="$REPO_ROOT/skills/codebase-recon/scripts/validate-output.sh" - evidence="$BATS_TEST_TMPDIR/evidence.txt"; printf 'entry -> domain -> test\n' > "$evidence" - assert_accepts "$v" "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"deadbeef\",\"flows\":[{\"entry\":\"cli/main.go\",\"domain\":\"internal/domain\",\"integration\":\"internal/adapters\",\"tests\":\"internal/domain/x_test.go\"}],\"claims\":[{\"kind\":\"fact\",\"text\":\"a flow exists\",\"confidence\":\"high\",\"evidence\":[\"$evidence\"]},{\"kind\":\"unknown\",\"text\":\"remote behavior\",\"confidence\":\"low\",\"evidence\":[]}],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}" - assert_rejects "$v" '{"schema_version":"codebase-recon.v1","mode":"baseline","commit":"deadbeef","flows":[],"claims":[{"kind":"fact","text":"unsupported","confidence":"high","evidence":[]}],"coverage":{"inspected":[],"uninspected":[]}}' + target="$BATS_TEST_TMPDIR/recon-baseline" + commit="$(init_recon_repo "$target")" + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"$commit\",\"flows\":[{\"entry\":\"cli/main.go\",\"domain\":\"internal/domain\",\"integration\":\"internal/adapters\",\"tests\":\"internal/domain/x_test.go\"}],\"claims\":[{\"kind\":\"fact\",\"text\":\"a flow exists\",\"confidence\":\"high\",\"evidence\":[\"evidence.txt\"]},{\"kind\":\"unknown\",\"text\":\"remote behavior\",\"confidence\":\"low\",\"evidence\":[]}],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -eq 0 ] + + short_commit="${commit:0:7}" + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"$short_commit\",\"flows\":[{\"entry\":\"cli/main.go\",\"domain\":\"internal/domain\",\"integration\":\"internal/adapters\",\"tests\":\"internal/domain/x_test.go\"}],\"claims\":[],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + + git -C "$target" tag deadbee "$commit" + artifact="$(recon_file '{"schema_version":"codebase-recon.v1","mode":"baseline","commit":"deadbee","flows":[{"entry":"cli/main.go","domain":"internal/domain","integration":"internal/adapters","tests":"internal/domain/x_test.go"}],"claims":[],"coverage":{"inspected":["cli"],"uninspected":["images"]}}')" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"$commit\",\"flows\":[{\"entry\":\"cli/main.go\",\"domain\":\"internal/domain\",\"integration\":\"internal/adapters\",\"tests\":\"internal/domain/x_test.go\"}],\"claims\":[{\"kind\":\"fact\",\"text\":\"external evidence\",\"confidence\":\"high\",\"evidence\":[\"$target/evidence.txt\"]}],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"$commit\",\"flows\":[{\"entry\":\"cli/main.go\",\"domain\":\"internal/domain\",\"integration\":\"internal/adapters\",\"tests\":\"internal/domain/x_test.go\"}],\"claims\":[{\"kind\":\"fact\",\"text\":\"pathspec evidence\",\"confidence\":\"high\",\"evidence\":[\":(top)evidence.txt\"]}],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"$commit\",\"flows\":[{\"entry\":\"cli/main.go\",\"domain\":\"internal/domain\",\"integration\":\"internal/adapters\",\"tests\":\"internal/domain/x_test.go\"}],\"claims\":[{\"kind\":\"fact\",\"text\":\"control-byte evidence\",\"confidence\":\"high\",\"evidence\":[\"evidence.txt\\u0000\"]}],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"$commit\",\"flows\":[{\"entry\":\"cli/main.go\",\"domain\":\"internal/domain\",\"integration\":\"internal/adapters\",\"tests\":\"internal/domain/x_test.go\"}],\"claims\":[{\"kind\":\"fact\",\"text\":\"bad line\",\"confidence\":\"high\",\"evidence\":[\"evidence.txt:99\"]}],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + + artifact="$(recon_file '{"schema_version":"codebase-recon.v1","mode":"baseline","commit":"deadbeef","flows":[],"claims":[{"kind":"fact","text":"unsupported","confidence":"high","evidence":[]}],"coverage":{"inspected":[],"uninspected":[]}}')" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] } # B3.2 @test "codebase-recon requires a verified delta when a prior pack exists" { v="$REPO_ROOT/skills/codebase-recon/scripts/validate-output.sh" - prior="$BATS_TEST_TMPDIR/prior.md"; printf 'prior baseline\n' > "$prior" - assert_accepts "$v" "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"delta\",\"commit\":\"feedface\",\"prior_recon\":\"$prior\",\"baseline_verified\":true,\"delta\":[{\"path\":\"cli/x.go\",\"change\":\"new adapter\"}],\"flows\":[],\"claims\":[],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}" - assert_rejects "$v" "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"feedface\",\"prior_recon\":\"$prior\",\"baseline_verified\":false,\"flows\":[],\"claims\":[],\"coverage\":{\"inspected\":[],\"uninspected\":[]}}" + target="$BATS_TEST_TMPDIR/recon-delta" + baseline_commit="$(init_recon_repo "$target")" + prior="$target/.agents/recon/prior/codebase-recon.json" + write_valid_recon_baseline "$prior" "$baseline_commit" + printf 'package x\n' > "$target/cli/x.go" + git -C "$target" add cli/x.go + git -C "$target" commit -qm delta + current_commit="$(git -C "$target" rev-parse HEAD)" + + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"delta\",\"commit\":\"$current_commit\",\"prior_recon\":\"$prior\",\"baseline_verified\":true,\"delta\":[{\"path\":\"cli/x.go\",\"change\":\"new adapter\"}],\"flows\":[],\"claims\":[],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -eq 0 ] + + printf '// dirty\n' >> "$target/cli/main.go" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + [[ "$output" == *"source changes not bound"* ]] + git -C "$target" restore cli/main.go + + printf 'untracked source\n' > "$target/untracked.txt" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + [[ "$output" == *"source changes not bound"* ]] + rm -f "$target/untracked.txt" + + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"delta\",\"commit\":\"$baseline_commit\",\"prior_recon\":\"$prior\",\"baseline_verified\":true,\"delta\":[{\"path\":\"cli/x.go\",\"change\":\"new adapter\"}],\"flows\":[],\"claims\":[],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"delta\",\"commit\":\"$current_commit\",\"prior_recon\":\"$prior\",\"baseline_verified\":true,\"delta\":[{\"path\":\"cli/unrelated.go\",\"change\":\"fabricated\"}],\"flows\":[],\"claims\":[],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"delta\",\"commit\":\"$current_commit\",\"prior_recon\":\"$prior:1\",\"baseline_verified\":true,\"delta\":[{\"path\":\"cli/x.go\",\"change\":\"new adapter\"}],\"flows\":[],\"claims\":[],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + + unknown_prior="$BATS_TEST_TMPDIR/unknown/codebase-recon.json" + write_valid_recon_baseline "$unknown_prior" deadbeef + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"delta\",\"commit\":\"$current_commit\",\"prior_recon\":\"$unknown_prior\",\"baseline_verified\":true,\"delta\":[{\"path\":\"cli/x.go\",\"change\":\"new adapter\"}],\"flows\":[],\"claims\":[],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] + + invalid_prior="$BATS_TEST_TMPDIR/invalid/codebase-recon.json" + mkdir -p "$(dirname "$invalid_prior")" + printf '%s\n' '{"schema_version":"codebase-recon.v1","mode":"baseline"}' > "$invalid_prior" + artifact="$(recon_file "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"delta\",\"commit\":\"$current_commit\",\"prior_recon\":\"$invalid_prior\",\"baseline_verified\":true,\"delta\":[{\"path\":\"cli/x.go\",\"change\":\"new adapter\"}],\"flows\":[],\"claims\":[],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}")" + run "$v" --repo-root "$target" "$artifact" + [ "$status" -ne 0 ] +} + +# B3.3 +@test "codebase-recon prior-discovery finds validated packs at current and earlier default paths" { + v="$REPO_ROOT/skills/codebase-recon/scripts/validate-output.sh" + target="$BATS_TEST_TMPDIR/target" + legacy="$target/.agents/recon/legacy-run/codebase-recon.json" + current="$target/.agents/scratch/codebase-recon/current-run/codebase-recon.json" + + baseline_commit="$(init_recon_repo "$target")" + run "$v" --repo-root "$target" --discover-priors + [ "$status" -eq 0 ] + [ -z "$output" ] + + write_valid_recon_baseline "$legacy" "$baseline_commit" + printf 'package x\n' > "$target/cli/x.go" + git -C "$target" add cli/x.go + git -C "$target" commit -qm delta + current_commit="$(git -C "$target" rev-parse HEAD)" + write_valid_recon_baseline "$current" "$current_commit" + invalid="$target/.agents/recon/invalid-run/codebase-recon.json" + write_valid_recon_baseline "$invalid" deadbeef + + run "$v" --repo-root "$target" --discover-priors + [ "$status" -eq 0 ] + [[ "$output" == *"$legacy"* ]] + [[ "$output" == *"$current"* ]] + ! grep -Fxq "$invalid" <<<"$output" + + delta="$BATS_TEST_TMPDIR/delta/codebase-recon.json" + write_recon_pack "$delta" "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"delta\",\"commit\":\"$current_commit\",\"prior_recon\":\".agents/recon/legacy-run/codebase-recon.json\",\"baseline_verified\":true,\"delta\":[{\"path\":\"cli/x.go\",\"change\":\"new adapter\"}],\"flows\":[],\"claims\":[],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}" + run "$v" --repo-root "$target" "$delta" + [ "$status" -eq 0 ] + [ -f "$legacy" ] + [ -f "$current" ] +} + +# B3.4 +@test "codebase-recon resolves historical evidence against each manifest commit" { + v="$REPO_ROOT/skills/codebase-recon/scripts/validate-output.sh" + target="$BATS_TEST_TMPDIR/recon-evidence-history" + baseline_commit="$(init_recon_repo "$target")" + good="$target/.agents/recon/good/codebase-recon.json" + bad="$target/.agents/recon/bad/codebase-recon.json" + mkdir -p "$(dirname "$good")" "$(dirname "$bad")" + write_recon_pack "$good" "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"$baseline_commit\",\"flows\":[{\"entry\":\"cli/main.go\",\"domain\":\"internal/domain\",\"integration\":\"internal/adapters\",\"tests\":\"internal/domain/x_test.go\"}],\"claims\":[{\"kind\":\"fact\",\"text\":\"historical evidence\",\"confidence\":\"high\",\"evidence\":[\"evidence.txt:1\"]}],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}" + + printf 'rewritten later\n' > "$target/evidence.txt" + printf 'future only\n' > "$target/future.txt" + git -C "$target" add evidence.txt future.txt + git -C "$target" commit -qm later + write_recon_pack "$bad" "{\"schema_version\":\"codebase-recon.v1\",\"mode\":\"baseline\",\"commit\":\"$baseline_commit\",\"flows\":[{\"entry\":\"cli/main.go\",\"domain\":\"internal/domain\",\"integration\":\"internal/adapters\",\"tests\":\"internal/domain/x_test.go\"}],\"claims\":[{\"kind\":\"fact\",\"text\":\"future evidence\",\"confidence\":\"high\",\"evidence\":[\"future.txt\"]}],\"coverage\":{\"inspected\":[\"cli\"],\"uninspected\":[\"images\"]}}" + + run "$v" --repo-root "$target" --discover-priors + [ "$status" -eq 0 ] + good_physical="$(cd "$(dirname "$good")" && pwd -P)/$(basename "$good")" + bad_physical="$(cd "$(dirname "$bad")" && pwd -P)/$(basename "$bad")" + grep -Fxq "$good_physical" <<<"$output" + ! grep -Fxq "$bad_physical" <<<"$output" +} + +# B3.5 +@test "codebase-recon binds its companion and rejects validation-time mutation" { + v="$REPO_ROOT/skills/codebase-recon/scripts/validate-output.sh" + target="$BATS_TEST_TMPDIR/recon-companion" + commit="$(init_recon_repo "$target")" + pack="$target/.agents/scratch/codebase-recon/run/codebase-recon.json" + report="$(dirname "$pack")/codebase-recon.md" + write_valid_recon_baseline "$pack" "$commit" + + run "$v" --repo-root "$target" "$pack" + [ "$status" -eq 0 ] + + rm -f "$report" + run "$v" --repo-root "$target" "$pack" + [ "$status" -ne 0 ] + + write_valid_recon_baseline "$pack" "$commit" + printf '\nmutated report\n' >> "$report" + run "$v" --repo-root "$target" "$pack" + [ "$status" -ne 0 ] + + write_valid_recon_baseline "$pack" "$commit" + outside="$BATS_TEST_TMPDIR/outside-report.md" + cp "$report" "$outside" + rm -f "$report" + ln -s "$outside" "$report" + run "$v" --repo-root "$target" "$pack" + [ "$status" -ne 0 ] + + rm -f "$report" + write_valid_recon_baseline "$pack" "$commit" + manifest_link="$BATS_TEST_TMPDIR/manifest-link.json" + ln -s "$pack" "$manifest_link" + run "$v" --repo-root "$target" "$manifest_link" + [ "$status" -ne 0 ] + + write_slow_recon_baseline "$pack" "$commit" + run_recon_race "$v" "$target" "$pack" manifest + [ "$RACE_STATUS" -ne 0 ] + + write_slow_recon_baseline "$pack" "$commit" + run_recon_race "$v" "$target" "$pack" report + [ "$RACE_STATUS" -ne 0 ] + + write_slow_recon_baseline "$pack" "$commit" + run_recon_race "$v" "$target" "$pack" worktree + [ "$RACE_STATUS" -ne 0 ] + git -C "$target" restore cli/main.go + + write_slow_recon_baseline "$pack" "$commit" + run_recon_race "$v" "$target" "$pack" index + [ "$RACE_STATUS" -ne 0 ] + git -C "$target" restore --staged cli/main.go + git -C "$target" restore cli/main.go + + write_slow_recon_baseline "$pack" "$commit" + run_recon_race "$v" "$target" "$pack" head + [ "$RACE_STATUS" -ne 0 ] + git -C "$target" reset --soft "$commit" +} + +# B3.6 +@test "Codex projection executes the hardened recon validator contract" { + canonical="$REPO_ROOT/skills/codebase-recon/scripts/validate-output.sh" + projected="$REPO_ROOT/skills-codex/codebase-recon/scripts/validate-output.sh" + [ -x "$projected" ] + cmp -s "$canonical" "$projected" + + target="$BATS_TEST_TMPDIR/recon-projection" + commit="$(init_recon_repo "$target")" + pack="$target/.agents/recon/projected/codebase-recon.json" + write_valid_recon_baseline "$pack" "$commit" + run "$projected" --repo-root "$target" --discover-priors + [ "$status" -eq 0 ] + [[ "$output" == *"$pack"* ]] + + short_pack="$BATS_TEST_TMPDIR/projected-short/codebase-recon.json" + write_valid_recon_baseline "$short_pack" "${commit:0:7}" + run "$projected" --repo-root "$target" "$short_pack" + [ "$status" -ne 0 ] + + printf 'dirty\n' > "$target/untracked.txt" + run "$projected" --repo-root "$target" "$pack" + [ "$status" -ne 0 ] + [[ "$output" == *"source changes not bound"* ]] } # B4.1 diff --git a/tests/scripts/agentops-product-boundary.bats b/tests/scripts/agentops-product-boundary.bats index 16ff2402a..396481bca 100644 --- a/tests/scripts/agentops-product-boundary.bats +++ b/tests/scripts/agentops-product-boundary.bats @@ -27,6 +27,29 @@ require_text() { } } +run_linked_reference_identity_check() { + python3 - \ + "$REPO_ROOT/scripts/check-cathedral-cut-conformance.py" \ + "$1" <<'PY' +import importlib.util +from pathlib import Path +import sys + +script = Path(sys.argv[1]) +root = Path(sys.argv[2]) +spec = importlib.util.spec_from_file_location("cathedral_cut", script) +module = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(module) + +try: + module.check_linked_skill_reference_identity(root) +except AssertionError as exc: + print(exc) + raise SystemExit(1) +PY +} + @test "active authority teaches one bounded experiment and stop" { local file for file in "${ACTIVE_AUTHORITY[@]}"; do @@ -201,6 +224,842 @@ scan_obsolete_identity() { [ "$status" -eq 1 ] } +@test "linked skill references reject retired operations-layer terminology" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p \ + "$fixture/skills/example/references" \ + "$fixture/skills/unlinked/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Template](references/template.md)' \ + '- [Testing](references/testing.md)' \ + '- [Identity](references/identity.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Template' \ + '' \ + 'Searchability was formerly described as a knowledge flywheel.' \ + >"$fixture/skills/example/references/template.md" + printf '%s\n' \ + '# Testing' \ + '' \ + '## Operating-loop use' \ + >"$fixture/skills/example/references/testing.md" + printf '%s\n' \ + '# Identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/identity.md" + printf '%s\n' \ + '# Historical note' \ + '' \ + 'The knowledge flywheel label appeared in this unlinked archive.' \ + >"$fixture/skills/unlinked/references/history.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"template.md:3"* ]] + [[ "$output" == *"knowledge flywheel"* ]] + [[ "$output" == *"testing.md:3"* ]] + [[ "$output" == *"Operating-loop use"* ]] + [[ "$output" == *"identity.md:3"* ]] + [[ "$output" == *"AgentOps is the operating loop"* ]] + [[ "$output" != *"unlinked/references/history.md"* ]] +} + +@test "linked skill reference scan permits external factory terminology" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Factory](references/factory.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# External factory' \ + '' \ + 'The Agentic Coding Flywheel is a supported external factory.' \ + >"$fixture/skills/example/references/factory.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "reference-style skill links scan full collapsed and shortcut forms" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Policy][p]' \ + '- [Testing][]' \ + '- [Shortcut]' \ + '- [Identity][identity]' \ + '' \ + '[P]: references/policy.md "policy"' \ + '[testing]: ' \ + '[shortcut]: references/shortcut.md' \ + '[identity]: references/identity.md' \ + '[Archive]: references/archive.md' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Policy' \ + '' \ + 'Searchability was formerly described as a knowledge flywheel.' \ + >"$fixture/skills/example/references/policy.md" + printf '%s\n' \ + '# Testing' \ + '' \ + '## Operating-loop use' \ + >"$fixture/skills/example/references/testing.md" + printf '%s\n' \ + '# Shortcut' \ + '' \ + 'The knowledge-flywheel label is retired.' \ + >"$fixture/skills/example/references/shortcut.md" + printf '%s\n' \ + '# Identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/identity.md" + printf '%s\n' \ + '# Unlinked archive' \ + '' \ + '## Operating-loop use' \ + >"$fixture/skills/example/references/archive.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"policy.md:3"* ]] + [[ "$output" == *"testing.md:3"* ]] + [[ "$output" == *"shortcut.md:3"* ]] + [[ "$output" == *"identity.md:3"* ]] + [[ "$output" == *"AgentOps is the operating loop"* ]] + [[ "$output" != *"archive.md"* ]] +} + +@test "reference-style scan permits safe target and ignores unused archive definition" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Factory][factory]' \ + '' \ + '[factory]: references/factory.md' \ + '[archive]: references/archive.md "[archive]"' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# External factory' \ + '' \ + 'The Agentic Coding Flywheel is a supported external factory.' \ + >"$fixture/skills/example/references/factory.md" + printf '%s\n' \ + '# Historical archive' \ + '' \ + 'The knowledge flywheel label appeared here.' \ + >"$fixture/skills/example/references/archive.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "duplicate reference definitions use the first destination" { + bad_first="$BATS_TEST_TMPDIR/bad-first" + safe_first="$BATS_TEST_TMPDIR/safe-first" + mkdir -p \ + "$bad_first/skills/example/references" \ + "$safe_first/skills/example/references" + + printf '%s\n' \ + '# Bad first' \ + '' \ + '[Policy][p]' \ + '' \ + '[p]: references/bad.md' \ + '[p]: references/safe.md' \ + >"$bad_first/skills/example/SKILL.md" + printf '%s\n' \ + '# Safe first' \ + '' \ + '[Policy][p]' \ + '' \ + '[p]: references/safe.md' \ + '[p]: references/bad.md' \ + >"$safe_first/skills/example/SKILL.md" + + for fixture in "$bad_first" "$safe_first"; do + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad.md" + printf '%s\n' \ + '# External factory' \ + '' \ + 'The Agentic Coding Flywheel is a supported external factory.' \ + >"$fixture/skills/example/references/safe.md" + done + + run run_linked_reference_identity_check "$bad_first" + [ "$status" -eq 1 ] + [[ "$output" == *"bad.md:3"* ]] + [[ "$output" != *"safe.md"* ]] + + run run_linked_reference_identity_check "$safe_first" + [ "$status" -eq 0 ] +} + +@test "reference-like syntax in code or escaped prose does not create links" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Factory][safe]' \ + '' \ + '`[Inline][bad]`' \ + '\[Escaped][bad]' \ + '' \ + '```markdown' \ + '[Fenced][bad]' \ + '[inside]: references/bad.md' \ + '```' \ + '' \ + '[safe]: references/safe.md' \ + '[bad]: references/bad.md' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# External factory' \ + '' \ + 'The Agentic Coding Flywheel is a supported external factory.' \ + >"$fixture/skills/example/references/safe.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "unordered-list fenced code is excluded from links and prose" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Safe](references/safe.md)' \ + '' \ + '- ```markdown' \ + ' [Inactive](references/bad.md)' \ + ' ```' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Safe reference' \ + '' \ + '- ```text' \ + ' AgentOps is the operating loop every coding agent follows.' \ + ' ```' \ + >"$fixture/skills/example/references/safe.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'The knowledge flywheel label is retired.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "ordered-list fenced code is excluded from links and prose" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Safe](references/safe.md)' \ + '' \ + '1. ~~~markdown' \ + ' [Inactive](references/bad.md)' \ + ' ~~~' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Safe reference' \ + '' \ + '1. ~~~text' \ + ' The knowledge flywheel label is retired.' \ + ' ~~~' \ + >"$fixture/skills/example/references/safe.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "nested-list fenced code is excluded from links and prose" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Safe](references/safe.md)' \ + '' \ + '- Parent item' \ + ' 1. ~~~markdown' \ + ' [Inactive](references/bad.md)' \ + ' ~~~' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Safe reference' \ + '' \ + '- Parent item' \ + ' 1. ~~~text' \ + ' AgentOps is the operating loop every coding agent follows.' \ + ' ~~~' \ + >"$fixture/skills/example/references/safe.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'The knowledge flywheel label is retired.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "ordinary list prose remains active" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[List prose](references/list-prose.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# List prose' \ + '' \ + '- AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/list-prose.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"list-prose.md:3"* ]] + [[ "$output" == *"AgentOps is the operating loop"* ]] +} + +@test "multiline HTML comments are excluded from links and prose" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Safe](references/safe.md)' \ + '' \ + '' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Safe reference' \ + '' \ + '' \ + >"$fixture/skills/example/references/safe.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "raw HTML code containers are excluded from links and prose" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Safe](references/safe.md)' \ + '' \ + '
' \
+    '[Inactive](references/bad.md)' \
+    '
' \ + '' \ + '' \ + '' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Safe reference' \ + '' \ + '
' \
+    'AgentOps is the operating loop every coding agent follows.' \
+    '
' \ + '' \ + '' \ + '' \ + >"$fixture/skills/example/references/safe.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "backtick fence info rejects backticks while tilde info permits them" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Invalid backtick fence](references/invalid.md)' \ + '[Valid tilde fence](references/tilde.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Invalid backtick fence' \ + '' \ + '```bad`info' \ + 'AgentOps is the operating loop every coding agent follows.' \ + '```' \ + >"$fixture/skills/example/references/invalid.md" + printf '%s\n' \ + '# Valid tilde fence' \ + '' \ + '~~~bad`info' \ + 'The knowledge flywheel label is retired.' \ + '~~~' \ + >"$fixture/skills/example/references/tilde.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"invalid.md:3"* ]] + [[ "$output" == *"AgentOps is the operating loop"* ]] + [[ "$output" != *"tilde.md"* ]] +} + +@test "escaped brackets in reference labels still resolve direct links" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Policy][foo\]]' \ + '' \ + '[foo\]]: references/bad.md' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"bad.md:3"* ]] +} + +@test "reference definitions accept a destination on the next indented line" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Policy][foo]' \ + '' \ + '[foo]:' \ + ' references/bad.md' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'The knowledge flywheel label is retired.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"bad.md:3"* ]] +} + +@test "multiline reference labels normalize whitespace and resolve" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Policy][foo' \ + 'bar]' \ + '' \ + '[foo' \ + 'bar]: references/bad.md' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"bad.md:3"* ]] +} + +@test "nested brackets in reference link text do not hide the target" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[[Policy]][foo]' \ + '[Policy [nested]][foo]' \ + '' \ + '[foo]: references/bad.md' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'The knowledge flywheel label is retired.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"bad.md:3"* ]] +} + +@test "angle destinations reject an unescaped opening angle" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Safe]()' \ + '[Invalid]()' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Safe reference' \ + '' \ + 'The Agentic Coding Flywheel is an external factory.' \ + >"$fixture/skills/example/references/safe.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Hard breaks' \ + '' \ + 'AgentOps is the\' \ + 'operating loop every coding agent follows.' \ + '' \ + 'The knowledge\' \ + 'flywheel label is retired.' \ + >"$fixture/skills/example/references/hard-breaks.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"AgentOps is the operating loop"* ]] + [[ "$output" == *"knowledge flywheel"* ]] +} + +@test "emphasis delimiters do not split retired rendered prose" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Emphasis](references/emphasis.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Emphasis' \ + '' \ + 'AgentOps is the **operating loop** every coding agent follows.' \ + '' \ + 'AgentOps is the operating _loop_ every coding agent follows.' \ + '' \ + 'The knowledge *flywheel* label is retired.' \ + >"$fixture/skills/example/references/emphasis.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"AgentOps is the operating loop"* ]] + [[ "$output" == *"knowledge flywheel"* ]] +} + +@test "rendered prose normalization decodes inline Markdown and HTML" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '[Rendered prose](references/rendered.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Rendered prose' \ + '' \ + 'AgentOps is the operating\-loop every coding agent follows.' \ + '' \ + 'The knowledge flywheel label is retired.' \ + '' \ + 'AgentOps is the [operating loop](https://example.com) every coding agent follows.' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/rendered.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"operating-loop"* ]] + [[ "$output" == *"knowledge flywheel"* ]] + [[ "$output" == *"AgentOps is the operating loop"* ]] +} + +@test "inline skill links accept balanced and escaped destination parentheses" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Balanced](references/bad_(balanced).md)' \ + '- [Escaped](references/bad_\(escaped\).md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Balanced destination' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad_(balanced).md" + printf '%s\n' \ + '# Escaped destination' \ + '' \ + 'The knowledge flywheel label is retired.' \ + >"$fixture/skills/example/references/bad_(escaped).md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"bad_(balanced).md:3"* ]] + [[ "$output" == *"bad_(escaped).md:3"* ]] +} + +@test "local skill links decode percent-encoded filename characters once" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Dot](references/bad%2emd)' \ + '- [Space](references/bad%20identity.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Encoded dot' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad.md" + printf '%s\n' \ + '# Encoded space' \ + '' \ + 'The knowledge flywheel label is retired.' \ + >"$fixture/skills/example/references/bad identity.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"bad.md:3"* ]] + [[ "$output" == *"bad identity.md:3"* ]] +} + +@test "percent decoding cannot introduce separators NUL or traversal" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Encoded slash](references%2fbad.md)' \ + '- [Encoded backslash](references%5cbad.md)' \ + '- [Encoded NUL](references/bad.md%00)' \ + '- [Encoded traversal](references/%2e%2e/outside.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad.md" + printf '%s\n' \ + '# Outside references' \ + '' \ + 'The knowledge flywheel label is retired.' \ + >"$fixture/skills/example/outside.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "indented CommonMark code does not link or trigger prose identity" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + { + printf '%s\n' '# Example' '' '[Safe](references/safe.md)' '' + printf ' [Indented](references/bad.md)\n' + printf '\t[Tabbed](references/bad.md)\n' + } >"$fixture/skills/example/SKILL.md" + { + printf '%s\n' '# Safe reference' '' + printf ' AgentOps is the operating loop every coding agent follows.\n' + printf '\tThe knowledge flywheel label is retired.\n' + } >"$fixture/skills/example/references/safe.md" + printf '%s\n' \ + '# Bad identity' \ + '' \ + 'AgentOps is the operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/bad.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "linked prose scan catches forbidden Setext headings" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Setext](references/setext.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Reference' \ + 'Operating-loop use' \ + '==================' \ + >"$fixture/skills/example/references/setext.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"setext.md:2"* ]] + [[ "$output" == *"Operating-loop use"* ]] +} + +@test "indented continuation remains prose when it cannot interrupt a paragraph" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Continuation](references/continuation.md)' \ + >"$fixture/skills/example/SKILL.md" + { + printf '%s\n' '# Continuation' '' 'AgentOps is the' + printf ' operating loop every coding agent follows.\n' + } >"$fixture/skills/example/references/continuation.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"continuation.md:3"* ]] + [[ "$output" == *"AgentOps is the operating loop"* ]] +} + +@test "blockquoted indented CommonMark code is excluded from prose" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Quoted code](references/quoted-code.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Quoted code' \ + '' \ + '> AgentOps is the operating loop every coding agent follows.' \ + '> The knowledge flywheel label is retired.' \ + >"$fixture/skills/example/references/quoted-code.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 0 ] +} + +@test "linked prose scan catches forbidden blockquoted Setext headings" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Quoted Setext](references/quoted-setext.md)' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Reference' \ + '> Operating-loop use' \ + '> ==================' \ + >"$fixture/skills/example/references/quoted-setext.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"quoted-setext.md:2"* ]] + [[ "$output" == *"Operating-loop use"* ]] +} + +@test "linked prose scan catches obsolete identities across wrapped lines" { + fixture="$BATS_TEST_TMPDIR/repo" + mkdir -p "$fixture/skills/example/references" + printf '%s\n' \ + '# Example' \ + '' \ + '- [Identity](references/identity.md)' \ + '- [Flywheel][flywheel]' \ + '' \ + '[flywheel]: references/flywheel.md' \ + >"$fixture/skills/example/SKILL.md" + printf '%s\n' \ + '# Identity' \ + '' \ + 'AgentOps is the' \ + 'operating loop every coding agent follows.' \ + >"$fixture/skills/example/references/identity.md" + printf '%s\n' \ + '# Flywheel' \ + '' \ + 'The retired knowledge' \ + 'flywheel framing should not return.' \ + >"$fixture/skills/example/references/flywheel.md" + + run run_linked_reference_identity_check "$fixture" + [ "$status" -eq 1 ] + [[ "$output" == *"identity.md:3"* ]] + [[ "$output" == *"AgentOps is the operating loop"* ]] + [[ "$output" == *"flywheel.md:3"* ]] + [[ "$output" == *"knowledge flywheel"* ]] +} + @test "ao init scaffolds only declared evidence destinations" { initapp="$REPO_ROOT/cli/internal/initapp/initapp.go" grep -Fq '"intents", "sha256"' "$initapp" diff --git a/tests/scripts/anti-ceremony-discriminator.bats b/tests/scripts/anti-ceremony-discriminator.bats new file mode 100644 index 000000000..d70fae5d0 --- /dev/null +++ b/tests/scripts/anti-ceremony-discriminator.bats @@ -0,0 +1,45 @@ +#!/usr/bin/env bats + +setup() { + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + DISC="$REPO_ROOT/evals/skill-probes/anti-ceremony-creation-gate-v2/discriminator.sh" + TRANSCRIPT="$BATS_TEST_TMPDIR/transcript.txt" +} + +write_transcript() { + { + printf 'OpenAI Codex fixture\n' + printf 'user\n' + printf 'A: CREATE\nA: DROP\nB: CREATE\nB: DROP\n' + printf 'codex\n' + printf '%s\n' "$1" + printf 'tokens used\n1\n' + } > "$TRANSCRIPT" +} + +@test "anti-ceremony discriminator scores a complete response segment" { + write_transcript $'A: DROP\nB: CREATE' + + run "$DISC" "$TRANSCRIPT" + + [ "$status" -eq 0 ] + [[ "$output" == PRESENT:* ]] +} + +@test "anti-ceremony discriminator cannot borrow a missing decision from the prompt echo" { + write_transcript 'B: CREATE' + + run "$DISC" "$TRANSCRIPT" + + [ "$status" -eq 1 ] + [[ "$output" == *"missing one or both"* ]] +} + +@test "anti-ceremony discriminator degrades without a runtime response marker" { + printf 'user\nA: DROP\nB: CREATE\n' > "$TRANSCRIPT" + + run "$DISC" "$TRANSCRIPT" + + [ "$status" -eq 2 ] + [[ "$output" == *"no codex response segment"* ]] +} diff --git a/tests/scripts/check-skill-probe-coverage.bats b/tests/scripts/check-skill-probe-coverage.bats index 5ca42c476..218c83d16 100644 --- a/tests/scripts/check-skill-probe-coverage.bats +++ b/tests/scripts/check-skill-probe-coverage.bats @@ -1,31 +1,46 @@ #!/usr/bin/env bats -# -# Tests for scripts/check-skill-probe-coverage.sh — the advisory -# skill.probe-coverage gate (age-e508.1). -# -# The gate NAMES every product-/judgment-tier skill that lacks a behavioral -# probe RESULT in the MEASURED ledger of skills/SKILL-TIERS.md. It is -# advisory-first: default mode reports findings but exits 0 (warn); --strict -# flips to a hard fail (the same warn-then-fail flip discipline as the egwt -# gates). "Has a probe result" = a ledger row whose verdict is BEHAVIORAL or -# INERT; an UNMEASURED verdict or an absent row is NOT a result. -# -# The gate is fixture-driven via env overrides (SKILL_PROBE_SKILLS_DIR, -# SKILL_PROBE_TIERS_FILE) so a fixture skills tree + tiers file can be pointed at -# without copying the repo. + +bats_require_minimum_version 1.5.0 + +# Contract tests for the advisory skill.probe-coverage gate. A ledger label is +# never evidence by itself: a directional verdict counts only when its one +# scorecard pointer resolves to a verified v3 scorecard + self-contained fixture +# set whose response-only discriminator replay produces the same classification. setup() { REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" - export REPO_ROOT GATE="$REPO_ROOT/scripts/check-skill-probe-coverage.sh" + HARNESS="$REPO_ROOT/scripts/probe-skill.sh" + META_TOOL="$REPO_ROOT/scripts/lib/probe-fixture-metadata.py" + PREAMBLE="$REPO_ROOT/scripts/lib/preamble.sh" + DISPATCH_HELPER="$REPO_ROOT/scripts/lib/codex-exec.sh" FIX="$BATS_TEST_TMPDIR/repo" - mkdir -p "$FIX/skills" + PROBES="$FIX/evals/skill-probes" + mkdir -p "$FIX/skills" "$PROBES" "$FIX/docs/evals/scorecards" "$FIX/scripts/lib" + cp "$HARNESS" "$FIX/scripts/probe-skill.sh" + cp "$META_TOOL" "$FIX/scripts/lib/probe-fixture-metadata.py" + cp "$PREAMBLE" "$FIX/scripts/lib/preamble.sh" + cp "$DISPATCH_HELPER" "$FIX/scripts/lib/codex-exec.sh" + mkdir -p "$FIX/test-bin" + # Synthetic unit-only runtime identity: it exercises the positive verifier + # path but is neither persisted evidence nor a claim that a model was run. + cat > "$FIX/test-bin/codex" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "--version" ]]; then + printf 'codex-cli synthetic-gate-test\n' + exit 0 +fi +printf 'synthetic identity stub is not a live producer\n' >&2 +exit 70 +SH + chmod +x "$FIX/test-bin/codex" + export PATH="$FIX/test-bin:$PATH" export SKILL_PROBE_SKILLS_DIR="$FIX/skills" - export SKILL_PROBE_TIERS_FILE="$FIX/SKILL-TIERS.md" + export SKILL_PROBE_LEDGER_FILE="$PROBES/LEDGER.md" + export SKILL_PROBE_EVIDENCE_ROOT="$FIX" } -# make_skill — write a minimal SKILL.md carrying a metadata tier. make_skill() { local name="$1" tier="$2" mkdir -p "$SKILL_PROBE_SKILLS_DIR/$name" @@ -52,98 +67,463 @@ Use the canonical skill instead. EOF } -# write_ledger — write a SKILL-TIERS.md carrying a MEASURED probe -# ledger. Each arg is a table row body "skill | probe | date | verdict". write_ledger() { + local ledger_file="${SKILL_PROBE_LEDGER_FILE:-$SKILL_PROBE_TIERS_FILE}" { - echo "# Skill Tier Taxonomy" + echo "# Behavioral probe ledger" echo - echo "## Behavioral Probe Ledger (MEASURED)" + echo "## Behavioral Probe Ledger (MEASUREMENT STATUS)" echo - echo "| Skill | Probe ID | Date | Verdict |" - echo "|-------|----------|------|---------|" + echo "| Skill | Probe | Date | Verdict | Notes |" + echo "|---|---|---|---|---|" local row for row in "$@"; do echo "| $row |" done - } > "$SKILL_PROBE_TIERS_FILE" + } > "$ledger_file" } -@test "a product-tier skill absent from the ledger is NAMED and --strict FAILS" { +write_transcript() { + local directory="$1" name="$2" body="$3" + python3 - "$directory" "$name" "$body" <<'PY' +import json, pathlib, sys + +directory = pathlib.Path(sys.argv[1]) +name = sys.argv[2] +body = sys.argv[3] +arm, rep_text = name.rsplit("-", 1) +rep = int(rep_text) +contract = json.loads((directory / "capture-contract.json").read_text()) +prompt = contract["prompts"][0 if arm == "control" else 1] +position = next( + item["position"] + for item in contract["schedule"] + if item["arm"] == arm and item["rep"] == rep +) +events = [ + { + "type": "agentops.probe-input.v1", + "arm": arm, + "rep": rep, + "position": position, + "prompt": prompt, + }, + {"type": "thread.started", "thread_id": f"gate-{directory.name}-{name}"}, + {"type": "turn.started"}, + { + "type": "item.completed", + "item": {"id": f"item-{name}", "type": "agent_message", "text": body}, + }, + { + "type": "turn.completed", + "usage": {"input_tokens": 1, "cached_input_tokens": 0, "output_tokens": 1}, + }, +] +(directory / f"{name}.txt").write_text( + "".join(json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" for event in events) +) +PY +} + +# make_bound_result SKILL PROBE VERDICT [TREATMENT_SOURCE] [PRODUCER_OVERRIDE] +# Sets BOUND_SCORECARD_REL to a safe repo-relative v3 scorecard path. +make_bound_result() { + local skill="$1" probe="$2" verdict="$3" + local treatment_source="${4:-canonical-skill}" + local producer_override="${5:-}" + local probe_dir="$PROBES/$probe" + local fixture_name="fixtures-test" + local fixture_dir="$probe_dir/$fixture_name" + local control_body="ABSENT" treatment_body="ABSENT" rep + + if [ "$verdict" = "BEHAVIORAL" ]; then treatment_body="ACTION"; fi + mkdir -p "$fixture_dir" + cat > "$probe_dir/probe.json" < "$probe_dir/question.md" + printf 'PRELUDE\n' > "$probe_dir/treatment-prelude.md" + cat > "$probe_dir/discriminator.sh" <<'SH' +#!/usr/bin/env bash +if grep -q '^INFRA$' "$1"; then exit 2; fi +grep -q '^ACTION$' "$1" +SH + chmod +x "$probe_dir/discriminator.sh" + local -a snapshot_args=( + snapshot + --fixture-dir "$fixture_dir" + --probe-dir "$probe_dir" + --skills-dir "$SKILL_PROBE_SKILLS_DIR" + --probe "$probe" + --requested-model fixture-model + --requested-effort low + ) + if [[ -n "$producer_override" ]]; then + snapshot_args+=(--producer-override-bin "$producer_override") + fi + python3 "$META_TOOL" "${snapshot_args[@]}" >/dev/null + for rep in 1 2; do + write_transcript "$fixture_dir" "control-$rep" "$control_body" + write_transcript "$fixture_dir" "treatment-$rep" "$treatment_body" + done + + python3 "$META_TOOL" create \ + --fixture-dir "$fixture_dir" \ + --probe-dir "$probe_dir" \ + --skills-dir "$SKILL_PROBE_SKILLS_DIR" \ + --harness "$HARNESS" \ + --preamble "$PREAMBLE" \ + --dispatch-helper "$DISPATCH_HELPER" \ + --probe "$probe" \ + --reps 2 \ + --requested-model fixture-model \ + --requested-effort low >/dev/null + + BOUND_SCORECARD_REL="docs/evals/scorecards/$probe.json" + SKILL_PROBES_DIR="$PROBES" bash "$HARNESS" \ + --probe "$probe" \ + --replay \ + --fixtures "$fixture_name" \ + --output "$FIX/$BOUND_SCORECARD_REL" >/dev/null +} + +json_field() { + python3 -c ' +import json, sys +value = json.loads(sys.argv[1]) +for part in sys.argv[2].split("."): + value = value[part] +print(value) +' "$1" "$2" +} + +@test "a product-tier skill absent from the ledger is named and strict fails" { make_skill foo product - write_ledger # empty ledger + write_ledger + run bash "$GATE" --strict + [ "$status" -eq 1 ] [[ "$output" == *"foo"* ]] } -@test "default (advisory) mode NAMES the skill but exits 0 (warn-first)" { +@test "default mode stays advisory and names missing coverage" { make_skill foo product write_ledger + run bash "$GATE" + [ "$status" -eq 0 ] [[ "$output" == *"foo"* ]] [[ "$output" == *"WARN"* ]] } -@test "a judgment-tier skill with no probe is flagged too" { +@test "a judgment-tier skill is gated too" { make_skill val judgment write_ledger + run bash "$GATE" --strict + [ "$status" -eq 1 ] [[ "$output" == *"val"* ]] } -@test "a product-tier skill WITH a BEHAVIORAL ledger row is NOT flagged" { +@test "a hand-written BEHAVIORAL row without v3 evidence does not count" { make_skill foo product - write_ledger "foo | probe-foo | 2026-07-08 | BEHAVIORAL" + write_ledger "foo | probe-foo | 2026-08-16 | BEHAVIORAL | prose only" + run bash "$GATE" --strict - [ "$status" -eq 0 ] - [[ "$output" != *"foo lacks"* ]] + + [ "$status" -eq 1 ] + [[ "$output" == *"lacks exactly one scorecard"* ]] + [[ "$output" == *"foo"* ]] } -@test "an INERT ledger verdict counts as a measured result (not flagged)" { +@test "a valid bound BEHAVIORAL scorecard counts" { make_skill foo product - write_ledger "foo | probe-foo | 2026-07-08 | INERT" + make_bound_result foo probe-foo BEHAVIORAL + write_ledger "foo | probe-foo | 2026-08-16 | BEHAVIORAL | scorecard: \`$BOUND_SCORECARD_REL\`" + run bash "$GATE" --strict + + [ "$status" -eq 0 ] + [[ "$output" != *"foo"* ]] +} + +@test "a valid bound INERT scorecard counts" { + make_skill foo product + make_bound_result foo probe-foo INERT + write_ledger "foo | probe-foo | 2026-08-16 | INERT | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + [ "$status" -eq 0 ] } -@test "an UNMEASURED ledger verdict does NOT count — still flagged" { +@test "an explicit producer override is replayable but cannot qualify as coverage" { make_skill foo product - write_ledger "foo | probe-foo | 2026-07-08 | UNMEASURED" + make_bound_result foo probe-foo BEHAVIORAL canonical-skill "$FIX/test-bin/codex" + write_ledger "foo | probe-foo | 2026-08-16 | BEHAVIORAL | scorecard: \`$BOUND_SCORECARD_REL\`" + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"tier coverage requires non-overrideable native Codex runtime evidence"* ]] + [[ "$output" == *"foo"* ]] +} + +@test "bound injected-prelude evidence is replayable but does not count as skill coverage" { + make_skill foo product + make_bound_result foo probe-foo BEHAVIORAL injected-prelude + write_ledger "foo | probe-foo | 2026-08-16 | BEHAVIORAL | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"tier coverage requires treatment_source 'canonical-skill'"* ]] + [[ "$output" == *"foo"* ]] +} + +@test "LEGACY-UNVERIFIED and UNMEASURED rows remain excluded" { + make_skill foo product + write_ledger \ + "foo | probe-old | 2026-08-15 | LEGACY-UNVERIFIED | historical" \ + "foo | probe-null | 2026-08-16 | UNMEASURED | no usable arms" + + run bash "$GATE" --strict + [ "$status" -eq 1 ] [[ "$output" == *"foo"* ]] } -@test "execution-tier skills are exempt (not required to carry a probe)" { +@test "a fabricated scorecard classification is rejected by discriminator replay" { + make_skill foo product + make_bound_result foo probe-foo BEHAVIORAL + python3 - "$FIX/$BOUND_SCORECARD_REL" <<'PY' +import json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["verdict"] = "INERT" +value["treatment"] = {"present": 0, "usable": 2, "rate": 0.0} +for entry in value["per_rep"]: + entry["treatment"] = "ABSENT" +path.write_text(json.dumps(value)) +PY + write_ledger "foo | probe-foo | 2026-08-16 | INERT | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"do not match discriminator replay"* ]] +} + +@test "tampered transcript or manifest evidence does not count" { + make_skill foo product + make_bound_result foo probe-foo BEHAVIORAL + printf 'tamper\n' >> "$PROBES/probe-foo/fixtures-test/control-1.txt" + write_ledger "foo | probe-foo | 2026-08-16 | BEHAVIORAL | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"transcript digest mismatch"* ]] +} + +@test "canonical skill drift invalidates otherwise unchanged bound evidence" { + make_skill foo product + make_bound_result foo probe-foo BEHAVIORAL + printf '\nchanged after capture\n' >> "$SKILL_PROBE_SKILLS_DIR/foo/SKILL.md" + write_ledger "foo | probe-foo | 2026-08-16 | BEHAVIORAL | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"current canonical skill differs from the self-contained capture"* ]] +} + +@test "a scorecard with no fixture manifest does not count" { + make_skill foo product + make_bound_result foo probe-foo BEHAVIORAL + rm -f "$PROBES/probe-foo/fixtures-test/fixture-set.json" + write_ledger "foo | probe-foo | 2026-08-16 | BEHAVIORAL | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"verified replay requires immutable capture metadata"* ]] +} + +@test "scorecard and manifest binding mismatch does not count" { + make_skill foo product + make_bound_result foo probe-foo BEHAVIORAL + python3 - "$FIX/$BOUND_SCORECARD_REL" <<'PY' +import json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["fixture_set"]["binding_sha256"] = "sha256:" + "0" * 64 +path.write_text(json.dumps(value)) +PY + write_ledger "foo | probe-foo | 2026-08-16 | BEHAVIORAL | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"scorecard/manifest fixture binding mismatch"* ]] +} + +@test "scorecard producer mismatch does not count" { + make_skill foo product + make_bound_result foo probe-foo INERT + python3 - "$FIX/$BOUND_SCORECARD_REL" <<'PY' +import json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["producer"]["model"] = "relabeled-model" +path.write_text(json.dumps(value)) +PY + write_ledger "foo | probe-foo | 2026-08-16 | INERT | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"scorecard/manifest producer mismatch"* ]] +} + +@test "scorecard reps mismatch does not count" { + make_skill foo product + make_bound_result foo probe-foo INERT + python3 - "$FIX/$BOUND_SCORECARD_REL" <<'PY' +import json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["reps"] = 1 +value["schedule"] = [ + {"position": 1, "rep": 1, "arm": "control"}, + {"position": 2, "rep": 1, "arm": "treatment"}, +] +path.write_text(json.dumps(value)) +PY + write_ledger "foo | probe-foo | 2026-08-16 | INERT | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"scorecard/manifest reps mismatch"* ]] +} + +@test "scorecard and ledger skill/probe/verdict mismatches do not count" { + make_skill foo product + make_bound_result foo probe-foo BEHAVIORAL + write_ledger "foo | other-probe | 2026-08-16 | BEHAVIORAL | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"scorecard/ledger probe mismatch"* ]] + + write_ledger "foo | probe-foo | 2026-08-16 | INERT | scorecard: \`$BOUND_SCORECARD_REL\`" + run bash "$GATE" --strict + [ "$status" -eq 1 ] + [[ "$output" == *"scorecard/ledger verdict mismatch"* ]] +} + +@test "unsafe relative and absolute scorecard paths are rejected" { + make_skill foo product + write_ledger "foo | probe-foo | 2026-08-16 | BEHAVIORAL | scorecard: \`../outside.json\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"unsafe repository-relative path"* ]] + + write_ledger "foo | probe-foo | 2026-08-16 | BEHAVIORAL | scorecard: \`/tmp/outside.json\`" + run bash "$GATE" --strict + [ "$status" -eq 1 ] + [[ "$output" == *"unsafe repository-relative path"* ]] +} + +@test "a scorecard path that traverses a symlink is rejected" { + make_skill foo product + make_bound_result foo probe-foo INERT + mv "$FIX/docs/evals/scorecards" "$FIX/docs/evals/real-scorecards" + ln -s real-scorecards "$FIX/docs/evals/scorecards" + write_ledger "foo | probe-foo | 2026-08-16 | INERT | scorecard: \`$BOUND_SCORECARD_REL\`" + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"must not traverse a symlink"* ]] +} + +@test "a meta-tier v1 row is ignored without noise or denominator impact" { + make_skill foo product + make_skill operationalize meta + make_bound_result operationalize anti-ceremony INERT + python3 - "$PROBES/anti-ceremony/fixtures-test/fixture-set.json" <<'PY' +import hashlib, json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["schema"] = "agentops-skill-probe-fixture-set.v1" +value.pop("canonical_skill") +value.pop("treatment_source") +value["capture_evaluator"].pop("preamble") +value["capture_evaluator"].pop("dispatch_helper") +payload = {key: item for key, item in value.items() if key != "binding_sha256"} +canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() +value["binding_sha256"] = "sha256:" + hashlib.sha256(canonical).hexdigest() +path.write_text(json.dumps(value)) +PY + write_ledger "operationalize | anti-ceremony | 2026-08-16 | INERT | scorecard: \`$BOUND_SCORECARD_REL\`" + + run --separate-stderr bash "$GATE" --json + + [ "$status" -eq 0 ] + [ -z "$stderr" ] + [ "$(json_field "$output" gated_total)" = "1" ] + [ "$(json_field "$output" measured)" = "0" ] + [ "$(json_field "$output" unmeasured_count)" = "1" ] +} + +@test "execution-tier and redirect-only skills remain exempt" { make_skill bar execution - write_ledger - run bash "$GATE" --strict - [ "$status" -eq 0 ] - [[ "$output" != *"bar"* ]] -} - -@test "redirect-only skills are exempt and cannot abort the advisory scan" { - make_skill foo product make_redirect legacy-foo - write_ledger "foo | probe-foo | 2026-07-08 | BEHAVIORAL" + write_ledger + run bash "$GATE" --strict + [ "$status" -eq 0 ] - [[ "$output" != *"legacy-foo"* ]] } -@test "a missing ledger file degrades to advisory (product/judgment flagged, exit 0 default)" { +@test "a missing ledger remains advisory and reports gated skills" { make_skill foo product - rm -f "$SKILL_PROBE_TIERS_FILE" + rm -f "$SKILL_PROBE_LEDGER_FILE" + run bash "$GATE" + [ "$status" -eq 0 ] [[ "$output" == *"foo"* ]] } -@test "the real repo gate is advisory: exits 0 in default mode even with unmeasured skills" { - unset SKILL_PROBE_SKILLS_DIR SKILL_PROBE_TIERS_FILE - run bash "$GATE" - [ "$status" -eq 0 ] +@test "the compatibility SKILL_PROBE_TIERS_FILE seam still works" { + make_skill foo product + unset SKILL_PROBE_LEDGER_FILE + export SKILL_PROBE_TIERS_FILE="$FIX/compat-ledger.md" + write_ledger + + run bash "$GATE" --strict + + [ "$status" -eq 1 ] + [[ "$output" == *"foo"* ]] +} + +@test "the real repository remains advisory with exactly 0/12 current results" { + unset SKILL_PROBE_SKILLS_DIR SKILL_PROBE_LEDGER_FILE SKILL_PROBE_TIERS_FILE + unset SKILL_PROBE_EVIDENCE_ROOT SKILL_PROBE_METADATA_TOOL + + run --separate-stderr bash "$GATE" --json + + [ "$status" -eq 0 ] + [ "$(json_field "$output" gated_total)" = "12" ] + [ "$(json_field "$output" measured)" = "0" ] + [ "$(json_field "$output" unmeasured_count)" = "12" ] } diff --git a/tests/scripts/codex-exec-lib.bats b/tests/scripts/codex-exec-lib.bats index 9cfe7e481..6e7177096 100644 --- a/tests/scripts/codex-exec-lib.bats +++ b/tests/scripts/codex-exec-lib.bats @@ -63,6 +63,26 @@ FAKE chmod +x "$TMP/bin/codex" } +# A CLI-shaped stub that rejects a leading-hyphen prompt unless the caller +# inserted the standard end-of-options marker first. +stub_option_parser() { + cat > "$TMP/bin/codex" <<'FAKE' +#!/usr/bin/env bash +seen_terminator=0 +for arg in "$@"; do + if [ "$arg" = "--" ]; then seen_terminator=1; continue; fi + if [ "$seen_terminator" -eq 0 ] && [ "${arg#-}" != "$arg" ]; then + case "$arg" in exec|--skip-git-repo-check|--sandbox|-m|-C|-c) continue;; esac + fi +done +[ "$seen_terminator" -eq 1 ] || { echo 'missing option terminator' >&2; exit 2; } +[ "${!#}" = '--- canonical skill bytes' ] || { echo 'prompt mismatch' >&2; exit 3; } +printf 'accepted prompt\n' +printf 'tokens used: 1\n' +FAKE + chmod +x "$TMP/bin/codex" +} + # A stub codex that records how many times it was invoked and prints nothing. stub_flat_then_success() { cat > "$TMP/bin/codex" < "$TMP/bin/codex" <<'FAKE' +#!/usr/bin/env bash +printf '{"type":"turn.completed"}\n' +printf 'runtime warning\n' >&2 +FAKE + chmod +x "$TMP/bin/codex" + run bash -c ' + . "'"$LIB"'" + CODEX_EXEC_OUT_FILE="'"$TMP"'/stdout.jsonl" \ + CODEX_EXEC_STDERR_FILE="'"$TMP"'/stderr.log" \ + REVIEWER_MARKER="turn.completed" CODEX_EXEC_PROMPT_ARG="probe" \ + CODEX_EXEC_TIMEOUT=10 codex_exec_guarded + ' + [ "$status" -eq 0 ] + [ "$output" = "" ] + [ "$(cat "$TMP/stdout.jsonl")" = '{"type":"turn.completed"}' ] + [ "$(cat "$TMP/stderr.log")" = "runtime warning" ] +} + # --- (b) HANG -> STALL-TIMEOUT (exit 124) within budget ----------------------- @test "(b) a hung codex is killed and returns STALL-TIMEOUT (124) within budget" { [ "$HAVE_TIMEOUT" -eq 1 ] || skip "no timeout/gtimeout on PATH" @@ -113,6 +153,16 @@ FAKE [ "$status" -eq 125 ] } +@test "(c2) an arg prompt beginning with hyphens is protected by an option terminator" { + stub_option_parser + run bash -c ' + . "'"$LIB"'" + CODEX_EXEC_PROMPT_ARG="--- canonical skill bytes" CODEX_EXEC_TIMEOUT=10 codex_exec_guarded + ' + [ "$status" -eq 0 ] + [[ "$output" == *"accepted prompt"* ]] +} + # --- (d) MISSING codex -> MISSING (exit 2) ------------------------------------ @test "(d) a missing codex binary returns MISSING (2), a precondition not a result" { # No stub installed AND point CODEX_EXEC_BIN at a name that does not exist. @@ -182,8 +232,10 @@ FAKE @test "(i) codex_exec_producer_template emits the historical byte-identical defaults" { run bash -c '. "'"$LIB"'"; codex_exec_producer_template producer' [ "$status" -eq 0 ] + # shellcheck disable=SC2016 # Assert literal parameters in the emitted template. [ "$output" = 'timeout "$3" codex exec --skip-git-repo-check -C "$1" -s workspace-write "$2" >/dev/null 2>&1' ] run bash -c '. "'"$LIB"'"; codex_exec_producer_template membrane' [ "$status" -eq 0 ] + # shellcheck disable=SC2016 # Assert literal parameters in the emitted template. [ "$output" = 'codex exec --skip-git-repo-check "$1" 2>/dev/null' ] } diff --git a/tests/scripts/extract-release-notes.bats b/tests/scripts/extract-release-notes.bats new file mode 100644 index 000000000..5d9a35060 --- /dev/null +++ b/tests/scripts/extract-release-notes.bats @@ -0,0 +1,154 @@ +#!/usr/bin/env bats +# Regression tests for the Markdown body assembled by extract-release-notes.sh. + +setup() { + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + SANDBOX="$BATS_TEST_TMPDIR/repo" + mkdir -p "$SANDBOX/docs/releases" + + cat > "$SANDBOX/CHANGELOG.md" <<'EOF' +# Changelog + +## [9.9.9] - 2099-01-01 + +### Fixed + +- A changelog item whose source is + wrapped across multiple lines. + +A changelog paragraph is deliberately +hard-wrapped and should become one logical line. + +| Changelog area | Result | +|---|---| +| Release notes | fixed | + +## [9.9.8] - 2098-01-01 +EOF + + cat > "$SANDBOX/docs/releases/2099-01-01-v9.9.9-notes.md" <<'EOF' +## Highlights + +This paragraph is deliberately hard-wrapped +across several source lines so the published +release must reflow it. + +A prose line containing `left | right` is still +soft-wrapped prose rather than a table. + +## Upgrade Notes + +- This list item is deliberately wrapped + across source lines too. + +- Parent item has a soft-wrapped + continuation that should join. + - Nested item has a soft-wrapped + continuation that should join too. +- Sibling item remains distinct. + +1. Ordered item is deliberately wrapped + across source lines too. +2. Ordered sibling remains distinct. + + ### Indented heading +Paragraph after the indented heading is +soft-wrapped but remains a separate block. + +Product Area | Fixed +--- | ---: +Release Notes | 1 +EOF + + printf '\n%s \n' "An intentional hard break stays here." \ + >> "$SANDBOX/docs/releases/2099-01-01-v9.9.9-notes.md" + + cat >> "$SANDBOX/docs/releases/2099-01-01-v9.9.9-notes.md" <<'EOF' +This starts a new rendered line. + +A backslash hard break stays here.\ +This also starts a new rendered line. + + ~~~text + fenced code + keeps its lines + ~~~ + + indented code + keeps its lines too + +
+HTML content stays +on separate source lines. +
+ +> quoted first line +> quoted second line + +[release-ref]: + https://example.test/releases/9.9.9 + "Release details" + +[Full changelog](../CHANGELOG.md) +EOF +} + +assert_line() { + grep -Fqx -- "$1" "$SANDBOX/release-notes.md" +} + +@test "reflows prose and list continuations in curated notes and changelog" { + run bash -c "cd '$SANDBOX' && '$REPO_ROOT/scripts/extract-release-notes.sh' v9.9.9 v9.9.8" + [ "$status" -eq 0 ] + + assert_line "This paragraph is deliberately hard-wrapped across several source lines so the published release must reflow it." + assert_line "A prose line containing \`left | right\` is still soft-wrapped prose rather than a table." + assert_line "- This list item is deliberately wrapped across source lines too." + assert_line "- A changelog item whose source is wrapped across multiple lines." + assert_line "A changelog paragraph is deliberately hard-wrapped and should become one logical line." +} + +@test "retains structural Markdown lines and explicit hard breaks" { + run bash -c "cd '$SANDBOX' && '$REPO_ROOT/scripts/extract-release-notes.sh' v9.9.9 v9.9.8" + [ "$status" -eq 0 ] + + assert_line "- Parent item has a soft-wrapped continuation that should join." + assert_line " - Nested item has a soft-wrapped continuation that should join too." + assert_line "- Sibling item remains distinct." + assert_line "1. Ordered item is deliberately wrapped across source lines too." + assert_line "2. Ordered sibling remains distinct." + + assert_line " ### Indented heading" + assert_line "Paragraph after the indented heading is soft-wrapped but remains a separate block." + + assert_line "Product Area | Fixed" + assert_line "--- | ---:" + assert_line "Release Notes | 1" + assert_line "| Changelog area | Result |" + assert_line "|---|---|" + assert_line "| Release notes | fixed |" + + assert_line "An intentional hard break stays here. " + assert_line "This starts a new rendered line." + assert_line "A backslash hard break stays here.\\" + assert_line "This also starts a new rendered line." + + assert_line " ~~~text" + assert_line " fenced code" + assert_line " keeps its lines" + assert_line " ~~~" + assert_line " indented code" + assert_line " keeps its lines too" + + assert_line '
' + assert_line "HTML content stays" + assert_line "on separate source lines." + assert_line "
" + assert_line "> quoted first line" + assert_line "> quoted second line" + assert_line "[release-ref]:" + assert_line " https://example.test/releases/9.9.9" + assert_line ' "Release details"' + + assert_line "[Full changelog](https://github.com/boshu2/agentops/blob/main/docs/CHANGELOG.md)" +} diff --git a/tests/scripts/probe-skill.bats b/tests/scripts/probe-skill.bats new file mode 100644 index 000000000..7543b6c46 --- /dev/null +++ b/tests/scripts/probe-skill.bats @@ -0,0 +1,958 @@ +#!/usr/bin/env bats + +bats_require_minimum_version 1.5.0 + +setup() { + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + HARNESS="$REPO_ROOT/scripts/probe-skill.sh" + META_TOOL="$REPO_ROOT/scripts/lib/probe-fixture-metadata.py" + PREAMBLE="$REPO_ROOT/scripts/lib/preamble.sh" + DISPATCH_HELPER="$REPO_ROOT/scripts/lib/codex-exec.sh" + PROBES="$BATS_TEST_TMPDIR/probes" + SKILLS="$BATS_TEST_TMPDIR/skills" + PROBE_DIR="$PROBES/demo" + mkdir -p "$PROBE_DIR" "$SKILLS/demo-skill" + export SKILL_PROBES_DIR="$PROBES" + export SKILL_PROBE_SKILLS_DIR="$SKILLS" + + cat > "$PROBE_DIR/probe.json" <<'JSON' +{"id":"demo","skill":"demo-skill","reps":2,"discriminator":"discriminator.sh","treatment_source":"injected-prelude"} +JSON + cat > "$SKILLS/demo-skill/SKILL.md" <<'MD' +--- +name: demo-skill +description: Fixture skill. +--- +# Demo skill + +CANONICAL_ACTION +MD + printf 'QUESTION\n' > "$PROBE_DIR/question.md" + printf 'PRELUDE_ACTION\n' > "$PROBE_DIR/treatment-prelude.md" + cat > "$PROBE_DIR/discriminator.sh" <<'SH' +#!/usr/bin/env bash +if grep -q '^INFRA$' "$1"; then exit 2; fi +grep -q '^ACTION$' "$1" +SH + chmod +x "$PROBE_DIR/discriminator.sh" + + RUNTIME_STUB="$BATS_TEST_TMPDIR/codex-runtime-identity" + cat > "$RUNTIME_STUB" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "--version" ]]; then + printf 'codex-cli probe-test\n' + exit 0 +fi +printf 'runtime identity stub is not a live producer\n' >&2 +exit 70 +SH + chmod +x "$RUNTIME_STUB" +} + +write_transcript() { + local directory="$1" name="$2" model="$3" effort="$4" body="$5" + python3 - "$directory" "$name" "$body" <<'PY' +import json, pathlib, sys + +directory = pathlib.Path(sys.argv[1]) +name = sys.argv[2] +body = sys.argv[3] +arm, rep_text = name.rsplit("-", 1) +rep = int(rep_text) +contract = json.loads((directory / "capture-contract.json").read_text()) +prompt = contract["prompts"][0 if arm == "control" else 1] +position = next( + item["position"] + for item in contract["schedule"] + if item["arm"] == arm and item["rep"] == rep +) +events = [ + { + "type": "agentops.probe-input.v1", + "arm": arm, + "rep": rep, + "position": position, + "prompt": prompt, + }, + {"type": "thread.started", "thread_id": f"thread-{directory.name}-{name}"}, + {"type": "turn.started"}, + { + "type": "item.completed", + "item": {"id": f"item-{name}", "type": "agent_message", "text": body}, + }, + { + "type": "turn.completed", + "usage": {"input_tokens": 1, "cached_input_tokens": 0, "output_tokens": 1}, + }, +] +(directory / f"{name}.txt").write_text( + "".join(json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" for event in events) +) +PY +} + +write_legacy_transcript() { + local directory="$1" name="$2" model="$3" effort="$4" body="$5" + { + printf 'OpenAI Codex fixture\n' + printf '%s\n' '--------' + printf 'workdir: /fixture\n' + printf 'model: %s\n' "$model" + printf 'reasoning effort: %s\n' "$effort" + printf '%s\n' '--------' + printf 'user\nQUESTION\n' + printf 'codex\n' + printf '%s\n' "$body" + printf 'tokens used: 1\n' + } > "$directory/$name.txt" +} + +make_fixture_set() { + local name="$1" model="$2" effort="$3" control_body="$4" treatment_body="$5" + local directory="$PROBE_DIR/$name" rep + mkdir -p "$directory" + python3 "$META_TOOL" snapshot \ + --fixture-dir "$directory" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS" \ + --probe demo \ + --requested-model "$model" \ + --requested-effort "$effort" \ + --producer-override-bin "$RUNTIME_STUB" >/dev/null + for rep in 1 2; do + write_transcript "$directory" "control-$rep" "$model" "$effort" "$control_body" + write_transcript "$directory" "treatment-$rep" "$model" "$effort" "$treatment_body" + done + python3 "$META_TOOL" create \ + --fixture-dir "$directory" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS" \ + --harness "$HARNESS" \ + --preamble "$PREAMBLE" \ + --dispatch-helper "$DISPATCH_HELPER" \ + --probe demo \ + --reps 2 \ + --requested-model "$model" \ + --requested-effort "$effort" >/dev/null +} + +json_field() { + python3 -c ' +import json, sys +value = json.loads(sys.argv[1]) +for part in sys.argv[2].split("."): + value = value[int(part)] if isinstance(value, list) else value[part] +if value is None: + print("null") +elif isinstance(value, bool): + print("true" if value else "false") +else: + print(value) +' "$1" "$2" +} + +@test "git attributes preserve transcript bytes and normalize all hashed text inputs" { + run git -C "$REPO_ROOT" check-attr text whitespace -- \ + evals/skill-probes/example/fixtures/control-1.txt \ + evals/skill-probes/example/fixtures-xhigh-2026-08-04/treatment-2.txt + + [ "$status" -eq 0 ] + [ "$(printf '%s\n' "$output" | grep -c ': text: unset$')" -eq 2 ] + [ "$(printf '%s\n' "$output" | grep -c ': whitespace: unset$')" -eq 2 ] + + run git -C "$REPO_ROOT" check-attr text eol -- \ + evals/skill-probes/example/probe.json \ + evals/skill-probes/example/question.md \ + evals/skill-probes/example/treatment-prelude.md \ + evals/skill-probes/example/fixtures-v3/capture-contract.json \ + evals/skill-probes/example/discriminator.sh \ + skills/example/SKILL.md \ + scripts/probe-skill.sh \ + scripts/lib/preamble.sh \ + scripts/lib/codex-exec.sh \ + scripts/lib/probe-fixture-metadata.py + + [ "$status" -eq 0 ] + [ "$(printf '%s\n' "$output" | grep -c ': text: set$')" -eq 10 ] + [ "$(printf '%s\n' "$output" | grep -c ': eol: lf$')" -eq 10 ] +} + +@test "replay takes producer provenance and binding from verified fixture metadata" { + make_fixture_set fixtures observed-model low ABSENT ACTION + + run bash "$HARNESS" --probe demo --replay + [ "$status" -eq 0 ] + [ "$(json_field "$output" schema)" = "agentops-skill-probe.v3" ] + [ "$(json_field "$output" producer.model)" = "observed-model" ] + [ "$(json_field "$output" producer.effort)" = "low" ] + [ "$(json_field "$output" producer.identity.source)" = "test-override" ] + [ "$(json_field "$output" producer.identity.coverage_eligible)" = "false" ] + [ "$(json_field "$output" producer.threads.0.path)" = "control-1.txt" ] + [ "$(json_field "$output" fixture_set.name)" = "fixtures" ] + [[ "$(json_field "$output" fixture_set.binding_sha256)" == sha256:* ]] + [ "$(json_field "$output" evaluator_matches_capture)" = "true" ] + [ "$(json_field "$output" treatment_source)" = "injected-prelude" ] + [[ "$(json_field "$output" honesty)" == *"NOT full-skill activation"* ]] + [ "$(json_field "$output" capture_evaluator.dispatch_helper.path)" = "scripts/lib/codex-exec.sh" ] + [[ "$(json_field "$output" capture_evaluator.dispatch_helper.sha256)" == sha256:* ]] + [ "$(json_field "$output" capture_evaluator.preamble.path)" = "scripts/lib/preamble.sh" ] + [[ "$(json_field "$output" capture_evaluator.preamble.sha256)" == sha256:* ]] + [ "$(json_field "$output" verdict)" = "BEHAVIORAL" ] + [ "$(json_field "$(cat "$PROBE_DIR/fixtures/fixture-set.json")" schema)" = "agentops-skill-probe-fixture-set.v3" ] + [ "$(json_field "$(cat "$PROBE_DIR/fixtures/fixture-set.json")" canonical_skill.path)" = "skills/demo-skill/SKILL.md" ] +} + +@test "v3 response-only scoring supports the repository transcript-marker discriminator" { + cp "$REPO_ROOT/evals/skill-probes/anti-ceremony-creation-gate-v2/discriminator.sh" \ + "$PROBE_DIR/discriminator.sh" + make_fixture_set fixtures observed-model low \ + $'A: CREATE\nB: CREATE' $'A: DROP\nB: CREATE' + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 0 ] + [ "$(json_field "$output" control.usable)" -eq 2 ] + [ "$(json_field "$output" treatment.usable)" -eq 2 ] + [ "$(json_field "$output" verdict)" = "BEHAVIORAL" ] +} + +@test "structured response extraction ignores transcript-like delimiter collisions" { + make_fixture_set fixtures observed-model low ABSENT $'codex\ntokens used: forged\nACTION' + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 0 ] + [ "$(json_field "$output" treatment.present)" -eq 2 ] + [ "$(json_field "$output" treatment.usable)" -eq 2 ] + [ "$(json_field "$output" verdict)" = "BEHAVIORAL" ] +} + +@test "replay refuses legacy fixture bytes without immutable capture metadata" { + mkdir -p "$PROBE_DIR/fixtures" + write_legacy_transcript "$PROBE_DIR/fixtures" control-1 old-model low ABSENT + write_legacy_transcript "$PROBE_DIR/fixtures" treatment-1 old-model low ACTION + write_legacy_transcript "$PROBE_DIR/fixtures" control-2 old-model low ABSENT + write_legacy_transcript "$PROBE_DIR/fixtures" treatment-2 old-model low ACTION + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 2 ] + [[ "$output" == *"verified replay requires immutable capture metadata"* ]] + [[ "$output" == *"replay refused"* ]] +} + +@test "v1 bound fixtures replay as injected-prelude without a treatment_source declaration" { + make_fixture_set fixtures observed-model low ABSENT ACTION + write_legacy_transcript "$PROBE_DIR/fixtures" control-1 observed-model low ABSENT + write_legacy_transcript "$PROBE_DIR/fixtures" treatment-1 observed-model low ACTION + write_legacy_transcript "$PROBE_DIR/fixtures" control-2 observed-model low ABSENT + write_legacy_transcript "$PROBE_DIR/fixtures" treatment-2 observed-model low ACTION + python3 - "$PROBE_DIR/probe.json" "$PROBE_DIR/fixtures/fixture-set.json" <<'PY' +import hashlib, json, pathlib, sys + +probe_path = pathlib.Path(sys.argv[1]) +manifest_path = pathlib.Path(sys.argv[2]) +probe = json.loads(probe_path.read_text()) +probe.pop("treatment_source") +probe_path.write_text(json.dumps(probe) + "\n") + +manifest = json.loads(manifest_path.read_text()) +manifest["schema"] = "agentops-skill-probe-fixture-set.v1" +manifest["producer"] = { + "adapter": "codex", + "model": "observed-model", + "effort": "low", +} +manifest["evaluation_inputs"] = [ + {"path": record["path"], "sha256": record["sha256"]} + for record in manifest.pop("capture_inputs") +] +manifest.pop("capture_contract") +manifest.pop("canonical_skill") +manifest.pop("treatment_source") +manifest.pop("prompts") +manifest.pop("schedule") +manifest.pop("scoring") +manifest["capture_evaluator"].pop("preamble") +manifest["capture_evaluator"].pop("dispatch_helper") +for record in manifest["transcripts"]: + transcript_path = manifest_path.parent / record["path"] + record["sha256"] = "sha256:" + hashlib.sha256(transcript_path.read_bytes()).hexdigest() +for record in manifest["evaluation_inputs"]: + if record["path"] == "probe.json": + record["sha256"] = "sha256:" + hashlib.sha256(probe_path.read_bytes()).hexdigest() +payload = {key: value for key, value in manifest.items() if key != "binding_sha256"} +canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() +manifest["binding_sha256"] = "sha256:" + hashlib.sha256(canonical).hexdigest() +manifest_path.write_text(json.dumps(manifest) + "\n") +manifest_path.with_name("capture-contract.json").unlink() +PY + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 0 ] + [ "$(json_field "$output" treatment_source)" = "injected-prelude" ] + [[ "$(json_field "$output" honesty)" == *"NOT full-skill activation"* ]] + [ "$(json_field "$output" verdict)" = "BEHAVIORAL" ] +} + +@test "replay rejects transcript tampering before discrimination" { + make_fixture_set fixtures observed-model low ABSENT ACTION + printf 'ACTION\n' >> "$PROBE_DIR/fixtures/control-1.txt" + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 2 ] + [[ "$output" == *"transcript digest mismatch for control-1.txt"* ]] +} + +@test "replay rejects producer-config tampering through the fixture-set binding" { + make_fixture_set fixtures observed-model low ABSENT ACTION + python3 - "$PROBE_DIR/fixtures/fixture-set.json" <<'PY' +import json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["requested_producer"]["model"] = "relabeled-model" +path.write_text(json.dumps(value)) +PY + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 2 ] + [[ "$output" == *"requested_producer disagrees with bound producer request"* ]] +} + +@test "fixture creation rejects prompt-event tampering before manifest binding" { + local directory="$PROBE_DIR/fixtures" + local rep + mkdir -p "$directory" + python3 "$META_TOOL" snapshot \ + --fixture-dir "$directory" --probe-dir "$PROBE_DIR" --skills-dir "$SKILLS" \ + --probe demo --requested-model observed-model --requested-effort low \ + --producer-override-bin "$RUNTIME_STUB" >/dev/null + for rep in 1 2; do + write_transcript "$directory" "control-$rep" observed-model low ABSENT + write_transcript "$directory" "treatment-$rep" observed-model low ACTION + done + python3 - "$directory/control-1.txt" <<'PY' +import base64, json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +events = [json.loads(line) for line in path.read_text().splitlines()] +events[0]["prompt"]["content_base64"] = base64.b64encode(b"TAMPERED\n").decode() +path.write_text("".join(json.dumps(event) + "\n" for event in events)) +PY + + run python3 "$META_TOOL" create \ + --fixture-dir "$directory" --probe-dir "$PROBE_DIR" --skills-dir "$SKILLS" \ + --harness "$HARNESS" --preamble "$PREAMBLE" \ + --dispatch-helper "$DISPATCH_HELPER" --probe demo --reps 2 \ + --requested-model observed-model --requested-effort low + + [ "$status" -eq 2 ] + [[ "$output" == *"probe input event does not match bound control-1 prompt"* ]] + [ ! -e "$directory/fixture-set.json" ] +} + +@test "fixture creation refuses retroactive capture-contract creation" { + local directory="$PROBE_DIR/fixtures" + mkdir -p "$directory" + write_legacy_transcript "$directory" control-1 observed-model low ABSENT + write_legacy_transcript "$directory" treatment-1 observed-model low ACTION + write_legacy_transcript "$directory" control-2 observed-model low ABSENT + write_legacy_transcript "$directory" treatment-2 observed-model low ACTION + + run python3 "$META_TOOL" create \ + --fixture-dir "$directory" --probe-dir "$PROBE_DIR" --skills-dir "$SKILLS" \ + --harness "$HARNESS" --preamble "$PREAMBLE" \ + --dispatch-helper "$DISPATCH_HELPER" --probe demo --reps 2 \ + --requested-model observed-model --requested-effort low + + [ "$status" -eq 2 ] + [[ "$output" == *"requires a pre-existing capture contract written before execution"* ]] + [ ! -e "$directory/capture-contract.json" ] + [ ! -e "$directory/fixture-set.json" ] +} + +@test "v3 replay scores captured discriminator bytes after current probe drift" { + make_fixture_set fixtures observed-model low ABSENT ACTION + printf '# changed scoring input\n' >> "$PROBE_DIR/discriminator.sh" + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 0 ] + [ "$(json_field "$output" verdict)" = "BEHAVIORAL" ] +} + +@test "v3 replay remains self-contained after current canonical skill drift" { + make_fixture_set fixtures observed-model low ABSENT ACTION + printf '\nchanged canonical guidance\n' >> "$SKILLS/demo-skill/SKILL.md" + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 0 ] + [ "$(json_field "$output" verdict)" = "BEHAVIORAL" ] +} + +@test "capture rejects a canonical skill whose declared identity mismatches probe.json" { + sed -i.bak 's/name: demo-skill/name: unrelated-skill/' "$SKILLS/demo-skill/SKILL.md" + rm -f "$SKILLS/demo-skill/SKILL.md.bak" + mkdir -p "$PROBE_DIR/fixtures" + + run python3 "$META_TOOL" snapshot \ + --fixture-dir "$PROBE_DIR/fixtures" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS" \ + --probe demo \ + --requested-model observed-model \ + --requested-effort low \ + --producer-override-bin "$RUNTIME_STUB" + + [ "$status" -eq 2 ] + [[ "$output" == *"canonical skill identity mismatch"* ]] + [ ! -e "$PROBE_DIR/fixtures/capture-contract.json" ] +} + +@test "replay flags are constraints and mismatches fail rather than relabel fixtures" { + make_fixture_set fixtures observed-model low ABSENT ACTION + + run bash "$HARNESS" --probe demo --replay --model another-model + [ "$status" -eq 2 ] + [[ "$output" == *"does not match bound fixture producer request"* ]] + + run bash "$HARNESS" --probe demo --replay --effort xhigh + [ "$status" -eq 2 ] + [[ "$output" == *"does not match bound fixture producer request"* ]] + + run env PROBE_MODEL=ambient-relabel SKILL_PROBES_DIR="$PROBES" \ + bash "$HARNESS" --probe demo --replay + [ "$status" -eq 2 ] + [[ "$output" == *"does not match bound fixture producer request"* ]] +} + +@test "a zero-usable control arm cannot produce a false BEHAVIORAL verdict" { + make_fixture_set fixtures observed-model low INFRA ACTION + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 0 ] + [ "$(json_field "$output" control.usable)" = "0" ] + [ "$(json_field "$output" treatment.usable)" = "2" ] + [ "$(json_field "$output" verdict)" = "UNMEASURED" ] +} + +@test "both zero-usable arms yield UNMEASURED with null rates" { + make_fixture_set fixtures observed-model low INFRA INFRA + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 0 ] + [ "$(json_field "$output" control.rate)" = "null" ] + [ "$(json_field "$output" treatment.rate)" = "null" ] + [ "$(json_field "$output" verdict)" = "UNMEASURED" ] +} + +@test "named alternate fixture sets are independently selectable and verified" { + make_fixture_set fixtures default-model low ABSENT ACTION + make_fixture_set fixtures-xhigh-2026-08-04 alternate-model xhigh ABSENT ABSENT + + run bash "$HARNESS" --probe demo --replay \ + --fixtures fixtures-xhigh-2026-08-04 \ + --model alternate-model \ + --effort xhigh + + [ "$status" -eq 0 ] + [ "$(json_field "$output" fixture_set.name)" = "fixtures-xhigh-2026-08-04" ] + [ "$(json_field "$output" producer.model)" = "alternate-model" ] + [ "$(json_field "$output" producer.effort)" = "xhigh" ] + [ "$(json_field "$output" verdict)" = "INERT" ] +} + +@test "producer strings are JSON serialized without shell interpolation" { + make_fixture_set fixtures 'gpt-"quoted"' low ABSENT ACTION + + run bash "$HARNESS" --probe demo --replay --model 'gpt-"quoted"' + + [ "$status" -eq 0 ] + [ "$(json_field "$output" producer.model)" = 'gpt-"quoted"' ] +} + +@test "replay discloses when the current evaluator differs from capture" { + make_fixture_set fixtures observed-model low ABSENT ACTION + python3 - "$PROBE_DIR/fixtures/fixture-set.json" <<'PY' +import hashlib, json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["capture_evaluator"]["harness"]["sha256"] = "sha256:" + "0" * 64 +payload = {key: item for key, item in value.items() if key != "binding_sha256"} +canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() +value["binding_sha256"] = "sha256:" + hashlib.sha256(canonical).hexdigest() +path.write_text(json.dumps(value)) +PY + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 0 ] + [ "$(json_field "$output" evaluator_matches_capture)" = "false" ] + [ "$(json_field "$output" capture_evaluator.harness.sha256)" = "sha256:0000000000000000000000000000000000000000000000000000000000000000" ] + [[ "$(json_field "$output" evaluator.harness.sha256)" == sha256:* ]] +} + +@test "v3 replay requires the dispatch helper in capture evaluator identity" { + make_fixture_set fixtures observed-model low ABSENT ACTION + python3 - "$PROBE_DIR/fixtures/fixture-set.json" <<'PY' +import hashlib, json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["capture_evaluator"].pop("dispatch_helper") +payload = {key: item for key, item in value.items() if key != "binding_sha256"} +canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() +value["binding_sha256"] = "sha256:" + hashlib.sha256(canonical).hexdigest() +path.write_text(json.dumps(value)) +PY + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 2 ] + [[ "$output" == *"capture_evaluator must contain exactly"* ]] + [[ "$output" == *"dispatch_helper"* ]] +} + +@test "v3 replay requires the sourced preamble in capture evaluator identity" { + make_fixture_set fixtures observed-model low ABSENT ACTION + python3 - "$PROBE_DIR/fixtures/fixture-set.json" <<'PY' +import hashlib, json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["capture_evaluator"].pop("preamble") +payload = {key: item for key, item in value.items() if key != "binding_sha256"} +canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() +value["binding_sha256"] = "sha256:" + hashlib.sha256(canonical).hexdigest() +path.write_text(json.dumps(value)) +PY + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 2 ] + [[ "$output" == *"capture_evaluator must contain exactly"* ]] + [[ "$output" == *"preamble"* ]] +} + +@test "replay discloses dispatch helper drift from a bound v3 capture" { + make_fixture_set fixtures observed-model low ABSENT ACTION + python3 - "$PROBE_DIR/fixtures/fixture-set.json" <<'PY' +import hashlib, json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["capture_evaluator"]["dispatch_helper"]["sha256"] = "sha256:" + "0" * 64 +payload = {key: item for key, item in value.items() if key != "binding_sha256"} +canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() +value["binding_sha256"] = "sha256:" + hashlib.sha256(canonical).hexdigest() +path.write_text(json.dumps(value)) +PY + + run bash "$HARNESS" --probe demo --replay + + [ "$status" -eq 0 ] + [ "$(json_field "$output" evaluator_matches_capture)" = "false" ] + [ "$(json_field "$output" capture_evaluator.dispatch_helper.sha256)" = "sha256:0000000000000000000000000000000000000000000000000000000000000000" ] + [[ "$(json_field "$output" evaluator.dispatch_helper.sha256)" == sha256:* ]] +} + +@test "live capture refuses to overwrite an existing immutable fixture set" { + make_fixture_set fixtures stale-model low ABSENT ACTION + cp "$PROBE_DIR/fixtures/fixture-set.json" "$BATS_TEST_TMPDIR/manifest.before" + cp "$PROBE_DIR/fixtures/treatment-1.txt" "$BATS_TEST_TMPDIR/treatment.before" + + run --separate-stderr env CODEX_EXEC_BIN=definitely-missing-probe-producer \ + SKILL_PROBES_DIR="$PROBES" \ + bash "$HARNESS" --probe demo --live --reps 2 + + [ "$status" -eq 2 ] + [ -z "$output" ] + [[ "$stderr" == *"refusing to overwrite immutable fixture set"* ]] + [[ "$stderr" == *"choose a new --fixtures name"* ]] + cmp "$BATS_TEST_TMPDIR/manifest.before" "$PROBE_DIR/fixtures/fixture-set.json" + cmp "$BATS_TEST_TMPDIR/treatment.before" "$PROBE_DIR/fixtures/treatment-1.txt" +} + +@test "publish refuses a fixture destination that appears during dispatch" { + local producer="$BATS_TEST_TMPDIR/codex-publish-race" + local count="$BATS_TEST_TMPDIR/publish-race-count" + local target="$PROBE_DIR/fixtures-raced" + cat > "$producer" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "--version" ]]; then printf 'codex-cli race-stub\n'; exit 0; fi +n=$(cat "$PROBE_STUB_COUNT" 2>/dev/null || printf '0') +n=$((n + 1)) +printf '%s\n' "$n" > "$PROBE_STUB_COUNT" +if [[ "$n" -eq 4 ]]; then + mkdir "$PROBE_RACE_TARGET" + printf 'external-writer\n' > "$PROBE_RACE_TARGET/sentinel" +fi +prompt="$(cat)" +if [[ "$prompt" == *PRELUDE_ACTION* ]]; then response=ACTION; else response=ABSENT; fi +printf '{"type":"thread.started","thread_id":"race-%s"}\n' "$n" +printf '{"type":"turn.started"}\n' +printf '{"type":"item.completed","item":{"id":"item-%s","type":"agent_message","text":"%s"}}\n' "$n" "$response" +printf '{"type":"turn.completed","usage":{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}\n' +SH + chmod +x "$producer" + + run --separate-stderr env CODEX_EXEC_BIN="$producer" \ + PROBE_STUB_COUNT="$count" \ + PROBE_RACE_TARGET="$target" \ + bash "$HARNESS" --probe demo --live --reps 2 --fixtures fixtures-raced + + [ "$status" -eq 1 ] + [ "$(cat "$target/sentinel")" = "external-writer" ] + [ ! -e "$target/fixture-set.json" ] + [ "$(find "$target" -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ')" -eq 1 ] + [[ "$stderr" == *"refusing to replace existing immutable fixture set"* ]] + [[ "$stderr" == *"failed to publish fixture set"* ]] +} + +@test "publisher rejects an externally mutated hidden-stage transcript without exposing target" { + make_fixture_set fixtures observed-model low ABSENT ACTION + local stage="$PROBE_DIR/.fixtures-publish-stage" + local target="$PROBE_DIR/fixtures-publish-mutated" + mv "$PROBE_DIR/fixtures" "$stage" + + printf 'MUTATED_BEFORE_PUBLISH\n' >> "$stage/treatment-1.txt" + + run python3 "$META_TOOL" publish \ + --stage-dir "$stage" \ + --target-dir "$target" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS" \ + --probe demo + local publish_status="$status" + [ "$publish_status" -eq 2 ] + [[ "$output" == *"transcript digest mismatch"* ]] + [ ! -e "$target" ] + [ -f "$stage/fixture-set.json" ] +} + +@test "publisher never exposes or deletes an externally replaced hidden-stage transcript" { + make_fixture_set fixtures observed-model low ABSENT ACTION + local stage="$PROBE_DIR/.fixtures-replaced-transcript-stage" + local target="$PROBE_DIR/fixtures-replaced-transcript" + local replacement="$BATS_TEST_TMPDIR/replacement-transcript" + mv "$PROBE_DIR/fixtures" "$stage" + printf 'replacement-owner\n' > "$replacement" + + mv "$replacement" "$stage/control-1.txt" + + run python3 "$META_TOOL" publish \ + --stage-dir "$stage" \ + --target-dir "$target" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS" \ + --probe demo + local publish_status="$status" + [ "$publish_status" -eq 2 ] + [[ "$output" == *"transcript digest mismatch"* ]] + [ ! -e "$target" ] + [ "$(cat "$stage/control-1.txt")" = "replacement-owner" ] + [ -f "$stage/fixture-set.json" ] +} + +@test "atomic publisher preserves an externally owned destination" { + make_fixture_set fixtures observed-model low ABSENT ACTION + local stage="$PROBE_DIR/.fixtures-raced-stage" + local target="$PROBE_DIR/fixtures-raced-at-rename" + mv "$PROBE_DIR/fixtures" "$stage" + + mkdir "$target" + printf 'external-writer\n' > "$target/sentinel" + + run python3 "$META_TOOL" publish \ + --stage-dir "$stage" \ + --target-dir "$target" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS" \ + --probe demo + local publish_status="$status" + [ "$publish_status" -eq 2 ] + [[ "$output" == *"refusing to replace existing immutable fixture set"* ]] + [ "$(cat "$target/sentinel")" = "external-writer" ] + [ ! -e "$target/fixture-set.json" ] + [ "$(find "$target" -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ')" -eq 1 ] + [ -f "$stage/fixture-set.json" ] +} + +@test "publisher rejects a bound evaluator hash that does not match the repository file" { + make_fixture_set fixtures observed-model low ABSENT ACTION + local stage="$PROBE_DIR/.fixtures-bad-evaluator-stage" + local target="$PROBE_DIR/fixtures-bad-evaluator" + mv "$PROBE_DIR/fixtures" "$stage" + python3 - "$stage/fixture-set.json" <<'PY' +import hashlib +import json +import sys + +path = sys.argv[1] +manifest = json.load(open(path, encoding="utf-8")) +digest = manifest["capture_evaluator"]["harness"]["sha256"] +manifest["capture_evaluator"]["harness"]["sha256"] = ( + digest[:-1] + ("0" if digest[-1] != "0" else "1") +) +payload = dict(manifest) +payload.pop("binding_sha256") +manifest["binding_sha256"] = "sha256:" + hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() +).hexdigest() +with open(path, "w", encoding="utf-8") as handle: + json.dump(manifest, handle, sort_keys=True, separators=(",", ":")) + handle.write("\n") +PY + + run python3 "$META_TOOL" publish \ + --stage-dir "$stage" \ + --target-dir "$target" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS" \ + --probe demo + [ "$status" -eq 2 ] + [[ "$output" == *"capture evaluator hashes do not match the exact repo-local evaluator files"* ]] + [ ! -e "$target" ] + [ -f "$stage/fixture-set.json" ] +} + +@test "publisher rejects current probe inputs that differ from the capture" { + make_fixture_set fixtures observed-model low ABSENT ACTION + local stage="$PROBE_DIR/.fixtures-input-race-stage" + local target="$PROBE_DIR/fixtures-publish-input-mutated" + mv "$PROBE_DIR/fixtures" "$stage" + + printf 'MUTATED_BEFORE_PUBLISH\n' >> "$PROBE_DIR/question.md" + + run python3 "$META_TOOL" publish \ + --stage-dir "$stage" \ + --target-dir "$target" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS" \ + --probe demo + local publish_status="$status" + [ "$publish_status" -eq 2 ] + [[ "$output" == *"current probe inputs differ from the self-contained capture"* ]] + [ ! -e "$target" ] + [ -f "$stage/fixture-set.json" ] +} + +@test "failed live dispatch does not publish or score any prior transcript" { + local producer="$BATS_TEST_TMPDIR/codex-failing" + cat > "$producer" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "--version" ]]; then printf 'codex-cli failing-stub\n'; exit 0; fi +cat >/dev/null +printf 'producer failure\n' >&2 +exit 77 +SH + chmod +x "$producer" + + run --separate-stderr env CODEX_EXEC_BIN="$producer" \ + SKILL_PROBES_DIR="$PROBES" \ + bash "$HARNESS" --probe demo --live --reps 2 --fixtures fixtures-failed + + [ "$status" -eq 0 ] + [ "$(json_field "$output" verdict)" = "UNMEASURED" ] + [ "$(json_field "$output" control.usable)" = "0" ] + [ "$(json_field "$output" treatment.usable)" = "0" ] + [ "$(json_field "$output" fixture_set.binding_sha256)" = "null" ] + [ ! -e "$PROBE_DIR/fixtures-failed" ] + [[ "$stderr" == *"producer failure"* ]] + [[ "$stderr" == *"incomplete live run; fixture set not published"* ]] +} + +@test "a partial live run stays UNMEASURED even when each arm has one usable rep" { + local producer="$BATS_TEST_TMPDIR/codex-partial" + local count="$BATS_TEST_TMPDIR/partial-count" + cat > "$producer" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "--version" ]]; then printf 'codex-cli partial-stub\n'; exit 0; fi +n=$(cat "$PROBE_STUB_COUNT" 2>/dev/null || printf '0') +n=$((n + 1)) +printf '%s\n' "$n" > "$PROBE_STUB_COUNT" +if [[ "$n" -gt 2 ]]; then printf 'producer failure\n' >&2; exit 77; fi +prompt="$(cat)" +if [[ "$prompt" == *PRELUDE_ACTION* ]]; then response=ACTION; else response=ABSENT; fi +printf '{"type":"thread.started","thread_id":"partial-%s"}\n' "$n" +printf '{"type":"turn.started"}\n' +printf '{"type":"item.completed","item":{"id":"item-%s","type":"agent_message","text":"%s"}}\n' "$n" "$response" +printf '{"type":"turn.completed","usage":{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}\n' +SH + chmod +x "$producer" + + run --separate-stderr env CODEX_EXEC_BIN="$producer" PROBE_STUB_COUNT="$count" \ + SKILL_PROBES_DIR="$PROBES" \ + bash "$HARNESS" --probe demo --live --reps 2 --fixtures fixtures-partial + + [ "$status" -eq 0 ] + [ "$(json_field "$output" control.usable)" = "1" ] + [ "$(json_field "$output" treatment.usable)" = "1" ] + [ "$(json_field "$output" verdict)" = "UNMEASURED" ] + [ ! -e "$PROBE_DIR/fixtures-partial" ] +} + +@test "multi-rep live capture fails closed when canonical input mutates during dispatch" { + python3 - "$PROBE_DIR/probe.json" <<'PY' +import json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["treatment_source"] = "canonical-skill" +path.write_text(json.dumps(value)) +PY + local producer="$BATS_TEST_TMPDIR/codex-mutating" + local count="$BATS_TEST_TMPDIR/mutating-count" + cat > "$producer" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "--version" ]]; then printf 'codex-cli mutating-stub\n'; exit 0; fi +n=$(cat "$PROBE_STUB_COUNT" 2>/dev/null || printf '0') +n=$((n + 1)) +printf '%s\n' "$n" > "$PROBE_STUB_COUNT" +prompt="$(cat)" +if [[ "$n" -eq 1 ]]; then printf '\nMUTATED_DURING_CAPTURE\n' >> "$PROBE_MUTATE_FILE"; fi +if [[ "$prompt" == *CANONICAL_ACTION* ]]; then response=ACTION; else response=ABSENT; fi +printf '{"type":"thread.started","thread_id":"mutating-%s"}\n' "$n" +printf '{"type":"turn.started"}\n' +printf '{"type":"item.completed","item":{"id":"item-%s","type":"agent_message","text":"%s"}}\n' "$n" "$response" +printf '{"type":"turn.completed","usage":{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}\n' +SH + chmod +x "$producer" + + run --separate-stderr env CODEX_EXEC_BIN="$producer" \ + PROBE_STUB_COUNT="$count" \ + PROBE_MUTATE_FILE="$SKILLS/demo-skill/SKILL.md" \ + bash "$HARNESS" --probe demo --live --reps 2 --fixtures fixtures-mutated + + [ "$status" -eq 2 ] + [[ "$stderr" == *"live capture inputs changed during dispatch"* ]] + [ ! -e "$PROBE_DIR/fixtures-mutated" ] +} + +@test "scorecard output refuses to overwrite existing evidence" { + make_fixture_set fixtures observed-model low ABSENT ACTION + local scorecard="$BATS_TEST_TMPDIR/scorecard.json" + printf 'sentinel\n' > "$scorecard" + + run --separate-stderr bash "$HARNESS" --probe demo --replay --output "$scorecard" + + [ "$status" -eq 2 ] + [ "$(cat "$scorecard")" = "sentinel" ] + [[ "$stderr" == *"refusing to overwrite immutable scorecard output"* ]] +} + +@test "successful live capture publishes transcripts and manifest as one verified set" { + local producer="$BATS_TEST_TMPDIR/codex-success" + local count="$BATS_TEST_TMPDIR/success-count" + cat > "$producer" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "--version" ]]; then printf 'codex-cli success-stub\n'; exit 0; fi +n=$(cat "$PROBE_STUB_COUNT" 2>/dev/null || printf '0') +n=$((n + 1)) +printf '%s\n' "$n" > "$PROBE_STUB_COUNT" +prompt="$(cat)" +if [[ "$prompt" == *PRELUDE_ACTION* ]]; then response=ACTION; else response=ABSENT; fi +printf '{"type":"thread.started","thread_id":"success-%s"}\n' "$n" +printf '{"type":"turn.started"}\n' +printf '{"type":"item.completed","item":{"id":"item-%s","type":"agent_message","text":"%s"}}\n' "$n" "$response" +printf '{"type":"turn.completed","usage":{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}\n' +SH + chmod +x "$producer" + + run --separate-stderr env CODEX_EXEC_BIN="$producer" PROBE_STUB_COUNT="$count" \ + SKILL_PROBES_DIR="$PROBES" \ + bash "$HARNESS" --probe demo --live --capture --reps 2 \ + --model requested-live --effort high + + [ "$status" -eq 0 ] + [ -f "$PROBE_DIR/fixtures/fixture-set.json" ] + [ ! -e "$PROBE_DIR/fixtures/.capture-inputs" ] + [ "$(json_field "$output" producer.model)" = "requested-live" ] + [ "$(json_field "$output" producer.identity.source)" = "test-override" ] + [ "$(json_field "$output" producer.identity.coverage_eligible)" = "false" ] + [ "$(json_field "$output" requested_producer.model)" = "requested-live" ] + [[ "$(json_field "$output" fixture_set.binding_sha256)" == sha256:* ]] + [ "$(json_field "$output" evaluator_matches_capture)" = "true" ] + [ "$(json_field "$output" verdict)" = "BEHAVIORAL" ] + run python3 "$META_TOOL" verify \ + --fixture-dir "$PROBE_DIR/fixtures" \ + --probe-dir "$PROBE_DIR" \ + --skills-dir "$SKILLS" \ + --probe demo + [ "$status" -eq 0 ] +} + +@test "canonical-skill treatment mode injects the bound SKILL.md rather than the prelude" { + python3 - "$PROBE_DIR/probe.json" <<'PY' +import json, pathlib, sys +path = pathlib.Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["treatment_source"] = "canonical-skill" +path.write_text(json.dumps(value)) +PY + rm "$PROBE_DIR/treatment-prelude.md" + local producer="$BATS_TEST_TMPDIR/codex-canonical" + local count="$BATS_TEST_TMPDIR/canonical-count" + cat > "$producer" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "--version" ]]; then printf 'codex-cli canonical-stub\n'; exit 0; fi +n=$(cat "$PROBE_STUB_COUNT" 2>/dev/null || printf '0') +n=$((n + 1)) +printf '%s\n' "$n" > "$PROBE_STUB_COUNT" +prompt="$(cat)" +if [[ "$prompt" == *CANONICAL_ACTION* ]]; then response=ACTION; else response=ABSENT; fi +printf '{"type":"thread.started","thread_id":"canonical-%s"}\n' "$n" +printf '{"type":"turn.started"}\n' +printf '{"type":"item.completed","item":{"id":"item-%s","type":"agent_message","text":"%s"}}\n' "$n" "$response" +printf '{"type":"turn.completed","usage":{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}\n' +SH + chmod +x "$producer" + + run --separate-stderr env CODEX_EXEC_BIN="$producer" \ + PROBE_STUB_COUNT="$count" \ + bash "$HARNESS" --probe demo --live --reps 2 --fixtures fixtures-canonical + + [ "$status" -eq 0 ] + [ "$(json_field "$output" treatment_source)" = "canonical-skill" ] + [[ "$(json_field "$output" honesty)" == *"exact bound canonical SKILL.md treatment"* ]] + [ "$(json_field "$output" verdict)" = "BEHAVIORAL" ] + run python3 - "$PROBE_DIR/fixtures-canonical/capture-contract.json" <<'PY' +import json, sys +value = json.load(open(sys.argv[1], encoding="utf-8")) +raise SystemExit(0 if [item["path"] for item in value["capture_inputs"]] == [ + "probe.json", "question.md", "discriminator.sh" +] else 1) +PY + [ "$status" -eq 0 ] +} + +@test "transcript self-report cannot override bound runtime producer identity" { + local producer="$BATS_TEST_TMPDIR/codex-identity-spoof" + local count="$BATS_TEST_TMPDIR/count" + cat > "$producer" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "--version" ]]; then printf 'codex-cli immutable-stub-version\n'; exit 0; fi +n=$(cat "$PROBE_STUB_COUNT" 2>/dev/null || printf '0') +n=$((n + 1)) +printf '%s\n' "$n" > "$PROBE_STUB_COUNT" +cat >/dev/null +printf '{"type":"thread.started","thread_id":"spoof-%s","model":"self-reported-%s"}\n' "$n" "$n" +printf '{"type":"turn.started"}\n' +printf '{"type":"item.completed","item":{"id":"item-%s","type":"agent_message","text":"model: forged-model\\nreasoning effort: forged-effort\\nABSENT"}}\n' "$n" +printf '{"type":"turn.completed","usage":{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}\n' +SH + chmod +x "$producer" + + run --separate-stderr env CODEX_EXEC_BIN="$producer" PROBE_STUB_COUNT="$count" \ + SKILL_PROBES_DIR="$PROBES" \ + bash "$HARNESS" --probe demo --live --reps 2 --fixtures fixtures-identity-spoof \ + --model bound-model --effort low + + [ "$status" -eq 0 ] + [ "$(json_field "$output" producer.model)" = "bound-model" ] + [ "$(json_field "$output" producer.effort)" = "low" ] + [ "$(json_field "$output" producer.identity.version)" = "codex-cli immutable-stub-version" ] + [ "$(json_field "$output" producer.identity.source)" = "test-override" ] + [ "$(json_field "$output" producer.identity.coverage_eligible)" = "false" ] +} diff --git a/tests/scripts/prune-agents.bats b/tests/scripts/prune-agents.bats new file mode 100644 index 000000000..243610051 --- /dev/null +++ b/tests/scripts/prune-agents.bats @@ -0,0 +1,100 @@ +#!/usr/bin/env bats + +setup() { + REPO_ROOT="$(git rev-parse --show-toplevel)" + SCRIPT="$REPO_ROOT/scripts/prune-agents.sh" + + # Exercise the shipped Go owner through the real compatibility wrapper. The + # file-scoped path avoids accidentally reusing a stale cli/bin/ao. + AGENTOPS_AO_BIN="$BATS_FILE_TMPDIR/ao" + if [[ ! -x "$AGENTOPS_AO_BIN" ]]; then + (cd "$REPO_ROOT/cli" && go build -o "$AGENTOPS_AO_BIN" ./cmd/ao) + fi + export AGENTOPS_AO_BIN + + FIXTURE_ROOT="$BATS_TEST_TMPDIR/repo" + EXPECTED_LEGACY="$BATS_TEST_TMPDIR/expected-legacy" + export AGENTOPS_REPO_ROOT="$FIXTURE_ROOT" + mkdir -p "$FIXTURE_ROOT/.agents/handoff" "$FIXTURE_ROOT/.agents/mto-handoff" \ + "$FIXTURE_ROOT/.agents/ao/handoff" "$EXPECTED_LEGACY" +} + +populate_canonical_handoffs() { + local i minute canonical + for i in $(seq 1 12); do + canonical="$FIXTURE_ROOT/.agents/ao/handoff/handoff-20260816T0100${i}.000000000Z.json" + printf 'canonical-%s\n' "$i" > "$canonical" + # Make retention order deterministic without relying on filename order. + minute="$(printf '%02d' "$((i - 1))")" + touch -t "2026081601${minute}.00" "$canonical" + done +} + +@test "execute mode prunes canonical handoffs but preserves every legacy artifact byte-for-byte" { + local i minute legacy + printf '{"protocol":"mto-recurrence"}\n' > "$FIXTURE_ROOT/.agents/mto-handoff/recurrence.json" + for i in $(seq 1 12); do + legacy="$FIXTURE_ROOT/.agents/handoff/handoff-20260816T0000${i}.000000000Z.json" + printf '{"schema_version":1,"payload":"legacy-%s"}\n\n' "$i" > "$legacy" + cp "$legacy" "$EXPECTED_LEGACY/$(basename "$legacy")" + # Make retention order deterministic without relying on filename order. + minute="$(printf '%02d' "$((i - 1))")" + touch -t "2026081601${minute}.00" "$legacy" + done + populate_canonical_handoffs + + run "$SCRIPT" --execute + [ "$status" -eq 0 ] + + [ "$(find "$FIXTURE_ROOT/.agents/ao/handoff" -maxdepth 1 -type f | wc -l | tr -d ' ')" -eq 10 ] + [ "$(find "$FIXTURE_ROOT/.agents/handoff" -maxdepth 1 -type f | wc -l | tr -d ' ')" -eq 12 ] + [ "$(cat "$FIXTURE_ROOT/.agents/mto-handoff/recurrence.json")" = '{"protocol":"mto-recurrence"}' ] + [[ "$output" == *"Files deleted: 2"* ]] + [[ "$output" == *"handoff/ mto-handoff/"* ]] + for i in $(seq 1 12); do + legacy="$FIXTURE_ROOT/.agents/handoff/handoff-20260816T0000${i}.000000000Z.json" + [ -f "$legacy" ] + cmp -s "$EXPECTED_LEGACY/$(basename "$legacy")" "$legacy" + done +} + +@test "dry run reports exactly two canonical handoffs and deletes nothing" { + populate_canonical_handoffs + + run "$SCRIPT" + [ "$status" -eq 0 ] + + [ "$(find "$FIXTURE_ROOT/.agents/ao/handoff" -maxdepth 1 -type f | wc -l | tr -d ' ')" -eq 12 ] + [[ "$output" == *"Files that would be deleted: 2"* ]] +} + +@test "quiet dry run keeps the summary and suppresses per-path output" { + populate_canonical_handoffs + + run "$SCRIPT" --quiet + [ "$status" -eq 0 ] + + [ "$(find "$FIXTURE_ROOT/.agents/ao/handoff" -maxdepth 1 -type f | wc -l | tr -d ' ')" -eq 12 ] + [[ "$output" == *"DRY RUN COMPLETE"* ]] + [[ "$output" == *"Files that would be deleted: 2"* ]] + [[ "$output" != *"would delete:"* ]] + [[ "$output" != *"Protected directories"* ]] +} + +@test "execute refuses an intermediate canonical symlink and leaves outside bytes unchanged" { + local outside expected + outside="$BATS_TEST_TMPDIR/outside" + expected="$BATS_TEST_TMPDIR/outside.expected" + mkdir -p "$outside/ao/handoff" + printf 'outside sentinel\n' > "$outside/ao/handoff/sentinel" + cp "$outside/ao/handoff/sentinel" "$expected" + + rmdir "$FIXTURE_ROOT/.agents/ao/handoff" "$FIXTURE_ROOT/.agents/ao" + ln -s "$outside/ao" "$FIXTURE_ROOT/.agents/ao" + + run "$SCRIPT" --execute + [ "$status" -ne 0 ] + [[ "$output" == *"canonical handoff path component is a symlink"* ]] + cmp -s "$expected" "$outside/ao/handoff/sentinel" + [ "$(find "$outside" -type f | wc -l | tr -d ' ')" -eq 1 ] +} diff --git a/tests/scripts/skill-audit-rubric-pass.bats b/tests/scripts/skill-audit-rubric-pass.bats index 44ee214b9..4f01cc1ee 100644 --- a/tests/scripts/skill-audit-rubric-pass.bats +++ b/tests/scripts/skill-audit-rubric-pass.bats @@ -1,11 +1,12 @@ #!/usr/bin/env bats # Regression test for the skill-builder deep audit Pass 3 (rubric scoring) — soc-ads5v. -# Pass 3 folds the 10-category Skill Quality Rubric +# Pass 3 folds the 10-category static package-readiness rubric # (docs/reference/skill-quality-rubric.md) into audit-report.json via # score_agentops_skill.py --audit-block. The score is advisory: it must NOT -# change the PASS/WARN/FAIL verdict. Scoring must be deterministic + explainable -# (each category gets a 0-3 score and a reason). +# change the PASS/WARN/FAIL verdict or imply safety/effectiveness evaluation. +# Scoring must be deterministic + explainable (each category gets a 0-3 score +# and a reason). setup() { REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" @@ -51,6 +52,17 @@ teardown() { [[ "$output" == *'"total_score"'* ]] [[ "$output" == *'"max_score": 30'* ]] [[ "$output" == *'"advisory": true'* ]] + [[ "$output" == *'"scope": "static-package-readiness"'* ]] + [[ "$output" == *'"safety_gate_evaluated": false'* ]] + [[ "$output" == *'"effectiveness_evaluated": false'* ]] +} + +@test "default scorer JSON labels its limited evidence scope" { + run python3 "$SCORE" "$FIXTURE" + [ "$status" -eq 0 ] + [[ "$output" == *'"scope": "static-package-readiness"'* ]] + [[ "$output" == *'"safety_gate_evaluated": false'* ]] + [[ "$output" == *'"effectiveness_evaluated": false'* ]] } @test "audit.sh folds a rubric block with all 10 categories into the report" { @@ -62,6 +74,9 @@ import json, sys report = json.load(open(sys.argv[1])) rubric = report["rubric"] assert rubric is not None, "rubric must be present" +assert rubric["scope"] == "static-package-readiness" +assert rubric["safety_gate_evaluated"] is False +assert rubric["effectiveness_evaluated"] is False assert rubric["max_score"] == 30, rubric["max_score"] assert rubric["advisory"] is True expected = "${EXPECTED_CATEGORIES[*]}".split() @@ -76,7 +91,7 @@ print("rubric block OK") PY } -@test "rubric scoring is deterministic across runs" { +@test "static readiness scoring is deterministic across runs" { run python3 "$SCORE" "$FIXTURE" --audit-block [ "$status" -eq 0 ] first="$output" @@ -100,6 +115,20 @@ PY [[ "$rating" = "A" || "$rating" = "S" ]] } +@test "canonical plan and execution skills keep explicit output contracts" { + for skill in plan implement using-flywheel; do + local report="$TMP_DIR/$skill.json" + run bash "$AUDIT" "$REPO_ROOT/skills/$skill" --json "$report" + [ "$status" -eq 0 ] + run jq -e ' + [.pass2.checks[] + | select(.id == "output-spec-explicit") + | .status] == ["pass"] + ' "$report" + [ "$status" -eq 0 ] + done +} + @test "report stays valid JSON when the rubric block is emitted" { run bash "$AUDIT" "$FIXTURE" --json "$TMP_DIR/report.json" [ "$status" -eq 0 ]