diff --git a/cli/internal/gates/checks/seed.go b/cli/internal/gates/checks/seed.go index 853d50829..b3932282d 100644 --- a/cli/internal/gates/checks/seed.go +++ b/cli/internal/gates/checks/seed.go @@ -48,6 +48,19 @@ var ( "scripts/check-skill-redirects.sh", "tests/scripts/check-skill-redirects.bats", } + // docs.cli-snippets resolves every `ao …` command cited in a live doc against + // the cobra tree; a rename/removal of a command silently strands a golden-path + // snippet. Runs on any docs change plus self-reference (the script, its baseline + // allowlist, the bats twin, and the shared resolution lib) so editing the gate + // re-runs it. (age-gate-the-ungated-egwt.4) + docsCliSnippetsPaths = []string{ + "docs/**", + "scripts/check-docs-cli-snippets.sh", + "scripts/.docs-cli-snippets-baseline", + "scripts/lib/ao-snippet-resolve.sh", + "scripts/lib/ao_snippet_resolve.py", + "tests/scripts/check-docs-cli-snippets.bats", + } // Claude workflows must use `br` (bd/Dolt is retired). operating-loop.js — // the most-viewed content artifact on the public repo — shipped a prompt // telling agents to run `bd ready` with no gate to catch it. @@ -218,6 +231,7 @@ func init() { {ID: "always.three-gap-supergate", Tiers: gates.Full, Match: goalsPaths, Blocking: true, Backing: "check-three-gap-supergate.sh"}, {ID: "always.sovereignty-proof-citations", Tiers: gates.Full, Match: docsPaths, Blocking: true, Backing: "validate-sovereignty-proof-citations.sh"}, {ID: "docs.no-retired-tech", Tiers: gates.Fast | gates.Full, Match: docsPaths, Blocking: true, Backing: "check-docs-no-retired-tech.sh", RepairHint: "convert to current truth, or add a RETIRED/HISTORICAL banner in the first 15 lines; see scripts/check-docs-no-retired-tech.sh"}, + {ID: "docs.cli-snippets", Tiers: gates.Full, Match: docsCliSnippetsPaths, Blocking: false, Backing: "check-docs-cli-snippets.sh", RepairHint: "fix the dead ao reference or prune the stale baseline entry; flips Blocking after one clean advisory cycle (age-gate-the-ungated-egwt.4)"}, {ID: "corpus.secret-scan", Tiers: gates.Full, Match: corpusPaths, Blocking: true, Backing: "check-corpus-secret-scan.sh"}, {ID: "corpus.witness-dolt-jsonl-crosscheck", Tiers: gates.Full, Match: corpusPaths, Blocking: true, Backing: "witness-dolt-jsonl-crosscheck.sh"}, {ID: "doctrine.memrl-health", Tiers: gates.Full, Blocking: true, Backing: "check-memrl-health.sh"}, diff --git a/docs/leverage-points.md b/docs/leverage-points.md index 81fc7328c..3ef205298 100644 --- a/docs/leverage-points.md +++ b/docs/leverage-points.md @@ -91,7 +91,7 @@ The knowledge stock `K` lives in `.agents/`. Its structure: - **Phase boundaries (R to P to I to V)** — The RPI lifecycle enforces sequential phases: Research, Plan, Implement, Validate. Each phase must complete before the next begins. This is a deliberate delay that prevents premature implementation. The cost of finding a bug increases 10x at each stage it survives. - **Circuit breaker (60 min)** — `/evolve` stops if no productive cycle occurred in the last 60 minutes. This prevents the system from oscillating between idle cycles indefinitely. Implemented as a timestamp check against `cycle-history.jsonl`. - **Confidence decay (10%/week)** — Learning freshness scores decay over time, creating a delay between when knowledge was created and when it becomes effectively invisible to retrieval. This matches Ebbinghaus's forgetting curve. -- **Stale run TTL** — RPI runs that do not close cleanly have their state cleaned up by `ao rpi cleanup` (and the `--auto-clean-stale-after` pre-run sweep) once older than the TTL. Prevents stale state from contaminating future sessions. +- **Stale run TTL** — RPI runs that do not close cleanly have their state cleaned up by the worktree-maintenance sweep (`ao worktree`) once older than the TTL. Prevents stale state from contaminating future sessions. - **Maturity lifecycle** — `ao maturity --expire` implements time-delayed eviction. Knowledge that is not retrieved within its TTL decays out of the active set. **dK/dt mapping:** Controls the lag between `I(t)` and usable `K`. Phase boundaries create healthy delays (validation before deployment). Confidence decay and maturity lifecycle create the `delta * K` drain term. diff --git a/docs/trust-factory.md b/docs/trust-factory.md index 462dd70a3..25edb4c2d 100644 --- a/docs/trust-factory.md +++ b/docs/trust-factory.md @@ -53,8 +53,8 @@ context is logged so a later reviewer can ask *what did the agent actually know when it acted?* without guessing. **Reproducibility.** The RPI workflow (Research → Plan → Implement → Validate) -runs each phase against a written contract, and `ao crank` isolates parallel -work in worktrees so the inputs to a phase can be replayed. The packet captured +runs each phase against a written contract, and the `/crank` skill isolates +parallel work in worktrees so the inputs to a phase can be replayed. The packet captured at planning time is the same packet a re-run consumes; deviations show up as diffs against a known input, not as drift. diff --git a/docs/workflows/README.md b/docs/workflows/README.md index 930fe0b99..fc191726f 100644 --- a/docs/workflows/README.md +++ b/docs/workflows/README.md @@ -14,7 +14,7 @@ tree. | New feature or broad change | [Complete Cycle](complete-cycle.md) | `/rpi "goal"` or `/plan` -> `/crank ` | | Existing tracked backlog | `ao reconcile --json`, then `BEADS_DIR="$(ao beads dir)" br ready` | `/implement ` or `/crank ` | | Review-only checkpoint | `/council validate this PR` | `/vibe recent` before merge | -| Terminal-native operation | `ao factory start --goal "goal"` | `ao rpi phased "goal"` and `ao rpi status` | +| Terminal-native operation | `ao session bootstrap` | `ao lookup --query ""`, then drive the operating loop and `ao gate check --fast --scope head` | ## Decision Tree diff --git a/scripts/.docs-cli-snippets-baseline b/scripts/.docs-cli-snippets-baseline new file mode 100644 index 000000000..46e29af91 --- /dev/null +++ b/scripts/.docs-cli-snippets-baseline @@ -0,0 +1,48 @@ +# scripts/.docs-cli-snippets-baseline +# FILENAME-pinned allowlist for check-docs-cli-snippets.sh (age-gate-the-ungated-egwt.4). +# +# Each line is a live doc that currently cites a removed/renamed ao command +# (ao rpi, ao evolve, ao factory, ao daemon, ao activate, ...). The gate +# fails if a NON-baselined live doc cites a dead command, AND fails if a +# baselined file no longer triggers ANY finding (prune it — the allowlist +# only shrinks). Burn a file down (use the live equivalent or historical +# wording), then delete its line here. +# +# dyr0-lane-owned golden paths (first-value-path, getting-started/**, +# examples/**) STAY baselined — do not edit those pages in this lane. +docs/3.0.md +docs/activation-profiles.md +docs/agentops-3-explainer-kit.md +docs/agentops-3-pmf-evidence-loop.md +docs/agentops-3-youtube-starter-series.md +docs/architecture/codebase-overview.md +docs/architecture/codex-hookless-lifecycle.md +docs/architecture/operating-loop.md +docs/architecture/primitive-chains.md +docs/cdlc.md +docs/CLEANUP-distill-to-core.md +docs/cli-skills-map.md +docs/contracts/agent-native-mechanism.md +docs/contracts/agents-write-surfaces.md +docs/doctrine/effective-feedback-compute.md +docs/domain-practice-packets.md +docs/domains/README.md +docs/examples/agentops-3-council-verdict-example.md +docs/examples/agentops-3-domain-practice-packet.md +docs/first-value-path.md +docs/getting-started/index.md +docs/GLOSSARY.md +docs/how-it-works.md +docs/knowledge-flywheel.md +docs/positioning/12-factors-validation-lens.md +docs/positioning/hn-devto-article.md +docs/reference/agentops-domain-evolution-plan.md +docs/release-e2e-checklist.md +docs/RELEASING.md +docs/ROADMAP.md +docs/runbooks/autonomy-runtime-cycle-1.md +docs/runbooks/nightly-evolution.md +docs/seed-definition.md +docs/SKILLS.md +docs/software-factory.md +docs/workflows/session-lifecycle.md diff --git a/scripts/check-docs-cli-snippets.sh b/scripts/check-docs-cli-snippets.sh new file mode 100755 index 000000000..09847b96e --- /dev/null +++ b/scripts/check-docs-cli-snippets.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# check-docs-cli-snippets.sh +# +# Resolve every `ao …` command cited in a LIVE doc against the live cobra tree +# and fail if a doc names a command that does not exist (a removed/renamed +# command like `ao factory start`, `ao rpi phased`, `ao evolve` on a golden +# path). Port-style sibling of scripts/validate-skill-cli-snippets.sh — it +# SHARES that gate's resolution core (scripts/lib/ao-snippet-resolve.*) rather +# than forking it (age-gate-the-ungated-egwt.4). +# +# Scope = the shared LIVE-doc set (scripts/lib/docs-scope.sh: docs/**/*.md minus +# dated archives and self-declared-historical docs). Extraction covers BOTH +# fenced code blocks and inline code spans; plain prose is NOT scanned (the +# false-positive guard). +# +# Resolution is SOUND: unlike the skills gate's byte-identical `help`-mode +# predicate, this uses `ao --help` and rejects cobra's "unknown command" +# / "Unknown help topic" — because `ao help ` ALWAYS exits 0, so the +# help-mode predicate cannot detect a removed command at all. The archive-tagged +# build (`-tags "flywheel legacy"`) keeps archived-but-revivable commands +# resolvable, so this only flags TRULY removed commands. +# +# Baseline ratchet (scripts/.docs-cli-snippets-baseline): FILENAME-pinned, seeds +# every current offender. Two-way enforcement: +# (a) a NON-baselined live doc with a dead ao ref → exit 1 +# (b) a baselined file that no longer triggers ANY finding → exit 1 (prune it) +# Allowlists only ever shrink. +# +# Exit: 0 clean · 1 offender / stale-baseline · 2 usage/setup error + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Shared machinery. Resolve via the pre-cd absolutized $SCRIPT_DIR (a relative +# ${BASH_SOURCE[0]} would resolve wrongly after a cd — the .1 lane's pawl catch). +# shellcheck source=scripts/lib/docs-scope.sh +. "$SCRIPT_DIR/lib/docs-scope.sh" +# shellcheck source=scripts/lib/ao-snippet-resolve.sh +. "$SCRIPT_DIR/lib/ao-snippet-resolve.sh" + +# Pin the docs scope root to THIS repo (the DOCS_ROOT env seam is for lib tests). +DOCS_ROOT="$ROOT" +export DOCS_ROOT + +BASELINE="${DOCS_CLI_SNIPPETS_BASELINE:-$ROOT/scripts/.docs-cli-snippets-baseline}" + +# Build (or reuse) the archive-tagged ao binary; sets + exports AO_BIN. +ao_snippet_resolve_bin "$ROOT" >/dev/null +if [[ -n "${AO_SNIPPET_TMP_DIR:-}" ]]; then + trap 'rm -rf "$AO_SNIPPET_TMP_DIR"' EXIT +fi +export REPO_ROOT="$ROOT" +export AO_RESOLVE_MODE=strict +export DOCS_CLI_BASELINE="$BASELINE" + +python3 - <<'PY' +import os +import pathlib +import re +import shlex +import subprocess +import sys + +sys.path.insert(0, os.environ["AO_SNIPPET_LIB_DIR"]) +from ao_snippet_resolve import iter_snippets, make_resolver_from_env + +repo_root = pathlib.Path(os.environ["REPO_ROOT"]) +docs_root = pathlib.Path(os.environ["DOCS_ROOT"]) +baseline_path = pathlib.Path(os.environ["DOCS_CLI_BASELINE"]) + +resolver = make_resolver_from_env() + +# ---- live-doc scope + historical exemption (shared bash lib, via subprocess) -- +def _sh(func): + """Call a docs-scope.sh function and return (rc, stdout).""" + lib = pathlib.Path(os.environ["AO_SNIPPET_LIB_DIR"]) / "docs-scope.sh" + script = f'. "{lib}"; {func}' + r = subprocess.run(["bash", "-c", script], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True, + env={**os.environ}) + return r.returncode, r.stdout + +def live_files(): + _, out = _sh("docs_scope_live_files") + return [line for line in out.splitlines() if line.strip()] + +def is_exempt(f): + rc, _ = _sh(f'docs_scope_is_exempt "{f}"') + return rc == 0 + +# ---- nearest-live-command suggestion ----------------------------------------- +def _levenshtein(a, b): + if a == b: + return 0 + if not a: + return len(b) + if not b: + return len(a) + prev = list(range(len(b) + 1)) + for i, ca in enumerate(a, 1): + cur = [i] + for j, cb in enumerate(b, 1): + cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))) + prev = cur + return prev[-1] + +def _top_level_commands(): + # Parse `ao --help` for the visible top-level command names. + _, out = resolver._probe([]) + names = [] + for line in out.splitlines(): + m = re.match(r"^ ([a-z][a-z0-9-]+)\s{2,}\S", line) + if m: + names.append(m.group(1)) + return sorted(set(names)) + +_TOP = None +def suggest(token): + global _TOP + if _TOP is None: + _TOP = _top_level_commands() + if not _TOP: + return None + best = min(_TOP, key=lambda c: _levenshtein(token, c)) + # only suggest a genuinely-near match + if _levenshtein(token, best) <= max(2, len(token) // 2): + return best + return None + +# A line that DESCRIBES a command's removal (negation / past-tense) is documenting +# the retirement, not prescribing the dead command — not an offender. Mirrors the +# REMOVAL_LANG exemption in check-docs-no-retired-tech.sh. +_REMOVAL_LANG = re.compile( + r"no `ao|removed|retired|deleted|deprecat|superseded|no longer|is gone|are gone|" + r"not a (?:selectable|valid)|deprecation pointer|gets? a deprecation", + re.IGNORECASE, +) + +# ---- resolve one snippet; return the offending token or None ----------------- +def offending_token(snippet): + try: + tokens = shlex.split(snippet) + except ValueError: + return None + tokens = resolver.trim_shell_tokens(tokens) + if not tokens or tokens[0] != "ao": + return None + if resolver.is_regex_like(tokens): + return None + command, _ = resolver.resolve_command(tokens) + if command: + return None + if len(tokens) == 1: + return None + if all(t.startswith("-") for t in tokens[1:]): + return None # all-flags form (e.g. `ao --version`); flags not scoped here + # first wordish subcommand token is the offending name + first = tokens[1] + # skip placeholders / non-command tokens (e.g. `ao ...`, `ao `) + if not re.match(r"^[a-z][a-z0-9-]*$", first): + return None + return first + +# ---- scan --------------------------------------------------------------------- +findings = {} # file -> list of (lineno, snippet, token, suggestion) +triggered = set() # files with >=1 finding +for f in live_files(): + if is_exempt(f): + continue + p = docs_root / f + try: + text = p.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + lines = text.splitlines() + for lineno, snippet in iter_snippets(text): + tok = offending_token(snippet) + if tok is None: + continue + src_line = lines[lineno - 1] if 0 <= lineno - 1 < len(lines) else "" + if _REMOVAL_LANG.search(src_line): + continue # describing the removal, not prescribing the dead command + findings.setdefault(f, []).append((lineno, snippet, tok, suggest(tok))) + triggered.add(f) + +# ---- baseline ratchet --------------------------------------------------------- +baselined = set() +if baseline_path.exists(): + for line in baseline_path.read_text(encoding="utf-8").splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + baselined.add(s) + +new_offenders = sorted(triggered - baselined) +stale_baseline = sorted(baselined - triggered) + +failed = False + +if new_offenders: + failed = True + print("check-docs-cli-snippets: FAIL — live doc(s) cite a removed/unknown ao command:", file=sys.stderr) + for f in new_offenders: + for lineno, snippet, tok, sug in findings[f]: + hint = f" (did you mean `ao {sug}`?)" if sug else "" + print(f" {f}:{lineno}: unknown ao command `ao {tok}` in `{snippet}`{hint}", file=sys.stderr) + print("", file=sys.stderr) + print("Fix the dead ao reference (use the live equivalent, or historical wording), " + "or — only if the page is a dyr0-lane golden path — add it to " + f"{baseline_path.relative_to(repo_root)}.", file=sys.stderr) + +if stale_baseline: + failed = True + print("check-docs-cli-snippets: FAIL — baseline entr(ies) no longer trigger any finding (prune them):", file=sys.stderr) + for f in stale_baseline: + print(f" {f}", file=sys.stderr) + print("", file=sys.stderr) + print(f"The allowlist only shrinks. Remove the above line(s) from " + f"{baseline_path.relative_to(repo_root)}.", file=sys.stderr) + +if failed: + sys.exit(1) + +n_baselined = len(baselined & triggered) +print(f"check-docs-cli-snippets: PASS — no un-baselined live doc cites a removed ao command " + f"({len(triggered)} file(s) with findings, all baselined; {n_baselined} baseline entr(ies) still active).") +PY diff --git a/scripts/lib/ao-snippet-resolve.sh b/scripts/lib/ao-snippet-resolve.sh new file mode 100644 index 000000000..3ba03c183 --- /dev/null +++ b/scripts/lib/ao-snippet-resolve.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# scripts/lib/ao-snippet-resolve.sh — sourced library: shared `ao` snippet +# resolution machinery for the skills-tree gate and the docs-tree gate. +# +# Source it (do NOT execute it): +# . "$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/ao-snippet-resolve.sh" +# OR (from a script that has already absolutized its own dir): +# . "$SCRIPT_DIR/lib/ao-snippet-resolve.sh" +# +# Extracted from scripts/validate-skill-cli-snippets.sh (age-gate-the-ungated-egwt.4) +# so both snippet gates resolve `ao` commands against the SAME live cobra tree +# from one place ("share the core, don't fork it"). The Python resolution core +# lives beside this file as scripts/lib/ao_snippet_resolve.py; this bash lib owns +# the build machinery + exposes the module directory so callers' inline Python +# can import it. +# +# Behavior-preserving contract: this lib does NOT `set -euo pipefail` on behalf +# of its callers (the extracting script sets strict mode itself). The functions +# are pure/idempotent under either mode. + +# ao_snippet_lib_dir — absolute path to this lib's directory (holds the Python +# module). Resolved via BASH_SOURCE at source time; safe even if the caller cd's +# afterward, because we absolutize immediately. +ao_snippet_lib_dir() { + # shellcheck disable=SC1007 # `CDPATH=` clears CDPATH for this one cd (env-prefix, not an assignment) + CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd +} + +# AO_SNIPPET_LIB_DIR — module dir, computed once at source time (pre-cd-safe). +AO_SNIPPET_LIB_DIR="$(ao_snippet_lib_dir)" +export AO_SNIPPET_LIB_DIR + +# ao_snippet_resolve_bin [REPO_ROOT] — echo the path to an `ao` binary suitable +# for snippet resolution and export it as AO_BIN. +# +# Honors an already-set AGENTOPS_AO_BIN (fast path for CI / tests). Otherwise +# builds `ao` from /cli with the ADR-0012 archive tags +# (`-tags "flywheel legacy"`) so snippets that document archived-but-revivable +# commands (e.g. `ao harvest`, `ao forge`, behind //go:build flywheel|legacy) +# still resolve — the default spine build omits them and would false-fail those +# (two prior escapes). The built binary lands in a mktemp dir; the CALLER is +# responsible for its own EXIT trap cleanup if it wants the temp removed. +# +# REPO_ROOT defaults to the parent of this lib's dir's parent (…/scripts/lib → +# repo root), matching the historical layout. +ao_snippet_resolve_bin() { + local repo_root="${1:-}" + if [[ -z "$repo_root" ]]; then + # shellcheck disable=SC1007 # `CDPATH=` clears CDPATH for this one cd (env-prefix, not an assignment) + repo_root="$(CDPATH= cd "$AO_SNIPPET_LIB_DIR/../.." && pwd)" + fi + + local ao_bin="${AGENTOPS_AO_BIN:-}" + if [[ -z "$ao_bin" ]]; then + local tmp_dir + tmp_dir="$(mktemp -d)" + ao_bin="$tmp_dir/ao" + ( + cd "$repo_root/cli" || exit 1 + go build -tags "flywheel legacy" -o "$ao_bin" ./cmd/ao + ) + # Surface the temp dir so the caller can trap-clean it. + AO_SNIPPET_TMP_DIR="$tmp_dir" + export AO_SNIPPET_TMP_DIR + fi + + [[ -x "$ao_bin" ]] || { + echo "Missing or non-executable ao binary: $ao_bin" >&2 + return 1 + } + AO_BIN="$ao_bin" + export AO_BIN + printf '%s\n' "$ao_bin" +} diff --git a/scripts/lib/ao_snippet_resolve.py b/scripts/lib/ao_snippet_resolve.py new file mode 100644 index 000000000..8e8628178 --- /dev/null +++ b/scripts/lib/ao_snippet_resolve.py @@ -0,0 +1,164 @@ +"""ao_snippet_resolve — shared resolution core for `ao …` CLI snippets in prose. + +Extracted from scripts/validate-skill-cli-snippets.sh (age-gate-the-ungated-egwt.4) +so that BOTH the skills-tree snippet gate and the docs-tree snippet gate resolve +`ao` commands against the SAME live cobra tree with ONE piece of machinery — the +"share the core, don't fork it" contract. + +What lives here (the reusable core): + * snippet extraction from a line (backtick spans + a bare `ao …` line) + * shell-token trimming at control operators / redirects + * the longest-leading-wordish-run candidate resolution against `ao`'s help tree + * flag checking against the resolved command's help text + +What is INJECTED by the caller (the one axis the two gates differ on): + * the *resolution predicate* — "does this command chain name a real command?" + - RESOLVE_MODE="help" → the ORIGINAL skills-validator semantics: + `ao help ` and trust returncode == 0. + (Byte-for-byte behavior-preserving for that gate.) + - RESOLVE_MODE="strict" → a SOUND check for the docs gate: + `ao --help` and reject output that carries + cobra's "unknown command" / "Unknown help topic". + The `help`-mode predicate is unsound here because + `ao help ` ALWAYS exits 0 (even for a + removed command like `ao factory` / `ao rpi`), so it + cannot detect the dead-command class the docs gate + exists to kill. + +The AO binary path comes from the AO_BIN env var (set by the bash driver, which +builds `ao` with `-tags "flywheel legacy"` so archived-but-revivable commands +still resolve — two prior escapes: the default spine build omits them). +""" + +import os +import re +import shlex +import subprocess + +_WORDISH = re.compile(r"^[a-z][a-z0-9-]*$") +_CONTROL_TOKENS = {"|", "||", "&&", ";", "&"} +_UNKNOWN_CMD = re.compile(r"unknown command|Unknown help topic", re.IGNORECASE) + + +class Resolver: + """Resolve `ao` command chains against the live cobra tree. + + mode: "help" (byte-identical skills-validator semantics) or "strict" + (sound `--help`-based check for the docs gate). + """ + + def __init__(self, ao_bin, mode="help"): + self.ao_bin = ao_bin + if mode not in ("help", "strict"): + raise ValueError(f"unknown resolve mode: {mode!r}") + self.mode = mode + self._help_cache = {} + + # ---- resolution primitive (the one injected axis) --------------------- + def _probe(self, command): + """Return (ok, help_text). `ok` is True when `command` names a real + command chain under `ao`. help_text is the command's help output (used + for flag validation) — for the `help` mode it is exactly the original + validator's `ao help ` stdout so flag checks are unchanged.""" + key = tuple(command) + if key in self._help_cache: + return self._help_cache[key] + if self.mode == "help": + result = subprocess.run( + [self.ao_bin, "help", *command], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + ok = result.returncode == 0 + out = result.stdout + else: # strict + result = subprocess.run( + [self.ao_bin, *command, "--help"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + ok = not _UNKNOWN_CMD.search(result.stdout) + out = result.stdout + self._help_cache[key] = (ok, out) + return ok, out + + def global_help(self): + # Global help is always the root command's help; identical in both modes + # (`ao help` == `ao --help` content for flag lookups). + return self._probe([])[1] + + # ---- shared machinery ------------------------------------------------- + @staticmethod + def trim_shell_tokens(tokens): + trimmed = [] + for token in tokens: + if token in _CONTROL_TOKENS: + break + if token.startswith(("|", ">", "<")): + break + if token.endswith((";", "&&", "||")): + trimmed.append(token.rstrip(";")) + break + trimmed.append(token) + return [token for token in trimmed if token] + + def resolve_command(self, tokens): + """Return (command, help_text) for the longest leading run of wordish + subcommand tokens that names a real command, else (None, None).""" + candidates = [] + for token in tokens[1:]: + if token.startswith("-"): + break + if any(ch in token for ch in "<>[]{}=$("): + break + if not _WORDISH.match(token): + break + candidates.append(token) + + for end in range(len(candidates), 0, -1): + candidate = candidates[:end] + ok, help_text = self._probe(candidate) + if ok: + return candidate, help_text + return None, None + + @staticmethod + def normalize_flag(token): + if "=" in token: + token = token.split("=", 1)[0] + return token + + @staticmethod + def is_regex_like(tokens): + return any(re.search(r"[\[\]\(\)\^\*\+\?]", token) for token in tokens[1:]) + + +def iter_snippets(text): + """Yield (lineno, snippet) for every `ao …` command in text. + + Covers BOTH backtick-delimited code spans (inline code) AND a line whose + stripped form starts with `ao ` (a bare fenced-code-block command line) — + the same extraction the skills validator uses. Plain prose that merely + mentions "ao" without a code span or a leading `ao ` is NOT scanned (the + false-positive guard the docs gate depends on). + """ + for lineno, line in enumerate(text.splitlines(), start=1): + if "ao " not in line: + continue + snippets = [] + for match in re.finditer(r"`([^`]*\bao\b[^`]*)`", line): + snippets.append(match.group(1).strip()) + stripped = line.strip() + if stripped.startswith("ao "): + snippets.append(stripped) + for snippet in snippets: + yield lineno, snippet + + +def make_resolver_from_env(): + """Build a Resolver from AO_BIN + AO_RESOLVE_MODE env vars.""" + ao_bin = os.environ["AO_BIN"] + mode = os.environ.get("AO_RESOLVE_MODE", "help") + return Resolver(ao_bin, mode=mode) diff --git a/scripts/validate-skill-cli-snippets.sh b/scripts/validate-skill-cli-snippets.sh index d7368c053..ff5d018a4 100755 --- a/scripts/validate-skill-cli-snippets.sh +++ b/scripts/validate-skill-cli-snippets.sh @@ -3,125 +3,47 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -AO_BIN="${AGENTOPS_AO_BIN:-}" -if [[ -z "$AO_BIN" ]]; then - TMP_DIR="$(mktemp -d)" - trap 'rm -rf "$TMP_DIR"' EXIT - AO_BIN="$TMP_DIR/ao" - ( - cd "$REPO_ROOT/cli" - # Build with the ADR-0012 archive tags so snippets that document - # archived-but-revivable commands (e.g. `ao harvest`, `ao forge`, behind - # //go:build flywheel|legacy) still validate against the full command - # surface. The default `ao` omits them, but a skill may legitimately - # reference any command; validating only the spine would false-fail those. - go build -tags "flywheel legacy" -o "$AO_BIN" ./cmd/ao - ) +# Shared `ao` snippet resolution machinery (build + resolver core), so this gate +# and the docs.cli-snippets gate resolve against the SAME cobra tree from one +# place. Resolve via the pre-cd absolutized $SCRIPT_DIR (a relative +# ${BASH_SOURCE[0]} would resolve wrongly if a caller cd'd first — the .1 lane's +# round-1 pawl catch). +# shellcheck source=scripts/lib/ao-snippet-resolve.sh +. "$SCRIPT_DIR/lib/ao-snippet-resolve.sh" + +# Build (or reuse) the archive-tagged ao binary; sets + exports AO_BIN. +AO_BIN="$(ao_snippet_resolve_bin "$REPO_ROOT")" +# Clean up a lib-built temp binary on exit (no-op when AGENTOPS_AO_BIN was set). +if [[ -n "${AO_SNIPPET_TMP_DIR:-}" ]]; then + trap 'rm -rf "$AO_SNIPPET_TMP_DIR"' EXIT fi -[[ -x "$AO_BIN" ]] || { - echo "Missing or non-executable ao binary: $AO_BIN" >&2 - exit 1 -} - export AO_BIN export REPO_ROOT +# Byte-identical resolution semantics for the skills gate: `ao help `, +# trust returncode==0 (the original validator's predicate). +export AO_RESOLVE_MODE=help python3 - <<'PY' import os import pathlib import re import shlex -import subprocess import sys +# Import the shared resolution core (extracted to scripts/lib/). +sys.path.insert(0, os.environ["AO_SNIPPET_LIB_DIR"]) +from ao_snippet_resolve import iter_snippets, make_resolver_from_env + repo_root = pathlib.Path(os.environ["REPO_ROOT"]) -ao_bin = os.environ["AO_BIN"] roots = [repo_root / "skills", repo_root / "skills-codex"] allowed_suffixes = {".md", ".sh"} -wordish = re.compile(r"^[a-z][a-z0-9-]*$") -control_tokens = {"|", "||", "&&", ";", "&"} stale_beads_resolver = re.compile(r"BEADS_DIR=\$PWD/_beads|git -C _beads|git add \.beads|git add _beads") stale_beads_allowed = re.compile(r"\b(anti-pattern|do not|don't|must not|never|reject|fails?|historical|retired)\b", re.IGNORECASE) +resolver = make_resolver_from_env() failures = [] -help_cache = {} - -def iter_snippets(path: pathlib.Path): - try: - text = path.read_text(encoding="utf-8") - except UnicodeDecodeError: - return - - for lineno, line in enumerate(text.splitlines(), start=1): - if "ao " not in line: - continue - - snippets = [] - for match in re.finditer(r"`([^`]*\bao\b[^`]*)`", line): - snippets.append(match.group(1).strip()) - - stripped = line.strip() - if stripped.startswith("ao "): - snippets.append(stripped) - - for snippet in snippets: - yield lineno, snippet - -def command_help(command): - key = tuple(command) - if key not in help_cache: - result = subprocess.run( - [ao_bin, "help", *command], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - help_cache[key] = result - return help_cache[key] - -def global_help(): - return command_help([]) - -def trim_shell_tokens(tokens): - trimmed = [] - for token in tokens: - if token in control_tokens: - break - if token.startswith(("|", ">", "<")): - break - if token.endswith((";", "&&", "||")): - trimmed.append(token.rstrip(";")) - break - trimmed.append(token) - return [token for token in trimmed if token] - -def resolve_command(tokens): - candidates = [] - for token in tokens[1:]: - if token.startswith("-"): - break - if any(ch in token for ch in "<>[]{}=$("): - break - if not wordish.match(token): - break - candidates.append(token) - - for end in range(len(candidates), 0, -1): - candidate = candidates[:end] - result = command_help(candidate) - if result.returncode == 0: - return candidate, result.stdout - return None, None - -def normalize_flag(token): - if "=" in token: - token = token.split("=", 1)[0] - return token - -def is_regex_like(tokens): - return any(re.search(r"[\[\]\(\)\^\*\+\?]", token) for token in tokens[1:]) def validate_snippet(path: pathlib.Path, lineno: int, snippet: str): try: @@ -129,21 +51,21 @@ def validate_snippet(path: pathlib.Path, lineno: int, snippet: str): except ValueError: return - tokens = trim_shell_tokens(tokens) + tokens = resolver.trim_shell_tokens(tokens) if not tokens or tokens[0] != "ao": return - if is_regex_like(tokens): + if resolver.is_regex_like(tokens): return - command, help_text = resolve_command(tokens) + command, help_text = resolver.resolve_command(tokens) if not command: if len(tokens) == 1: return if all(token.startswith("-") for token in tokens[1:]): - help_text = global_help().stdout + help_text = resolver.global_help() for flag in tokens[1:]: - normalized = normalize_flag(flag) + normalized = resolver.normalize_flag(flag) if normalized not in help_text: failures.append( f"{path.relative_to(repo_root)}:{lineno}: flag {normalized} not found in help for ao" @@ -158,7 +80,7 @@ def validate_snippet(path: pathlib.Path, lineno: int, snippet: str): continue if len(token) > 1 and token[1:].isdigit(): continue - flags.append(normalize_flag(token)) + flags.append(resolver.normalize_flag(token)) for flag in flags: if flag not in help_text: failures.append( @@ -182,7 +104,7 @@ for root in roots: failures.append( f"{path.relative_to(repo_root)}:{lineno}: stale beads resolver; use BEADS_DIR=\"$(ao beads dir)\" and git -C \"$(ao beads dir)\"" ) - for lineno, snippet in iter_snippets(path): + for lineno, snippet in iter_snippets(text): validate_snippet(path, lineno, snippet) if failures: diff --git a/tests/scripts/check-docs-cli-snippets.bats b/tests/scripts/check-docs-cli-snippets.bats new file mode 100644 index 000000000..57a46de6c --- /dev/null +++ b/tests/scripts/check-docs-cli-snippets.bats @@ -0,0 +1,131 @@ +#!/usr/bin/env bats +# +# Tests for scripts/check-docs-cli-snippets.sh — the docs.cli-snippets gate that +# resolves every `ao …` command cited in a LIVE doc against the live cobra tree +# and fails on a removed/renamed command, with a FILENAME-pinned shrink-only +# baseline (age-gate-the-ungated-egwt.4). +# +# Pattern: build a fixture "repo" in BATS_TEST_TMPDIR that carries its own copy +# of the check script + shared libs (so the script's ROOT resolves to the +# fixture, exactly as docs-scope.bats does), seed a docs/ tree + a baseline, and +# assert the two-way ratchet. The archive-tagged `ao` binary is built once and +# injected via AGENTOPS_AO_BIN (the script's documented fast path). + +setup_file() { + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + export REPO_ROOT + # Build the archive-tagged ao once for the whole file; inject via env. + AO_BIN_FILE="$BATS_FILE_TMPDIR/ao" + ( cd "$REPO_ROOT/cli" && go build -tags "flywheel legacy" -o "$AO_BIN_FILE" ./cmd/ao ) + export AGENTOPS_AO_BIN="$AO_BIN_FILE" +} + +setup() { + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + export AGENTOPS_AO_BIN="$BATS_FILE_TMPDIR/ao" + + # Fixture repo: script + shared libs, so ROOT="$SCRIPT_DIR/.." == the fixture. + FIX="$BATS_TEST_TMPDIR/repo" + mkdir -p "$FIX/scripts/lib" "$FIX/docs" "$FIX/cli" + cp "$REPO_ROOT/scripts/check-docs-cli-snippets.sh" "$FIX/scripts/" + cp "$REPO_ROOT/scripts/lib/docs-scope.sh" "$FIX/scripts/lib/" + cp "$REPO_ROOT/scripts/lib/ao-snippet-resolve.sh" "$FIX/scripts/lib/" + cp "$REPO_ROOT/scripts/lib/ao_snippet_resolve.py" "$FIX/scripts/lib/" + chmod +x "$FIX/scripts/check-docs-cli-snippets.sh" + export FIX + BASELINE="$FIX/scripts/.docs-cli-snippets-baseline" + export BASELINE +} + +# Run the fixture's check with a given baseline content ($1 = baseline lines, +# may be empty). Populates $status / $output. +run_check() { + printf '%s' "$1" > "$BASELINE" + run env DOCS_CLI_SNIPPETS_BASELINE="$BASELINE" bash "$FIX/scripts/check-docs-cli-snippets.sh" +} + +# ---- dead-command detection -------------------------------------------------- + +@test "a live doc citing a removed ao command FAILS, naming file + token" { + printf '# Guide\n\nRun `ao factory start --goal "x"` to begin.\n' > "$FIX/docs/guide.md" + run_check "" + [ "$status" -eq 1 ] + [[ "$output" == *"docs/guide.md"* ]] + [[ "$output" == *"ao factory"* ]] +} + +@test "a live doc citing only LIVE ao commands PASSES" { + printf '# Guide\n\nRun `ao gate check --fast` and `ao session bootstrap`.\n' > "$FIX/docs/guide.md" + run_check "" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +@test "extraction covers a fenced code block line, not just inline spans" { + { printf '# Guide\n\n'; printf '```bash\n'; printf 'ao rpi phased "goal"\n'; printf '```\n'; } > "$FIX/docs/guide.md" + run_check "" + [ "$status" -eq 1 ] + [[ "$output" == *"docs/guide.md"* ]] + [[ "$output" == *"ao rpi"* ]] +} + +@test "plain prose mentioning ao (no code span, no leading 'ao ') is NOT scanned" { + printf '# Guide\n\nThe factory command was removed; use the loop instead.\n' > "$FIX/docs/guide.md" + run_check "" + [ "$status" -eq 0 ] +} + +@test "a line DESCRIBING a command removal is exempt (removal-lang)" { + printf '# Guide\n\nThere is no `ao hooks` command any more; 3.0 is hookless.\n' > "$FIX/docs/guide.md" + run_check "" + [ "$status" -eq 0 ] +} + +# ---- banner / historical exemption (shared docs-scope lib) -------------------- + +@test "a doc with a RETIRED banner in the first 15 lines is exempt from scanning" { + printf '# Old Runbook (RETIRED)\n\nRun `ao factory start` here (historical).\n' > "$FIX/docs/old.md" + run_check "" + [ "$status" -eq 0 ] +} + +# ---- baseline ratchet (two-way) ---------------------------------------------- + +@test "a baselined offender PASSES (allowlisted)" { + printf '# Guide\n\nRun `ao factory start --goal "x"`.\n' > "$FIX/docs/guide.md" + run_check $'docs/guide.md\n' + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +@test "a NON-baselined offender FAILS even when other files ARE baselined" { + printf '# A\n\nRun `ao factory start`.\n' > "$FIX/docs/a.md" + printf '# B\n\nRun `ao rpi phased`.\n' > "$FIX/docs/b.md" + run_check $'docs/a.md\n' # only a.md allowlisted; b.md is a new offender + [ "$status" -eq 1 ] + [[ "$output" == *"docs/b.md"* ]] +} + +@test "a STALE baseline entry (no longer triggers any finding) FAILS demanding prune" { + printf '# Clean\n\nRun `ao gate check --fast`.\n' > "$FIX/docs/clean.md" + run_check $'docs/clean.md\n' # clean.md has no dead command, but is baselined + [ "$status" -eq 1 ] + [[ "$output" == *"no longer trigger"* ]] + [[ "$output" == *"docs/clean.md"* ]] +} + +@test "a baseline entry for a DELETED file is stale and FAILS" { + # No docs/gone.md exists at all. + printf '# Live\n\nRun `ao lookup --query x`.\n' > "$FIX/docs/live.md" + run_check $'docs/gone.md\n' + [ "$status" -eq 1 ] + [[ "$output" == *"docs/gone.md"* ]] +} + +# ---- the REAL repo passes with its seeded baseline --------------------------- + +@test "the real repo passes with its committed baseline (gate lands green)" { + run env AGENTOPS_AO_BIN="$BATS_FILE_TMPDIR/ao" bash "$REPO_ROOT/scripts/check-docs-cli-snippets.sh" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +}