feat(gates): docs.cli-snippets gate — resolve ao commands in live docs against the cobra tree (age-gate-the-ungated-egwt.4)

This commit is contained in:
boshu
2026-07-01 17:42:11 -04:00
parent c9dd7e37cc
commit b7503b4519
10 changed files with 692 additions and 110 deletions
+14
View File
@@ -48,6 +48,19 @@ var (
"scripts/check-skill-redirects.sh",
"tests/scripts/check-skill-redirects.bats",
}
// docs.cli-snippets resolves every `ao …` command cited in a live doc against
// the cobra tree; a rename/removal of a command silently strands a golden-path
// snippet. Runs on any docs change plus self-reference (the script, its baseline
// allowlist, the bats twin, and the shared resolution lib) so editing the gate
// re-runs it. (age-gate-the-ungated-egwt.4)
docsCliSnippetsPaths = []string{
"docs/**",
"scripts/check-docs-cli-snippets.sh",
"scripts/.docs-cli-snippets-baseline",
"scripts/lib/ao-snippet-resolve.sh",
"scripts/lib/ao_snippet_resolve.py",
"tests/scripts/check-docs-cli-snippets.bats",
}
// Claude workflows must use `br` (bd/Dolt is retired). operating-loop.js —
// the most-viewed content artifact on the public repo — shipped a prompt
// telling agents to run `bd ready` with no gate to catch it.
@@ -218,6 +231,7 @@ func init() {
{ID: "always.three-gap-supergate", Tiers: gates.Full, Match: goalsPaths, Blocking: true, Backing: "check-three-gap-supergate.sh"},
{ID: "always.sovereignty-proof-citations", Tiers: gates.Full, Match: docsPaths, Blocking: true, Backing: "validate-sovereignty-proof-citations.sh"},
{ID: "docs.no-retired-tech", Tiers: gates.Fast | gates.Full, Match: docsPaths, Blocking: true, Backing: "check-docs-no-retired-tech.sh", RepairHint: "convert to current truth, or add a RETIRED/HISTORICAL banner in the first 15 lines; see scripts/check-docs-no-retired-tech.sh"},
{ID: "docs.cli-snippets", Tiers: gates.Full, Match: docsCliSnippetsPaths, Blocking: false, Backing: "check-docs-cli-snippets.sh", RepairHint: "fix the dead ao reference or prune the stale baseline entry; flips Blocking after one clean advisory cycle (age-gate-the-ungated-egwt.4)"},
{ID: "corpus.secret-scan", Tiers: gates.Full, Match: corpusPaths, Blocking: true, Backing: "check-corpus-secret-scan.sh"},
{ID: "corpus.witness-dolt-jsonl-crosscheck", Tiers: gates.Full, Match: corpusPaths, Blocking: true, Backing: "witness-dolt-jsonl-crosscheck.sh"},
{ID: "doctrine.memrl-health", Tiers: gates.Full, Blocking: true, Backing: "check-memrl-health.sh"},
+1 -1
View File
@@ -91,7 +91,7 @@ The knowledge stock `K` lives in `.agents/`. Its structure:
- **Phase boundaries (R to P to I to V)** — The RPI lifecycle enforces sequential phases: Research, Plan, Implement, Validate. Each phase must complete before the next begins. This is a deliberate delay that prevents premature implementation. The cost of finding a bug increases 10x at each stage it survives.
- **Circuit breaker (60 min)** — `/evolve` stops if no productive cycle occurred in the last 60 minutes. This prevents the system from oscillating between idle cycles indefinitely. Implemented as a timestamp check against `cycle-history.jsonl`.
- **Confidence decay (10%/week)** — Learning freshness scores decay over time, creating a delay between when knowledge was created and when it becomes effectively invisible to retrieval. This matches Ebbinghaus's forgetting curve.
- **Stale run TTL** — RPI runs that do not close cleanly have their state cleaned up by `ao rpi cleanup` (and the `--auto-clean-stale-after` pre-run sweep) once older than the TTL. Prevents stale state from contaminating future sessions.
- **Stale run TTL** — RPI runs that do not close cleanly have their state cleaned up by the worktree-maintenance sweep (`ao worktree`) once older than the TTL. Prevents stale state from contaminating future sessions.
- **Maturity lifecycle** — `ao maturity --expire` implements time-delayed eviction. Knowledge that is not retrieved within its TTL decays out of the active set.
**dK/dt mapping:** Controls the lag between `I(t)` and usable `K`. Phase boundaries create healthy delays (validation before deployment). Confidence decay and maturity lifecycle create the `delta * K` drain term.
+2 -2
View File
@@ -53,8 +53,8 @@ context is logged so a later reviewer can ask *what did the agent actually
know when it acted?* without guessing.
**Reproducibility.** The RPI workflow (Research → Plan → Implement → Validate)
runs each phase against a written contract, and `ao crank` isolates parallel
work in worktrees so the inputs to a phase can be replayed. The packet captured
runs each phase against a written contract, and the `/crank` skill isolates
parallel work in worktrees so the inputs to a phase can be replayed. The packet captured
at planning time is the same packet a re-run consumes; deviations show up as
diffs against a known input, not as drift.
+1 -1
View File
@@ -14,7 +14,7 @@ tree.
| New feature or broad change | [Complete Cycle](complete-cycle.md) | `/rpi "goal"` or `/plan` -> `/crank <epic-id>` |
| Existing tracked backlog | `ao reconcile --json`, then `BEADS_DIR="$(ao beads dir)" br ready` | `/implement <issue-id>` or `/crank <epic-id>` |
| Review-only checkpoint | `/council validate this PR` | `/vibe recent` before merge |
| Terminal-native operation | `ao factory start --goal "goal"` | `ao rpi phased "goal"` and `ao rpi status` |
| Terminal-native operation | `ao session bootstrap` | `ao lookup --query "<topic>"`, then drive the operating loop and `ao gate check --fast --scope head` |
## Decision Tree
+48
View File
@@ -0,0 +1,48 @@
# scripts/.docs-cli-snippets-baseline
# FILENAME-pinned allowlist for check-docs-cli-snippets.sh (age-gate-the-ungated-egwt.4).
#
# Each line is a live doc that currently cites a removed/renamed ao command
# (ao rpi, ao evolve, ao factory, ao daemon, ao activate, ...). The gate
# fails if a NON-baselined live doc cites a dead command, AND fails if a
# baselined file no longer triggers ANY finding (prune it — the allowlist
# only shrinks). Burn a file down (use the live equivalent or historical
# wording), then delete its line here.
#
# dyr0-lane-owned golden paths (first-value-path, getting-started/**,
# examples/**) STAY baselined — do not edit those pages in this lane.
docs/3.0.md
docs/activation-profiles.md
docs/agentops-3-explainer-kit.md
docs/agentops-3-pmf-evidence-loop.md
docs/agentops-3-youtube-starter-series.md
docs/architecture/codebase-overview.md
docs/architecture/codex-hookless-lifecycle.md
docs/architecture/operating-loop.md
docs/architecture/primitive-chains.md
docs/cdlc.md
docs/CLEANUP-distill-to-core.md
docs/cli-skills-map.md
docs/contracts/agent-native-mechanism.md
docs/contracts/agents-write-surfaces.md
docs/doctrine/effective-feedback-compute.md
docs/domain-practice-packets.md
docs/domains/README.md
docs/examples/agentops-3-council-verdict-example.md
docs/examples/agentops-3-domain-practice-packet.md
docs/first-value-path.md
docs/getting-started/index.md
docs/GLOSSARY.md
docs/how-it-works.md
docs/knowledge-flywheel.md
docs/positioning/12-factors-validation-lens.md
docs/positioning/hn-devto-article.md
docs/reference/agentops-domain-evolution-plan.md
docs/release-e2e-checklist.md
docs/RELEASING.md
docs/ROADMAP.md
docs/runbooks/autonomy-runtime-cycle-1.md
docs/runbooks/nightly-evolution.md
docs/seed-definition.md
docs/SKILLS.md
docs/software-factory.md
docs/workflows/session-lifecycle.md
+228
View File
@@ -0,0 +1,228 @@
#!/usr/bin/env bash
# check-docs-cli-snippets.sh
#
# Resolve every `ao …` command cited in a LIVE doc against the live cobra tree
# and fail if a doc names a command that does not exist (a removed/renamed
# command like `ao factory start`, `ao rpi phased`, `ao evolve` on a golden
# path). Port-style sibling of scripts/validate-skill-cli-snippets.sh — it
# SHARES that gate's resolution core (scripts/lib/ao-snippet-resolve.*) rather
# than forking it (age-gate-the-ungated-egwt.4).
#
# Scope = the shared LIVE-doc set (scripts/lib/docs-scope.sh: docs/**/*.md minus
# dated archives and self-declared-historical docs). Extraction covers BOTH
# fenced code blocks and inline code spans; plain prose is NOT scanned (the
# false-positive guard).
#
# Resolution is SOUND: unlike the skills gate's byte-identical `help`-mode
# predicate, this uses `ao <chain> --help` and rejects cobra's "unknown command"
# / "Unknown help topic" — because `ao help <anything>` ALWAYS exits 0, so the
# help-mode predicate cannot detect a removed command at all. The archive-tagged
# build (`-tags "flywheel legacy"`) keeps archived-but-revivable commands
# resolvable, so this only flags TRULY removed commands.
#
# Baseline ratchet (scripts/.docs-cli-snippets-baseline): FILENAME-pinned, seeds
# every current offender. Two-way enforcement:
# (a) a NON-baselined live doc with a dead ao ref → exit 1
# (b) a baselined file that no longer triggers ANY finding → exit 1 (prune it)
# Allowlists only ever shrink.
#
# Exit: 0 clean · 1 offender / stale-baseline · 2 usage/setup error
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
# Shared machinery. Resolve via the pre-cd absolutized $SCRIPT_DIR (a relative
# ${BASH_SOURCE[0]} would resolve wrongly after a cd — the .1 lane's pawl catch).
# shellcheck source=scripts/lib/docs-scope.sh
. "$SCRIPT_DIR/lib/docs-scope.sh"
# shellcheck source=scripts/lib/ao-snippet-resolve.sh
. "$SCRIPT_DIR/lib/ao-snippet-resolve.sh"
# Pin the docs scope root to THIS repo (the DOCS_ROOT env seam is for lib tests).
DOCS_ROOT="$ROOT"
export DOCS_ROOT
BASELINE="${DOCS_CLI_SNIPPETS_BASELINE:-$ROOT/scripts/.docs-cli-snippets-baseline}"
# Build (or reuse) the archive-tagged ao binary; sets + exports AO_BIN.
ao_snippet_resolve_bin "$ROOT" >/dev/null
if [[ -n "${AO_SNIPPET_TMP_DIR:-}" ]]; then
trap 'rm -rf "$AO_SNIPPET_TMP_DIR"' EXIT
fi
export REPO_ROOT="$ROOT"
export AO_RESOLVE_MODE=strict
export DOCS_CLI_BASELINE="$BASELINE"
python3 - <<'PY'
import os
import pathlib
import re
import shlex
import subprocess
import sys
sys.path.insert(0, os.environ["AO_SNIPPET_LIB_DIR"])
from ao_snippet_resolve import iter_snippets, make_resolver_from_env
repo_root = pathlib.Path(os.environ["REPO_ROOT"])
docs_root = pathlib.Path(os.environ["DOCS_ROOT"])
baseline_path = pathlib.Path(os.environ["DOCS_CLI_BASELINE"])
resolver = make_resolver_from_env()
# ---- live-doc scope + historical exemption (shared bash lib, via subprocess) --
def _sh(func):
"""Call a docs-scope.sh function and return (rc, stdout)."""
lib = pathlib.Path(os.environ["AO_SNIPPET_LIB_DIR"]) / "docs-scope.sh"
script = f'. "{lib}"; {func}'
r = subprocess.run(["bash", "-c", script], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True,
env={**os.environ})
return r.returncode, r.stdout
def live_files():
_, out = _sh("docs_scope_live_files")
return [line for line in out.splitlines() if line.strip()]
def is_exempt(f):
rc, _ = _sh(f'docs_scope_is_exempt "{f}"')
return rc == 0
# ---- nearest-live-command suggestion -----------------------------------------
def _levenshtein(a, b):
if a == b:
return 0
if not a:
return len(b)
if not b:
return len(a)
prev = list(range(len(b) + 1))
for i, ca in enumerate(a, 1):
cur = [i]
for j, cb in enumerate(b, 1):
cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
prev = cur
return prev[-1]
def _top_level_commands():
# Parse `ao --help` for the visible top-level command names.
_, out = resolver._probe([])
names = []
for line in out.splitlines():
m = re.match(r"^ ([a-z][a-z0-9-]+)\s{2,}\S", line)
if m:
names.append(m.group(1))
return sorted(set(names))
_TOP = None
def suggest(token):
global _TOP
if _TOP is None:
_TOP = _top_level_commands()
if not _TOP:
return None
best = min(_TOP, key=lambda c: _levenshtein(token, c))
# only suggest a genuinely-near match
if _levenshtein(token, best) <= max(2, len(token) // 2):
return best
return None
# A line that DESCRIBES a command's removal (negation / past-tense) is documenting
# the retirement, not prescribing the dead command — not an offender. Mirrors the
# REMOVAL_LANG exemption in check-docs-no-retired-tech.sh.
_REMOVAL_LANG = re.compile(
r"no `ao|removed|retired|deleted|deprecat|superseded|no longer|is gone|are gone|"
r"not a (?:selectable|valid)|deprecation pointer|gets? a deprecation",
re.IGNORECASE,
)
# ---- resolve one snippet; return the offending token or None -----------------
def offending_token(snippet):
try:
tokens = shlex.split(snippet)
except ValueError:
return None
tokens = resolver.trim_shell_tokens(tokens)
if not tokens or tokens[0] != "ao":
return None
if resolver.is_regex_like(tokens):
return None
command, _ = resolver.resolve_command(tokens)
if command:
return None
if len(tokens) == 1:
return None
if all(t.startswith("-") for t in tokens[1:]):
return None # all-flags form (e.g. `ao --version`); flags not scoped here
# first wordish subcommand token is the offending name
first = tokens[1]
# skip placeholders / non-command tokens (e.g. `ao ...`, `ao <bead>`)
if not re.match(r"^[a-z][a-z0-9-]*$", first):
return None
return first
# ---- scan ---------------------------------------------------------------------
findings = {} # file -> list of (lineno, snippet, token, suggestion)
triggered = set() # files with >=1 finding
for f in live_files():
if is_exempt(f):
continue
p = docs_root / f
try:
text = p.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
continue
lines = text.splitlines()
for lineno, snippet in iter_snippets(text):
tok = offending_token(snippet)
if tok is None:
continue
src_line = lines[lineno - 1] if 0 <= lineno - 1 < len(lines) else ""
if _REMOVAL_LANG.search(src_line):
continue # describing the removal, not prescribing the dead command
findings.setdefault(f, []).append((lineno, snippet, tok, suggest(tok)))
triggered.add(f)
# ---- baseline ratchet ---------------------------------------------------------
baselined = set()
if baseline_path.exists():
for line in baseline_path.read_text(encoding="utf-8").splitlines():
s = line.strip()
if not s or s.startswith("#"):
continue
baselined.add(s)
new_offenders = sorted(triggered - baselined)
stale_baseline = sorted(baselined - triggered)
failed = False
if new_offenders:
failed = True
print("check-docs-cli-snippets: FAIL — live doc(s) cite a removed/unknown ao command:", file=sys.stderr)
for f in new_offenders:
for lineno, snippet, tok, sug in findings[f]:
hint = f" (did you mean `ao {sug}`?)" if sug else ""
print(f" {f}:{lineno}: unknown ao command `ao {tok}` in `{snippet}`{hint}", file=sys.stderr)
print("", file=sys.stderr)
print("Fix the dead ao reference (use the live equivalent, or historical wording), "
"or — only if the page is a dyr0-lane golden path — add it to "
f"{baseline_path.relative_to(repo_root)}.", file=sys.stderr)
if stale_baseline:
failed = True
print("check-docs-cli-snippets: FAIL — baseline entr(ies) no longer trigger any finding (prune them):", file=sys.stderr)
for f in stale_baseline:
print(f" {f}", file=sys.stderr)
print("", file=sys.stderr)
print(f"The allowlist only shrinks. Remove the above line(s) from "
f"{baseline_path.relative_to(repo_root)}.", file=sys.stderr)
if failed:
sys.exit(1)
n_baselined = len(baselined & triggered)
print(f"check-docs-cli-snippets: PASS — no un-baselined live doc cites a removed ao command "
f"({len(triggered)} file(s) with findings, all baselined; {n_baselined} baseline entr(ies) still active).")
PY
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
# shellcheck shell=bash
# scripts/lib/ao-snippet-resolve.sh — sourced library: shared `ao` snippet
# resolution machinery for the skills-tree gate and the docs-tree gate.
#
# Source it (do NOT execute it):
# . "$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/ao-snippet-resolve.sh"
# OR (from a script that has already absolutized its own dir):
# . "$SCRIPT_DIR/lib/ao-snippet-resolve.sh"
#
# Extracted from scripts/validate-skill-cli-snippets.sh (age-gate-the-ungated-egwt.4)
# so both snippet gates resolve `ao` commands against the SAME live cobra tree
# from one place ("share the core, don't fork it"). The Python resolution core
# lives beside this file as scripts/lib/ao_snippet_resolve.py; this bash lib owns
# the build machinery + exposes the module directory so callers' inline Python
# can import it.
#
# Behavior-preserving contract: this lib does NOT `set -euo pipefail` on behalf
# of its callers (the extracting script sets strict mode itself). The functions
# are pure/idempotent under either mode.
# ao_snippet_lib_dir — absolute path to this lib's directory (holds the Python
# module). Resolved via BASH_SOURCE at source time; safe even if the caller cd's
# afterward, because we absolutize immediately.
ao_snippet_lib_dir() {
# shellcheck disable=SC1007 # `CDPATH=` clears CDPATH for this one cd (env-prefix, not an assignment)
CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd
}
# AO_SNIPPET_LIB_DIR — module dir, computed once at source time (pre-cd-safe).
AO_SNIPPET_LIB_DIR="$(ao_snippet_lib_dir)"
export AO_SNIPPET_LIB_DIR
# ao_snippet_resolve_bin [REPO_ROOT] — echo the path to an `ao` binary suitable
# for snippet resolution and export it as AO_BIN.
#
# Honors an already-set AGENTOPS_AO_BIN (fast path for CI / tests). Otherwise
# builds `ao` from <REPO_ROOT>/cli with the ADR-0012 archive tags
# (`-tags "flywheel legacy"`) so snippets that document archived-but-revivable
# commands (e.g. `ao harvest`, `ao forge`, behind //go:build flywheel|legacy)
# still resolve — the default spine build omits them and would false-fail those
# (two prior escapes). The built binary lands in a mktemp dir; the CALLER is
# responsible for its own EXIT trap cleanup if it wants the temp removed.
#
# REPO_ROOT defaults to the parent of this lib's dir's parent (…/scripts/lib →
# repo root), matching the historical layout.
ao_snippet_resolve_bin() {
local repo_root="${1:-}"
if [[ -z "$repo_root" ]]; then
# shellcheck disable=SC1007 # `CDPATH=` clears CDPATH for this one cd (env-prefix, not an assignment)
repo_root="$(CDPATH= cd "$AO_SNIPPET_LIB_DIR/../.." && pwd)"
fi
local ao_bin="${AGENTOPS_AO_BIN:-}"
if [[ -z "$ao_bin" ]]; then
local tmp_dir
tmp_dir="$(mktemp -d)"
ao_bin="$tmp_dir/ao"
(
cd "$repo_root/cli" || exit 1
go build -tags "flywheel legacy" -o "$ao_bin" ./cmd/ao
)
# Surface the temp dir so the caller can trap-clean it.
AO_SNIPPET_TMP_DIR="$tmp_dir"
export AO_SNIPPET_TMP_DIR
fi
[[ -x "$ao_bin" ]] || {
echo "Missing or non-executable ao binary: $ao_bin" >&2
return 1
}
AO_BIN="$ao_bin"
export AO_BIN
printf '%s\n' "$ao_bin"
}
+164
View File
@@ -0,0 +1,164 @@
"""ao_snippet_resolve — shared resolution core for `ao …` CLI snippets in prose.
Extracted from scripts/validate-skill-cli-snippets.sh (age-gate-the-ungated-egwt.4)
so that BOTH the skills-tree snippet gate and the docs-tree snippet gate resolve
`ao` commands against the SAME live cobra tree with ONE piece of machinery — the
"share the core, don't fork it" contract.
What lives here (the reusable core):
* snippet extraction from a line (backtick spans + a bare `ao …` line)
* shell-token trimming at control operators / redirects
* the longest-leading-wordish-run candidate resolution against `ao`'s help tree
* flag checking against the resolved command's help text
What is INJECTED by the caller (the one axis the two gates differ on):
* the *resolution predicate* — "does this command chain name a real command?"
- RESOLVE_MODE="help" → the ORIGINAL skills-validator semantics:
`ao help <chain>` and trust returncode == 0.
(Byte-for-byte behavior-preserving for that gate.)
- RESOLVE_MODE="strict" → a SOUND check for the docs gate:
`ao <chain> --help` and reject output that carries
cobra's "unknown command" / "Unknown help topic".
The `help`-mode predicate is unsound here because
`ao help <anything>` ALWAYS exits 0 (even for a
removed command like `ao factory` / `ao rpi`), so it
cannot detect the dead-command class the docs gate
exists to kill.
The AO binary path comes from the AO_BIN env var (set by the bash driver, which
builds `ao` with `-tags "flywheel legacy"` so archived-but-revivable commands
still resolve — two prior escapes: the default spine build omits them).
"""
import os
import re
import shlex
import subprocess
_WORDISH = re.compile(r"^[a-z][a-z0-9-]*$")
_CONTROL_TOKENS = {"|", "||", "&&", ";", "&"}
_UNKNOWN_CMD = re.compile(r"unknown command|Unknown help topic", re.IGNORECASE)
class Resolver:
"""Resolve `ao` command chains against the live cobra tree.
mode: "help" (byte-identical skills-validator semantics) or "strict"
(sound `--help`-based check for the docs gate).
"""
def __init__(self, ao_bin, mode="help"):
self.ao_bin = ao_bin
if mode not in ("help", "strict"):
raise ValueError(f"unknown resolve mode: {mode!r}")
self.mode = mode
self._help_cache = {}
# ---- resolution primitive (the one injected axis) ---------------------
def _probe(self, command):
"""Return (ok, help_text). `ok` is True when `command` names a real
command chain under `ao`. help_text is the command's help output (used
for flag validation) — for the `help` mode it is exactly the original
validator's `ao help <chain>` stdout so flag checks are unchanged."""
key = tuple(command)
if key in self._help_cache:
return self._help_cache[key]
if self.mode == "help":
result = subprocess.run(
[self.ao_bin, "help", *command],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
ok = result.returncode == 0
out = result.stdout
else: # strict
result = subprocess.run(
[self.ao_bin, *command, "--help"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
ok = not _UNKNOWN_CMD.search(result.stdout)
out = result.stdout
self._help_cache[key] = (ok, out)
return ok, out
def global_help(self):
# Global help is always the root command's help; identical in both modes
# (`ao help` == `ao --help` content for flag lookups).
return self._probe([])[1]
# ---- shared machinery -------------------------------------------------
@staticmethod
def trim_shell_tokens(tokens):
trimmed = []
for token in tokens:
if token in _CONTROL_TOKENS:
break
if token.startswith(("|", ">", "<")):
break
if token.endswith((";", "&&", "||")):
trimmed.append(token.rstrip(";"))
break
trimmed.append(token)
return [token for token in trimmed if token]
def resolve_command(self, tokens):
"""Return (command, help_text) for the longest leading run of wordish
subcommand tokens that names a real command, else (None, None)."""
candidates = []
for token in tokens[1:]:
if token.startswith("-"):
break
if any(ch in token for ch in "<>[]{}=$("):
break
if not _WORDISH.match(token):
break
candidates.append(token)
for end in range(len(candidates), 0, -1):
candidate = candidates[:end]
ok, help_text = self._probe(candidate)
if ok:
return candidate, help_text
return None, None
@staticmethod
def normalize_flag(token):
if "=" in token:
token = token.split("=", 1)[0]
return token
@staticmethod
def is_regex_like(tokens):
return any(re.search(r"[\[\]\(\)\^\*\+\?]", token) for token in tokens[1:])
def iter_snippets(text):
"""Yield (lineno, snippet) for every `ao …` command in text.
Covers BOTH backtick-delimited code spans (inline code) AND a line whose
stripped form starts with `ao ` (a bare fenced-code-block command line) —
the same extraction the skills validator uses. Plain prose that merely
mentions "ao" without a code span or a leading `ao ` is NOT scanned (the
false-positive guard the docs gate depends on).
"""
for lineno, line in enumerate(text.splitlines(), start=1):
if "ao " not in line:
continue
snippets = []
for match in re.finditer(r"`([^`]*\bao\b[^`]*)`", line):
snippets.append(match.group(1).strip())
stripped = line.strip()
if stripped.startswith("ao "):
snippets.append(stripped)
for snippet in snippets:
yield lineno, snippet
def make_resolver_from_env():
"""Build a Resolver from AO_BIN + AO_RESOLVE_MODE env vars."""
ao_bin = os.environ["AO_BIN"]
mode = os.environ.get("AO_RESOLVE_MODE", "help")
return Resolver(ao_bin, mode=mode)
+28 -106
View File
@@ -3,125 +3,47 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
AO_BIN="${AGENTOPS_AO_BIN:-}"
if [[ -z "$AO_BIN" ]]; then
TMP_DIR="$(mktemp -d)"
trap 'rm -rf "$TMP_DIR"' EXIT
AO_BIN="$TMP_DIR/ao"
(
cd "$REPO_ROOT/cli"
# Build with the ADR-0012 archive tags so snippets that document
# archived-but-revivable commands (e.g. `ao harvest`, `ao forge`, behind
# //go:build flywheel|legacy) still validate against the full command
# surface. The default `ao` omits them, but a skill may legitimately
# reference any command; validating only the spine would false-fail those.
go build -tags "flywheel legacy" -o "$AO_BIN" ./cmd/ao
)
# Shared `ao` snippet resolution machinery (build + resolver core), so this gate
# and the docs.cli-snippets gate resolve against the SAME cobra tree from one
# place. Resolve via the pre-cd absolutized $SCRIPT_DIR (a relative
# ${BASH_SOURCE[0]} would resolve wrongly if a caller cd'd first — the .1 lane's
# round-1 pawl catch).
# shellcheck source=scripts/lib/ao-snippet-resolve.sh
. "$SCRIPT_DIR/lib/ao-snippet-resolve.sh"
# Build (or reuse) the archive-tagged ao binary; sets + exports AO_BIN.
AO_BIN="$(ao_snippet_resolve_bin "$REPO_ROOT")"
# Clean up a lib-built temp binary on exit (no-op when AGENTOPS_AO_BIN was set).
if [[ -n "${AO_SNIPPET_TMP_DIR:-}" ]]; then
trap 'rm -rf "$AO_SNIPPET_TMP_DIR"' EXIT
fi
[[ -x "$AO_BIN" ]] || {
echo "Missing or non-executable ao binary: $AO_BIN" >&2
exit 1
}
export AO_BIN
export REPO_ROOT
# Byte-identical resolution semantics for the skills gate: `ao help <chain>`,
# trust returncode==0 (the original validator's predicate).
export AO_RESOLVE_MODE=help
python3 - <<'PY'
import os
import pathlib
import re
import shlex
import subprocess
import sys
# Import the shared resolution core (extracted to scripts/lib/).
sys.path.insert(0, os.environ["AO_SNIPPET_LIB_DIR"])
from ao_snippet_resolve import iter_snippets, make_resolver_from_env
repo_root = pathlib.Path(os.environ["REPO_ROOT"])
ao_bin = os.environ["AO_BIN"]
roots = [repo_root / "skills", repo_root / "skills-codex"]
allowed_suffixes = {".md", ".sh"}
wordish = re.compile(r"^[a-z][a-z0-9-]*$")
control_tokens = {"|", "||", "&&", ";", "&"}
stale_beads_resolver = re.compile(r"BEADS_DIR=\$PWD/_beads|git -C _beads|git add \.beads|git add _beads")
stale_beads_allowed = re.compile(r"\b(anti-pattern|do not|don't|must not|never|reject|fails?|historical|retired)\b", re.IGNORECASE)
resolver = make_resolver_from_env()
failures = []
help_cache = {}
def iter_snippets(path: pathlib.Path):
try:
text = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
return
for lineno, line in enumerate(text.splitlines(), start=1):
if "ao " not in line:
continue
snippets = []
for match in re.finditer(r"`([^`]*\bao\b[^`]*)`", line):
snippets.append(match.group(1).strip())
stripped = line.strip()
if stripped.startswith("ao "):
snippets.append(stripped)
for snippet in snippets:
yield lineno, snippet
def command_help(command):
key = tuple(command)
if key not in help_cache:
result = subprocess.run(
[ao_bin, "help", *command],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
help_cache[key] = result
return help_cache[key]
def global_help():
return command_help([])
def trim_shell_tokens(tokens):
trimmed = []
for token in tokens:
if token in control_tokens:
break
if token.startswith(("|", ">", "<")):
break
if token.endswith((";", "&&", "||")):
trimmed.append(token.rstrip(";"))
break
trimmed.append(token)
return [token for token in trimmed if token]
def resolve_command(tokens):
candidates = []
for token in tokens[1:]:
if token.startswith("-"):
break
if any(ch in token for ch in "<>[]{}=$("):
break
if not wordish.match(token):
break
candidates.append(token)
for end in range(len(candidates), 0, -1):
candidate = candidates[:end]
result = command_help(candidate)
if result.returncode == 0:
return candidate, result.stdout
return None, None
def normalize_flag(token):
if "=" in token:
token = token.split("=", 1)[0]
return token
def is_regex_like(tokens):
return any(re.search(r"[\[\]\(\)\^\*\+\?]", token) for token in tokens[1:])
def validate_snippet(path: pathlib.Path, lineno: int, snippet: str):
try:
@@ -129,21 +51,21 @@ def validate_snippet(path: pathlib.Path, lineno: int, snippet: str):
except ValueError:
return
tokens = trim_shell_tokens(tokens)
tokens = resolver.trim_shell_tokens(tokens)
if not tokens or tokens[0] != "ao":
return
if is_regex_like(tokens):
if resolver.is_regex_like(tokens):
return
command, help_text = resolve_command(tokens)
command, help_text = resolver.resolve_command(tokens)
if not command:
if len(tokens) == 1:
return
if all(token.startswith("-") for token in tokens[1:]):
help_text = global_help().stdout
help_text = resolver.global_help()
for flag in tokens[1:]:
normalized = normalize_flag(flag)
normalized = resolver.normalize_flag(flag)
if normalized not in help_text:
failures.append(
f"{path.relative_to(repo_root)}:{lineno}: flag {normalized} not found in help for ao"
@@ -158,7 +80,7 @@ def validate_snippet(path: pathlib.Path, lineno: int, snippet: str):
continue
if len(token) > 1 and token[1:].isdigit():
continue
flags.append(normalize_flag(token))
flags.append(resolver.normalize_flag(token))
for flag in flags:
if flag not in help_text:
failures.append(
@@ -182,7 +104,7 @@ for root in roots:
failures.append(
f"{path.relative_to(repo_root)}:{lineno}: stale beads resolver; use BEADS_DIR=\"$(ao beads dir)\" and git -C \"$(ao beads dir)\""
)
for lineno, snippet in iter_snippets(path):
for lineno, snippet in iter_snippets(text):
validate_snippet(path, lineno, snippet)
if failures:
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env bats
#
# Tests for scripts/check-docs-cli-snippets.sh — the docs.cli-snippets gate that
# resolves every `ao …` command cited in a LIVE doc against the live cobra tree
# and fails on a removed/renamed command, with a FILENAME-pinned shrink-only
# baseline (age-gate-the-ungated-egwt.4).
#
# Pattern: build a fixture "repo" in BATS_TEST_TMPDIR that carries its own copy
# of the check script + shared libs (so the script's ROOT resolves to the
# fixture, exactly as docs-scope.bats does), seed a docs/ tree + a baseline, and
# assert the two-way ratchet. The archive-tagged `ao` binary is built once and
# injected via AGENTOPS_AO_BIN (the script's documented fast path).
setup_file() {
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)"
export REPO_ROOT
# Build the archive-tagged ao once for the whole file; inject via env.
AO_BIN_FILE="$BATS_FILE_TMPDIR/ao"
( cd "$REPO_ROOT/cli" && go build -tags "flywheel legacy" -o "$AO_BIN_FILE" ./cmd/ao )
export AGENTOPS_AO_BIN="$AO_BIN_FILE"
}
setup() {
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)"
export AGENTOPS_AO_BIN="$BATS_FILE_TMPDIR/ao"
# Fixture repo: script + shared libs, so ROOT="$SCRIPT_DIR/.." == the fixture.
FIX="$BATS_TEST_TMPDIR/repo"
mkdir -p "$FIX/scripts/lib" "$FIX/docs" "$FIX/cli"
cp "$REPO_ROOT/scripts/check-docs-cli-snippets.sh" "$FIX/scripts/"
cp "$REPO_ROOT/scripts/lib/docs-scope.sh" "$FIX/scripts/lib/"
cp "$REPO_ROOT/scripts/lib/ao-snippet-resolve.sh" "$FIX/scripts/lib/"
cp "$REPO_ROOT/scripts/lib/ao_snippet_resolve.py" "$FIX/scripts/lib/"
chmod +x "$FIX/scripts/check-docs-cli-snippets.sh"
export FIX
BASELINE="$FIX/scripts/.docs-cli-snippets-baseline"
export BASELINE
}
# Run the fixture's check with a given baseline content ($1 = baseline lines,
# may be empty). Populates $status / $output.
run_check() {
printf '%s' "$1" > "$BASELINE"
run env DOCS_CLI_SNIPPETS_BASELINE="$BASELINE" bash "$FIX/scripts/check-docs-cli-snippets.sh"
}
# ---- dead-command detection --------------------------------------------------
@test "a live doc citing a removed ao command FAILS, naming file + token" {
printf '# Guide\n\nRun `ao factory start --goal "x"` to begin.\n' > "$FIX/docs/guide.md"
run_check ""
[ "$status" -eq 1 ]
[[ "$output" == *"docs/guide.md"* ]]
[[ "$output" == *"ao factory"* ]]
}
@test "a live doc citing only LIVE ao commands PASSES" {
printf '# Guide\n\nRun `ao gate check --fast` and `ao session bootstrap`.\n' > "$FIX/docs/guide.md"
run_check ""
[ "$status" -eq 0 ]
[[ "$output" == *"PASS"* ]]
}
@test "extraction covers a fenced code block line, not just inline spans" {
{ printf '# Guide\n\n'; printf '```bash\n'; printf 'ao rpi phased "goal"\n'; printf '```\n'; } > "$FIX/docs/guide.md"
run_check ""
[ "$status" -eq 1 ]
[[ "$output" == *"docs/guide.md"* ]]
[[ "$output" == *"ao rpi"* ]]
}
@test "plain prose mentioning ao (no code span, no leading 'ao ') is NOT scanned" {
printf '# Guide\n\nThe factory command was removed; use the loop instead.\n' > "$FIX/docs/guide.md"
run_check ""
[ "$status" -eq 0 ]
}
@test "a line DESCRIBING a command removal is exempt (removal-lang)" {
printf '# Guide\n\nThere is no `ao hooks` command any more; 3.0 is hookless.\n' > "$FIX/docs/guide.md"
run_check ""
[ "$status" -eq 0 ]
}
# ---- banner / historical exemption (shared docs-scope lib) --------------------
@test "a doc with a RETIRED banner in the first 15 lines is exempt from scanning" {
printf '# Old Runbook (RETIRED)\n\nRun `ao factory start` here (historical).\n' > "$FIX/docs/old.md"
run_check ""
[ "$status" -eq 0 ]
}
# ---- baseline ratchet (two-way) ----------------------------------------------
@test "a baselined offender PASSES (allowlisted)" {
printf '# Guide\n\nRun `ao factory start --goal "x"`.\n' > "$FIX/docs/guide.md"
run_check $'docs/guide.md\n'
[ "$status" -eq 0 ]
[[ "$output" == *"PASS"* ]]
}
@test "a NON-baselined offender FAILS even when other files ARE baselined" {
printf '# A\n\nRun `ao factory start`.\n' > "$FIX/docs/a.md"
printf '# B\n\nRun `ao rpi phased`.\n' > "$FIX/docs/b.md"
run_check $'docs/a.md\n' # only a.md allowlisted; b.md is a new offender
[ "$status" -eq 1 ]
[[ "$output" == *"docs/b.md"* ]]
}
@test "a STALE baseline entry (no longer triggers any finding) FAILS demanding prune" {
printf '# Clean\n\nRun `ao gate check --fast`.\n' > "$FIX/docs/clean.md"
run_check $'docs/clean.md\n' # clean.md has no dead command, but is baselined
[ "$status" -eq 1 ]
[[ "$output" == *"no longer trigger"* ]]
[[ "$output" == *"docs/clean.md"* ]]
}
@test "a baseline entry for a DELETED file is stale and FAILS" {
# No docs/gone.md exists at all.
printf '# Live\n\nRun `ao lookup --query x`.\n' > "$FIX/docs/live.md"
run_check $'docs/gone.md\n'
[ "$status" -eq 1 ]
[[ "$output" == *"docs/gone.md"* ]]
}
# ---- the REAL repo passes with its seeded baseline ---------------------------
@test "the real repo passes with its committed baseline (gate lands green)" {
run env AGENTOPS_AO_BIN="$BATS_FILE_TMPDIR/ao" bash "$REPO_ROOT/scripts/check-docs-cli-snippets.sh"
[ "$status" -eq 0 ]
[[ "$output" == *"PASS"* ]]
}