From 7dce71599871907efa5e2234d0344b775ff344d3 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:47:52 -0400 Subject: [PATCH] =?UTF-8?q?feat(gates):=20docs.demoted-claims=20honesty=20?= =?UTF-8?q?gate=20=E2=80=94=20banned=20unproven-claim=20lexicon=20over=20l?= =?UTF-8?q?ive=20docs=20(age-gate-the-ungated-egwt.6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cli/internal/gates/checks/seed.go | 11 ++ docs/agentops-brief.md | 4 +- docs/architecture/canonical-loop-model.md | 2 +- docs/brownian-ratchet.md | 8 +- docs/origin-story.md | 18 +- docs/philosophy.md | 12 +- docs/profiles/META_PATTERNS.md | 8 +- scripts/.docs-demoted-claims-baseline | 25 +++ scripts/check-docs-demoted-claims.sh | 181 +++++++++++++++++++ tests/scripts/check-docs-demoted-claims.bats | 176 ++++++++++++++++++ 10 files changed, 421 insertions(+), 24 deletions(-) create mode 100644 scripts/.docs-demoted-claims-baseline create mode 100755 scripts/check-docs-demoted-claims.sh create mode 100644 tests/scripts/check-docs-demoted-claims.bats diff --git a/cli/internal/gates/checks/seed.go b/cli/internal/gates/checks/seed.go index 743383d6e..53a49655e 100644 --- a/cli/internal/gates/checks/seed.go +++ b/cli/internal/gates/checks/seed.go @@ -77,6 +77,16 @@ var ( "scripts/check-workflow-no-retired-tracker.sh", "tests/scripts/check-workflow-no-retired-tracker.bats", } + // ADR-0004 / ADR-0011 demoted the corpus-moat + escape-corpus-compounding + // claims to a named UNPROVEN hypothesis; live narrative docs must not assert + // them (or ship an uncited multiplier / peer-review claim) as proven. Runs on + // any docs change plus self-reference (script / baseline / bats). + demotedClaimsPaths = []string{ + "docs/**", + "scripts/check-docs-demoted-claims.sh", + "scripts/.docs-demoted-claims-baseline", + "tests/scripts/check-docs-demoted-claims.bats", + } archDocDriftPaths = []string{ "docs/architecture/ports-and-adapters.md", "docs/contracts/bounded-contexts.yaml", @@ -240,6 +250,7 @@ func init() { {ID: "always.sovereignty-proof-citations", Tiers: gates.Full, Match: docsPaths, Blocking: true, Backing: "validate-sovereignty-proof-citations.sh"}, {ID: "docs.no-retired-tech", Tiers: gates.Fast | gates.Full, Match: docsPaths, Blocking: true, Backing: "check-docs-no-retired-tech.sh", RepairHint: "convert to current truth, or add a RETIRED/HISTORICAL banner in the first 15 lines; see scripts/check-docs-no-retired-tech.sh"}, {ID: "docs.cli-snippets", Tiers: gates.Full, Match: docsCliSnippetsPaths, Blocking: false, Backing: "check-docs-cli-snippets.sh", RepairHint: "fix the dead ao reference or prune the stale baseline entry; flips Blocking after one clean advisory cycle (age-gate-the-ungated-egwt.4)"}, + {ID: "docs.demoted-claims", Tiers: gates.Full, Match: demotedClaimsPaths, Blocking: false, Backing: "check-docs-demoted-claims.sh", RepairHint: "hedge the claim to match ADR-0004/ADR-0011 or add a citation; advisory one clean cycle then flips Blocking (age-gate-the-ungated-egwt.6)"}, {ID: "corpus.secret-scan", Tiers: gates.Full, Match: corpusPaths, Blocking: true, Backing: "check-corpus-secret-scan.sh"}, {ID: "corpus.witness-dolt-jsonl-crosscheck", Tiers: gates.Full, Match: corpusPaths, Blocking: true, Backing: "witness-dolt-jsonl-crosscheck.sh"}, {ID: "doctrine.memrl-health", Tiers: gates.Full, Blocking: true, Backing: "check-memrl-health.sh"}, diff --git a/docs/agentops-brief.md b/docs/agentops-brief.md index fe893c736..f6333ae91 100644 --- a/docs/agentops-brief.md +++ b/docs/agentops-brief.md @@ -4,7 +4,7 @@ AgentOps is an SDLC control plane for agentic software development. It keeps the `.agents/` is the substrate: a wiki of markdown files in your repo, version-controlled with your code, that agents read, traverse, and contribute to. The kind of wiki your team should already have. AgentOps automates the discipline of building one. -*The only verifiable moat in this uncertain time is context. Models will get smarter, harnesses will commoditize, agents will get cheaper. Your accumulated context — the lessons learned about your individual problems, the patterns that worked, the decisions that survived review — is the one asset that compounds and doesn't get eaten by the next vendor release. That's what your company actually is.* +*The bet in this uncertain time is that context is the thing that lasts. Models will get smarter, harnesses will commoditize, agents will get cheaper. Your accumulated context — the lessons learned about your individual problems, the patterns that worked, the decisions that survived review — is the asset least likely to get eaten by the next vendor release. That it compounds into a durable **moat** is an [explicitly unproven hypothesis](adr/ADR-0004-corpus-moat-unproven-position-on-the-system.md), not a settled claim; the proven asset is the verification loop. Still — that context is what your company actually is.* AgentOps is the shovel. Start digging. @@ -120,7 +120,7 @@ three-gap contract remains the internal proof model. ## What if the labs ship this natively? -They will. Anthropic's Managed Agents is the first move; others will follow. That's fine — the value isn't in this tool, it's in the corpus you build with it. AgentOps is bridge infrastructure: your `.agents/` directory is plain markdown in your repo, so if a frontier vendor ships native equivalents in 12 months, your corpus carries forward unchanged. +They will. Anthropic's Managed Agents is the first move; others will follow. That's fine — the durable value is meant to live in the corpus you build, not the tool (whether the corpus *compounds* into that durable value is the [unproven hypothesis](adr/ADR-0011-escape-corpus-compounding-unproven-structural-starvation.md), not a guarantee). AgentOps is bridge infrastructure: your `.agents/` directory is plain markdown in your repo, so if a frontier vendor ships native equivalents in 12 months, your corpus carries forward unchanged. --- diff --git a/docs/architecture/canonical-loop-model.md b/docs/architecture/canonical-loop-model.md index 3b875422e..71f104ec3 100644 --- a/docs/architecture/canonical-loop-model.md +++ b/docs/architecture/canonical-loop-model.md @@ -64,7 +64,7 @@ A loop tick takes context in and emits context out. The exhaust of one tick is t - **Out:** evidence, decisions, citations, and verdicts land in `.agents/` and on the bead, under the promotion ratchet. - **`ao compile`** rebuilds the corpus periodically so the next inject is fresh. -Context never flows through loop plumbing return values; it flows through the corpus and the bead. That is why the corpus is the moat: it is the compounding artifact, not a byproduct. +Context never flows through loop plumbing return values; it flows through the corpus and the bead. That is why the corpus is treated as the central artifact rather than a byproduct. Whether it compounds into a *moat* is an [explicitly unproven hypothesis](../adr/ADR-0004-corpus-moat-unproven-position-on-the-system.md) — the architecture bets on it, but the proven asset is the verification loop, not the moat. ## The DDD seam: what owns the loop versus what orchestrates it diff --git a/docs/brownian-ratchet.md b/docs/brownian-ratchet.md index a23eece52..039fb132c 100644 --- a/docs/brownian-ratchet.md +++ b/docs/brownian-ratchet.md @@ -390,9 +390,11 @@ Without the flywheel, the ratchet doesn't accelerate: - Flywheel amplifies future cycles - System accelerates over time -### The Compounding Effect +### The Compounding Effect (hypothesis, not a measurement) -| Cycle | Ratchet Output | Flywheel State | Effective Speed | +The table below is an **illustrative model, not measured data**. The specific multipliers are fabricated for shape; whether the flywheel actually compounds this way is an [explicitly unproven hypothesis facing a structural data-starvation headwind](adr/ADR-0011-escape-corpus-compounding-unproven-structural-starvation.md). Do not cite these numbers as evidence. + +| Cycle | Ratchet Output | Flywheel State | Illustrative Speed (model, uncited) | |-------|----------------|----------------|-----------------| | 1 | 10 units | baseline | 1.0x | | 2 | 10 units | +patterns | 1.2x | @@ -400,7 +402,7 @@ Without the flywheel, the ratchet doesn't accelerate: | 4 | 10 units | +memories | 1.6x | | N | 10 units | compounded | 2.0x+ | -The ratchet produces constant raw output. The flywheel multiplies it. +The ratchet produces constant raw output. *If* the flywheel hypothesis holds, it multiplies that output — that is the claim under test, not a settled result. ### Practical Integration diff --git a/docs/origin-story.md b/docs/origin-story.md index 8f01ec7ea..fb272d336 100644 --- a/docs/origin-story.md +++ b/docs/origin-story.md @@ -62,9 +62,9 @@ The methodology became a product. Knowledge OS became the knowledge flywheel. Th ## What Five Months of Production Taught Us -### The Five Principles (Validated by Evidence) +### The Five Principles (Learned in Production) -Every principle was learned through failure, refined through iteration, and validated against five months of production use. +Every principle was learned through failure and refined through iteration across five months of production use. Read them as hard-won operating heuristics, not as a validated theory of compounding — that larger claim is an [explicitly unproven hypothesis](adr/ADR-0004-corpus-moat-unproven-position-on-the-system.md). **1. Context Timing Beats Context Volume.** The instinct is to front-load everything. This fails. Agents drown in irrelevant context and hallucinate connections. What works: deliver the right context at the right time. AgentOps evolved from monolithic prompt files to skill-scoped references and session intelligence packets. @@ -78,8 +78,8 @@ Agents claim success without running tests. They report "all passing" after part **4. Parallel Agents Need Ownership Boundaries.** File collisions are the #1 swarm failure mode. Without ownership boundaries: ~40% failure rate. With pre-flight file-overlap checks and wave-based execution: near zero. -**5. The Flywheel Is The Product.** -The compounding effect is not "the model gets smarter." It's "your environment gets smarter." Early commits are skill scaffolding. Later commits are meta-capabilities: session intelligence, quality signals, closure integrity audits. The system spends more time improving itself and less time on raw features. +**5. The Environment, Not The Model, Is Where The Work Went.** +The bet is not "the model gets smarter" — it's "your environment gets smarter." (Whether that environment *compounds* into a durable advantage is the [unproven flywheel hypothesis](adr/ADR-0011-escape-corpus-compounding-unproven-structural-starvation.md), not a settled result.) Early commits are skill scaffolding. Later commits are meta-capabilities: session intelligence, quality signals, closure integrity audits. The system spends more time improving itself and less time on raw features. ### The Numbers @@ -154,25 +154,25 @@ Three independent voices arrived at the same place from different directions: **AgentOps** (devlog 1, January 2026): "The job isn't to write code anymore. The job is to build and run AI coding foundries." -Same answer from three directions. The bookkeeping is the moat. None of this is novel — bookkeeping has compounded human knowledge since Sumerian clay tablets. What's new: agents make the bookkeeping nearly free, and the race is on. +Same answer from three directions: the bet is that the bookkeeping is where the durable advantage lives. That the accumulated corpus is a *moat* is the [unproven hypothesis](adr/ADR-0004-corpus-moat-unproven-position-on-the-system.md), not a demonstrated result — but the conviction is old. Bookkeeping has compounded human knowledge since Sumerian clay tablets. What's new: agents make the bookkeeping nearly free, and the race is on. --- ## Where It Goes -The flywheel thesis is validated. The context compiler framing clarifies the architecture. The next phase is structural: +The verification loop is proven; the flywheel thesis (that the corpus *compounds*) remains an [explicitly unproven hypothesis](adr/ADR-0011-escape-corpus-compounding-unproven-structural-starvation.md). The context compiler framing clarifies the architecture. The next phase is structural: - **CLI decomposition** — Extract business logic from the 179-file monolith into domain packages. Make the compiler's pipeline visible in the code structure: ingest → analyze → compile → enforce → validate. - **Compiler maturity** — Expand the finding registry from ~50 entries to hundreds. Every domain the team works in should have compiled enforcement gates. -- **Cross-runtime parity** — The same knowledge compounds regardless of which AI runtime executes the session. +- **Cross-runtime parity** — The same corpus is designed to carry across runtimes, whichever AI executes the session (the compounding benefit of that is the hypothesis under test, not a proven claim). -The formula hasn't changed since October 2025: write things down, organize them, feed them back. The tooling got better. The thesis got proven. The bookkeeping stays the same. +The formula hasn't changed since October 2025: write things down, organize them, feed them back. The tooling got better. The verification loop got proven; the compounding thesis is still under test. The bookkeeping stays the same. --- ## See Also -- [Philosophy](philosophy.md) — The five validated principles in detail +- [Philosophy](philosophy.md) — The five production-learned principles in detail - [How It Works](how-it-works.md) — Brownian Ratchet, Ralph Wiggum Pattern - [Knowledge Flywheel](knowledge-flywheel.md) — The extraction and compounding pipeline - [README](https://github.com/boshu2/agentops/blob/main/README.md) — Quick start and product overview diff --git a/docs/philosophy.md b/docs/philosophy.md index 625ff3059..b04b2070b 100644 --- a/docs/philosophy.md +++ b/docs/philosophy.md @@ -66,7 +66,7 @@ Not every knowledge operation needs a frontier model. AgentOps uses three tiers: The dream loop (retired as a standalone `/dream` command; folded into the out-of-session substrate) uses the local tier when one is configured (otherwise your frontier model) for continuous compounding; when that compounding runs out of session — always-on, scheduled, unattended — it runs on an out-of-session substrate (reference: NTM + MCP + managed-agents), not an AgentOps daemon. `/council` and `/pre-mortem` use the frontier tier for high-stakes validation. The human reviews promotions from learning → finding → rule. -The ratio is intentional. Validation and curation cost 3-5x implementation time. This is not overhead — it is the ratchet. Without it, the flywheel runs backward. +The ratio is intentional. On this repo, validation and curation have run several times the implementation time. This is not overhead — it is the ratchet. Without it, the flywheel runs backward. ## The Ratchet @@ -84,15 +84,17 @@ It is not trying to replace thinking. The model thinks. AgentOps manages what th It is not a SaaS product or a managed service. All state lives locally. All operations are reversible. The product is the compounding environment — the `skills/`, the `ao` CLI, and the discipline enforced by the local pre-push gate. AgentOps 3.0 is hookless: skills and the CLI guide the workflow and the local pre-push Go gate (`ao gate check`) is the release authority (CI is a backstop). Hooks are opt-in — you author your own via the `hooks-authoring` skill — never installed by default. That environment is yours to own, version-control, and take with you. -## The Validated Thesis +## The Thesis, and What Is Actually Proven -As of April 2026, the flywheel thesis is empirically confirmed on a single production repo (this one): +The proven part is the verification: every change reaches *done* only with an independent proof artifact. Whether the knowledge corpus *compounds* over time is a separate, **explicitly-named unproven hypothesis** — demoted to that status in [ADR-0004](adr/ADR-0004-corpus-moat-unproven-position-on-the-system.md) and [ADR-0011](adr/ADR-0011-escape-corpus-compounding-unproven-structural-starvation.md), which found the compounding claim faces a structural data-starvation headwind. Do not read the numbers below as a validated flywheel. + +What this repo has accumulated (descriptive, not a compounding proof): - 163 learnings extracted, scored, and curated - 13 planning rules enforced at pre-mortem gates - 12 patterns promoted from repeated findings - 10/12 `ao doctor` checks passing, with the full CI gate suite green -The compound growth is measurable. Session 1 started cold. Session 100+ starts with a knowledge corpus that catches known failure modes before implementation begins. +Session 1 started cold. Session 100+ starts with a knowledge corpus that *can* catch known failure modes before implementation begins — that the corpus keeps paying off as it grows is the hypothesis, not a settled result. -That is the point. Not a bigger prompt. A repo that remembers. +That is the aim. Not a bigger prompt. A repo that remembers. diff --git a/docs/profiles/META_PATTERNS.md b/docs/profiles/META_PATTERNS.md index 677aa0130..84ca741c9 100644 --- a/docs/profiles/META_PATTERNS.md +++ b/docs/profiles/META_PATTERNS.md @@ -73,7 +73,7 @@ Constitution enforced universally **Example**: - ADHD patterns discovered in personal → Inform 40% rule in sre-devops -- AgentOps Laws proven in research → Apply to web-developer docs +- AgentOps Laws surfaced in research → Apply to web-developer docs --- @@ -154,7 +154,7 @@ Cycle repeats... 5. **researcher**: Analyzes new commit, refines pattern 6. **Loop**: Pattern improves over time through production usage -**Insight**: Knowledge OS is self-improving through role interaction. +**Insight**: Knowledge OS is *designed to* improve through role interaction (whether it self-improves in practice is an unproven hypothesis, per ADR-0011). --- @@ -346,7 +346,7 @@ Total: 113k (56.5%) ⚠️ Over 40% ``` sre-devops (work/gitops/) → Build AgentOps framework ↓ -Git metrics: 40x speedup, 95% success rate +Git metrics: an uncited speedup figure (removed — no citation), 95% success rate ↓ researcher (personal/12-factor-agentops/) → Extract patterns ↓ @@ -378,7 +378,7 @@ Loop repeats... | **sre-devops** | Monitoring, alerts, production health | Prometheus, ArgoCD | Zero downtime, SLO met | | **platform-engineer** | Tests, builds, manifests | `make test-app`, yamllint | CI passes, app deploys | | **web-developer** | Visual testing, cross-browser | Browser DevTools, Lighthouse | Renders correctly, accessible | -| **researcher** | Peer review, production usage | Git history analysis | Pattern adopted in production | +| **researcher** | Colleague review, production usage | Git history analysis | Pattern adopted in production | | **personal** | Career outcomes, opportunities | Resume, interviews, offers | Goal achieved, growth measured | **Insight**: Validation is role-contextual, not universal. diff --git a/scripts/.docs-demoted-claims-baseline b/scripts/.docs-demoted-claims-baseline new file mode 100644 index 000000000..ff5b4f022 --- /dev/null +++ b/scripts/.docs-demoted-claims-baseline @@ -0,0 +1,25 @@ +# .docs-demoted-claims-baseline — FILENAME-pinned baseline for +# scripts/check-docs-demoted-claims.sh (age-gate-the-ungated-egwt.6). +# +# Each non-comment line is a LIVE docs/** path that currently carries a banned +# unproven-claim-lexicon phrase (peer-review-uncited / multiplier-uncited / +# compounding-as-proven). The gate is a SHRINK RATCHET: this list may only get +# smaller. A baselined file that now has ZERO findings FAILS the gate demanding +# a prune; a NON-baselined live doc with a banned phrase FAILS as a regression. +# +# The six long-tail burn-down pages (philosophy, origin-story, brownian-ratchet, +# canonical-loop-model, agentops-brief, META_PATTERNS) were hedged/cited to match +# ADR-0004/ADR-0011 and pruned from this list by .6 itself. What remains: +# - docs/comparisons/** — owned by another lane; never edited here. +# - the-science / trust-factory / standards / workflows — out of .6's scope. +docs/comparisons/competitive-radar.md +docs/comparisons/vs-claude-flow.md +docs/comparisons/vs-gsd.md +docs/comparisons/vs-sdd.md +docs/comparisons/vs-superpowers.md +docs/standards/python-style-guide.md +docs/the-science.md +docs/trust-factory.md +docs/workflows/complete-cycle.md +docs/workflows/meta-observer/example-today.md +docs/workflows/meta-observer/SHOWCASE.md diff --git a/scripts/check-docs-demoted-claims.sh b/scripts/check-docs-demoted-claims.sh new file mode 100755 index 000000000..6a681776c --- /dev/null +++ b/scripts/check-docs-demoted-claims.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# check-docs-demoted-claims.sh +# +# Honesty gate (age-gate-the-ungated-egwt.6). SIBLING of +# check-docs-no-retired-tech.sh — a docs-scoped lexicon gate with its OWN +# lifecycle. ADR-0004 and ADR-0011 DEMOTED the "corpus moat" / "escape-corpus +# compounds" claims to a named UNPROVEN hypothesis facing a structural +# data-starvation headwind. This gate fails when a LIVE narrative doc still +# asserts one of the demoted claims *indicatively* (as proven), or ships an +# uncited multiplier / peer-review claim, unless the line carries an explicit +# hedge. +# +# Three banned lexicon classes (built by scanning the actual offenders on the +# burn-down pages, not from theory): +# (a) "Peer-reviewed" claims with NO adjacent citation +# (a same-line author-year, URL, footnote marker, or named source exempts). +# (b) Uncited multiplier claims: `\b[0-9]+(\.[0-9]+)?x\b` on a line that also +# carries speedup/faster/improvement language. +# (c) Compounding / moat / flywheel-thesis phrasings asserted indicatively as +# proven (e.g. "the corpus is the moat", "the flywheel thesis is validated", +# "empirically confirmed", "The Flywheel Is The Product", +# "self-improving through"). Conditional/mechanism phrasings — "knowledge +# compounds WHEN retrieval × usage beats decay" — are NOT offenders; the +# hedge/conditional exemption lets them through. +# +# Exemptions (a doc / line opts OUT): +# - ADRs (docs/adr/**) and dated-archive dirs — via docs_scope (never in scope). +# - Evidence / eval files (docs/evals/**, docs/evidence/**) — a doc that +# REPORTS a measured delta is allowed to state its number. +# - Banner-exempt docs (RETIRED/HISTORICAL/SUPERSEDED banner) — via docs_scope. +# - Any matched LINE that carries an explicit hedge: +# unproven / hypothesis / unvalidated / not yet proven / demoted, OR a +# conditional ("When …", "If …"). +# +# Baseline + shrink ratchet (scripts/.docs-demoted-claims-baseline, +# FILENAME-pinned): +# - a NON-baselined live doc with a banned phrase → exit 1. +# - a BASELINED file that now has ZERO findings → exit 1 demanding a prune +# (the shrink ratchet; the baseline may only get smaller). +# +# Blocking: currently ADVISORY in seed.go (Blocking:false). One clean advisory +# cycle then it flips Blocking. The exit codes below are already correct for the +# future flip — the bats twin proves the blocking case. +# +# Exit: 0 clean · 1 offender(s) / stale baseline entry · 2 usage/setup error + +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +# Shared LIVE-doc scope + historical-exemption resolution (age-gate-the-ungated-egwt.1). +# Resolve via $ROOT (absolutized BEFORE the cd above) — a relative +# ${BASH_SOURCE[0]} (e.g. `cd scripts && ./check-...`) would resolve wrongly +# after the cd (a prior pawl REFUTED that on the sibling's first land attempt). +. "$ROOT/scripts/lib/docs-scope.sh" +# Pin the scope root: this gate always scans ITS repo's docs/, never a tree an +# inherited DOCS_ROOT env var points at (the injection seam is for tests). +# shellcheck disable=SC2034 # DOCS_ROOT is read by the sourced docs-scope.sh lib. +DOCS_ROOT="$ROOT" + +BASELINE="$ROOT/scripts/.docs-demoted-claims-baseline" + +# ---- lexicon regexes ------------------------------------------------------- + +# (a) bare "peer-review(ed)" assertion. +CLASS_A='peer[- ]?review' +# A same-line citation exempts a peer-review claim: an author-year "(1995)", a +# URL, a footnote marker "[^", "et al.", or a named classical source. +CITE='\([12][0-9]{3}\)|https?://|\[\^|et al\.|\bDarr\b|\bBoone\b|\bEbbinghaus\b' + +# (b) N(.N)x multiplier … +CLASS_B='\b[0-9]+(\.[0-9]+)?x\b' +# … on a line that also carries speedup/improvement language. +CLASS_B_CTX='speedup|faster|improvement|\bimprove' + +# (c) demoted-claim phrasings asserted indicatively as proven. +CLASS_C='(is|are) the moat|(flywheel|corpus) thesis is (empirically )?(validated|confirmed|proven)|the flywheel thesis is (validated|confirmed|proven)|empirically confirmed|Validated Thesis|Validated by Evidence|Laws proven|self-improving through|knowledge compounds regardless|flywheel multiplies it|[Ff]lywheel [Ii]s [Tt]he [Pp]roduct' + +# A matched LINE that ALSO carries an explicit hedge (or is conditional) is +# honest / mechanism-describing, not an over-claim — exempt at the line level. +HEDGE='unproven|hypothesis|unvalidated|not yet proven|demoted|[Ww]hen |[Ii]f ' + +# Evidence/eval doc trees: a doc that REPORTS a measured delta may state numbers. +is_evidence_doc() { + case "$1" in + docs/evals/*|*/docs/evals/*|docs/evidence/*|*/docs/evidence/*) return 0 ;; + esac + return 1 +} + +# find_offenders FILE — emit "CLASS|line" for each banned, un-hedged line. +find_offenders() { + local f="$1" ln txt + # (a) + while IFS= read -r ln; do + txt="${ln#*:}" + printf '%s' "$txt" | grep -qEi "$CITE" && continue + printf '%s' "$txt" | grep -qEi "$HEDGE" && continue + echo "peer-review-uncited|$ln" + done < <(grep -nEi "$CLASS_A" "$f" 2>/dev/null || true) + # (b) + while IFS= read -r ln; do + txt="${ln#*:}" + printf '%s' "$txt" | grep -qEi "$CLASS_B_CTX" || continue + printf '%s' "$txt" | grep -qEi "$HEDGE" && continue + echo "multiplier-uncited|$ln" + done < <(grep -nEi "$CLASS_B" "$f" 2>/dev/null || true) + # (c) + while IFS= read -r ln; do + txt="${ln#*:}" + printf '%s' "$txt" | grep -qEi "$HEDGE" && continue + echo "compounding-as-proven|$ln" + done < <(grep -nEi "$CLASS_C" "$f" 2>/dev/null || true) +} + +# ---- load baseline (FILENAME-pinned; one live-doc path per non-comment line) -- +declare -A BASELINED=() +if [ -f "$BASELINE" ]; then + while IFS= read -r bl; do + bl="${bl%%$'\r'}" + case "$bl" in ''|'#'*) continue ;; esac + BASELINED["$bl"]=1 + done < "$BASELINE" +fi + +mapfile -t DOCS < <(docs_scope_live_files) + +declare -i scanned=0 exempt=0 +new_offenders=() # non-baselined live doc with a banned phrase +declare -A OFFENDING=() # file -> 1 for every file that currently offends +for f in "${DOCS[@]}"; do + scanned=$((scanned + 1)) + if docs_scope_is_exempt "$f"; then exempt=$((exempt + 1)); continue; fi + if is_evidence_doc "$f"; then exempt=$((exempt + 1)); continue; fi + hits="$(find_offenders "$f")" + [ -z "$hits" ] && continue + OFFENDING["$f"]=1 + if [ -z "${BASELINED[$f]:-}" ]; then + while IFS= read -r h; do + [ -z "$h" ] && continue + new_offenders+=("$f|$h") + done <<< "$hits" + fi +done + +# ---- shrink ratchet: a baselined file with ZERO findings must be pruned ------ +stale_baseline=() +for bl in "${!BASELINED[@]}"; do + if [ -z "${OFFENDING[$bl]:-}" ]; then + stale_baseline+=("$bl") + fi +done + +echo "check-docs-demoted-claims: scanned $scanned live docs ($exempt exempt), baseline pins ${#BASELINED[@]} file(s)" + +rc=0 +if [ "${#new_offenders[@]}" -gt 0 ]; then + rc=1 + echo "FAIL — ${#new_offenders[@]} demoted/unproven claim(s) in non-baselined live docs:" >&2 + for o in "${new_offenders[@]}"; do + file="${o%%|*}"; rest="${o#*|}"; cls="${rest%%|*}"; line="${rest#*|}" + echo " - $file [$cls]: $line" >&2 + done + echo >&2 + echo "Fix: hedge the claim to match ADR-0004 (corpus moat unproven) / ADR-0011" >&2 + echo " (escape-corpus compounding unproven — structural data-starvation), or" >&2 + echo " add a citation. Then leave the file OUT of the baseline (or prune it)." >&2 +fi + +if [ "${#stale_baseline[@]}" -gt 0 ]; then + rc=1 + echo "FAIL — ${#stale_baseline[@]} baseline entry(ies) now have zero findings — PRUNE them (shrink ratchet):" >&2 + printf ' - %s\n' "${stale_baseline[@]}" >&2 + echo >&2 + echo "Remove the cleaned file(s) from $BASELINE — the baseline may only shrink." >&2 +fi + +if [ "$rc" -eq 0 ]; then + echo "PASS — no non-baselined live doc over-claims a demoted hypothesis; baseline is tight." +fi +exit "$rc" diff --git a/tests/scripts/check-docs-demoted-claims.bats b/tests/scripts/check-docs-demoted-claims.bats new file mode 100644 index 000000000..7db55c653 --- /dev/null +++ b/tests/scripts/check-docs-demoted-claims.bats @@ -0,0 +1,176 @@ +#!/usr/bin/env bats +# +# Tests for scripts/check-docs-demoted-claims.sh — the docs.demoted-claims +# honesty gate (age-gate-the-ungated-egwt.6). SIBLING of the retired-tech gate; +# the banned lexicon is the ADR-0004/ADR-0011 demoted-claim vocabulary +# (peer-review-uncited / multiplier-uncited / compounding-as-proven). +# +# Fixture-repo pattern per tests/scripts/docs-scope.bats: stand up a self- +# contained repo (scripts/ + scripts/lib/docs-scope.sh + docs/ + a baseline) in +# BATS_TEST_TMPDIR and run the real script inside it. The script resolves its +# lib and baseline from its own $ROOT, so a fixture repo exercises the true +# code paths — including the exit codes that the future Blocking flip depends on. + +setup() { + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + SCRIPT="$REPO_ROOT/scripts/check-docs-demoted-claims.sh" + LIB="$REPO_ROOT/scripts/lib/docs-scope.sh" + [ -r "$SCRIPT" ] + [ -r "$LIB" ] + + # Build a fresh fixture repo per test. + FIX="$BATS_TEST_TMPDIR/repo" + mkdir -p "$FIX/scripts/lib" "$FIX/docs" "$FIX/docs/adr" "$FIX/docs/evals" + cp "$SCRIPT" "$FIX/scripts/" + cp "$LIB" "$FIX/scripts/lib/" + chmod +x "$FIX/scripts/check-docs-demoted-claims.sh" + RUN="$FIX/scripts/check-docs-demoted-claims.sh" +} + +# Helper: write the fixture baseline (FILENAME-pinned; one path per line). +seed_baseline() { + printf '%s\n' "$@" > "$FIX/scripts/.docs-demoted-claims-baseline" +} + +# ---- regression / clean ------------------------------------------------------ + +@test "empty baseline + a clean live doc passes" { + printf '# Clean\n\nThe verification loop is proven.\n' > "$FIX/docs/clean.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +# ---- class (c): compounding-as-proven --------------------------------------- + +@test "non-baselined doc asserting the flywheel thesis proven FAILS (names file + class + ADR)" { + printf '# Doc\n\nThe flywheel thesis is validated.\n' > "$FIX/docs/overclaim.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 1 ] + [[ "$output" == *"docs/overclaim.md"* ]] + [[ "$output" == *"compounding-as-proven"* ]] + # the repair hint points at the demoting ADRs + [[ "$output" == *"ADR-0004"* ]] + [[ "$output" == *"ADR-0011"* ]] +} + +@test "corpus-is-the-moat asserted indicatively FAILS" { + printf '# Doc\n\nThe corpus is the moat. The tool is replaceable.\n' > "$FIX/docs/moat.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 1 ] + [[ "$output" == *"docs/moat.md"* ]] +} + +# ---- class (b): uncited multiplier ------------------------------------------ + +@test "uncited multiplier with speedup language FAILS" { + printf '# Doc\n\nGit metrics: 40x speedup from parallelization.\n' > "$FIX/docs/mult.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 1 ] + [[ "$output" == *"docs/mult.md"* ]] + [[ "$output" == *"multiplier-uncited"* ]] +} + +# ---- class (a): uncited peer-review ----------------------------------------- + +@test "bare peer-review claim without a citation FAILS" { + printf '# Doc\n\nScientific foundation: Peer-reviewed.\n' > "$FIX/docs/pr.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 1 ] + [[ "$output" == *"docs/pr.md"* ]] + [[ "$output" == *"peer-review-uncited"* ]] +} + +@test "peer-review claim WITH an adjacent author-year citation PASSES" { + printf '# Doc\n\n| 17%%/week decay | Darr et al. (1995) | Empirical, peer-reviewed |\n' > "$FIX/docs/cited.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +# ---- line-level hedge exemption --------------------------------------------- + +@test "a hedged line (names it an unproven hypothesis) PASSES" { + printf '# Doc\n\nThat the corpus is the moat is an explicitly unproven hypothesis.\n' > "$FIX/docs/hedged.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +@test "a conditional compounding claim (When X, knowledge compounds) PASSES" { + printf '# Doc\n\nWhen retrieval times usage beats decay, knowledge compounds.\n' > "$FIX/docs/cond.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +# ---- evidence/eval-file exemption ------------------------------------------- + +@test "a docs/evals file reporting a measured delta is exempt (PASSES)" { + printf '# Measured\n\nThe treatment ran 40x faster than control in the A/B.\n' > "$FIX/docs/evals/ab.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +# ---- ADR exemption (out of scope entirely) ---------------------------------- + +@test "an ADR file asserting compounding is exempt (out of scope, PASSES)" { + printf '# ADR-0099\n\nThe flywheel thesis is validated. The corpus is the moat.\n' \ + > "$FIX/docs/adr/ADR-0099-thing.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +# ---- baselined offender is tolerated ---------------------------------------- + +@test "a baselined offending file PASSES (grandfathered)" { + printf '# Doc\n\nThe corpus is the moat.\n' > "$FIX/docs/legacy.md" + seed_baseline "docs/legacy.md" + run bash "$RUN" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +# ---- shrink ratchet: stale baseline entry ----------------------------------- + +@test "a baselined file with ZERO findings FAILS demanding a prune (shrink ratchet)" { + printf '# Clean now\n\nThe verification loop is proven.\n' > "$FIX/docs/wascleaned.md" + seed_baseline "docs/wascleaned.md" + run bash "$RUN" + [ "$status" -eq 1 ] + [[ "$output" == *"PRUNE"* ]] + [[ "$output" == *"docs/wascleaned.md"* ]] +} + +# ---- future blocking flip: exit code is already correct ---------------------- + +# The gate is registered advisory (Blocking:false) for one clean cycle. This +# proves the SCRIPT's own exit code is already 1 on a seeded offender, so when +# the seed.go row flips Blocking:true it will actually block — no code change to +# the script needed for the flip. +@test "seeded offender: script exit code is ready for the future Blocking flip" { + printf '# Doc\n\nThe flywheel thesis is empirically confirmed.\n' > "$FIX/docs/future.md" + : > "$FIX/scripts/.docs-demoted-claims-baseline" + run bash "$RUN" + [ "$status" -eq 1 ] # non-zero today ⇒ blocks the day Blocking flips true +} + +# ---- real-repo run with the real baseline ----------------------------------- + +@test "real repo run with the shipped baseline exits 0" { + run bash "$REPO_ROOT/scripts/check-docs-demoted-claims.sh" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +}