From e29568e1960b93dfc2c0f52b6802a0450cc6b623 Mon Sep 17 00:00:00 2001 From: boshu <241868352+boshu2@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:54:28 -0400 Subject: [PATCH] =?UTF-8?q?fix(docs):=20doc-lifecycle=20sweep=20=E2=80=94?= =?UTF-8?q?=20delete=20byte-dup,=20duplicates=20gate,=20BC6=20label,=20car?= =?UTF-8?q?eer-leak=20excision,=20hookless=20reconcile=20(age-gate-the-ung?= =?UTF-8?q?ated-egwt.12)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cli/internal/gates/checks/seed.go | 11 + docs/documentation-index.md | 2 +- docs/profiles/META_PATTERNS.md | 32 - docs/release-e2e-checklist.md | 1 - docs/templates/intent-issue.md | 2 +- docs/workflows/meta-observer/README.md | 8 +- docs/workflows/meta-observer/pattern-guide.md | 555 ------------------ scripts/check-docs-duplicates.sh | 104 ++++ tests/scripts/check-docs-duplicates.bats | 85 +++ 9 files changed, 206 insertions(+), 594 deletions(-) delete mode 100644 docs/workflows/meta-observer/pattern-guide.md create mode 100755 scripts/check-docs-duplicates.sh create mode 100644 tests/scripts/check-docs-duplicates.bats diff --git a/cli/internal/gates/checks/seed.go b/cli/internal/gates/checks/seed.go index 4764ac20c..d4fe73d63 100644 --- a/cli/internal/gates/checks/seed.go +++ b/cli/internal/gates/checks/seed.go @@ -96,6 +96,16 @@ var ( "scripts/.docs-demoted-claims-baseline", "tests/scripts/check-docs-demoted-claims.bats", } + // docs.duplicates shasums every live doc and fails on a byte-identical pair + // over the line threshold — an anti-regrowth guard after a docs-lifecycle + // sweep deleted a wholesale byte-dup (age-gate-the-ungated-egwt.12). Runs on + // any docs change plus self-reference (script / shared scope lib / bats). + docsDuplicatesPaths = []string{ + "docs/**", + "scripts/check-docs-duplicates.sh", + "scripts/lib/docs-scope.sh", + "tests/scripts/check-docs-duplicates.bats", + } archDocDriftPaths = []string{ "docs/architecture/ports-and-adapters.md", "docs/contracts/bounded-contexts.yaml", @@ -308,6 +318,7 @@ func init() { {ID: "docs.cli-snippets", Tiers: gates.Full, Match: docsCliSnippetsPaths, Blocking: false, Backing: "check-docs-cli-snippets.sh", RepairHint: "fix the dead ao reference or prune the stale baseline entry; flips Blocking after one clean advisory cycle (age-gate-the-ungated-egwt.4)"}, {ID: "docs.demoted-claims", Tiers: gates.Full, Match: demotedClaimsPaths, Blocking: false, Backing: "check-docs-demoted-claims.sh", RepairHint: "hedge the claim to match ADR-0004/ADR-0011 or add a citation; advisory one clean cycle then flips Blocking (age-gate-the-ungated-egwt.6)"}, {ID: "docs.adr-registry", Tiers: gates.Full, Match: adrRegistryPaths, Blocking: true, Backing: "check-adr-registry.sh", RepairHint: "duplicate/mismatched ADR number — renumber the newer ADR and sweep citations (age-gate-the-ungated-egwt.11)"}, + {ID: "docs.duplicates", Tiers: gates.Full, Match: docsDuplicatesPaths, Blocking: true, Backing: "check-docs-duplicates.sh", RepairHint: "byte-identical live docs — delete the copy, repoint links (age-gate-the-ungated-egwt.12)"}, {ID: "corpus.secret-scan", Tiers: gates.Full, Match: corpusPaths, Blocking: true, Backing: "check-corpus-secret-scan.sh"}, {ID: "corpus.witness-dolt-jsonl-crosscheck", Tiers: gates.Full, Match: corpusPaths, Blocking: true, Backing: "witness-dolt-jsonl-crosscheck.sh"}, {ID: "doctrine.memrl-health", Tiers: gates.Full, Blocking: true, Backing: "check-memrl-health.sh"}, diff --git a/docs/documentation-index.md b/docs/documentation-index.md index 25fd10801..92f1d6e0d 100644 --- a/docs/documentation-index.md +++ b/docs/documentation-index.md @@ -117,7 +117,7 @@ Bridge / framing docs: ### Meta-Observer - [Meta-Observer README](workflows/meta-observer/README.md) — Complete workflow package overview -- [Pattern Guide](workflows/meta-observer/pattern-guide.md) — Autonomous multi-session coordination guide +- [Pattern Guide](workflows/meta-observer-pattern.md) — Autonomous multi-session coordination guide - [Example Session](workflows/meta-observer/example-today.md) — Real example from 2025-11-09 - [Showcase](workflows/meta-observer/SHOWCASE.md) — Distributed intelligence for multi-session work diff --git a/docs/profiles/META_PATTERNS.md b/docs/profiles/META_PATTERNS.md index 84ca741c9..6f362a7cf 100644 --- a/docs/profiles/META_PATTERNS.md +++ b/docs/profiles/META_PATTERNS.md @@ -337,38 +337,6 @@ Total: 113k (56.5%) ⚠️ Over 40% --- -## Pattern 14: Cross-Flavor Feedback Loops - -**Discovery**: Technical work (work/) feeds personal development (personal/). - -**Feedback Loop**: - -``` -sre-devops (work/gitops/) → Build AgentOps framework - ↓ -Git metrics: an uncited speedup figure (removed — no citation), 95% success rate - ↓ -researcher (personal/12-factor-agentops/) → Extract patterns - ↓ -Document framework, compliance audits - ↓ -personal (personal/life/) → Track in MCI, update resume - ↓ -Career leverage: NVIDIA application with proof - ↓ -Visibility campaign: LinkedIn, YouTube - ↓ -Community adoption → New use cases - ↓ -researcher → Analyze new use cases → Improve framework - ↓ -Loop repeats... -``` - -**Insight**: Work and personal are not separate - they're symbiotic feedback loops. - ---- - ## Pattern 15: Role-Specific Validation Strategies **Discovery**: Each role validates work differently. diff --git a/docs/release-e2e-checklist.md b/docs/release-e2e-checklist.md index 5c215636c..200a0f157 100644 --- a/docs/release-e2e-checklist.md +++ b/docs/release-e2e-checklist.md @@ -29,7 +29,6 @@ Expect: - `Codex runtime sections` - `Codex artifact metadata` - `Skill runtime parity` - - `Hook install smoke (minimal + full)` - `ao init`/bootstrap + `ao rpi` smoke Reference test: `tests/integration/test-release-e2e-validation.sh`. diff --git a/docs/templates/intent-issue.md b/docs/templates/intent-issue.md index eecfd4358..1e5455186 100644 --- a/docs/templates/intent-issue.md +++ b/docs/templates/intent-issue.md @@ -15,7 +15,7 @@ ## Bounded context > Which bounded context from [`docs/contracts/context-map.md`](../contracts/context-map.md) does this work belong to? If it crosses contexts, this is two issues, not one. -> The bead must carry exactly one matching label: `bc-corpus`, `bc-validation`, `bc-loop`, `bc-factory`, or `bc-runtime`. +> The bead must carry exactly one matching label: `bc-corpus`, `bc-validation`, `bc-loop`, `bc-factory`, `bc-runtime`, or `bc-orchestration`. ## Hexagonal boundary diff --git a/docs/workflows/meta-observer/README.md b/docs/workflows/meta-observer/README.md index 7867b7bd3..003929b9d 100644 --- a/docs/workflows/meta-observer/README.md +++ b/docs/workflows/meta-observer/README.md @@ -130,11 +130,11 @@ Located in `.claude/workflows/meta-observer/` - Quick start guides - All included artifacts -2. **`pattern-guide.md`** +2. **`../meta-observer-pattern.md`** - Full pattern documentation - Architecture, principles, usage - Examples, troubleshooting - - [View guide →](./pattern-guide.md) + - [View guide →](../meta-observer-pattern.md) 3. **`example-today.md`** - Today's experiment (2025-11-09) @@ -394,9 +394,9 @@ This pattern validates and uses: │ ├── meta-observer.md # Observer agent protocol │ └── autonomous-worker.md # Worker template └── workflows/ + ├── meta-observer-pattern.md # Full pattern documentation └── meta-observer/ ├── README.md # This file - overview - ├── pattern-guide.md # Full pattern documentation ├── example-today.md # Real experiment example └── SHOWCASE.md # Public demonstration doc ``` @@ -464,7 +464,7 @@ This pattern validates and uses: ## Support **Questions?** -- Read: `pattern-guide.md` (comprehensive) +- Read: `../meta-observer-pattern.md` (comprehensive) - Example: `example-today.md` (real usage) - Showcase: `SHOWCASE.md` (for presentations) diff --git a/docs/workflows/meta-observer/pattern-guide.md b/docs/workflows/meta-observer/pattern-guide.md deleted file mode 100644 index c3476237c..000000000 --- a/docs/workflows/meta-observer/pattern-guide.md +++ /dev/null @@ -1,555 +0,0 @@ -# Meta-Observer Pattern: Autonomous Multi-Session Coordination - -**Version:** 1.0.0 -**Status:** Production-ready ✅ -**Discovered:** 2025-11-09 -**Pattern Type:** Multi-session coordination via stigmergy - ---- - -## Overview - -**Meta-Observer Pattern** enables N autonomous workers to coordinate through shared memory (Memory MCP) without central orchestration. - -**Key Innovation:** Emergent coordination > Central control - -**Inspiration:** Ant colonies (pheromone trails/stigmergy), not military (command/control) - ---- - -## Problem - -**Traditional multi-session coordination:** -- ❌ Central orchestrator becomes bottleneck -- ❌ Micromanagement slows workers -- ❌ Orchestrator doesn't know domain context -- ❌ Single point of failure -- ❌ Doesn't scale (coordination overhead increases with N) - -**Result:** Slower than serial work, despite parallelization - ---- - -## Solution - -**Autonomous workers + Shared memory + Minimal intervention observer** - -``` -WORKERS (N sessions) MEMORY MCP META-OBSERVER -─────────────────── ────────────── ────────────── -Work independently ←→ Shared knowledge ←─ Monitor passively -Update when complete ─→ Worker discoveries ─→ Synthesize findings -Self-organize via MCP ←─ Cross-worker data ─→ Document learnings -No coordination needed Persistent state Intervene if blocking -``` - -**Result:** Parallel work + Emergent insights + No bottlenecks - ---- - -## Core Principles - -### 1. Worker Autonomy - -**Workers are domain experts who self-organize.** - -- Make independent decisions -- No permission needed -- Trust domain expertise -- Work at own pace -- Update Memory MCP when complete - -### 2. Stigmergy Coordination - -**Coordination through shared environment (Memory MCP), not commands.** - -- Workers leave "pheromone trails" (observations in Memory MCP) -- Other workers detect trails and adapt -- Emergent patterns arise naturally -- No central coordinator needed - -### 3. Minimal Intervention - -**Observer watches, synthesizes, documents - intervenes rarely.** - -- Monitor worker activity passively (every 2-4h) -- Synthesize N worker streams into coherent narrative -- Intervene ONLY if blocking conflicts -- Trust worker autonomy 99% of time - -### 4. Emergent Intelligence - -**Valuable insights emerge from worker combinations.** - -- Cross-domain patterns -- Unexpected synergies -- Novel solutions -- Distributed problem-solving - -### 5. Natural Scaling - -**Pattern scales to N workers without overhead.** - -- Add workers: Just create new entity, no coordination changes -- Remove workers: No impact on others -- 10 workers = same overhead as 2 workers -- Coordination cost: O(1), not O(N²) - ---- - -## Architecture - -### Components - -**1. Autonomous Workers (N sessions)** -- Domain: Specific repo, codebase, or task area -- Goal: Domain-specific objectives -- Entity: Unique Memory MCP entity (`Worker Session: {domain}`) -- Behavior: Work independently, update Memory MCP when complete -- Coordination: Minimal (only if blocking another worker) - -**2. Memory MCP (Shared Knowledge Base)** -- Function: Stigmergy (coordination through environment) -- Data: Worker observations, discoveries, blockers, status -- Persistence: Survives /clear, session restarts, IDE restarts -- Access: All sessions read/write -- Structure: Entities + Relations + Observations - -**3. Meta-Observer (1 session)** -- Role: Monitor, synthesize, document -- Location: Workspace root (neutral territory) -- Behavior: Passive observation, active synthesis -- Intervention: Minimal (only blocking conflicts) -- Output: Synthesis documents, pattern learnings - ---- - -## Implementation - -### Step 1: Launch Meta-Observer Pattern - -```bash -/launch-meta-observer -``` - -**Agent prompts for:** -- Number of workers -- Domain for each worker -- Overall goal -- Observer location (default: workspace root) - -**Creates:** -- N worker entities in Memory MCP -- N worker briefs (`.agents/briefs/worker-N-{domain}.md`) -- 1 observer brief (`.agents/briefs/meta-observer-{timestamp}.md`) -- Meta-Observer session initialized - -### Step 2: Brief Workers - -**Each worker receives custom brief with:** -- Domain assignment -- Autonomous work protocol -- Memory MCP update instructions -- Unique entity name -- Context management guidelines - -**Workers understand:** -- ✅ Work autonomously (you're the expert) -- ✅ Update Memory MCP after major progress -- ✅ Check Memory MCP for other worker discoveries (optional) -- ✅ Coordinate only if blocking -- ✅ Manage context (sub-agents, bundling at 40%) - -### Step 3: Observer Monitors - -**Every 2-4 hours, observer:** -1. Queries Memory MCP for worker updates -2. Reads new discoveries -3. Identifies cross-worker patterns -4. Checks for blockers/conflicts -5. Updates synthesis -6. Intervenes if necessary (rare) -7. Documents learnings - -### Step 4: Workers Execute - -**Workers autonomously:** -1. Do domain work -2. Use sub-agents for complex tasks -3. Monitor context (stay <40%) -4. Update Memory MCP after phases -5. Check for blockers from others (optional) -6. Continue until goal complete - -### Step 5: End-of-Day Synthesis - -**Observer creates comprehensive synthesis:** -- What each worker completed -- Emergent insights across workers -- Patterns validated -- Learnings captured -- Overall progress toward goal - ---- - -## Usage Patterns - -### Pattern 1: Repository Parallelization - -**Use case:** Work spans multiple repos - -**Example:** -```bash -/launch-meta-observer --workers 3 \ - --domains "backend-api,frontend-ui,infrastructure" \ - --goal "Implement authentication feature" -``` - -**Workers:** -- Worker 1 (backend-api): Auth endpoints, JWT logic -- Worker 2 (frontend-ui): Login UI, protected routes -- Worker 3 (infrastructure): Database schema, secrets management - -**Observer:** Synthesizes complete feature, ensures consistency - -**Coordination:** Workers update Memory MCP when APIs ready, schemas deployed, etc. - -### Pattern 2: Domain Specialization - -**Use case:** Different expertise areas - -**Example:** -```bash -/launch-meta-observer --workers 4 \ - --domains "documentation,testing,deployment,monitoring" \ - --goal "Production-ready release" -``` - -**Workers:** -- Worker 1: API docs, user guides, tutorials -- Worker 2: Unit tests, integration tests, E2E tests -- Worker 3: CI/CD pipelines, deployment automation -- Worker 4: Metrics, logging, alerting, dashboards - -**Observer:** Synthesizes release readiness across all domains - -### Pattern 3: Phase Parallelization - -**Use case:** Independent phases of same project - -**Example:** -```bash -/launch-meta-observer --workers 3 \ - --domains "research,implementation,validation" \ - --goal "New feature development" -``` - -**Workers:** -- Worker 1: Research approaches, evaluate options, document findings -- Worker 2: Implement solution based on research -- Worker 3: Create validation suite, test implementation - -**Coordination:** Worker 1 hands off to Worker 2 via Memory MCP - -### Pattern 4: Launch Preparation (Today's Experiment) - -**Use case:** Multi-domain launch readiness - -**Example:** -```bash -/launch-meta-observer --workers 3 \ - --domains "12-factor-agentops,agentops-showcase,launch-content" \ - --goal "Q1 2025 public launch" -``` - -**Workers:** -- Worker 1: Framework documentation, factor-mapping, compliance -- Worker 2: VitePress website build, deployment, validation -- Worker 3: SEO blog posts, launch strategy, social content - -**Result:** All completed autonomously, emergent insights discovered - ---- - -## Memory MCP Patterns - -### Worker Update Pattern - -```typescript -// After completing major work -mcp__memory__add_observations({ - observations: [{ - entityName: "Worker Session: {domain}", - contents: [ - "Completed: {what}", - "Discoveries: {insights}", - "Impact: {contribution to goal}", - "Blockers: {none or description}", - "Context: {%}", - "Next: {steps}", - "Files: {modified}", - "Commits: {if applicable}" - ] - }] -}) -``` - -### Observer Query Pattern - -```typescript -// Check all worker updates -mcp__memory__search_nodes({ - query: "Worker Session completed discoveries" -}) - -// Get specific workers -mcp__memory__open_nodes({ - names: [ - "Worker Session: domain-1", - "Worker Session: domain-2" - ] -}) - -// Check for blockers -mcp__memory__search_nodes({ - query: "BLOCKER Worker Session" -}) -``` - -### Handoff Pattern - -```typescript -// Worker 1 completes, hands off to Worker 2 -mcp__memory__add_observations({ - observations: [{ - entityName: "Worker Session: domain-1", - contents: [ - "Work complete", - "Handoff to Worker 2:", - "- Artifacts: {files}", - "- Context: {what they need to know}", - "- Ready for: {next phase}", - "Status: COMPLETE, HANDED OFF" - ] - }] -}) - -// Worker 2 picks up -mcp__memory__search_nodes({ query: "Handoff to Worker 2" }) -``` - ---- - -## Success Metrics - -**Pattern succeeds when:** -- ✅ Workers complete work autonomously (no constant guidance) -- ✅ Emergent insights arise (worker combinations create value) -- ✅ Observer synthesis valuable (creates coherent narrative) -- ✅ Intervention minimal (only blocking conflicts) -- ✅ Faster than serial (parallelization actually helps) -- ✅ No context collapse (all sessions <40%) -- ✅ Scales naturally (adding workers doesn't slow down) - -**Pattern needs adjustment when:** -- ⚠️ Workers asking for constant guidance (be more autonomous) -- ⚠️ No emergent insights (workers in silos, not sharing via MCP) -- ⚠️ Observer intervening frequently (micromanaging) -- ⚠️ Slower than serial (parallelization overhead too high) -- ⚠️ Blocking conflicts undetected (observer not monitoring) -- ⚠️ Context collapse (workers not using sub-agents/bundling) - ---- - -## Advantages - -**vs Central Orchestration:** -- ✅ No bottleneck (workers don't wait for orchestrator) -- ✅ Domain expertise (workers know their domain best) -- ✅ Scales naturally (O(1) coordination, not O(N²)) -- ✅ Resilient (no single point of failure) -- ✅ Emergent insights (patterns arise from combinations) - -**vs No Coordination:** -- ✅ Shared knowledge (Memory MCP provides context) -- ✅ Conflict detection (observer watches for blockers) -- ✅ Synthesis (coherent narrative from distributed work) -- ✅ Learning capture (patterns documented) - ---- - -## Limitations - -**Not suitable for:** -- ❌ Single-session work (overhead not worth it) -- ❌ Tightly coupled tasks (constant sync needed) -- ❌ Simple linear workflow (serial is simpler) -- ❌ Real-time coordination required (async by design) - -**Challenges:** -- Workers must be disciplined about Memory MCP updates -- Observer must resist urge to micromanage -- Requires trust in worker autonomy -- Emergent patterns may be unexpected (feature or bug?) - ---- - -## Integration with 12-Factor AgentOps - -**This pattern validates:** - -**Factor II (JIT Context Loading):** -- Workers bundle at 40%, stay lean -- Observer stays <30% (just reading/synthesizing) -- Sub-agents keep worker context low -- Memory MCP eliminates need to load full context - -**Factor VI (Session Continuity):** -- Memory MCP persists across /clear -- Workers can bundle and resume seamlessly -- Observer synthesizes even after session restarts -- Work continues despite interruptions - -**Factor VII (Intelligent Routing):** -- Observer synthesizes, doesn't command -- Workers self-route based on domain expertise -- Memory MCP routes information between workers -- Emergent routing (not planned routing) - -**Factor IX (Pattern Extraction):** -- Observer captures emergent patterns -- Workers document discoveries in Memory MCP -- Learnings extracted automatically -- Pattern library grows organically - ---- - -## Advanced Usage - -### Nested Observers - -**For very large N (10+ workers):** - -``` -Meta-Meta-Observer -├── Domain Observer 1 (watches 5 workers) -├── Domain Observer 2 (watches 5 workers) -└── Domain Observer 3 (watches 5 workers) -``` - -**Use when:** 10+ workers, group into domains - -### Dynamic Worker Addition - -**Add worker mid-stream:** - -```bash -/worker-brief --domain "new-domain" --goal "additional work" --number 4 -``` - -**Observer automatically incorporates** new worker into monitoring. - -### Worker Subtraction - -**Worker completes and exits:** - -```typescript -mcp__memory__add_observations({ - observations: [{ - entityName: "Worker Session: domain-1", - contents: [ - "Status: COMPLETE", - "Exiting: Work done", - "Handoff: None needed", - "Final report: {summary}" - ] - }] -}) -``` - -**Other workers and observer continue** unaffected. - ---- - -## Experiment Results (2025-11-09) - -**Hypothesis:** Central orchestration best for multi-session work - -**Actual Discovery:** Autonomous coordination superior - -**Evidence:** -- 3 workers completed complex work independently -- Zero active coordination needed -- Emergent insights discovered (recursive validation) -- Observer synthesis highly valuable -- Pattern scales to N workers naturally -- Faster than would have been with central control - -**Conclusion:** Pattern validated ✅ - -**Status:** Production-ready for general use - ---- - -## Files - -**.claude/commands/** -- `launch-meta-observer.md` - Initialize pattern -- `worker-brief.md` - Generate worker instructions - -**.claude/agents/** -- `meta-observer.md` - Observer agent -- `autonomous-worker.md` - Worker template - -**.claude/workflows/** -- `meta-observer-pattern.md` - This file (full pattern docs) - ---- - -## See Also - -**Related Patterns:** -- Context Bundling Protocol -- Sub-Agent Delegation Pattern -- Stigmergy Coordination (ant colonies) - -**Related Factors:** -- Factor II: Context Loading → The 40% Rule as Overload Prevention -- Factor VI: Resume Work → Validation Continuity Across Sessions -- Factor VII: Smart Routing → Directing Work to Appropriate Validation Paths -- Factor IX: Mine Patterns → Learning What Passes Validation - -**Inspiration:** -- Ant colony optimization -- Swarm intelligence -- Distributed systems (eventual consistency) -- Self-organizing systems - ---- - -## Quick Start - -```bash -# Initialize pattern -/launch-meta-observer - -# Follow prompts for: -# - Number of workers -# - Domain per worker -# - Overall goal - -# Workers work autonomously -# Observer monitors and synthesizes -# You get comprehensive synthesis at end - -# That's it! -``` - ---- - -**Pattern:** Meta-Observer -**Principle:** Emergent coordination > Central control -**Scales to:** N autonomous workers -**Status:** Production-ready ✅ -**Discovered:** 2025-11-09 through experiment -**Maintained by:** AgentOps Community diff --git a/scripts/check-docs-duplicates.sh b/scripts/check-docs-duplicates.sh new file mode 100755 index 000000000..3fc7a5721 --- /dev/null +++ b/scripts/check-docs-duplicates.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# check-docs-duplicates.sh +# +# Anti-regrowth gate (age-gate-the-ungated-egwt.12). A docs-lifecycle sweep found +# docs/workflows/meta-observer/pattern-guide.md was a byte-identical copy of +# docs/workflows/meta-observer-pattern.md (556 lines). One was deleted and the +# inbound links repointed. This gate keeps a byte-identical duplicate from ever +# growing back: it shasums every LIVE doc and FAILS, naming BOTH files, on any +# byte-identical pair where the file exceeds the size threshold below. +# +# Why a size floor: tiny docs (a one-line stub, an empty section placeholder, a +# short redirect) can be legitimately identical without being a maintenance +# hazard. The concern is a large document duplicated wholesale — two copies of a +# 500-line guide that will drift out of sync. MIN_DUP_LINES sets that floor: a +# pair is only reported when BOTH files have MORE than this many lines. It is a +# named variable, documented here, so the threshold is one obvious knob. +# +# Scope = the shared LIVE-doc set (scripts/lib/docs-scope.sh: docs/**/*.md minus +# dated/historical-archive dirs). ADRs and archive trees are never in scope, so +# an intentional historical snapshot that happens to match a live doc is exempt +# by construction. +# +# Exit: 0 clean (no byte-identical over-threshold pair) · 1 duplicate pair(s) +# found · 2 usage/setup error. + +# Strict mode + hijack-proof REPO_ROOT come from the shared hardened preamble +# (scripts/lib/preamble.sh — the egwt.10 ratchet). `CDPATH=` is an intentional +# env-prefix (clears CDPATH for that one cd), not an assignment — hence SC1007. +# shellcheck disable=SC1007 +. "$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/preamble.sh" + +# Shared LIVE-doc scope resolution — resolved via the preamble's absolute +# REPO_ROOT before any cd (a relative ${BASH_SOURCE[0]} would resolve wrongly +# after the cd below; the .1 lane's pawl caught that on a sibling gate). +# shellcheck source=scripts/lib/docs-scope.sh +. "$REPO_ROOT/scripts/lib/docs-scope.sh" +# Pin the docs scope root to THIS repo (the DOCS_ROOT env seam is for lib tests). +# shellcheck disable=SC2034 # DOCS_ROOT is read by the sourced docs-scope.sh lib. +DOCS_ROOT="$REPO_ROOT" + +cd "$REPO_ROOT" || exit 2 + +# ---- threshold ------------------------------------------------------------- +# A byte-identical pair is only reported when BOTH files have MORE than this many +# lines. ~50 lines: below it, an incidental short-doc match is not a duplication +# hazard worth blocking on. +MIN_DUP_LINES="${MIN_DUP_LINES:-50}" + +# ---- scan ------------------------------------------------------------------ +mapfile -t DOCS < <(docs_scope_live_files) + +# Bucket live docs by content hash; only files over the line threshold are +# eligible to form a reported pair. +declare -A HASH_TO_FILES=() +declare -i scanned=0 eligible=0 +for f in "${DOCS[@]}"; do + scanned=$((scanned + 1)) + [ -r "$f" ] || continue + # Line count gate: skip small docs entirely (they can never form a reported + # pair, so no need to hash them). + lines="$(wc -l < "$f" | tr -d ' ')" + [ "$lines" -gt "$MIN_DUP_LINES" ] || continue + eligible=$((eligible + 1)) + h="$(shasum "$f" | awk '{print $1}')" + if [ -n "${HASH_TO_FILES[$h]:-}" ]; then + HASH_TO_FILES["$h"]="${HASH_TO_FILES[$h]}"$'\n'"$f" + else + HASH_TO_FILES["$h"]="$f" + fi +done + +# ---- report ---------------------------------------------------------------- +dup_pairs=() +for h in "${!HASH_TO_FILES[@]}"; do + # A hash bucket with >1 file is a byte-identical set. + files="${HASH_TO_FILES[$h]}" + count="$(printf '%s\n' "$files" | grep -c .)" + if [ "$count" -gt 1 ]; then + # Emit every unordered pair in the bucket, naming both files. + mapfile -t members < <(printf '%s\n' "$files") + n="${#members[@]}" + for ((i = 0; i < n; i++)); do + for ((j = i + 1; j < n; j++)); do + dup_pairs+=("${members[i]} == ${members[j]}") + done + done + fi +done + +echo "check-docs-duplicates: scanned $scanned live docs ($eligible over ${MIN_DUP_LINES}-line threshold)" + +if [ "${#dup_pairs[@]}" -gt 0 ]; then + echo "FAIL — ${#dup_pairs[@]} byte-identical live-doc pair(s) over the ${MIN_DUP_LINES}-line threshold:" >&2 + for p in "${dup_pairs[@]}"; do + echo " - $p" >&2 + done + echo >&2 + echo "Fix: delete one copy and repoint every inbound link to the survivor" >&2 + echo " (rg -n '' to find them). A large doc must not exist twice." >&2 + exit 1 +fi + +echo "PASS — no byte-identical live-doc pair over the ${MIN_DUP_LINES}-line threshold." +exit 0 diff --git a/tests/scripts/check-docs-duplicates.bats b/tests/scripts/check-docs-duplicates.bats new file mode 100644 index 000000000..856226e74 --- /dev/null +++ b/tests/scripts/check-docs-duplicates.bats @@ -0,0 +1,85 @@ +#!/usr/bin/env bats +# +# Tests for scripts/check-docs-duplicates.sh — the docs.duplicates anti-regrowth +# gate (age-gate-the-ungated-egwt.12). The gate shasums every LIVE doc and fails, +# naming BOTH files, on any byte-identical pair where both files exceed the +# MIN_DUP_LINES threshold. +# +# Fixture-repo pattern per tests/scripts/check-docs-demoted-claims.bats: stand up +# a self-contained repo (scripts/ + scripts/lib/{preamble,docs-scope}.sh + docs/) +# in BATS_TEST_TMPDIR and run the real script inside it. The script sources the +# hardened preamble (REPO_ROOT) then docs-scope, and pins DOCS_ROOT to REPO_ROOT, +# so a fixture repo exercises the true code paths. + +setup() { + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + SCRIPT="$REPO_ROOT/scripts/check-docs-duplicates.sh" + LIB="$REPO_ROOT/scripts/lib/docs-scope.sh" + PREAMBLE="$REPO_ROOT/scripts/lib/preamble.sh" + [ -r "$SCRIPT" ] + [ -r "$LIB" ] + [ -r "$PREAMBLE" ] + + # Build a fresh fixture repo per test. `git init` makes the preamble's + # REPO_ROOT resolution deterministic ($FIX), never a surrounding checkout. + FIX="$BATS_TEST_TMPDIR/repo" + mkdir -p "$FIX/scripts/lib" "$FIX/docs/workflows" + git -C "$FIX" init -q + cp "$SCRIPT" "$FIX/scripts/" + cp "$LIB" "$FIX/scripts/lib/" + cp "$PREAMBLE" "$FIX/scripts/lib/" + chmod +x "$FIX/scripts/check-docs-duplicates.sh" + RUN="$FIX/scripts/check-docs-duplicates.sh" +} + +# Helper: write a doc of N lines (each a distinct numbered line so content is +# deterministic and reproducible across a copy). +write_lines() { + local path="$1" n="$2" i + : > "$path" + for ((i = 1; i <= n; i++)); do + printf 'line %d of the guide with some body text to make it substantive\n' "$i" >> "$path" + done +} + +# ---- over-threshold byte-identical pair FAILS naming both -------------------- + +@test "a byte-identical pair over the line threshold FAILS naming both files" { + write_lines "$FIX/docs/workflows/original.md" 80 + cp "$FIX/docs/workflows/original.md" "$FIX/docs/workflows/copy.md" + run bash "$RUN" + [ "$status" -eq 1 ] + [[ "$output" == *"FAIL"* ]] + [[ "$output" == *"docs/workflows/original.md"* ]] + [[ "$output" == *"docs/workflows/copy.md"* ]] +} + +# ---- under-threshold identical pair PASSES ---------------------------------- + +@test "an identical pair UNDER the line threshold PASSES (small-doc floor)" { + write_lines "$FIX/docs/small-a.md" 10 + cp "$FIX/docs/small-a.md" "$FIX/docs/small-b.md" + run bash "$RUN" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +# ---- distinct docs PASS ----------------------------------------------------- + +@test "two distinct large docs PASS (not byte-identical)" { + write_lines "$FIX/docs/one.md" 80 + write_lines "$FIX/docs/two.md" 80 + # Make two.md differ by one line so it is not byte-identical to one.md. + printf 'a distinguishing final line\n' >> "$FIX/docs/two.md" + run bash "$RUN" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +} + +# ---- real-repo run: the deleted duplicate stays deleted (anti-regrowth) ------ + +@test "real repo run exits 0 (proves the byte-dup stays deleted)" { + run bash "$REPO_ROOT/scripts/check-docs-duplicates.sh" + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] +}