mirror of
https://github.com/boshu2/agentops.git
synced 2026-09-14 15:08:13 +08:00
prune goals 83→25, add trust-and-prove track specs
Council consensus: feature saturation reached. Removed 58 grep-for-string, file-exists, and keyword-in-README goals. Kept only goals guarding real mechanical invariants. Added Track 1 (RPI orchestrator stabilization) and Track 2 (flywheel proof) evolution cycle specs. Also includes pending security hardening, workflow fixes, and script cleanup.
This commit is contained in:
Boden Fuller
@@ -79,13 +79,16 @@ jobs:
|
||||
run: |
|
||||
chmod +x scripts/security-gate.sh
|
||||
./scripts/security-gate.sh --mode full
|
||||
env:
|
||||
SECURITY_GATE_OUTPUT_DIR: ${{ runner.temp }}/agentops-security
|
||||
TOOLCHAIN_OUTPUT_DIR: ${{ runner.temp }}/agentops-tooling
|
||||
|
||||
- name: Upload security gate artifacts
|
||||
uses: actions/upload-artifact@v6
|
||||
if: always()
|
||||
with:
|
||||
name: nightly-security-gate
|
||||
path: .agents/security/
|
||||
path: ${{ runner.temp }}/agentops-security/
|
||||
retention-days: 7
|
||||
|
||||
summary:
|
||||
|
||||
@@ -204,13 +204,16 @@ jobs:
|
||||
run: |
|
||||
chmod +x scripts/security-gate.sh
|
||||
./scripts/security-gate.sh --mode quick
|
||||
env:
|
||||
SECURITY_GATE_OUTPUT_DIR: ${{ runner.temp }}/agentops-security
|
||||
TOOLCHAIN_OUTPUT_DIR: ${{ runner.temp }}/agentops-tooling
|
||||
|
||||
- name: Upload security gate artifacts
|
||||
uses: actions/upload-artifact@v6
|
||||
if: always()
|
||||
with:
|
||||
name: security-gate
|
||||
path: .agents/security/
|
||||
path: ${{ runner.temp }}/agentops-security/
|
||||
retention-days: 7
|
||||
|
||||
plugin-load-test:
|
||||
|
||||
+1
-5
@@ -5,10 +5,6 @@ title = "agentops gitleaks config"
|
||||
|
||||
[allowlist]
|
||||
paths = [
|
||||
# Security scan artifacts contain tool output (semgrep/gosec JSON)
|
||||
# that references secret patterns — not actual secrets
|
||||
'''^\.agents/security/''',
|
||||
'''^\.agents/tooling/''',
|
||||
# Temp directories used by tests and reverse-engineer scripts
|
||||
# Temp directories used by tests, reverse-engineer scripts, and scanner output
|
||||
'''^\.tmp/''',
|
||||
]
|
||||
|
||||
+177
-490
@@ -7,9 +7,11 @@ pillars:
|
||||
- validated-acceleration
|
||||
- goal-driven-automation
|
||||
- zero-friction-workflow
|
||||
# Total: 83 goals
|
||||
# Pillar goals define WHAT the system does (user-visible value).
|
||||
# Infrastructure goals define HOW the system stays healthy (build, test, docs).
|
||||
# Total: 25 goals
|
||||
# Pruned 2026-02-21 per council consensus: feature saturation reached.
|
||||
# Removed 58 grep-for-string-in-file-we-control goals, file-exists checks,
|
||||
# keyword-in-README/SKILL.md goals, and duplicates (go-race-clean = go-cli-tests).
|
||||
# Kept only goals that guard real mechanical invariants.
|
||||
#
|
||||
# Theoretical foundation (4 pillars → product pillars):
|
||||
# Systems Theory (Meadows) → targets leverage points #3-#6
|
||||
@@ -20,493 +22,8 @@ pillars:
|
||||
# Knowledge Flywheel (σ×ρ > δ) → escape velocity → knowledge-compounding
|
||||
goals:
|
||||
# ============================================================================
|
||||
# PILLAR GOALS (25 total)
|
||||
# ============================================================================
|
||||
|
||||
# === Pillar: Knowledge Compounding (10 goals, weight 3-5) ===
|
||||
- id: readme-compounding-hero
|
||||
description: "README hero subtitle communicates the core value prop"
|
||||
check: "head -10 README.md | grep -qi 'forget.*sessions\\|sessions.*smarter\\|memory.*compounds'"
|
||||
weight: 5
|
||||
pillar: knowledge-compounding
|
||||
added: "2026-02-19"
|
||||
- id: readme-compounding-story
|
||||
description: "README leads with the compounding story (session 1 vs session N improvement narrative)"
|
||||
check: "head -50 README.md | grep -qi 'session.*smarter\\|session.*better\\|session.*learns\\|session.*compounds'"
|
||||
weight: 4
|
||||
pillar: knowledge-compounding
|
||||
added: "2026-02-19"
|
||||
- id: no-sigma-rho-user-facing
|
||||
description: "No sigma/rho/delta Greek notation in README or user-facing skill SKILL.md descriptions"
|
||||
check: "grep -rl 'σ\\|ρ\\|δ' README.md skills/*/SKILL.md 2>/dev/null | grep -v references/ | grep -v shared/ | wc -l | tr -d ' ' | xargs test 0 -eq"
|
||||
weight: 3
|
||||
pillar: knowledge-compounding
|
||||
added: "2026-02-19"
|
||||
- id: no-stale-framing
|
||||
description: "No stale factor names in docs, README, CONTRIBUTING, or SVG assets (factors must use 'Factor X: Name → Purpose' format)"
|
||||
check: "bash -c \"find README.md CONTRIBUTING.md docs/ docs/assets/ -type f \\( -name '*.md' -o -name '*.svg' \\) 2>/dev/null | xargs grep -l 'Factor [IVX]\\+:' 2>/dev/null | xargs grep 'Factor [IVX]\\+:' 2>/dev/null | grep -v '→' | wc -l | tr -d ' ' | xargs test 0 -eq\""
|
||||
weight: 3
|
||||
pillar: knowledge-compounding
|
||||
added: "2026-02-19"
|
||||
- id: positioning-no-vibe-coding-primary
|
||||
description: "Positioning doc does not use 'DevOps for Vibe-Coding' as primary tagline"
|
||||
check: "head -15 docs/positioning/devops-for-vibe-coding.md | grep -c 'Primary.*DevOps for Vibe-Coding' | xargs test 0 -eq"
|
||||
weight: 3
|
||||
pillar: knowledge-compounding
|
||||
added: "2026-02-19"
|
||||
- id: mission-in-goals-yaml
|
||||
description: "GOALS.yaml mission matches deliberation consensus"
|
||||
check: "yq -e '.mission == \"Make every coding agent session smarter than the last\"' GOALS.yaml >/dev/null 2>&1"
|
||||
weight: 5
|
||||
pillar: knowledge-compounding
|
||||
added: "2026-02-19"
|
||||
- id: three-ways-in-readme
|
||||
description: "README references DevOps Three Ways (flow, feedback, continual learning) as design foundation"
|
||||
check: "grep -q 'Three Ways' README.md && grep -qi 'flow.*feedback.*learning\\|flow.*feedback.*continual' README.md"
|
||||
weight: 4
|
||||
pillar: knowledge-compounding
|
||||
added: "2026-02-19"
|
||||
- id: meadows-in-readme
|
||||
description: "README references Meadows' hierarchy as design rationale for targeting information flows and rules"
|
||||
check: "grep -q 'Meadows' README.md && grep -q 'information flows' README.md"
|
||||
weight: 4
|
||||
pillar: knowledge-compounding
|
||||
added: "2026-02-19"
|
||||
- id: brownian-ratchet-in-docs
|
||||
description: "Brownian Ratchet is documented as a theoretical pillar (chaos + filter + ratchet = progress)"
|
||||
check: "grep -q 'Brownian' README.md && test -f docs/brownian-ratchet.md && grep -qi 'chaos.*filter.*ratchet\\|filter.*ratchet\\|one-way gate' docs/brownian-ratchet.md"
|
||||
weight: 4
|
||||
pillar: validated-acceleration
|
||||
added: "2026-02-19"
|
||||
- id: flywheel-escape-velocity-in-docs
|
||||
description: "Knowledge Flywheel escape velocity condition is documented (σ×ρ > δ)"
|
||||
check: "test -f docs/the-science.md && grep -q 'Escape Velocity' docs/the-science.md && grep -q 'retrieval.*usage.*decay\\|σ.*ρ.*δ' docs/the-science.md"
|
||||
weight: 4
|
||||
pillar: knowledge-compounding
|
||||
added: "2026-02-19"
|
||||
- id: four-pillars-in-product
|
||||
description: "PRODUCT.md Design Principles section has at least 4 numbered theoretical pillars"
|
||||
check: "awk '/^## Design Principles/,0 {print}' PRODUCT.md | grep -cE '^[0-9]+\\.' | xargs test 4 -le"
|
||||
weight: 5
|
||||
pillar: knowledge-compounding
|
||||
added: "2026-02-19"
|
||||
# === Pillar: Validated Acceleration (5 goals, weight 3-4) ===
|
||||
- id: status-skill-shows-health
|
||||
description: "/status skill references knowledge health or flywheel metrics"
|
||||
check: "grep -qi 'flywheel\\|knowledge.*health\\|compounding\\|retrieval.*rate' skills/status/SKILL.md"
|
||||
weight: 3
|
||||
pillar: validated-acceleration
|
||||
added: "2026-02-19"
|
||||
- id: council-structured-verdicts
|
||||
description: "/council outputs structured PASS/WARN/FAIL verdicts"
|
||||
check: "grep -q 'PASS.*WARN.*FAIL\\|verdict.*PASS\\|PASS | WARN | FAIL' skills/council/SKILL.md"
|
||||
weight: 4
|
||||
pillar: validated-acceleration
|
||||
added: "2026-02-19"
|
||||
- id: vibe-runs-complexity
|
||||
description: "/vibe skill runs complexity analysis before council"
|
||||
check: "grep -qi 'complexity' skills/vibe/SKILL.md"
|
||||
weight: 3
|
||||
pillar: validated-acceleration
|
||||
added: "2026-02-19"
|
||||
- id: crank-wave-vibe-gate
|
||||
description: "/crank enforces wave vibe gates between waves"
|
||||
check: "grep -qi 'wave.*vibe\\|vibe.*gate' skills/crank/SKILL.md"
|
||||
weight: 3
|
||||
pillar: validated-acceleration
|
||||
added: "2026-02-19"
|
||||
# === Pillar: Goal-Driven Automation (5 goals, weight 3-4) ===
|
||||
- id: evolve-dormancy-messaging
|
||||
description: "/evolve skill documents dormancy as a valid success state"
|
||||
check: "grep -qi 'dormancy\\|dormant\\|nothing to do.*valid\\|all goals met.*stop' skills/evolve/SKILL.md"
|
||||
weight: 3
|
||||
pillar: goal-driven-automation
|
||||
added: "2026-02-19"
|
||||
- id: no-autonomous-user-facing
|
||||
description: "README does not use the word 'autonomous' (use 'hands-free' or 'goal-driven' instead)"
|
||||
check: "grep -ci 'autonomous' README.md | xargs test 0 -eq"
|
||||
weight: 3
|
||||
pillar: goal-driven-automation
|
||||
added: "2026-02-19"
|
||||
- id: evolve-kill-switch
|
||||
description: "/evolve has documented kill switch (KILL and STOP files)"
|
||||
check: "grep -q 'KILL' skills/evolve/SKILL.md && grep -q 'STOP' skills/evolve/SKILL.md"
|
||||
weight: 4
|
||||
pillar: goal-driven-automation
|
||||
added: "2026-02-19"
|
||||
- id: rpi-gate-retry-logic
|
||||
description: "/rpi has retry gates for pre-mortem FAIL and vibe FAIL"
|
||||
check: "grep -qi 'retry.*loop\\|max.*attempts\\|FAIL.*retry\\|retry.*FAIL' skills/rpi/SKILL.md"
|
||||
weight: 3
|
||||
pillar: goal-driven-automation
|
||||
added: "2026-02-19"
|
||||
- id: goals-yaml-pillar-coverage
|
||||
description: "GOALS.yaml has goals for all 4 pillars"
|
||||
check: "bash scripts/check-pillar-coverage.sh"
|
||||
weight: 4
|
||||
pillar: goal-driven-automation
|
||||
added: "2026-02-19"
|
||||
# === Pillar: Zero-Friction Workflow (10 goals, weight 3-4) ===
|
||||
- id: quickstart-under-5min
|
||||
description: "/quickstart skill promises value in under 5 minutes"
|
||||
check: "grep -qi '5 min\\|five min\\|under.*minutes\\|minutes.*or less' skills/quickstart/SKILL.md"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: session-start-injects-knowledge
|
||||
description: "Session start hook injects prior knowledge automatically"
|
||||
check: "grep -qi 'inject\\|ao inject\\|knowledge' hooks/session-start.sh"
|
||||
weight: 4
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: handoff-skill-exists
|
||||
description: "/handoff skill exists for seamless session continuity"
|
||||
check: "test -f skills/handoff/SKILL.md && grep -q '^name:' skills/handoff/SKILL.md"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: implement-single-issue
|
||||
description: "/implement skill exists for single-issue execution"
|
||||
check: "test -f skills/implement/SKILL.md && grep -q '^name:' skills/implement/SKILL.md"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: product-olympus-relationship
|
||||
description: "PRODUCT.md has 'Relationship to Olympus' section distinguishing standalone AgentOps from Olympus daemon layer"
|
||||
check: "head -70 PRODUCT.md | grep -q '^## Relationship to Olympus'"
|
||||
weight: 4
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: product-under-line-limit
|
||||
description: "PRODUCT.md stays under 200 lines (prevents bloat creep)"
|
||||
check: "test $(wc -l < PRODUCT.md | tr -d ' ') -le 200"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: quickstart-tiered-catalog
|
||||
description: "Quickstart has tiered skill catalog (full catalog in references/, starter set in SKILL.md)"
|
||||
check: "test -f skills/quickstart/references/full-catalog.md && grep -qi 'starter\\|core\\|begin\\|first' skills/quickstart/SKILL.md"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: verb-aliases-exist
|
||||
description: "Core skills have verb aliases in descriptions (review→vibe, execute→crank, parallel→swarm)"
|
||||
check: "grep -qi 'review\\|check' skills/vibe/SKILL.md && grep -qi 'execute\\|run epic' skills/crank/SKILL.md && grep -qi 'parallel\\|concurrent' skills/swarm/SKILL.md"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: learn-skill-exists
|
||||
description: "/learn skill exists for manual knowledge capture into the flywheel"
|
||||
check: "test -f skills/learn/SKILL.md && grep -q '^name:' skills/learn/SKILL.md"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
# ============================================================================
|
||||
# INFRASTRUCTURE GOALS (28 total)
|
||||
# These are not tied to a pillar — they ensure the system itself is healthy.
|
||||
# Sub-categories below are ordered by severity (fix Critical before Hygiene).
|
||||
# ============================================================================
|
||||
|
||||
# === Critical (weight 5) — Build & test integrity ===
|
||||
# If these fail, nothing works. Fix immediately.
|
||||
- id: go-cli-builds
|
||||
description: "Go CLI compiles without errors"
|
||||
check: "cd cli && go build -o /tmp/test-ao ./cmd/ao >/dev/null 2>&1"
|
||||
weight: 5
|
||||
added: "2026-02-19"
|
||||
- id: go-cli-tests
|
||||
description: "Go CLI unit tests pass"
|
||||
check: "cd cli && go test -race ./... >/dev/null 2>&1"
|
||||
weight: 5
|
||||
added: "2026-02-19"
|
||||
# === Important (weight 3-4) — Suite & safety ===
|
||||
# Full suite, hooks, manifests. Should stay green; fix within a cycle.
|
||||
- id: full-test-suite
|
||||
description: "Full test suite passes (all tiers)"
|
||||
check: "./tests/run-all.sh"
|
||||
weight: 4
|
||||
added: "2026-02-19"
|
||||
- id: hook-preflight
|
||||
description: "All hooks pass safety checks (kill switches, paths)"
|
||||
check: "./scripts/validate-hook-preflight.sh >/dev/null 2>&1"
|
||||
weight: 4
|
||||
added: "2026-02-19"
|
||||
- id: toolchain-security-gate
|
||||
description: "Unified security gate script is executable and testable"
|
||||
check: "test -x scripts/security-gate.sh && bash tests/scripts/test-security-gate.sh >/dev/null 2>&1"
|
||||
weight: 4
|
||||
added: "2026-02-19"
|
||||
- id: ci-security-toolchain-gate
|
||||
description: "Validate workflow enforces security toolchain gate"
|
||||
check: "grep -q 'security-toolchain-gate' .github/workflows/validate.yml && grep -q 'scripts/security-gate.sh' .github/workflows/validate.yml"
|
||||
weight: 4
|
||||
added: "2026-02-19"
|
||||
- id: release-security-gate
|
||||
description: "Release workflow requires security gate before build/publish"
|
||||
check: "grep -q '^ security-gate:' .github/workflows/release.yml && grep -q 'needs: \\[doc-release-gate, security-gate\\]' .github/workflows/release.yml && grep -q 'scripts/security-gate.sh' .github/workflows/release.yml"
|
||||
weight: 4
|
||||
added: "2026-02-19"
|
||||
- id: nightly-security-toolchain
|
||||
description: "Nightly workflow runs security toolchain checks"
|
||||
check: "grep -q '^ security-toolchain:' .github/workflows/nightly.yml && grep -q 'scripts/security-gate.sh' .github/workflows/nightly.yml"
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
- id: smoke-tests
|
||||
description: "Smoke tests pass (toolchain, skills, manifests)"
|
||||
check: "./tests/smoke-test.sh"
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
- id: go-vet-clean
|
||||
description: "Go CLI passes vet checks (no common bugs)"
|
||||
check: "cd cli && go vet ./... 2>&1"
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
- id: manifest-versions-match
|
||||
description: "Plugin and marketplace versions are in sync"
|
||||
check: "test \"$(jq -r '.metadata.version' .claude-plugin/marketplace.json)\" = \"$(jq -r '.version' .claude-plugin/plugin.json)\""
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
- id: go-race-clean
|
||||
description: "Go CLI passes race detector (no data races)"
|
||||
check: "cd cli && go test -race ./... >/dev/null 2>&1"
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
# === Mission Fitness (weight 2-3) — AO ownership areas ===
|
||||
# Knowledge quality, schema validity, contract health. Slower to degrade.
|
||||
- id: knowledge-flywheel-health
|
||||
description: "Knowledge flywheel has ≥5 learnings and newest is <30 days old"
|
||||
check: "test $(ls .agents/learnings/*.md 2>/dev/null | wc -l | tr -d ' ') -ge 5 && test $(find .agents/learnings/ -name '*.md' -mtime -30 2>/dev/null | wc -l | tr -d ' ') -ge 1"
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
- id: skill-semantic-stability
|
||||
description: "Core skills have name, tier, and description in YAML frontmatter"
|
||||
check: "for s in council crank swarm rpi vibe pre-mortem post-mortem plan research implement evolve; do grep -q '^name:' skills/$s/SKILL.md && grep -q 'tier:' skills/$s/SKILL.md && grep -q '^description:' skills/$s/SKILL.md || exit 1; done"
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
- id: bridge-contract-schemas
|
||||
description: "Bridge contracts doc defines all 3 canonical surfaces (INVOCATION_ENVELOPE, STATE_CHECKPOINT_HANDOFF, OBSERVABILITY_EVENTS)"
|
||||
check: "grep -q 'INVOCATION_ENVELOPE' docs/ol-bridge-contracts.md && grep -q 'STATE_CHECKPOINT_HANDOFF' docs/ol-bridge-contracts.md && grep -q 'OBSERVABILITY_EVENTS' docs/ol-bridge-contracts.md"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: learning-format-compliance
|
||||
description: "All learnings conform to interchange spec (YAML frontmatter with id, type, created_at)"
|
||||
check: "bash -c 'for f in $(ls .agents/learnings/*.md 2>/dev/null | grep -v INDEX.md); do head -10 \"$f\" | grep -q \"^---\" && head -10 \"$f\" | grep -q \"^id:\" && head -10 \"$f\" | grep -q \"^type:\" && head -10 \"$f\" | grep -q \"^created_at:\" || { echo FAIL:$f; exit 1; }; done'"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: council-schema-valid
|
||||
description: "Council verdict schema has additionalProperties:false at all levels and required covers core properties (v2 optional fields: fix, why, ref)"
|
||||
check: "jq -e '. as $s | ($s.additionalProperties == false) and (($s.required | length) == ($s.properties | keys | length)) and ($s.properties.findings.items.additionalProperties == false) and (($s.properties.findings.items.required | length) >= 5) and (($s.properties.findings.items.properties | keys | length) <= ($s.properties.findings.items.required | length) + 3)' skills/council/schemas/verdict.json >/dev/null 2>&1"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
# === Hygiene (weight 1-3) — Docs & consistency ===
|
||||
# Lint, coverage, doc counts. Nice to have green; low urgency.
|
||||
- id: shellcheck-clean
|
||||
description: "Hook scripts pass shellcheck with no warnings"
|
||||
check: "shellcheck -x -P SCRIPTDIR hooks/*.sh 2>&1 | grep -c '^In ' | xargs test 0 -eq"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: go-coverage-floor
|
||||
description: "Go CLI average test coverage stays above 80%"
|
||||
check: "cd cli && go test -cover ./... 2>&1 | grep '^ok' | sed -n 's/.*coverage: \\([0-9.]*\\)%.*/\\1/p' | awk '{s+=$1;n++} END{if(n>0 && s/n>=80) exit 0; else exit 1}'"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: rpi-phased-context-discipline
|
||||
description: "ao rpi phased includes context discipline and handoff instructions in every phase prompt"
|
||||
check: "cd cli && go build -o /tmp/test-ao-rpi ./cmd/ao >/dev/null 2>&1 && grep -q 'phaseContextBudgets' cmd/ao/rpi_phased.go"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: rpi-context-window-contract
|
||||
description: "RPI large-repo context sharding contract is healthy and runnable."
|
||||
check: "./scripts/rpi/context-window-contract.sh"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: doc-coverage
|
||||
description: "All public skills have references/ documentation"
|
||||
check: "test $(ls -d skills/*/references/ 2>/dev/null | wc -l | tr -d ' ') -ge 16"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: skill-tier-coverage
|
||||
description: "All skills have tier: defined in YAML frontmatter"
|
||||
check: "test $(grep -rl 'tier:' skills/*/SKILL.md 2>/dev/null | wc -l | tr -d ' ') -eq $(ls -d skills/*/ 2>/dev/null | wc -l | tr -d ' ')"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: skill-validation
|
||||
description: "All skills have valid SKILL.md with frontmatter"
|
||||
check: "./tests/skills/run-all.sh"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: orchestration-skills-validated
|
||||
description: "All orchestration-tier skills have validate.sh scripts"
|
||||
check: "for d in council crank swarm codex-team rpi evolve; do test -x skills/$d/scripts/validate.sh || exit 1; done"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: behavioral-skill-contracts
|
||||
description: "All skills have behavioral validate.sh checks that pass"
|
||||
check: "for d in skills/*/scripts/validate.sh; do \"$d\" >/dev/null 2>&1 || exit 1; done"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: hook-tests-pass
|
||||
description: "All 100 hook integration tests pass"
|
||||
check: "bash tests/hooks/test-hooks.sh 2>&1 | tail -1 | grep -q 'ALL PASSED'"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: export-constraints-readiness
|
||||
description: "ao export-constraints subcommand is stubbed in CLI"
|
||||
check: "grep -rq 'export.constraints\\|export-constraints\\|exportConstraints' cli/cmd/ao/ 2>/dev/null"
|
||||
weight: 1
|
||||
added: "2026-02-19"
|
||||
- id: incident-runbook-exists
|
||||
description: "Consumer incident runbook exists with recovery procedures"
|
||||
check: "test -f docs/INCIDENT-RUNBOOK.md && grep -q 'AGENTOPS_HOOKS_DISABLED' docs/INCIDENT-RUNBOOK.md"
|
||||
weight: 1
|
||||
added: "2026-02-19"
|
||||
- id: readme-goal-count
|
||||
description: "README goal count matches actual GOALS.yaml goal count"
|
||||
check: "bash tests/docs/validate-goal-count.sh"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: passive-citation-tracking
|
||||
description: "PostToolUse(Read) hook records passive citations for .agents/ reads"
|
||||
check: "grep -q '\"Read\"' hooks/hooks.json && grep -q 'passive_read' hooks/citation-tracker.sh"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: cache-eviction-wired
|
||||
description: "SessionEnd eviction hook archives stale low-utility learnings"
|
||||
check: "grep -q 'maturity --evict' hooks/hooks.json"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
- id: flywheel-cache-health
|
||||
description: "Flywheel skill reports cache hit rate and eviction metrics"
|
||||
check: "grep -qi 'cite-report' skills/flywheel/SKILL.md && grep -qi 'eviction' skills/flywheel/SKILL.md"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
# ============================================================================
|
||||
# CROSS-RUNTIME COMPATIBILITY GOALS (10 total)
|
||||
# Ensure skills work across Claude Code, Codex CLI, OpenCode, and Cursor.
|
||||
# ============================================================================
|
||||
|
||||
# === Backend Detection (weight 4) — Skills detect and use correct runtime ===
|
||||
- id: shared-backend-4tier
|
||||
description: "Shared contract has 4-tier backend selection (Codex → Claude teams → OpenCode → Inline)"
|
||||
check: "grep -q 'Codex experimental sub-agents' skills/shared/SKILL.md && grep -q 'Claude native teams' skills/shared/SKILL.md && grep -q 'OpenCode subagents' skills/shared/SKILL.md && grep -q 'Inline fallback' skills/shared/SKILL.md"
|
||||
weight: 4
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: shared-skill-chaining-contract
|
||||
description: "Shared contract has cross-runtime skill chaining rules (Claude exec vs OpenCode read-only)"
|
||||
check: "grep -q 'Skill Invocation Across Runtimes' skills/shared/SKILL.md && grep -qi 'read-only' skills/shared/SKILL.md && grep -q 'NEVER use slashcommand' skills/shared/SKILL.md"
|
||||
weight: 4
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: orchestration-skills-opencode-tier
|
||||
description: "All orchestration skills include OpenCode in backend selection"
|
||||
check: "grep -qi 'opencode' skills/council/SKILL.md && grep -qi 'opencode' skills/research/SKILL.md && grep -qi 'opencode' skills/swarm/SKILL.md && grep -qi 'opencode' skills/codex-team/SKILL.md && grep -qi 'opencode' skills/trace/SKILL.md"
|
||||
weight: 4
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
# === OpenCode Plugin (weight 3-4) — Plugin bridge is functional ===
|
||||
- id: opencode-plugin-exists
|
||||
description: "OpenCode plugin exists with system transform and tool guards"
|
||||
check: "test -f .opencode/plugins/agentops.js && grep -q 'chat.system.transform' .opencode/plugins/agentops.js && grep -q 'tool.execute.before' .opencode/plugins/agentops.js && grep -q 'tool.definition' .opencode/plugins/agentops.js"
|
||||
weight: 4
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: opencode-plugin-audit-hook
|
||||
description: "OpenCode plugin has tool.execute.after audit logging"
|
||||
check: "grep -q 'tool.execute.after' .opencode/plugins/agentops.js && grep -q 'audit' .opencode/plugins/agentops.js"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: opencode-plugin-compaction-hook
|
||||
description: "OpenCode plugin has session.compacting hook for context preservation"
|
||||
check: "grep -q 'session.compacting' .opencode/plugins/agentops.js && grep -q 'AgentOps' .opencode/plugins/agentops.js"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: opencode-plugin-syntax-valid
|
||||
description: "OpenCode plugin passes Node.js syntax check"
|
||||
check: "node -c .opencode/plugins/agentops.js 2>/dev/null"
|
||||
weight: 4
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
- id: opencode-install-script
|
||||
description: "OpenCode one-liner install script exists and is executable"
|
||||
check: "test -x scripts/install-opencode.sh && grep -q 'curl' .opencode/INSTALL.md"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
# === Headless Test Harness (weight 3) — Can verify skills in OpenCode ===
|
||||
- id: opencode-test-harness
|
||||
description: "OpenCode headless test harness exists with tier and ratchet support"
|
||||
check: "test -x tests/opencode/run-headless-tests.sh && grep -q 'tier' tests/opencode/run-headless-tests.sh && grep -q 'MAX_ATTEMPTS' tests/opencode/run-headless-tests.sh"
|
||||
weight: 3
|
||||
pillar: validated-acceleration
|
||||
added: "2026-02-19"
|
||||
- id: opencode-assertions-framework
|
||||
description: "OpenCode test assertion framework exists with per-skill validators"
|
||||
check: "test -f tests/opencode/assertions.sh && test -f tests/opencode/skill-assertions.sh && grep -q 'assert_contains' tests/opencode/assertions.sh"
|
||||
weight: 3
|
||||
pillar: validated-acceleration
|
||||
added: "2026-02-19"
|
||||
- id: opencode-test-prompts-external
|
||||
description: "OpenCode test prompts externalized to files for prompt tuning"
|
||||
check: "test -d tests/opencode/prompts && ls tests/opencode/prompts/*.txt 2>/dev/null | wc -l | grep -qE '[5-9]|[1-9][0-9]'"
|
||||
weight: 2
|
||||
pillar: validated-acceleration
|
||||
added: "2026-02-19"
|
||||
- id: opencode-live-tier1
|
||||
description: "At least 4/6 Tier 1 skills pass headless testing with 3 attempts"
|
||||
check: "bash scripts/check-opencode-live.sh"
|
||||
weight: 5
|
||||
pillar: validated-acceleration
|
||||
added: "2026-02-19"
|
||||
# === Universal Skill Format (weight 3) — Skills discoverable by all agents ===
|
||||
- id: all-skills-have-frontmatter
|
||||
description: "Every SKILL.md has YAML frontmatter with name and description (required by all runtimes)"
|
||||
check: "bash -c 'for f in skills/*/SKILL.md; do head -5 \"$f\" | grep -q \"^---\" && head -10 \"$f\" | grep -q \"^name:\" && head -10 \"$f\" | grep -q \"^description:\" || { echo FAIL:$f; exit 1; }; done'"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
# --- Meta-Goals (type: meta) ---
|
||||
- id: go-complexity-gate
|
||||
description: "Changed non-test Go code under cli/ stays below complexity failure threshold"
|
||||
check: "bash scripts/check-go-complexity.sh"
|
||||
weight: 5
|
||||
type: meta
|
||||
pillar: infrastructure
|
||||
added: "2026-02-20"
|
||||
- id: wiring-closure
|
||||
description: "All scripts, skills, and hooks are referenced by their parent registries"
|
||||
check: "bash scripts/check-wiring-closure.sh"
|
||||
weight: 6
|
||||
type: meta
|
||||
pillar: infrastructure
|
||||
added: "2026-02-19"
|
||||
- id: telemetry-coverage
|
||||
description: "Orchestration skills log to skill-telemetry.jsonl"
|
||||
check: "grep -q 'log-telemetry.sh' skills/rpi/SKILL.md && grep -q 'log-telemetry.sh' skills/crank/SKILL.md && grep -q 'log-telemetry.sh' skills/evolve/SKILL.md"
|
||||
weight: 5
|
||||
type: meta
|
||||
pillar: infrastructure
|
||||
added: "2026-02-19"
|
||||
- id: compaction-proofing
|
||||
description: "Orchestration skills commit checkpoints mid-workflow"
|
||||
check: "grep -q 'checkpoint-commit.sh' skills/rpi/SKILL.md && grep -q 'checkpoint-commit.sh' skills/crank/SKILL.md"
|
||||
weight: 5
|
||||
type: meta
|
||||
pillar: infrastructure
|
||||
added: "2026-02-19"
|
||||
- id: ao-goals-builds
|
||||
description: "ao goals subcommand compiles and runs"
|
||||
check: "cd cli && go build -o /tmp/test-ao ./cmd/ao && /tmp/test-ao goals validate --json >/dev/null 2>&1"
|
||||
weight: 5
|
||||
pillar: infrastructure
|
||||
added: "2026-02-19"
|
||||
|
||||
# ============================================================================
|
||||
# SELF-HEALING GOALS (3 total)
|
||||
# Prevent the exact failure modes that required manual pruning.
|
||||
# SELF-HEALING (3 goals, weight 7)
|
||||
# Highest-weight goals. Prevent the failure modes that required manual pruning.
|
||||
# ============================================================================
|
||||
|
||||
- id: no-stale-goals
|
||||
@@ -526,3 +43,173 @@ goals:
|
||||
check: "bash scripts/check-product-freshness.sh"
|
||||
weight: 7
|
||||
added: "2026-02-19"
|
||||
|
||||
# ============================================================================
|
||||
# BUILD INTEGRITY (4 goals, weight 3-5)
|
||||
# If these fail, nothing works. Fix immediately.
|
||||
# ============================================================================
|
||||
|
||||
- id: go-cli-builds
|
||||
description: "Go CLI compiles without errors"
|
||||
check: "cd cli && go build -o /tmp/test-ao ./cmd/ao >/dev/null 2>&1"
|
||||
weight: 5
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: go-cli-tests
|
||||
description: "Go CLI unit tests pass (with race detector)"
|
||||
check: "cd cli && go test -race ./... >/dev/null 2>&1"
|
||||
weight: 5
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: full-test-suite
|
||||
description: "Full test suite passes (all tiers)"
|
||||
check: "./tests/run-all.sh"
|
||||
weight: 4
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: go-vet-clean
|
||||
description: "Go CLI passes vet checks (no common bugs)"
|
||||
check: "cd cli && go vet ./... 2>&1"
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
|
||||
# ============================================================================
|
||||
# SECURITY (4 goals, weight 4)
|
||||
# Safety gates across all pipelines.
|
||||
# ============================================================================
|
||||
|
||||
- id: hook-preflight
|
||||
description: "All hooks pass safety checks (kill switches, paths)"
|
||||
check: "./scripts/validate-hook-preflight.sh >/dev/null 2>&1"
|
||||
weight: 4
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: toolchain-security-gate
|
||||
description: "Unified security gate script is executable and testable"
|
||||
check: "test -x scripts/security-gate.sh && bash tests/scripts/test-security-gate.sh >/dev/null 2>&1"
|
||||
weight: 4
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: ci-security-toolchain-gate
|
||||
description: "Validate workflow enforces security toolchain gate"
|
||||
check: "grep -q 'security-toolchain-gate' .github/workflows/validate.yml && grep -q 'scripts/security-gate.sh' .github/workflows/validate.yml"
|
||||
weight: 4
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: release-security-gate
|
||||
description: "Release workflow requires security gate before build/publish"
|
||||
check: "grep -q '^ security-gate:' .github/workflows/release.yml && grep -q 'needs: \\[doc-release-gate, security-gate\\]' .github/workflows/release.yml && grep -q 'scripts/security-gate.sh' .github/workflows/release.yml"
|
||||
weight: 4
|
||||
added: "2026-02-19"
|
||||
|
||||
# ============================================================================
|
||||
# QUALITY GATES (3 goals, weight 2-5)
|
||||
# Complexity, coverage, and lint checks that catch regressions.
|
||||
# ============================================================================
|
||||
|
||||
- id: go-complexity-gate
|
||||
description: "Changed non-test Go code under cli/ stays below complexity failure threshold"
|
||||
check: "bash scripts/check-go-complexity.sh"
|
||||
weight: 5
|
||||
type: meta
|
||||
added: "2026-02-20"
|
||||
|
||||
- id: go-coverage-floor
|
||||
description: "Go CLI average test coverage stays above 80%"
|
||||
check: "cd cli && go test -cover ./... 2>&1 | grep '^ok' | sed -n 's/.*coverage: \\([0-9.]*\\)%.*/\\1/p' | awk '{s+=$1;n++} END{if(n>0 && s/n>=80) exit 0; else exit 1}'"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: smoke-tests
|
||||
description: "Smoke tests pass (toolchain, skills, manifests)"
|
||||
check: "./tests/smoke-test.sh"
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
|
||||
# ============================================================================
|
||||
# SYSTEM HEALTH (4 goals, weight 3-6)
|
||||
# Goals subsystem, plugin versions, wiring integrity.
|
||||
# ============================================================================
|
||||
|
||||
- id: ao-goals-builds
|
||||
description: "ao goals subcommand compiles and runs"
|
||||
check: "cd cli && go build -o /tmp/test-ao ./cmd/ao && /tmp/test-ao goals validate --json >/dev/null 2>&1"
|
||||
weight: 5
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: manifest-versions-match
|
||||
description: "Plugin and marketplace versions are in sync"
|
||||
check: "test \"$(jq -r '.metadata.version' .claude-plugin/marketplace.json)\" = \"$(jq -r '.version' .claude-plugin/plugin.json)\""
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: wiring-closure
|
||||
description: "All scripts, skills, and hooks are referenced by their parent registries"
|
||||
check: "bash scripts/check-wiring-closure.sh"
|
||||
weight: 6
|
||||
type: meta
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: skill-validation
|
||||
description: "All skills have valid SKILL.md with frontmatter"
|
||||
check: "./tests/skills/run-all.sh"
|
||||
weight: 2
|
||||
added: "2026-02-19"
|
||||
|
||||
# ============================================================================
|
||||
# SKILLS HEALTH (2 goals, weight 3)
|
||||
# Core skill contracts and universal discoverability.
|
||||
# ============================================================================
|
||||
|
||||
- id: skill-semantic-stability
|
||||
description: "Core skills have name, tier, and description in YAML frontmatter"
|
||||
check: "for s in council crank swarm rpi vibe pre-mortem post-mortem plan research implement evolve; do grep -q '^name:' skills/$s/SKILL.md && grep -q 'tier:' skills/$s/SKILL.md && grep -q '^description:' skills/$s/SKILL.md || exit 1; done"
|
||||
weight: 3
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: all-skills-have-frontmatter
|
||||
description: "Every SKILL.md has YAML frontmatter with name and description (required by all runtimes)"
|
||||
check: "bash -c 'for f in skills/*/SKILL.md; do head -5 \"$f\" | grep -q \"^---\" && head -10 \"$f\" | grep -q \"^name:\" && head -10 \"$f\" | grep -q \"^description:\" || { echo FAIL:$f; exit 1; }; done'"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
|
||||
# ============================================================================
|
||||
# FEATURE-CRITICAL (5 goals, weight 3-5)
|
||||
# Guard the core features that make the system work.
|
||||
# ============================================================================
|
||||
|
||||
- id: session-start-injects-knowledge
|
||||
description: "Session start hook injects prior knowledge automatically"
|
||||
check: "grep -qi 'inject\\|ao inject\\|knowledge' hooks/session-start.sh"
|
||||
weight: 4
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: evolve-kill-switch
|
||||
description: "/evolve has documented kill switch (KILL and STOP files)"
|
||||
check: "grep -q 'KILL' skills/evolve/SKILL.md && grep -q 'STOP' skills/evolve/SKILL.md"
|
||||
weight: 4
|
||||
pillar: goal-driven-automation
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: rpi-phased-context-discipline
|
||||
description: "ao rpi phased includes context discipline and handoff instructions in every phase prompt"
|
||||
check: "cd cli && go build -o /tmp/test-ao-rpi ./cmd/ao >/dev/null 2>&1 && grep -q 'phaseContextBudgets' cmd/ao/rpi_phased.go"
|
||||
weight: 3
|
||||
pillar: zero-friction-workflow
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: telemetry-coverage
|
||||
description: "Orchestration skills log to skill-telemetry.jsonl"
|
||||
check: "grep -q 'log-telemetry.sh' skills/rpi/SKILL.md && grep -q 'log-telemetry.sh' skills/crank/SKILL.md && grep -q 'log-telemetry.sh' skills/evolve/SKILL.md"
|
||||
weight: 5
|
||||
type: meta
|
||||
added: "2026-02-19"
|
||||
|
||||
- id: compaction-proofing
|
||||
description: "Orchestration skills commit checkpoints mid-workflow"
|
||||
check: "grep -q 'checkpoint-commit.sh' skills/rpi/SKILL.md && grep -q 'checkpoint-commit.sh' skills/crank/SKILL.md"
|
||||
weight: 5
|
||||
type: meta
|
||||
added: "2026-02-19"
|
||||
|
||||
@@ -345,6 +345,8 @@ generate_sbom_artifacts() {
|
||||
|
||||
run_security_gate() {
|
||||
local output_file="$ARTIFACT_DIR/security-gate-${SECURITY_MODE}.json"
|
||||
SECURITY_GATE_OUTPUT_DIR="$ARTIFACT_DIR/security" \
|
||||
TOOLCHAIN_OUTPUT_DIR="$ARTIFACT_DIR/tooling" \
|
||||
./scripts/security-gate.sh --mode "$SECURITY_MODE" --require-tools --json > "$output_file"
|
||||
jq -e '.gate_status' "$output_file" >/dev/null
|
||||
echo "Security report: $output_file"
|
||||
|
||||
Executable
+322
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# claude-freeze-repro.sh — Capture diagnostic evidence of Claude Code
|
||||
# CPU freeze triggered by TaskStop (or any tool interruption).
|
||||
#
|
||||
# PROBLEM: Once the freeze hits, the system can't fork new processes.
|
||||
# SOLUTION: Start all monitors BEFORE the trigger so they're already
|
||||
# capturing when the event loop wedges.
|
||||
#
|
||||
# Usage:
|
||||
# Terminal A: Run a Claude Code session normally
|
||||
# Terminal B: ./scripts/claude-freeze-repro.sh [duration_seconds]
|
||||
# (monitors start, waits for you to trigger)
|
||||
# Terminal A: In Claude, run a background task then stop it
|
||||
# Terminal B: Press ENTER after freeze (or let it auto-detect)
|
||||
#
|
||||
# Output: /tmp/claude-freeze-diag-<timestamp>/
|
||||
# Attach the whole directory to the GitHub issue.
|
||||
|
||||
DURATION="${1:-180}" # seconds to monitor (default 3 min)
|
||||
POLL_INTERVAL=1 # seconds between snapshots
|
||||
DIAG_DIR="/tmp/claude-freeze-diag-$(date +%Y%m%dT%H%M%S)"
|
||||
mkdir -p "$DIAG_DIR"
|
||||
|
||||
cleanup_pids=()
|
||||
cleanup() {
|
||||
for pid in "${cleanup_pids[@]}"; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
wait 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
# ── Step 1: Find Claude Code process ────────────────────────────
|
||||
log "Searching for Claude Code process..."
|
||||
|
||||
# macOS pgrep doesn't reliably match Bun/Node binaries that override argv[0].
|
||||
# Use ps + grep instead: find processes whose comm is exactly "claude".
|
||||
CLAUDE_PID=""
|
||||
CLAUDE_CPU="0"
|
||||
BEST_CPU_INT=0
|
||||
|
||||
while IFS= read -r line; do
|
||||
pid=$(echo "$line" | awk '{print $1}')
|
||||
cpu=$(echo "$line" | awk '{print $2}')
|
||||
cpu_int=${cpu%%.*}
|
||||
if [[ "${cpu_int:-0}" -gt "$BEST_CPU_INT" ]] || [[ -z "$CLAUDE_PID" ]]; then
|
||||
CLAUDE_PID="$pid"
|
||||
CLAUDE_CPU="$cpu"
|
||||
BEST_CPU_INT="${cpu_int:-0}"
|
||||
fi
|
||||
done < <(ps -eo pid=,pcpu=,comm=,args= 2>/dev/null | awk '$3 == "claude" && $4 == "claude"')
|
||||
|
||||
if [[ -z "$CLAUDE_PID" ]]; then
|
||||
echo "ERROR: No Claude Code process found."
|
||||
echo "Start a Claude Code session in another terminal first."
|
||||
echo ""
|
||||
echo "Processes with 'claude' in name:"
|
||||
ps -eo pid,pcpu,comm 2>/dev/null | grep -i claude || echo " (none)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Found PID $CLAUDE_PID (claude) at ${CLAUDE_CPU}% CPU"
|
||||
|
||||
# ── Step 2: Capture environment baseline ────────────────────────
|
||||
log "Capturing baseline..."
|
||||
|
||||
{
|
||||
echo "=== Environment ==="
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Hostname: $(hostname)"
|
||||
echo "OS: $(sw_vers -productName 2>/dev/null || uname -s) $(sw_vers -productVersion 2>/dev/null || uname -r)"
|
||||
echo "Kernel: $(uname -r)"
|
||||
echo "Arch: $(uname -m)"
|
||||
echo "CPUs: $(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo unknown)"
|
||||
echo "RAM: $(sysctl -n hw.memsize 2>/dev/null | awk '{printf "%.0f GB", $1/1073741824}' 2>/dev/null || echo unknown)"
|
||||
echo ""
|
||||
echo "=== Claude Code ==="
|
||||
echo "PID: $CLAUDE_PID"
|
||||
timeout 5 claude --version 2>/dev/null || echo "claude --version: not available"
|
||||
echo ""
|
||||
echo "=== Claude processes (sanitized — comm only, no args) ==="
|
||||
ps -eo pid,ppid,stat,pcpu,pmem,rss,comm 2>/dev/null | head -1 || true
|
||||
ps -eo pid,ppid,stat,pcpu,pmem,rss,comm 2>/dev/null | grep -i claude | grep -v grep || true
|
||||
echo ""
|
||||
echo "=== Open FDs (baseline) ==="
|
||||
timeout 10 lsof -p "$CLAUDE_PID" 2>/dev/null | head -50 || true
|
||||
} > "$DIAG_DIR/00-baseline.txt" 2>&1
|
||||
|
||||
log "Baseline saved"
|
||||
|
||||
# ── Step 3: Start monitors ──────────────────────────────────────
|
||||
log "Starting monitors (${DURATION}s window)..."
|
||||
|
||||
# Monitor A: macOS CPU profiler — samples call stacks
|
||||
# This is the money shot: shows exactly what function is spinning
|
||||
if command -v sample &>/dev/null; then
|
||||
log " [A] sample (call-stack profiler, ${DURATION}s)"
|
||||
sample "$CLAUDE_PID" "$DURATION" -f "$DIAG_DIR/01-sample.txt" 2>/dev/null &
|
||||
cleanup_pids+=($!)
|
||||
fi
|
||||
|
||||
# Monitor B: spindump — macOS hang detector (needs root)
|
||||
if command -v spindump &>/dev/null && [[ $(id -u) -eq 0 ]]; then
|
||||
log " [B] spindump (hang detector, ${DURATION}s)"
|
||||
spindump "$CLAUDE_PID" "$DURATION" -file "$DIAG_DIR/02-spindump.txt" 2>/dev/null &
|
||||
cleanup_pids+=($!)
|
||||
else
|
||||
log " [B] spindump skipped (needs: sudo $0)"
|
||||
fi
|
||||
|
||||
# Monitor C: CPU% timeline — 1 sample/sec via ps polling
|
||||
log " [C] CPU timeline (ps polling, ${POLL_INTERVAL}s interval)"
|
||||
(
|
||||
echo "timestamp,pid,pcpu,pmem,rss_kb,vsz_kb,state,threads"
|
||||
end=$((SECONDS + DURATION))
|
||||
while [[ $SECONDS -lt $end ]]; do
|
||||
ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
ps -p "$CLAUDE_PID" -o pid=,pcpu=,pmem=,rss=,vsz=,state= 2>/dev/null | while read -r pid pcpu pmem rss vsz state; do
|
||||
nlwp=$(ps -M -p "$CLAUDE_PID" 2>/dev/null | tail -n +2 | wc -l | tr -d ' ')
|
||||
echo "${ts},${pid},${pcpu},${pmem},${rss},${vsz},${state},${nlwp}"
|
||||
done
|
||||
sleep "$POLL_INTERVAL"
|
||||
done
|
||||
) > "$DIAG_DIR/03-cpu-timeline.csv" 2>&1 &
|
||||
cleanup_pids+=($!)
|
||||
|
||||
# Monitor D: Child process tracker — catches zombie/orphan children
|
||||
log " [D] Child process tracker (2s interval)"
|
||||
(
|
||||
end=$((SECONDS + DURATION))
|
||||
while [[ $SECONDS -lt $end ]]; do
|
||||
echo "--- $(date -u +%Y-%m-%dT%H:%M:%SZ) ---"
|
||||
pgrep -P "$CLAUDE_PID" 2>/dev/null | while read -r child; do
|
||||
ps -p "$child" -o pid,ppid,stat,pcpu,rss,comm 2>/dev/null || true
|
||||
pgrep -P "$child" 2>/dev/null | while read -r grandchild; do
|
||||
echo " └─ $(ps -p "$grandchild" -o pid,ppid,stat,pcpu,rss,comm 2>/dev/null || true)"
|
||||
done
|
||||
done
|
||||
echo ""
|
||||
sleep 2
|
||||
done
|
||||
) > "$DIAG_DIR/04-children.txt" 2>&1 &
|
||||
cleanup_pids+=($!)
|
||||
|
||||
# Monitor E: Open file descriptors over time (pipes are the likely culprit)
|
||||
log " [E] FD/pipe tracker (5s interval)"
|
||||
(
|
||||
end=$((SECONDS + DURATION))
|
||||
while [[ $SECONDS -lt $end ]]; do
|
||||
echo "--- $(date -u +%Y-%m-%dT%H:%M:%SZ) ---"
|
||||
timeout 5 lsof -p "$CLAUDE_PID" 2>/dev/null | grep -E 'PIPE|FIFO|pipe|CHR' || echo "(no pipes)"
|
||||
echo ""
|
||||
sleep 5
|
||||
done
|
||||
) > "$DIAG_DIR/05-fd-pipes.txt" 2>&1 &
|
||||
cleanup_pids+=($!)
|
||||
|
||||
# Monitor F: System load (proves the freeze starves other processes)
|
||||
log " [F] System load tracker (${POLL_INTERVAL}s interval)"
|
||||
(
|
||||
end=$((SECONDS + DURATION))
|
||||
while [[ $SECONDS -lt $end ]]; do
|
||||
ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
load=$(sysctl -n vm.loadavg 2>/dev/null | awk '{print $2, $3, $4}' || uptime | awk -F'load average:' '{print $2}')
|
||||
echo "${ts} load: ${load}"
|
||||
sleep "$POLL_INTERVAL"
|
||||
done
|
||||
) > "$DIAG_DIR/06-load.txt" 2>&1 &
|
||||
cleanup_pids+=($!)
|
||||
|
||||
# Brief pause to let monitors initialize
|
||||
sleep 1
|
||||
|
||||
# ── Step 4: Wait for trigger ────────────────────────────────────
|
||||
cat <<'BANNER'
|
||||
|
||||
╔══════════════════════════════════════════════════════════════╗
|
||||
║ MONITORS RUNNING ║
|
||||
║ ║
|
||||
║ Now trigger the freeze in your Claude Code session: ║
|
||||
║ ║
|
||||
║ 1. Ask Claude to run a background Bash task ║
|
||||
║ e.g. "run sleep 60 in background" ║
|
||||
║ ║
|
||||
║ 2. Then ask Claude to stop that task ║
|
||||
║ (or it may call TaskStop automatically) ║
|
||||
║ ║
|
||||
║ Press ENTER after the freeze happens. ║
|
||||
║ (Auto-detects if CPU exceeds 95% for 10s) ║
|
||||
╚══════════════════════════════════════════════════════════════╝
|
||||
|
||||
BANNER
|
||||
|
||||
# Auto-detect freeze: poll CPU% and trigger capture if sustained spike
|
||||
(
|
||||
spike_count=0
|
||||
while true; do
|
||||
cpu=$(ps -p "$CLAUDE_PID" -o pcpu= 2>/dev/null | tr -d ' ' || echo "0")
|
||||
cpu_int=${cpu%%.*}
|
||||
if [[ "${cpu_int:-0}" -ge 95 ]]; then
|
||||
spike_count=$((spike_count + 1))
|
||||
if [[ "$spike_count" -ge 10 ]]; then
|
||||
echo "FREEZE_DETECTED" > "$DIAG_DIR/.freeze-flag"
|
||||
log "AUTO-DETECTED: CPU at ${cpu}% for ${spike_count}s"
|
||||
{
|
||||
echo "=== Freeze detected at $(date -u +%Y-%m-%dT%H:%M:%SZ) ==="
|
||||
echo "CPU: ${cpu}%"
|
||||
echo "Sustained spike: ${spike_count}s"
|
||||
echo ""
|
||||
echo "=== Process state ==="
|
||||
ps -eo pid,ppid,stat,pcpu,pmem,rss,comm 2>/dev/null | grep -i claude | grep -v grep || true
|
||||
echo ""
|
||||
echo "=== Thread states ==="
|
||||
ps -M -p "$CLAUDE_PID" 2>/dev/null || true
|
||||
echo ""
|
||||
echo "=== Open FDs ==="
|
||||
timeout 10 lsof -p "$CLAUDE_PID" 2>/dev/null | head -100 || true
|
||||
} > "$DIAG_DIR/07-freeze-snapshot.txt" 2>&1
|
||||
break
|
||||
fi
|
||||
else
|
||||
spike_count=0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
) &
|
||||
DETECT_PID=$!
|
||||
cleanup_pids+=($DETECT_PID)
|
||||
|
||||
# Wait for user ENTER or auto-detection
|
||||
while true; do
|
||||
if read -t 2; then
|
||||
break
|
||||
fi
|
||||
if [[ -f "$DIAG_DIR/.freeze-flag" ]]; then
|
||||
echo ""
|
||||
log "Freeze auto-detected! Continuing capture for 30 more seconds..."
|
||||
sleep 30
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Step 5: Final capture ───────────────────────────────────────
|
||||
log "Capturing final state..."
|
||||
|
||||
{
|
||||
echo "=== Final process state ==="
|
||||
echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo ""
|
||||
ps -eo pid,ppid,stat,pcpu,pmem,rss,comm 2>/dev/null | grep -i claude | grep -v grep || echo "(no claude processes)"
|
||||
echo ""
|
||||
echo "=== Thread dump ==="
|
||||
ps -M -p "$CLAUDE_PID" 2>/dev/null || echo "(process gone)"
|
||||
echo ""
|
||||
echo "=== Final FDs ==="
|
||||
timeout 10 lsof -p "$CLAUDE_PID" 2>/dev/null | head -100 || echo "(process gone)"
|
||||
echo ""
|
||||
echo "=== System load ==="
|
||||
uptime
|
||||
} > "$DIAG_DIR/08-final-state.txt" 2>&1
|
||||
|
||||
# ── Step 6: Stop monitors and package ───────────────────────────
|
||||
log "Stopping monitors..."
|
||||
cleanup
|
||||
trap - EXIT
|
||||
|
||||
rm -f "$DIAG_DIR/.freeze-flag"
|
||||
|
||||
# Generate summary
|
||||
cat > "$DIAG_DIR/README.md" <<EOF
|
||||
# Claude Code TaskStop Freeze — Diagnostic Bundle
|
||||
|
||||
**Generated:** $(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
**PID:** $CLAUDE_PID
|
||||
**Duration:** ${DURATION}s capture window
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| 00-baseline.txt | Environment + process state before trigger |
|
||||
| 01-sample.txt | macOS CPU profiler call stacks (**KEY EVIDENCE**) |
|
||||
| 02-spindump.txt | macOS hang detector output (if run as root) |
|
||||
| 03-cpu-timeline.csv | CPU% sampled every ${POLL_INTERVAL}s (shows spike onset) |
|
||||
| 04-children.txt | Child process lifecycle (zombie/orphan detection) |
|
||||
| 05-fd-pipes.txt | Pipe/FIFO file descriptors (unclosed pipe = likely cause) |
|
||||
| 06-load.txt | System load average (proves system-wide starvation) |
|
||||
| 07-freeze-snapshot.txt | Snapshot captured at moment of detected freeze |
|
||||
| 08-final-state.txt | Process state after capture window |
|
||||
|
||||
## How to read the evidence
|
||||
|
||||
1. **01-sample.txt** — Look for a single function consuming >90% of samples.
|
||||
This is the infinite loop. The function name/address identifies the bug.
|
||||
|
||||
2. **03-cpu-timeline.csv** — Import into a spreadsheet. The CPU% column
|
||||
should show a sharp jump from normal (<30%) to 100% at the freeze moment.
|
||||
|
||||
3. **05-fd-pipes.txt** — Compare before and after freeze. If a PIPE fd
|
||||
appears before and persists after the child is killed, that's the
|
||||
unclosed pipe causing the event loop to spin.
|
||||
|
||||
4. **04-children.txt** — If the stopped background task's process is still
|
||||
alive (zombie state Z) after TaskStop, cleanup failed.
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════════════════════════╗"
|
||||
echo "║ CAPTURE COMPLETE ║"
|
||||
echo "╚══════════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
echo "Output: $DIAG_DIR"
|
||||
echo ""
|
||||
ls -lh "$DIAG_DIR"
|
||||
echo ""
|
||||
echo "To attach to GitHub issue:"
|
||||
echo " tar czf /tmp/claude-freeze-diag.tar.gz -C /tmp $(basename "$DIAG_DIR")"
|
||||
echo ""
|
||||
echo "Key file to check first: $DIAG_DIR/01-sample.txt"
|
||||
@@ -3,7 +3,7 @@
|
||||
#
|
||||
# Usage: ./scripts/compute-triage-accuracy.sh
|
||||
#
|
||||
# Reads: .agents/tooling/triage-decisions.jsonl
|
||||
# Reads: $TOOLCHAIN_OUTPUT_DIR/triage-decisions.jsonl (default: $TMPDIR/agentops-tooling/)
|
||||
# Output: Accuracy statistics by agent and tool
|
||||
#
|
||||
# Ground truth is set when:
|
||||
@@ -13,12 +13,14 @@
|
||||
#
|
||||
# To add ground truth:
|
||||
# # Read file, update ground_truth, write back
|
||||
# TDIR="${TOOLCHAIN_OUTPUT_DIR:-${TMPDIR:-/tmp}/agentops-tooling}"
|
||||
# jq 'if .file_line == "src/auth.go:42" then .ground_truth = "TRUE_POS" else . end' \
|
||||
# .agents/tooling/triage-decisions.jsonl > tmp && mv tmp .agents/tooling/triage-decisions.jsonl
|
||||
# "$TDIR/triage-decisions.jsonl" > tmp && mv tmp "$TDIR/triage-decisions.jsonl"
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LOG_FILE=".agents/tooling/triage-decisions.jsonl"
|
||||
TOOLING_DIR="${TOOLCHAIN_OUTPUT_DIR:-${TMPDIR:-/tmp}/agentops-tooling}"
|
||||
LOG_FILE="$TOOLING_DIR/triage-decisions.jsonl"
|
||||
|
||||
if [[ ! -f "$LOG_FILE" ]]; then
|
||||
echo "No triage decisions logged yet"
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
# verdict - Triage verdict: TRUE_POS or FALSE_POS
|
||||
# agent - Agent that made the decision (e.g., security-reviewer)
|
||||
#
|
||||
# Output: Appends JSONL entry to .agents/tooling/triage-decisions.jsonl
|
||||
# Output: Appends JSONL entry to $TOOLCHAIN_OUTPUT_DIR/triage-decisions.jsonl (default: $TMPDIR/agentops-tooling/)
|
||||
#
|
||||
# Ground truth can be added later via:
|
||||
# jq '. | select(.file_line == "src/auth.go:42") | .ground_truth = "TRUE_POS"'
|
||||
@@ -38,8 +38,9 @@ if [[ "$VERDICT" != "TRUE_POS" && "$VERDICT" != "FALSE_POS" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LOG_FILE=".agents/tooling/triage-decisions.jsonl"
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
TOOLING_DIR="${TOOLCHAIN_OUTPUT_DIR:-${TMPDIR:-/tmp}/agentops-tooling}"
|
||||
LOG_FILE="$TOOLING_DIR/triage-decisions.jsonl"
|
||||
mkdir -p "$TOOLING_DIR"
|
||||
|
||||
jq -n \
|
||||
--arg file_line "$FILE_LINE" \
|
||||
|
||||
+15
-24
@@ -125,30 +125,23 @@ numfmt_size() {
|
||||
prune_keep_newest "$AGENTS_DIR/council" 30 "council"
|
||||
[[ "$QUIET" == false ]] && echo ""
|
||||
|
||||
# --- Policy: tooling/ — keep last run only (newest date prefix) ---
|
||||
if [[ -d "$AGENTS_DIR/tooling" ]]; then
|
||||
tooling_count=$(find "$AGENTS_DIR/tooling" -maxdepth 1 -type f 2>/dev/null | wc -l | tr -d ' ')
|
||||
if [[ "$tooling_count" -gt 0 ]]; then
|
||||
# All tooling files from the same run — keep newest by mtime, prune rest
|
||||
# Since tooling has no date-prefix convention, keep files from last 1 day
|
||||
old_tooling=$(find "$AGENTS_DIR/tooling" -maxdepth 1 -type f -mtime +1 2>/dev/null | wc -l | tr -d ' ')
|
||||
if [[ "$old_tooling" -gt 0 ]]; then
|
||||
[[ "$QUIET" == false ]] && echo "[tooling] $tooling_count total files — $old_tooling older than 1 day"
|
||||
find "$AGENTS_DIR/tooling" -maxdepth 1 -type f -mtime +1 -print0 2>/dev/null \
|
||||
| while IFS= read -r -d '' f; do
|
||||
size=$(stat -f%z "$f" 2>/dev/null || stat --format=%s "$f" 2>/dev/null || echo 0)
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
[[ "$QUIET" == false ]] && echo " would delete: $f"
|
||||
else
|
||||
rm -f "$f"
|
||||
[[ "$QUIET" == false ]] && echo " deleted: $f"
|
||||
fi
|
||||
done
|
||||
else
|
||||
[[ "$QUIET" == false ]] && echo "[tooling] $tooling_count files — all from recent run. Nothing to prune."
|
||||
# --- tooling/ and security/ no longer live in .agents/ (moved to $TMPDIR) ---
|
||||
# Clean up any legacy directories left from older versions
|
||||
for legacy_dir in "$AGENTS_DIR/tooling" "$AGENTS_DIR/security"; do
|
||||
if [[ -d "$legacy_dir" ]]; then
|
||||
legacy_count=$(find "$legacy_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
|
||||
if [[ "$legacy_count" -gt 0 ]]; then
|
||||
[[ "$QUIET" == false ]] && echo "[legacy] $legacy_dir has $legacy_count files (scanner output moved to \$TMPDIR)"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
[[ "$QUIET" == false ]] && echo " would delete: $legacy_dir/ ($legacy_count files)"
|
||||
else
|
||||
rm -rf "$legacy_dir"
|
||||
mkdir -p "$legacy_dir"
|
||||
[[ "$QUIET" == false ]] && echo " deleted: $legacy_dir/ ($legacy_count files)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
[[ "$QUIET" == false ]] && echo ""
|
||||
|
||||
# --- Policy: knowledge/pending/ — older than 14 days ---
|
||||
@@ -210,6 +203,4 @@ if [[ "$QUIET" == false ]]; then
|
||||
echo ""
|
||||
echo "Protected directories (never pruned):"
|
||||
echo " learnings/ patterns/ plans/ research/ retros/"
|
||||
echo ""
|
||||
echo "Recommendation: Add .agents/tooling/ to .gitignore (1.1GB of regenerable scanner output)"
|
||||
fi
|
||||
|
||||
@@ -62,7 +62,8 @@ if [[ ! -x "$TOOLCHAIN_SCRIPT" ]]; then
|
||||
fi
|
||||
|
||||
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-${MODE}"
|
||||
SECURITY_DIR="$REPO_ROOT/.agents/security/$RUN_ID"
|
||||
SECURITY_BASE="${SECURITY_GATE_OUTPUT_DIR:-${TMPDIR:-/tmp}/agentops-security}"
|
||||
SECURITY_DIR="$SECURITY_BASE/$RUN_ID"
|
||||
mkdir -p "$SECURITY_DIR"
|
||||
|
||||
TOOLCHAIN_ARGS=(--gate --json)
|
||||
@@ -78,8 +79,9 @@ set -e
|
||||
SUMMARY_JSON="$SECURITY_DIR/summary.json"
|
||||
printf '%s\n' "$TOOLCHAIN_OUTPUT" > "$SUMMARY_JSON"
|
||||
|
||||
if [[ -d "$REPO_ROOT/.agents/tooling" ]]; then
|
||||
cp -a "$REPO_ROOT/.agents/tooling/." "$SECURITY_DIR/" 2>/dev/null || true
|
||||
TOOLING_SRC="${TOOLCHAIN_OUTPUT_DIR:-${TMPDIR:-/tmp}/agentops-tooling}"
|
||||
if [[ -d "$TOOLING_SRC" ]]; then
|
||||
cp -a "$TOOLING_SRC/." "$SECURITY_DIR/" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if command -v jq >/dev/null 2>&1 && jq empty "$SUMMARY_JSON" >/dev/null 2>&1; then
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
# Toolchain Validate - Run all available linters/scanners
|
||||
# Outputs structured findings to .agents/tooling/
|
||||
# Outputs structured findings to $TOOLCHAIN_OUTPUT_DIR (default: $TMPDIR/agentops-tooling/)
|
||||
#
|
||||
# Usage: ./scripts/toolchain-validate.sh [OPTIONS]
|
||||
#
|
||||
@@ -20,7 +20,7 @@ set -euo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
OUTPUT_DIR="$REPO_ROOT/.agents/tooling"
|
||||
OUTPUT_DIR="${TOOLCHAIN_OUTPUT_DIR:-${TMPDIR:-/tmp}/agentops-tooling}"
|
||||
|
||||
# Parse arguments
|
||||
QUICK=false
|
||||
|
||||
@@ -34,7 +34,7 @@ Single run (default dynamic command is `--help`):
|
||||
```bash
|
||||
python3 skills/security-suite/scripts/security_suite.py run \
|
||||
--binary "$(command -v ao)" \
|
||||
--out-dir .agents/security-suite/ao-current
|
||||
--out-dir .tmp/security-suite/ao-current
|
||||
```
|
||||
|
||||
Baseline regression gate:
|
||||
@@ -42,8 +42,8 @@ Baseline regression gate:
|
||||
```bash
|
||||
python3 skills/security-suite/scripts/security_suite.py run \
|
||||
--binary "$(command -v ao)" \
|
||||
--out-dir .agents/security-suite/ao-current \
|
||||
--baseline-dir .agents/security-suite/ao-baseline \
|
||||
--out-dir .tmp/security-suite/ao-current \
|
||||
--baseline-dir .tmp/security-suite/ao-baseline \
|
||||
--fail-on-removed
|
||||
```
|
||||
|
||||
@@ -52,7 +52,7 @@ Policy gate:
|
||||
```bash
|
||||
python3 skills/security-suite/scripts/security_suite.py run \
|
||||
--binary "$(command -v ao)" \
|
||||
--out-dir .agents/security-suite/ao-current \
|
||||
--out-dir .tmp/security-suite/ao-current \
|
||||
--policy-file skills/security-suite/references/policy-example.json \
|
||||
--fail-on-policy-fail
|
||||
```
|
||||
@@ -125,7 +125,7 @@ python3 skills/security-suite/scripts/security_suite.py run \
|
||||
|
||||
### Scenario: Capture a Baseline and Gate a New Release
|
||||
|
||||
**User says:** `/security-suite run --binary $(command -v ao) --out-dir .agents/security-suite/ao-v2.4`
|
||||
**User says:** `/security-suite run --binary $(command -v ao) --out-dir .tmp/security-suite/ao-v2.4`
|
||||
|
||||
**What happens:**
|
||||
1. The suite runs static analysis (file metadata, linked libraries, embedded archive signatures), dynamic tracing (sandboxed `--help` execution observing processes, file changes, network endpoints), and contract capture against the `ao` binary.
|
||||
@@ -135,7 +135,7 @@ python3 skills/security-suite/scripts/security_suite.py run \
|
||||
|
||||
### Scenario: CI Regression Gate With Baseline and Policy
|
||||
|
||||
**User says:** `/security-suite run --binary ./bin/ao-candidate --out-dir .tmp/ao-candidate --baseline-dir .agents/security-suite/ao-v2.4 --policy-file skills/security-suite/references/policy-example.json --fail-on-removed --fail-on-policy-fail`
|
||||
**User says:** `/security-suite run --binary ./bin/ao-candidate --out-dir .tmp/ao-candidate --baseline-dir .tmp/security-suite/ao-v2.4 --policy-file skills/security-suite/references/policy-example.json --fail-on-removed --fail-on-policy-fail`
|
||||
|
||||
**What happens:**
|
||||
1. The suite runs all three collection primitives on the candidate binary, then compares the resulting contract against the v2.4 baseline to produce `compare/baseline-diff.json` with any added, removed, or changed commands.
|
||||
|
||||
@@ -33,7 +33,7 @@ scripts/security-gate.sh --mode quick
|
||||
|
||||
Expected behavior:
|
||||
- Fails on high/critical findings from available scanners.
|
||||
- Writes artifacts under `.agents/security/<run-id>/`.
|
||||
- Writes artifacts under `$TMPDIR/agentops-security/<run-id>/`.
|
||||
|
||||
### 2) Pre-Release (strict)
|
||||
|
||||
@@ -62,7 +62,7 @@ Expected behavior:
|
||||
## Triage Guidance
|
||||
|
||||
When gate fails:
|
||||
1. Open latest artifact in `.agents/security/` and identify scanner + file.
|
||||
1. Open latest artifact in `$TMPDIR/agentops-security/` and identify scanner + file.
|
||||
2. Classify severity (critical/high/medium).
|
||||
3. Fix immediately for critical/high or create tracked follow-up issue with owner.
|
||||
4. Re-run `scripts/security-gate.sh` until gate passes.
|
||||
@@ -98,7 +98,7 @@ Actions:
|
||||
|
||||
**What happens:**
|
||||
1. The skill runs `scripts/security-gate.sh --mode quick`, which executes available scanners (semgrep, gosec, gitleaks) against the current working tree and flags high/critical findings.
|
||||
2. Scan artifacts are written to `.agents/security/<run-id>/` for review, and the gate reports a pass/blocked verdict.
|
||||
2. Scan artifacts are written to `$TMPDIR/agentops-security/<run-id>/` for review, and the gate reports a pass/blocked verdict.
|
||||
|
||||
**Result:** The gate passes with no high/critical findings, confirming the branch is safe to open a PR.
|
||||
|
||||
@@ -108,7 +108,7 @@ Actions:
|
||||
|
||||
**What happens:**
|
||||
1. The skill runs `scripts/security-gate.sh --mode full`, which performs a comprehensive scan including all scanner passes, test-inclusive toolchain checks, and stricter severity thresholds.
|
||||
2. Artifacts are retained under `.agents/security/<run-id>/` for audit trail and incident response, and a structured report is generated.
|
||||
2. Artifacts are retained under `$TMPDIR/agentops-security/<run-id>/` for audit trail and incident response, and a structured report is generated.
|
||||
|
||||
**Result:** The full gate blocks the release on two medium-severity findings in `cli/internal/config.go`; the operator triages and fixes them before re-running the gate to get a clean pass.
|
||||
|
||||
@@ -117,7 +117,7 @@ Actions:
|
||||
| Problem | Cause | Solution |
|
||||
|---------|-------|----------|
|
||||
| Gate reports "scanner not found" and skips checks | Required scanner (semgrep, gosec, or gitleaks) is not installed | Install the missing scanner: `brew install semgrep`, `go install github.com/securego/gosec/v2/cmd/gosec@latest`, or `brew install gitleaks`. |
|
||||
| Gate passes locally but fails in CI | CI environment has additional scanners or stricter config | Compare `.agents/security/` artifacts from both environments; align scanner versions and config files across local and CI. |
|
||||
| Gate passes locally but fails in CI | CI environment has additional scanners or stricter config | Compare `$TMPDIR/agentops-security/` artifacts from both environments; align scanner versions and config files across local and CI. |
|
||||
| False positive blocking the gate | Scanner flags a non-issue as high/critical severity | Add a scanner-specific inline suppression comment (e.g., `# nosemgrep: rule-id`) or update the scanner config to exclude the pattern, then document the suppression reason. |
|
||||
| Artifacts directory `.agents/security/` not created | Script lacks write permissions or the `.agents/` directory does not exist | Ensure `.agents/` exists and is writable; create it with `mkdir -p .agents/security` if needed. |
|
||||
| Artifacts directory `$TMPDIR/agentops-security/` not created | Script lacks write permissions or `$TMPDIR` is not writable | Verify `$TMPDIR` is set and writable; the script auto-creates subdirectories on each run. |
|
||||
| Nightly scan not detecting regressions | Nightly workflow is not configured or is pointing at stale branch | Verify `.github/workflows/nightly.yml` runs `scripts/security-gate.sh --mode full` against the correct branch (typically `main`). |
|
||||
|
||||
@@ -100,7 +100,7 @@ setup_test_project() {
|
||||
mkdir -p .agents/ao/sessions
|
||||
mkdir -p .agents/pool/pending
|
||||
mkdir -p .agents/pool/staged
|
||||
mkdir -p .agents/tooling
|
||||
mkdir -p .agents/tooling # test fixture — legacy path kept for e2e isolation
|
||||
|
||||
# Create .beads directory for issue tracking
|
||||
mkdir -p .beads/issues
|
||||
@@ -559,7 +559,7 @@ No security issues detected.
|
||||
VIBE
|
||||
|
||||
# Create tooling output simulation
|
||||
cat > .agents/tooling/summary.json << 'TOOLSUMMARY'
|
||||
cat > .agents/tooling/summary.json << 'TOOLSUMMARY' # test fixture in isolated tmp dir
|
||||
{
|
||||
"timestamp": "2026-02-03T12:00:00Z",
|
||||
"exit_code": 0,
|
||||
|
||||
@@ -53,7 +53,7 @@ cat <<'JSON'
|
||||
"low": 0
|
||||
},
|
||||
"gate_status": "PASS",
|
||||
"output_dir": ".agents/tooling"
|
||||
"output_dir": "/tmp/agentops-tooling"
|
||||
}
|
||||
JSON
|
||||
exit 0
|
||||
@@ -111,13 +111,19 @@ test_json_output() {
|
||||
|
||||
test_artifacts() {
|
||||
create_mock_toolchain
|
||||
SECURITY_GATE_TOOLCHAIN_SCRIPT="$MOCK_TOOLCHAIN" scripts/security-gate.sh --mode quick >/dev/null 2>&1 || true
|
||||
local test_output_dir
|
||||
test_output_dir="$(mktemp -d)"
|
||||
SECURITY_GATE_TOOLCHAIN_SCRIPT="$MOCK_TOOLCHAIN" \
|
||||
SECURITY_GATE_OUTPUT_DIR="$test_output_dir/security" \
|
||||
TOOLCHAIN_OUTPUT_DIR="$test_output_dir/tooling" \
|
||||
scripts/security-gate.sh --mode quick >/dev/null 2>&1 || true
|
||||
|
||||
local latest
|
||||
latest=$(ls -td .agents/security/* 2>/dev/null | head -1 || true)
|
||||
latest=$(ls -td "$test_output_dir/security"/* 2>/dev/null | head -1 || true)
|
||||
|
||||
if [[ -z "$latest" ]]; then
|
||||
fail "no .agents/security artifacts created"
|
||||
fail "no security artifacts created"
|
||||
rm -rf "$test_output_dir"
|
||||
return
|
||||
fi
|
||||
|
||||
@@ -126,6 +132,7 @@ test_artifacts() {
|
||||
else
|
||||
fail "missing security-gate-summary.json"
|
||||
fi
|
||||
rm -rf "$test_output_dir"
|
||||
}
|
||||
|
||||
echo "================================"
|
||||
|
||||
@@ -125,13 +125,16 @@ test_tool_count() {
|
||||
|
||||
# Test 8: Output directory is created
|
||||
test_output_dir() {
|
||||
./scripts/toolchain-validate.sh --quick > /dev/null 2>&1 || true
|
||||
local test_dir
|
||||
test_dir="$(mktemp -d)"
|
||||
TOOLCHAIN_OUTPUT_DIR="$test_dir/tooling" ./scripts/toolchain-validate.sh --quick > /dev/null 2>&1 || true
|
||||
|
||||
if [[ -d ".agents/tooling" ]]; then
|
||||
pass "Output directory .agents/tooling exists"
|
||||
if [[ -d "$test_dir/tooling" ]]; then
|
||||
pass "Output directory created at TOOLCHAIN_OUTPUT_DIR"
|
||||
else
|
||||
fail "Output directory .agents/tooling not created"
|
||||
fail "Output directory not created"
|
||||
fi
|
||||
rm -rf "$test_dir"
|
||||
}
|
||||
|
||||
# Run all tests
|
||||
|
||||
Reference in New Issue
Block a user