mirror of
https://github.com/boshu2/agentops.git
synced 2026-09-14 15:08:13 +08:00
feat(bdd): scenario→test linkage runner + CI gate (soc-63xfx #scenario-test-runner) (#503)
## What Mechanizes the scenario→test linkage gap. 71 `.feature` files existed under `skills/*/references/` but only a handful had an executing test; the rest were unverified documentation. This adds a linkage **convention + checker + blocking CI gate** — a linkage check, not a full Gherkin step-interpreter (that scope was deliberately avoided per the bead's pragmatic-scope guidance). ## Convention `@covered-by:<test-path>[::<Name>]` Gherkin tag placed directly above a `Scenario:` (or above `Feature:` for file-wide coverage). Lightest convention that resolves "scenario X is covered by test Y" without maintaining a separate registry. The `::<Name>` form additionally asserts a named test (Go func / bats `@test` / bash func) exists in the file. ## Deliverables - **`scripts/check-scenario-test-linkage.sh`** — parses every `.feature` scenario, resolves `@covered-by` targets, and FAILS on: unlinked-and-unallowlisted scenarios, dangling links (missing path or named test), allowlist/tag conflicts, and stale allowlist entries. Flags: `--warn-only`, `--json`, `--help`. Exit 0/1/2. - **`scripts/.scenario-linkage-allow`** — 68 doc-only feature files, a *draining backlog* (drain as skills gain executing tests). The gate is real: a scenario not here and not tagged fails CI. - **Starter set** — `rpi`, `goals`, `scenario` features fully linked (11 scenarios) to their `tests/e2e/*` tests, proving the mechanism end to end. - **`validate.yml`** — `validate-scenario-test-linkage` T1 blocking job + summary `needs` + summary echo; manifest entry in `docs/contracts/ci-jobs.yaml` + regenerated `AGENTS-CI.md` table (parity gate green). - **`docs/contracts/scenario-test-linkage.md`** + `documentation-index.md` catalog entry. - **`tests/scripts/check-scenario-test-linkage.bats`** — 13 regression tests (pass/fail/warn/json/file-level-tag/misuse). Sibling to `executable-spec-link-integrity` (directive→scenario). Together they close the GOALS-directive → scenario → test evidence chain. ## Verification (all green) - `shellcheck scripts/check-scenario-test-linkage.sh` — clean - gate run locally — PASS (256 scenarios, 71 files; 11 linked, 245 allowlisted) - 5 failure paths manually exercised → exit 1; `--warn-only` → exit 0; misuse → exit 2 - `bats tests/scripts/check-scenario-test-linkage.bats` — 13/13 ok - `check-contracts-structural-floor.sh` — PASS (new contract catalogued) - `validate-ci-policy-parity.sh` — PASS (manifest + AGENTS table synced) - `validate.yml` + `ci-jobs.yaml` YAML parse OK - diff stat: +639/−2, all-additions, no collateral data loss Closes-scenario: soc-63xfx#scenario-test-runner Bounded-context: BC4-Evidence Evidence: scripts/check-scenario-test-linkage.sh
This commit is contained in:
Bo
@@ -1820,6 +1820,26 @@ jobs:
|
||||
echo "=== ao goals trace --orphans ==="
|
||||
cli/bin/ao goals trace --orphans || echo "WARN: trace --orphans found issues (warn-only, F1.6 / soc-58nt.1.9)"
|
||||
|
||||
validate-scenario-test-linkage:
|
||||
# T1 (≤5min). Sibling to executable-spec-link-integrity: that gate links
|
||||
# GOALS directives → scenarios; this one links scenarios → tests. Blocking.
|
||||
# Every Gherkin Scenario in skills/*/references/*.feature must carry a
|
||||
# @covered-by:<test-path> tag OR have its feature file allowlisted in
|
||||
# scripts/.scenario-linkage-allow (doc-only). Fails on unlinked-and-
|
||||
# unlisted scenarios, dangling links, and stale allowlist entries.
|
||||
# (soc-63xfx)
|
||||
needs: [changes]
|
||||
if: needs.changes.outputs.skills == 'true' || needs.changes.outputs.shell == 'true' || needs.changes.outputs.ci == 'true'
|
||||
name: validate-scenario-test-linkage (T1, soc-63xfx)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Scenario→test linkage gate (scripts/check-scenario-test-linkage.sh)
|
||||
run: |
|
||||
chmod +x scripts/check-scenario-test-linkage.sh
|
||||
./scripts/check-scenario-test-linkage.sh
|
||||
|
||||
validate-pr-evidence-claims:
|
||||
# AP#7 mechanical enforcement (soc-eqjd, replaces local pre-push check #39
|
||||
# retired by soc-g2r9). For each `Evidence:` line in the PR body, assert
|
||||
@@ -1827,7 +1847,7 @@ jobs:
|
||||
# with `summary` and depends on the same broad set so all sibling-job logs
|
||||
# are present when we fetch. `summary` also depends on this job so its
|
||||
# verdict surfaces in the CI summary.
|
||||
needs: [changes, doc-release-gate, smoke-test, hook-preflight, agentops-eval-baseline-audit, validate-hooks-doc-parity, hook-output-schema-lint, validate-ci-policy-parity, validate-codex-runtime-sections, validate-codex-generated-artifacts, validate-codex-backbone-prompts, validate-codex-override-coverage, validate-codex-rpi-contract, validate-codex-lifecycle-guards, validate-codex-parity-drift, validate-quarantine-empty, validate-registry-drift, validate-bounded-contexts-drift, validate-skill-domain-map-golden, validate-flywheel-proof, validate-goals-validate, validate-wiring-closure, validate-corpus-freshness, validate-flywheel-compounding-snapshot, validate-factory-yield-ledger, validate-finding-registry, validate-agents-split, validate-sovereignty-proof-citations, validate-factory-admission, validate-contracts-structural-floor, validate-docs-learning-references, validate-three-gap-supergate, validate-headless-runtime-skills, validate-skill-frontmatter, validate-context-map-drift, embedded-sync, cli-docs-parity, registry-check, agentops-contract-canaries, eval-workbench-verify, factory-claim-ledger-strict, eval-skill-delta, shellcheck, markdownlint, security-scan, security-toolchain-gate, skill-integrity, skill-schema, skill-dependency-check, contract-compatibility-gate, memrl-health, plugin-load-test, retrieval-quality, go-build, windows-smoke, cli-integration, json-flag-consistency, file-manifest-overlap, doctor-check, skill-lint, bats-tests, check-test-staleness, swarm-evidence, executable-spec-link-integrity]
|
||||
needs: [changes, doc-release-gate, smoke-test, hook-preflight, agentops-eval-baseline-audit, validate-hooks-doc-parity, hook-output-schema-lint, validate-ci-policy-parity, validate-codex-runtime-sections, validate-codex-generated-artifacts, validate-codex-backbone-prompts, validate-codex-override-coverage, validate-codex-rpi-contract, validate-codex-lifecycle-guards, validate-codex-parity-drift, validate-quarantine-empty, validate-registry-drift, validate-bounded-contexts-drift, validate-skill-domain-map-golden, validate-flywheel-proof, validate-goals-validate, validate-wiring-closure, validate-corpus-freshness, validate-flywheel-compounding-snapshot, validate-factory-yield-ledger, validate-finding-registry, validate-agents-split, validate-sovereignty-proof-citations, validate-factory-admission, validate-contracts-structural-floor, validate-docs-learning-references, validate-three-gap-supergate, validate-headless-runtime-skills, validate-skill-frontmatter, validate-context-map-drift, embedded-sync, cli-docs-parity, registry-check, agentops-contract-canaries, eval-workbench-verify, factory-claim-ledger-strict, eval-skill-delta, shellcheck, markdownlint, security-scan, security-toolchain-gate, skill-integrity, skill-schema, skill-dependency-check, contract-compatibility-gate, memrl-health, plugin-load-test, retrieval-quality, go-build, windows-smoke, cli-integration, json-flag-consistency, file-manifest-overlap, doctor-check, skill-lint, bats-tests, check-test-staleness, swarm-evidence, executable-spec-link-integrity, validate-scenario-test-linkage]
|
||||
if: always() && github.event_name == 'pull_request'
|
||||
name: AP#7 — verify Evidence claims in PR body
|
||||
runs-on: ubuntu-latest
|
||||
@@ -1936,7 +1956,7 @@ jobs:
|
||||
echo "::notice::AP#7 verification skipped — no completed sibling jobs returned logs. The PR's Evidence: line was extracted but cannot be cross-checked against per-job logs in this CI shape (likely: only this job ran, or all siblings were skipped)."
|
||||
|
||||
summary:
|
||||
needs: [changes, doc-release-gate, smoke-test, hook-preflight, agentops-eval-baseline-audit, validate-hooks-doc-parity, hook-output-schema-lint, validate-ci-policy-parity, validate-codex-runtime-sections, validate-codex-generated-artifacts, validate-codex-backbone-prompts, validate-codex-override-coverage, validate-codex-rpi-contract, validate-codex-lifecycle-guards, validate-codex-parity-drift, validate-pr-evidence-claims, validate-quarantine-empty, validate-registry-drift, validate-bounded-contexts-drift, validate-skill-domain-map-golden, validate-flywheel-proof, validate-goals-validate, validate-wiring-closure, validate-corpus-freshness, validate-flywheel-compounding-snapshot, validate-factory-yield-ledger, validate-finding-registry, validate-agents-split, validate-sovereignty-proof-citations, validate-factory-admission, validate-contracts-structural-floor, validate-docs-learning-references, validate-three-gap-supergate, validate-headless-runtime-skills, validate-skill-frontmatter, validate-context-map-drift, embedded-sync, cli-docs-parity, registry-check, agentops-contract-canaries, eval-workbench-verify, factory-claim-ledger-strict, eval-skill-delta, shellcheck, markdownlint, security-scan, security-toolchain-gate, skill-integrity, skill-schema, skill-dependency-check, contract-compatibility-gate, memrl-health, plugin-load-test, retrieval-quality, go-build, windows-smoke, cli-integration, json-flag-consistency, file-manifest-overlap, doctor-check, skill-lint, bats-tests, check-test-staleness, swarm-evidence, executable-spec-link-integrity, lint-evidence-lines-advisory, check-skill-catalog-drift-advisory]
|
||||
needs: [changes, doc-release-gate, smoke-test, hook-preflight, agentops-eval-baseline-audit, validate-hooks-doc-parity, hook-output-schema-lint, validate-ci-policy-parity, validate-codex-runtime-sections, validate-codex-generated-artifacts, validate-codex-backbone-prompts, validate-codex-override-coverage, validate-codex-rpi-contract, validate-codex-lifecycle-guards, validate-codex-parity-drift, validate-pr-evidence-claims, validate-quarantine-empty, validate-registry-drift, validate-bounded-contexts-drift, validate-skill-domain-map-golden, validate-flywheel-proof, validate-goals-validate, validate-wiring-closure, validate-corpus-freshness, validate-flywheel-compounding-snapshot, validate-factory-yield-ledger, validate-finding-registry, validate-agents-split, validate-sovereignty-proof-citations, validate-factory-admission, validate-contracts-structural-floor, validate-docs-learning-references, validate-three-gap-supergate, validate-headless-runtime-skills, validate-skill-frontmatter, validate-context-map-drift, embedded-sync, cli-docs-parity, registry-check, agentops-contract-canaries, eval-workbench-verify, factory-claim-ledger-strict, eval-skill-delta, shellcheck, markdownlint, security-scan, security-toolchain-gate, skill-integrity, skill-schema, skill-dependency-check, contract-compatibility-gate, memrl-health, plugin-load-test, retrieval-quality, go-build, windows-smoke, cli-integration, json-flag-consistency, file-manifest-overlap, doctor-check, skill-lint, bats-tests, check-test-staleness, swarm-evidence, executable-spec-link-integrity, validate-scenario-test-linkage, lint-evidence-lines-advisory, check-skill-catalog-drift-advisory]
|
||||
runs-on: ubuntu-latest
|
||||
if: always()
|
||||
steps:
|
||||
@@ -2007,6 +2027,7 @@ jobs:
|
||||
echo "check-test-staleness: ${{ needs.check-test-staleness.result }}"
|
||||
echo "swarm-evidence: ${{ needs.swarm-evidence.result }}"
|
||||
echo "executable-spec-link-integrity: ${{ needs.executable-spec-link-integrity.result }}"
|
||||
echo "validate-scenario-test-linkage: ${{ needs.validate-scenario-test-linkage.result }}"
|
||||
|
||||
# Fail on actual failures. On release tags, skipped jobs are also a
|
||||
# failure because validate.yml must be a full exact-SHA release
|
||||
|
||||
@@ -109,6 +109,7 @@ These CI 1-40 items are intentionally not being hardened in this wave. Revisit o
|
||||
| **check-test-staleness** (non-blocking) | Detects stale/abandoned test files | Non-blocking (`continue-on-error: true`) |
|
||||
| **swarm-evidence** (non-blocking) | Swarm evidence files and file manifests are valid | Non-blocking (`continue-on-error: true`); informational artifact validation only |
|
||||
| **executable-spec-link-integrity** (non-blocking) | Runs `ao goals scenarios --lint` (directive↔scenario link lint) and `ao goals trace --orphans` (whole-chain orphan/gap audit) warn-only (F1.6, soc-58nt.1.9). Non-blocking (`continue-on-error: true`). Promotion criterion: remove `continue-on-error: true` from the CI job and replace `warn` with `fail` in pre-push check 38 after two consecutive clean runs on main | Broken directive↔scenario link or orphaned directive/scenario/bead in the trace chain (surfaces as warning, not failure, until promoted) |
|
||||
| **validate-scenario-test-linkage** | Every Gherkin scenario in skills/*/references/*.feature must declare its covering test via a `@covered-by:<test-path>` tag, or have its feature file allowlisted as documentation-only in scripts/.scenario-linkage-allow (soc-63xfx). Sibling to executable-spec-link-integrity: that gate links GOALS directives→scenarios; this one links scenarios→tests. Blocking (T1). | A scenario has no @covered-by tag and is not allowlisted, a @covered-by link is dangling (test path or named test missing), or the allowlist references a deleted feature file |
|
||||
| **lint-evidence-lines-advisory** (non-blocking) | I0 advisory wiring for `scripts/lint-evidence-lines.sh` (soc-dspz). Pre-AP#7 check on PR-body `Evidence` lines so authors get an annotation before the strict `validate-pr-evidence-claims` gate runs. Promotion path I0 advisory then T2 required after one clean week. | Non-blocking (`continue-on-error: true`); emits `::warning::` annotation with claim summary |
|
||||
| **check-skill-catalog-drift-advisory** (non-blocking) | I0 advisory wiring for `scripts/check-skill-catalog-drift.sh` (soc-dspz). Annotates when `skills/catalog.json` drifts from `skills/*/SKILL.md` frontmatter. Promotion path I0 advisory then T2 required after one clean week. | Non-blocking (`continue-on-error: true`); emits `::warning::` annotation with drift summary |
|
||||
|
||||
|
||||
@@ -86,6 +86,9 @@ jobs:
|
||||
- name: executable-spec-link-integrity
|
||||
reason: 'Runs `ao goals scenarios --lint` (directive↔scenario link lint) and `ao goals trace --orphans` (whole-chain orphan/gap audit) warn-only (F1.6, soc-58nt.1.9). Non-blocking (`continue-on-error: true`). Promotion criterion: remove `continue-on-error: true` from the CI job and replace `warn` with `fail` in pre-push check 38 after two consecutive clean runs on main'
|
||||
failure: Broken directive↔scenario link or orphaned directive/scenario/bead in the trace chain (surfaces as warning, not failure, until promoted)
|
||||
- name: validate-scenario-test-linkage
|
||||
reason: 'Every Gherkin scenario in skills/*/references/*.feature must declare its covering test via a `@covered-by:<test-path>` tag, or have its feature file allowlisted as documentation-only in scripts/.scenario-linkage-allow (soc-63xfx). Sibling to executable-spec-link-integrity: that gate links GOALS directives→scenarios; this one links scenarios→tests. Blocking (T1).'
|
||||
failure: A scenario has no @covered-by tag and is not allowlisted, a @covered-by link is dangling (test path or named test missing), or the allowlist references a deleted feature file
|
||||
- name: validate-ci-policy-parity
|
||||
reason: AGENTS CI table and blocking policy match workflow summary enforcement
|
||||
failure: Docs say non-blocking/required but workflow differs
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# Scenario → Test Linkage Contract
|
||||
|
||||
Status: active · Bead: soc-63xfx · Gate: `validate-scenario-test-linkage` (T1, blocking)
|
||||
|
||||
## Why
|
||||
|
||||
71 `.feature` files live under `skills/*/references/`. Historically only a
|
||||
handful had an executing test; the rest were documentation that drifted free of
|
||||
any check. A scenario nobody runs is a claim nobody verifies.
|
||||
|
||||
This contract makes the scenario→test relationship **mechanical**: every Gherkin
|
||||
`Scenario:` in the corpus must either name the test that covers it, or its
|
||||
feature file must be explicitly declared documentation-only. The gate fails on
|
||||
any scenario that is neither.
|
||||
|
||||
It is the sibling of `executable-spec-link-integrity` (soc-58nt / soc-m8tdn),
|
||||
which links **GOALS directives → scenarios**. This one links **scenarios →
|
||||
tests**. Together they close the spec-to-evidence chain:
|
||||
|
||||
```
|
||||
GOALS directive ──► scenario (.feature) ──► test (executing)
|
||||
(exec-spec-link-integrity) (scenario-test-linkage)
|
||||
```
|
||||
|
||||
## The convention
|
||||
|
||||
Tag a scenario with the test that covers it using a Gherkin tag:
|
||||
|
||||
```gherkin
|
||||
@covered-by:tests/e2e/rpi-phased-domain.sh
|
||||
Scenario: Phases run in order and never compress
|
||||
When /rpi executes
|
||||
Then it runs Research, then Plan, then Implement in order
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- The tag goes on its **own line directly above** the `Scenario:` it covers —
|
||||
the standard Gherkin tag position. A non-tag line (a step, `Background`, a
|
||||
blank then content) between the tag and the scenario breaks the association.
|
||||
- **File-level tags** (placed above the `Feature:` line) apply to **every**
|
||||
scenario in the file.
|
||||
- Multiple tags may stack; a scenario passes only if **all** of its
|
||||
`@covered-by:` targets resolve.
|
||||
- Two target forms:
|
||||
- `@covered-by:<test-path>` — the test file must exist (repo-relative path).
|
||||
- `@covered-by:<test-path>::<Name>` — additionally, `<Name>` (a Go func, a
|
||||
bats `@test` label, a bash function, etc.) must appear in that file. The
|
||||
name match is a substring check, which keeps the gate language-agnostic
|
||||
while still catching a renamed or deleted test.
|
||||
|
||||
`<test-path>` is any executing test surface: `tests/e2e/*.sh`,
|
||||
`tests/scripts/*.bats`, `cli/**/*_test.go`, and so on. The gate only asserts the
|
||||
target **exists** — it does not run it. Running is the test suite's job; linking
|
||||
is this gate's job.
|
||||
|
||||
## The allowlist
|
||||
|
||||
`scripts/.scenario-linkage-allow` lists feature files (repo-relative, one per
|
||||
line, `#` comments allowed) that are intentionally documentation-only for now.
|
||||
The gate skips scenarios in those files.
|
||||
|
||||
The allowlist is a **draining backlog**, not a parking lot:
|
||||
|
||||
- When a skill gains an executing test, tag its scenarios with `@covered-by:`
|
||||
and **remove the file from the allowlist**.
|
||||
- A file may **not** be both allowlisted **and** carry `@covered-by:` tags —
|
||||
that ambiguous intent is a gate error. Pick one.
|
||||
- A stale allowlist entry (a listed file that no longer exists) is a gate error.
|
||||
- Do not add new feature files to the allowlist without a tracking bead; new
|
||||
specs should ship with their covering test.
|
||||
|
||||
## Gate behavior
|
||||
|
||||
`scripts/check-scenario-test-linkage.sh`:
|
||||
|
||||
| Condition | Result |
|
||||
|---|---|
|
||||
| Scenario has a resolving `@covered-by:` tag | pass |
|
||||
| Scenario's feature file is allowlisted | pass (counted as doc-only) |
|
||||
| Scenario has no tag and file is not allowlisted | **FAIL** |
|
||||
| `@covered-by:` path does not exist | **FAIL** (dangling) |
|
||||
| `@covered-by:...::Name` not found in the file | **FAIL** (dangling) |
|
||||
| Allowlisted file also carries a `@covered-by:` tag | **FAIL** (ambiguous) |
|
||||
| Allowlist entry points at a deleted file | **FAIL** (stale) |
|
||||
|
||||
Flags: `--warn-only` (advisory, exit 0), `--json` (machine-readable summary),
|
||||
`-h`/`--help`. Exit codes: `0` pass, `1` fail, `2` misuse.
|
||||
|
||||
Wired into `.github/workflows/validate.yml` as `validate-scenario-test-linkage`,
|
||||
gated on changes to `skills/**`, `**/*.sh`, or `.github/**`.
|
||||
|
||||
## Starter set (at landing, soc-63xfx)
|
||||
|
||||
3 of 71 feature files are fully linked to executing tests:
|
||||
|
||||
| Feature | Covering test |
|
||||
|---|---|
|
||||
| `skills/rpi/references/rpi.feature` | `tests/e2e/rpi-phased-domain.sh` |
|
||||
| `skills/goals/references/goals.feature` | `tests/e2e/goals-measure-scenarios.sh`, `goals-steer-auto.sh`, `goals-trace-chain.sh` |
|
||||
| `skills/scenario/references/scenario.feature` | `tests/e2e/goals-scenarios-link.sh` |
|
||||
|
||||
The remaining 68 files are allowlisted in `scripts/.scenario-linkage-allow` with
|
||||
the standing goal to drain the list as skills gain executing tests.
|
||||
@@ -272,6 +272,7 @@ Bridge / framing docs:
|
||||
- [Bounded Contexts (yaml)](contracts/bounded-contexts.yaml) — Canonical BC1-BC5 definitions (id/name/responsibility/ports/center-of-gravity); registry doc prose must match this yaml (drift-checked by `scripts/check-bounded-contexts-drift.sh`, soc-zxia.2)
|
||||
- [add-validate-job scaffolder](https://github.com/boshu2/agentops/blob/main/scripts/add-validate-job.sh) — CI integration scaffolder; emits all 5 touch-points (workflow + summary needs + summary echo + pre-push + bats stub + AGENTS table) atomically when adding a new `validate-*` job (soc-3oij)
|
||||
- [CI Jobs Manifest (yaml)](contracts/ci-jobs.yaml) — Canonical reason+failure for every validate.yml CI job; AGENTS.md `### CI Jobs and What They Check` table is rendered from this yaml + workflow `summary.needs` via `scripts/generate-ci-jobs-table.sh` (golden-file gate enforced by `validate-ci-policy-parity`, soc-3oij)
|
||||
- [Scenario → Test Linkage](contracts/scenario-test-linkage.md) — Every Gherkin scenario in `skills/*/references/*.feature` must declare its covering test via a `@covered-by:<test-path>` tag or be allowlisted as doc-only in `scripts/.scenario-linkage-allow`; gate is `scripts/check-scenario-test-linkage.sh` / `validate-scenario-test-linkage` (sibling to executable-spec-link-integrity; links scenarios→tests, soc-63xfx)
|
||||
- [@claude Bot Delegation](contracts/claude-bot-delegation.md) — Operational runbook for the `@claude` GitHub App: permissions, triggers, status decoding, gotchas, when to delegate
|
||||
- [Local Pre-Push Gate Retirement](contracts/local-pre-push-gate-retirement.md) — ADR (soc-g2r9): CI is the sole authoritative push gate; `scripts/pre-push-gate.sh` and its helpers are retired in follow-up waves; AP#7 mechanical enforcement migrates from pre-push to a validate.yml job
|
||||
- [Skill Dispositions (yaml)](contracts/skill-dispositions.yaml) — Canonical per-skill domain/disposition/rationale data; source-of-truth for `agentops-skill-domain-map.md`. Hand-edits to the .md forbidden — edit yaml and run `scripts/generate-skill-domain-map.sh` (golden-file gate, soc-zxia.3)
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# .scenario-linkage-allow — feature files intentionally documentation-only.
|
||||
#
|
||||
# Each line is a repo-relative path to a .feature file whose scenarios are NOT
|
||||
# yet linked to executing tests via @covered-by: tags. The scenario→test
|
||||
# linkage gate (scripts/check-scenario-test-linkage.sh) skips scenarios in
|
||||
# these files. The gate is real: any scenario NOT here and NOT @covered-by-
|
||||
# tagged fails CI.
|
||||
#
|
||||
# Tracking: soc-63xfx seeded this allowlist with the corpus that existed when
|
||||
# the gate landed (3 of 71 feature files have executing tests: rpi, goals,
|
||||
# scenario). The standing goal is to DRAIN this list — as a skill gains an
|
||||
# executing test, tag its scenarios with @covered-by: and remove the file here.
|
||||
# Do NOT add new feature files to this allowlist without a tracking bead; new
|
||||
# specs should ship with their covering test.
|
||||
#
|
||||
# A file may NOT be both allowlisted AND carry @covered-by: tags (ambiguous
|
||||
# intent) — the gate rejects that. To link a file, delete its line below.
|
||||
#
|
||||
# Lines starting with '#' and blank lines are ignored. Trailing '# comment' on
|
||||
# an entry is stripped.
|
||||
|
||||
skills/autodev/references/autodev.feature
|
||||
skills/beads/references/beads.feature
|
||||
skills/bootstrap/references/bootstrap.feature
|
||||
skills/brainstorm/references/brainstorm.feature
|
||||
skills/bug-hunt/references/bug-hunt.feature
|
||||
skills/codex-team/references/codex-team.feature
|
||||
skills/compile/references/compile.feature
|
||||
skills/complexity/references/complexity.feature
|
||||
skills/council/references/council-modes.feature
|
||||
skills/crank/references/crank.feature
|
||||
skills/curate/references/curate.feature
|
||||
skills/deps/references/deps.feature
|
||||
skills/design/references/design.feature
|
||||
skills/discovery/references/discovery.feature
|
||||
skills/doc/references/doc.feature
|
||||
skills/domain/references/domain.feature
|
||||
skills/dream/references/dream.feature
|
||||
skills/evolve/references/evolve.feature
|
||||
skills/forge/references/forge.feature
|
||||
skills/grafana-platform-dashboard/references/grafana-platform-dashboard.feature
|
||||
skills/handoff/references/handoff.feature
|
||||
skills/harvest/references/harvest.feature
|
||||
skills/heal-skill/references/heal-skill.feature
|
||||
skills/hooks-authoring/references/hooks-authoring.feature
|
||||
skills/implement/references/implement.feature
|
||||
skills/knowledge-activation/references/knowledge-activation.feature
|
||||
skills/openai-docs/references/openai-docs.feature
|
||||
skills/oss-docs/references/oss-docs.feature
|
||||
skills/perf/references/perf.feature
|
||||
skills/plan/references/plan.feature
|
||||
skills/post-mortem/references/post-mortem.feature
|
||||
skills/pr-implement/references/pr-implement.feature
|
||||
skills/pr-plan/references/pr-plan.feature
|
||||
skills/pr-prep/references/pr-prep.feature
|
||||
skills/pr-research/references/pr-research.feature
|
||||
skills/pr-retro/references/pr-retro.feature
|
||||
skills/pr-validate/references/pr-validate.feature
|
||||
skills/pre-mortem/references/pre-mortem.feature
|
||||
skills/product/references/product.feature
|
||||
skills/push/references/push.feature
|
||||
skills/quickstart/references/quickstart.feature
|
||||
skills/ratchet/references/ratchet.feature
|
||||
skills/readme/references/readme.feature
|
||||
skills/recover/references/recover.feature
|
||||
skills/red-team/references/red-team.feature
|
||||
skills/refactor/references/refactor.feature
|
||||
skills/release/references/release.feature
|
||||
skills/research/references/research.feature
|
||||
skills/retro/references/retro.feature
|
||||
skills/reverse-engineer-rpi/references/reverse-engineer-rpi.feature
|
||||
skills/review/references/review.feature
|
||||
skills/scaffold/references/scaffold.feature
|
||||
skills/scope/references/scope.feature
|
||||
skills/security-suite/references/security-suite.feature
|
||||
skills/security/references/security.feature
|
||||
skills/session-bootstrap/references/session-bootstrap.feature
|
||||
skills/ship-loop/references/ship-loop.feature
|
||||
skills/skill-auditor/references/skill-auditor.feature
|
||||
skills/skill-builder/references/skill-builder.feature
|
||||
skills/status/references/status.feature
|
||||
skills/swarm/references/swarm.feature
|
||||
skills/system-tuning/references/system-tuning.feature
|
||||
skills/test/references/test.feature
|
||||
skills/trace/references/trace.feature
|
||||
skills/update/references/update.feature
|
||||
skills/validate/references/validate.feature
|
||||
skills/validation/references/validation.feature
|
||||
skills/vibe/references/vibe.feature
|
||||
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env bash
|
||||
# check-scenario-test-linkage.sh — mechanize the scenario→test linkage gate.
|
||||
#
|
||||
# 71 .feature files live under skills/*/references/ but historically only the
|
||||
# /rpi spec was actually executed; the rest were documentation. This gate makes
|
||||
# the linkage mechanical: every Gherkin `Scenario:` in the corpus must EITHER
|
||||
# declare the test that covers it with a `@covered-by:` tag, OR its feature file
|
||||
# must be explicitly allowlisted as intentionally documentation-only.
|
||||
#
|
||||
# Sibling to the directive→scenario gate (soc-m8tdn / executable-spec-link-
|
||||
# integrity): that one links GOALS directives → scenarios; this one links
|
||||
# scenarios → tests. Together they close the spec-to-evidence chain.
|
||||
#
|
||||
# Convention (the lightest one that resolves "scenario X is covered by test Y"):
|
||||
#
|
||||
# @covered-by:<test-path> tag the whole file as the cover, OR
|
||||
# @covered-by:<test-path>::<Name> name a specific test function (Go etc.)
|
||||
#
|
||||
# Place the tag on its own line directly above the `Scenario:` it covers (the
|
||||
# standard Gherkin tag position). Multiple tags may stack. Tags on the line(s)
|
||||
# above `Feature:` apply to every scenario in the file. The script:
|
||||
#
|
||||
# 1. parses every .feature scenario across skills/*/references/
|
||||
# 2. resolves each scenario's @covered-by target(s)
|
||||
# 3. FAILS on:
|
||||
# - a scenario with no @covered-by tag whose file is not allowlisted
|
||||
# - a dangling link: the test path does not exist, or the ::Name
|
||||
# function/definition is not found in that file
|
||||
#
|
||||
# Allowlist: scripts/.scenario-linkage-allow lists feature files (repo-relative)
|
||||
# that are intentionally documentation-only in this slice. A file may NOT be
|
||||
# both allowlisted AND carry @covered-by tags (ambiguous intent) — that errors.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/check-scenario-test-linkage.sh # blocking gate
|
||||
# bash scripts/check-scenario-test-linkage.sh --warn-only # advisory
|
||||
# bash scripts/check-scenario-test-linkage.sh --json # machine-readable summary
|
||||
#
|
||||
# Exits 0 on pass, 1 on fail (unless --warn-only), 2 on misuse.
|
||||
#
|
||||
# practices: [continuous-integration, design-by-contract, code-complete]
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
ALLOWLIST="$REPO_ROOT/scripts/.scenario-linkage-allow"
|
||||
WARN_ONLY=0
|
||||
JSON=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--warn-only) WARN_ONLY=1; shift;;
|
||||
--json) JSON=1; shift;;
|
||||
-h|--help)
|
||||
grep '^#' "$0" | sed 's/^# \?//'
|
||||
exit 0
|
||||
;;
|
||||
*) echo "Unknown arg: $1" >&2; exit 2;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Load allowlisted feature files (repo-relative paths, one per line; '#' comments
|
||||
# and blank lines ignored).
|
||||
declare -A ALLOWED=()
|
||||
if [[ -f "$ALLOWLIST" ]]; then
|
||||
while IFS= read -r entry; do
|
||||
entry="${entry%%#*}" # strip trailing comment
|
||||
entry="$(printf '%s' "$entry" | xargs)" # trim whitespace
|
||||
[[ -z "$entry" ]] && continue
|
||||
ALLOWED["$entry"]=1
|
||||
done < "$ALLOWLIST"
|
||||
fi
|
||||
|
||||
# Resolve a @covered-by target to an error string, or empty on success.
|
||||
# Arg: the target spec after "@covered-by:" — "path" or "path::Name".
|
||||
resolve_target() {
|
||||
local target="$1" path name
|
||||
if [[ "$target" == *"::"* ]]; then
|
||||
path="${target%%::*}"
|
||||
name="${target##*::}"
|
||||
else
|
||||
path="$target"
|
||||
name=""
|
||||
fi
|
||||
|
||||
local abs="$REPO_ROOT/$path"
|
||||
if [[ ! -f "$abs" ]]; then
|
||||
printf 'test path does not exist: %s' "$path"
|
||||
return
|
||||
fi
|
||||
if [[ -n "$name" ]]; then
|
||||
# The named symbol must appear in the file (Go func, bats @test, bash
|
||||
# function, or step label). Substring match keeps the gate language-
|
||||
# agnostic while still catching a renamed/deleted test.
|
||||
if ! grep -qF "$name" "$abs"; then
|
||||
printf 'test "%s" not found in %s' "$name" "$path"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
printf '' # success
|
||||
}
|
||||
|
||||
errors=0
|
||||
scenarios_total=0
|
||||
scenarios_linked=0
|
||||
scenarios_allowlisted=0
|
||||
files_total=0
|
||||
files_allowlisted=0
|
||||
|
||||
declare -a ERROR_LINES=()
|
||||
|
||||
while IFS= read -r -d '' feature; do
|
||||
rel="${feature#"$REPO_ROOT"/}"
|
||||
files_total=$((files_total + 1))
|
||||
|
||||
is_allowed=0
|
||||
[[ -n "${ALLOWED[$rel]:-}" ]] && is_allowed=1
|
||||
|
||||
# Collect file-level tags (tags appearing before the Feature: line) and,
|
||||
# per scenario, the tags on the contiguous block of lines above it.
|
||||
file_tags=""
|
||||
pending_tags=""
|
||||
seen_feature=0
|
||||
has_any_tag=0
|
||||
|
||||
while IFS= read -r line; do
|
||||
trimmed="$(printf '%s' "$line" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')"
|
||||
|
||||
if [[ "$trimmed" == @* ]]; then
|
||||
# A tag line. Accumulate any @covered-by: targets it carries.
|
||||
for tok in $trimmed; do
|
||||
if [[ "$tok" == @covered-by:* ]]; then
|
||||
has_any_tag=1
|
||||
pending_tags+="${tok#@covered-by:} "
|
||||
fi
|
||||
done
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$trimmed" == Feature:* && $seen_feature -eq 0 ]]; then
|
||||
seen_feature=1
|
||||
file_tags="$pending_tags"
|
||||
pending_tags=""
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$trimmed" == Scenario:* || "$trimmed" == "Scenario Outline:"* ]]; then
|
||||
scenarios_total=$((scenarios_total + 1))
|
||||
scen_name="${trimmed#Scenario: }"
|
||||
scen_name="${scen_name#Scenario Outline: }"
|
||||
|
||||
# Effective targets = file-level tags + this scenario's pending tags.
|
||||
effective="$file_tags $pending_tags"
|
||||
pending_tags=""
|
||||
# Normalize whitespace.
|
||||
effective="$(printf '%s' "$effective" | xargs || true)"
|
||||
|
||||
if [[ $is_allowed -eq 1 ]]; then
|
||||
scenarios_allowlisted=$((scenarios_allowlisted + 1))
|
||||
# An allowlisted file with @covered-by tags is ambiguous intent.
|
||||
if [[ -n "$effective" ]]; then
|
||||
ERROR_LINES+=("$rel: scenario \"$scen_name\" is in an allowlisted (doc-only) file but also declares @covered-by — remove the allowlist entry or the tag")
|
||||
errors=$((errors + 1))
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -z "$effective" ]]; then
|
||||
ERROR_LINES+=("$rel: scenario \"$scen_name\" has no @covered-by tag and the file is not allowlisted")
|
||||
ERROR_LINES+=(" fix: add '@covered-by:<test-path>' above the Scenario, OR add '$rel' to scripts/.scenario-linkage-allow")
|
||||
errors=$((errors + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
scenario_ok=1
|
||||
for tgt in $effective; do
|
||||
err="$(resolve_target "$tgt")"
|
||||
if [[ -n "$err" ]]; then
|
||||
ERROR_LINES+=("$rel: scenario \"$scen_name\" has a dangling @covered-by:$tgt — $err")
|
||||
errors=$((errors + 1))
|
||||
scenario_ok=0
|
||||
fi
|
||||
done
|
||||
[[ $scenario_ok -eq 1 ]] && scenarios_linked=$((scenarios_linked + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Any other non-tag, non-blank line clears pending scenario tags only if
|
||||
# we have not yet reached the scenario it would annotate. In Gherkin,
|
||||
# tags must be immediately above their scenario, so a Background or step
|
||||
# line between a tag and a Scenario invalidates the tag association.
|
||||
if [[ -n "$trimmed" && $seen_feature -eq 1 ]]; then
|
||||
pending_tags=""
|
||||
fi
|
||||
done < "$feature"
|
||||
|
||||
# Tally an allowlisted file even when it has zero @covered-by tags (expected).
|
||||
if [[ $is_allowed -eq 1 ]]; then
|
||||
files_allowlisted=$((files_allowlisted + 1))
|
||||
if [[ $has_any_tag -eq 1 && ${#ERROR_LINES[@]} -eq 0 ]]; then
|
||||
: # handled per-scenario above
|
||||
fi
|
||||
fi
|
||||
done < <(find "$REPO_ROOT/skills" -path '*/references/*.feature' -type f -print0 | sort -z)
|
||||
|
||||
# Detect stale allowlist entries: a listed file that no longer exists.
|
||||
stale_allow=0
|
||||
for rel in "${!ALLOWED[@]}"; do
|
||||
if [[ ! -f "$REPO_ROOT/$rel" ]]; then
|
||||
ERROR_LINES+=("allowlist: '$rel' is listed in scripts/.scenario-linkage-allow but the file no longer exists — remove the stale entry")
|
||||
errors=$((errors + 1))
|
||||
stale_allow=$((stale_allow + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $JSON -eq 1 ]]; then
|
||||
printf '{"feature_files":%d,"allowlisted_files":%d,"scenarios_total":%d,"scenarios_linked":%d,"scenarios_allowlisted":%d,"errors":%d,"stale_allowlist_entries":%d,"result":"%s"}\n' \
|
||||
"$files_total" "$files_allowlisted" "$scenarios_total" "$scenarios_linked" "$scenarios_allowlisted" "$errors" "$stale_allow" \
|
||||
"$([[ $errors -eq 0 ]] && echo pass || echo fail)"
|
||||
fi
|
||||
|
||||
if [[ $errors -eq 0 ]]; then
|
||||
[[ $JSON -eq 0 ]] && echo "check-scenario-test-linkage: PASS (${scenarios_total} scenarios across ${files_total} feature files; ${scenarios_linked} linked, ${scenarios_allowlisted} allowlisted in ${files_allowlisted} doc-only file(s))"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ $JSON -eq 0 ]]; then
|
||||
for e in "${ERROR_LINES[@]}"; do
|
||||
echo "$e" >&2
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ "$WARN_ONLY" -eq 1 ]]; then
|
||||
[[ $JSON -eq 0 ]] && echo "check-scenario-test-linkage: WARN ($errors issue(s); --warn-only)" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ $JSON -eq 0 ]] && echo "check-scenario-test-linkage: FAIL ($errors issue(s))" >&2
|
||||
exit 1
|
||||
@@ -10,16 +10,19 @@ Feature: Goals maintains and measures the fitness specification
|
||||
I want declared gates measured and directives steered against GOALS.md
|
||||
So that the loop has an objective, operator-owned target to compound toward
|
||||
|
||||
@covered-by:tests/e2e/goals-measure-scenarios.sh
|
||||
Scenario: measure runs the declared fitness gates into a verdict
|
||||
When /goals measure runs
|
||||
Then each declared gate is evaluated to PASS or FAIL
|
||||
And the result is written to result.json
|
||||
|
||||
@covered-by:tests/e2e/goals-steer-auto.sh
|
||||
Scenario: directives are the steering layer the loop measures against
|
||||
When directives are managed via /goals steer (add/remove/prioritize)
|
||||
Then they live in GOALS.md and surface through `ao goals measure --directives`
|
||||
And /evolve selects work against the failing goals + directive gaps
|
||||
|
||||
@covered-by:tests/e2e/goals-trace-chain.sh
|
||||
Scenario: GOALS.md is the source of truth
|
||||
Then /goals reads and writes GOALS.md (output_contract: GOALS.md)
|
||||
And it consumes no other skill — it is the measurement root that /evolve consumes
|
||||
|
||||
@@ -13,21 +13,25 @@ Feature: RPI runs one turn's lifecycle without skipping moves
|
||||
Background:
|
||||
Given a goal, bead, or execution packet entering the rpi lifecycle
|
||||
|
||||
@covered-by:tests/e2e/rpi-phased-domain.sh
|
||||
Scenario: Phases run in order and never compress
|
||||
When /rpi executes
|
||||
Then it runs Research, then Plan, then Implement in order
|
||||
And no phase is skipped or merged into another (strict delegation is on by default)
|
||||
|
||||
@covered-by:tests/e2e/rpi-phased-domain.sh
|
||||
Scenario: Validation cannot be skipped
|
||||
When /rpi reaches the end of Implement
|
||||
Then validation runs before the turn is considered done
|
||||
And the lifecycle objective is preserved, not silently dropped at a phase boundary
|
||||
|
||||
@covered-by:tests/e2e/rpi-phased-domain.sh
|
||||
Scenario: Context density survives phase handoffs
|
||||
When one phase hands off to the next
|
||||
Then intent, boundary, evidence, decision, constraint, and next-action carry forward
|
||||
And anything else is omitted or linked, not pasted wholesale
|
||||
|
||||
@covered-by:tests/e2e/rpi-phased-domain.sh
|
||||
Scenario: Autonomous run produces durable phase artifacts
|
||||
Given /rpi --auto
|
||||
Then the full lifecycle runs without per-phase human approval
|
||||
|
||||
@@ -12,18 +12,22 @@ Feature: Scenario manages holdout scenarios for behavioral validation
|
||||
Background:
|
||||
Given a holdout directory under .agents/holdout/
|
||||
|
||||
@covered-by:tests/e2e/goals-scenarios-link.sh
|
||||
Scenario: Scenarios are authored into the holdout directory
|
||||
When /scenario authors a scenario
|
||||
Then it writes a scenario artifact under .agents/holdout/
|
||||
|
||||
@covered-by:tests/e2e/goals-scenarios-link.sh
|
||||
Scenario: Scenarios are validated against the schema
|
||||
When /scenario validates
|
||||
Then each scenario is checked against the scenario schema and failures are reported
|
||||
|
||||
@covered-by:tests/e2e/goals-scenarios-link.sh
|
||||
Scenario: Scenarios are listable and linkable to GOALS directives
|
||||
When /scenario lists
|
||||
Then it shows the holdout scenarios and their links to GOALS.md directives
|
||||
|
||||
@covered-by:tests/e2e/goals-scenarios-link.sh
|
||||
Scenario: Holdout scenarios feed validation
|
||||
When /validation runs
|
||||
Then it consumes the holdout scenarios for behavioral scoring
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
# Regression tests for scripts/check-scenario-test-linkage.sh (soc-63xfx).
|
||||
# Builds a self-contained fake repo (scripts/, skills/<name>/references/, and a
|
||||
# tests/ tree for @covered-by targets), then runs the gate inside it. The script
|
||||
# resolves REPO_ROOT relative to its own location, so copying it into the fake
|
||||
# repo's scripts/ dir scopes every check to the fixture.
|
||||
|
||||
setup() {
|
||||
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)"
|
||||
SCRIPT="$REPO_ROOT/scripts/check-scenario-test-linkage.sh"
|
||||
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
FAKE_REPO="$TMP_DIR/repo"
|
||||
mkdir -p "$FAKE_REPO/scripts" \
|
||||
"$FAKE_REPO/skills/foo/references" \
|
||||
"$FAKE_REPO/skills/bar/references" \
|
||||
"$FAKE_REPO/tests/e2e"
|
||||
/bin/cp "$SCRIPT" "$FAKE_REPO/scripts/check-scenario-test-linkage.sh"
|
||||
chmod +x "$FAKE_REPO/scripts/check-scenario-test-linkage.sh"
|
||||
FAKE_SCRIPT="$FAKE_REPO/scripts/check-scenario-test-linkage.sh"
|
||||
|
||||
# A real test file that @covered-by can resolve to, with a named function.
|
||||
cat > "$FAKE_REPO/tests/e2e/foo.sh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
TestFooCoverage() { :; }
|
||||
EOF
|
||||
}
|
||||
|
||||
teardown() {
|
||||
rm -rf "$TMP_DIR"
|
||||
}
|
||||
|
||||
# Write a feature file. $1=skill, $2=tag-line-or-empty, rest=scenario names.
|
||||
write_feature() {
|
||||
local skill="$1" tag="$2"; shift 2
|
||||
local f="$FAKE_REPO/skills/$skill/references/$skill.feature"
|
||||
{
|
||||
printf '# fake spec\n\nFeature: %s feature\n\n' "$skill"
|
||||
for scen in "$@"; do
|
||||
[ -n "$tag" ] && printf ' %s\n' "$tag"
|
||||
printf ' Scenario: %s\n When x\n Then y\n\n' "$scen"
|
||||
done
|
||||
} > "$f"
|
||||
}
|
||||
|
||||
@test "script exists and is executable" {
|
||||
[ -f "$SCRIPT" ]
|
||||
[ -x "$SCRIPT" ]
|
||||
}
|
||||
|
||||
@test "PASS: scenario with a resolving @covered-by path" {
|
||||
write_feature foo "@covered-by:tests/e2e/foo.sh" "covered scenario"
|
||||
printf 'skills/bar/references/bar.feature\n' > "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature bar "" "doc only"
|
||||
run bash "$FAKE_SCRIPT"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"PASS"* ]]
|
||||
[[ "$output" == *"1 linked"* ]]
|
||||
}
|
||||
|
||||
@test "PASS: scenario with @covered-by path::Name when the name exists" {
|
||||
write_feature foo "@covered-by:tests/e2e/foo.sh::TestFooCoverage" "named cover"
|
||||
printf 'skills/bar/references/bar.feature\n' > "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature bar "" "doc only"
|
||||
run bash "$FAKE_SCRIPT"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "PASS: allowlisted doc-only feature with no tags" {
|
||||
printf 'skills/foo/references/foo.feature\nskills/bar/references/bar.feature\n' \
|
||||
> "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature foo "" "untagged one" "untagged two"
|
||||
write_feature bar "" "untagged three"
|
||||
run bash "$FAKE_SCRIPT"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"allowlisted"* ]]
|
||||
}
|
||||
|
||||
@test "FAIL: scenario with no tag and file not allowlisted" {
|
||||
write_feature foo "" "orphan scenario"
|
||||
printf 'skills/bar/references/bar.feature\n' > "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature bar "" "doc only"
|
||||
run bash "$FAKE_SCRIPT"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"no @covered-by tag and the file is not allowlisted"* ]]
|
||||
}
|
||||
|
||||
@test "FAIL: dangling @covered-by path (file missing)" {
|
||||
write_feature foo "@covered-by:tests/e2e/missing.sh" "dangling scenario"
|
||||
printf 'skills/bar/references/bar.feature\n' > "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature bar "" "doc only"
|
||||
run bash "$FAKE_SCRIPT"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"dangling"* ]]
|
||||
[[ "$output" == *"test path does not exist"* ]]
|
||||
}
|
||||
|
||||
@test "FAIL: dangling @covered-by path::Name (name missing in file)" {
|
||||
write_feature foo "@covered-by:tests/e2e/foo.sh::TestDoesNotExist" "named dangling"
|
||||
printf 'skills/bar/references/bar.feature\n' > "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature bar "" "doc only"
|
||||
run bash "$FAKE_SCRIPT"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"not found in"* ]]
|
||||
}
|
||||
|
||||
@test "FAIL: allowlisted file that also carries a @covered-by tag (ambiguous)" {
|
||||
write_feature foo "@covered-by:tests/e2e/foo.sh" "tagged but allowlisted"
|
||||
printf 'skills/foo/references/foo.feature\nskills/bar/references/bar.feature\n' \
|
||||
> "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature bar "" "doc only"
|
||||
run bash "$FAKE_SCRIPT"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"allowlisted (doc-only) file but also declares @covered-by"* ]]
|
||||
}
|
||||
|
||||
@test "FAIL: stale allowlist entry (listed file does not exist)" {
|
||||
write_feature foo "@covered-by:tests/e2e/foo.sh" "covered"
|
||||
printf 'skills/foo/references/foo.feature\n' > /dev/null # foo is linked, not allowlisted
|
||||
printf 'skills/ghost/references/ghost.feature\nskills/bar/references/bar.feature\n' \
|
||||
> "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature bar "" "doc only"
|
||||
run bash "$FAKE_SCRIPT"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"no longer exists"* ]]
|
||||
}
|
||||
|
||||
@test "--warn-only downgrades a failure to exit 0" {
|
||||
write_feature foo "" "orphan scenario"
|
||||
printf 'skills/bar/references/bar.feature\n' > "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature bar "" "doc only"
|
||||
run bash "$FAKE_SCRIPT" --warn-only
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"WARN"* ]]
|
||||
}
|
||||
|
||||
@test "--json emits a machine-readable summary" {
|
||||
write_feature foo "@covered-by:tests/e2e/foo.sh" "covered"
|
||||
printf 'skills/bar/references/bar.feature\n' > "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature bar "" "doc only"
|
||||
run bash "$FAKE_SCRIPT" --json
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *'"result":"pass"'* ]]
|
||||
[[ "$output" == *'"scenarios_linked":1'* ]]
|
||||
}
|
||||
|
||||
@test "file-level tag (above Feature) covers all scenarios" {
|
||||
local f="$FAKE_REPO/skills/foo/references/foo.feature"
|
||||
{
|
||||
printf '# fake spec\n\n'
|
||||
printf '@covered-by:tests/e2e/foo.sh\n'
|
||||
printf 'Feature: foo feature\n\n'
|
||||
printf ' Scenario: one\n When x\n Then y\n\n'
|
||||
printf ' Scenario: two\n When x\n Then y\n'
|
||||
} > "$f"
|
||||
printf 'skills/bar/references/bar.feature\n' > "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
write_feature bar "" "doc only"
|
||||
run bash "$FAKE_SCRIPT"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"2 linked"* ]]
|
||||
}
|
||||
|
||||
@test "misuse: unknown flag exits 2" {
|
||||
write_feature bar "" "x"
|
||||
printf 'skills/bar/references/bar.feature\n' > "$FAKE_REPO/scripts/.scenario-linkage-allow"
|
||||
run bash "$FAKE_SCRIPT" --bogus
|
||||
[ "$status" -eq 2 ]
|
||||
}
|
||||
Reference in New Issue
Block a user