From ce9ae2e2dc2de7ea05f4a8a6e636ccf576f83c79 Mon Sep 17 00:00:00 2001 From: Dan Guido Date: Mon, 14 Sep 2026 01:49:16 -0400 Subject: [PATCH] Add post-patch-validation plugin (#302) * Add post-patch-validation plugin * Use public contact email for post-patch-validation * Fix post-patch validation CLI and artifact edge cases --- .claude-plugin/marketplace.json | 11 + .github/workflows/lint.yml | 5 +- CODEOWNERS | 1 + Makefile | 2 +- README.md | 1 + .../.claude-plugin/plugin.json | 10 + plugins/post-patch-validation/README.md | 164 ++ .../evals/behavior-regression/case.yaml | 36 + .../evals/behavior-regression/scaffold.sh | 25 + .../evals/complete-fix/case.yaml | 39 + .../evals/complete-fix/scaffold.sh | 25 + .../evals/missed-variant/case.yaml | 40 + .../evals/missed-variant/scaffold.sh | 25 + .../skills/post-patch-validation/SKILL.md | 236 ++ .../post-patch-validation/agents/openai.yaml | 4 + .../references/evidence-model.md | 182 ++ .../scripts/post_patch_validation.py | 1957 +++++++++++++++++ .../scripts/pyproject.toml | 6 + .../post-patch-validation/tests/conftest.py | 22 + .../post-patch-validation/tests/extract.mjs | 35 + .../post-patch-validation/tests/run-all.sh | 7 + .../tests/test_cli_artifacts.py | 69 + .../tests/test_eval_cases.py | 130 ++ .../tests/test_runner.py | 1120 ++++++++++ .../tests/test_workflow_contract.py | 84 + .../tests/workflow_logic.test.mjs | 77 + .../workflows/validate-patch.js | 377 ++++ 27 files changed, 4687 insertions(+), 3 deletions(-) create mode 100644 plugins/post-patch-validation/.claude-plugin/plugin.json create mode 100644 plugins/post-patch-validation/README.md create mode 100644 plugins/post-patch-validation/evals/behavior-regression/case.yaml create mode 100755 plugins/post-patch-validation/evals/behavior-regression/scaffold.sh create mode 100644 plugins/post-patch-validation/evals/complete-fix/case.yaml create mode 100755 plugins/post-patch-validation/evals/complete-fix/scaffold.sh create mode 100644 plugins/post-patch-validation/evals/missed-variant/case.yaml create mode 100755 plugins/post-patch-validation/evals/missed-variant/scaffold.sh create mode 100644 plugins/post-patch-validation/skills/post-patch-validation/SKILL.md create mode 100644 plugins/post-patch-validation/skills/post-patch-validation/agents/openai.yaml create mode 100644 plugins/post-patch-validation/skills/post-patch-validation/references/evidence-model.md create mode 100755 plugins/post-patch-validation/skills/post-patch-validation/scripts/post_patch_validation.py create mode 100644 plugins/post-patch-validation/skills/post-patch-validation/scripts/pyproject.toml create mode 100644 plugins/post-patch-validation/tests/conftest.py create mode 100644 plugins/post-patch-validation/tests/extract.mjs create mode 100755 plugins/post-patch-validation/tests/run-all.sh create mode 100644 plugins/post-patch-validation/tests/test_cli_artifacts.py create mode 100644 plugins/post-patch-validation/tests/test_eval_cases.py create mode 100644 plugins/post-patch-validation/tests/test_runner.py create mode 100644 plugins/post-patch-validation/tests/test_workflow_contract.py create mode 100644 plugins/post-patch-validation/tests/workflow_logic.test.mjs create mode 100644 plugins/post-patch-validation/workflows/validate-patch.js diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 3658dcb..403ea58 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -135,6 +135,17 @@ }, "source": "./plugins/mutation-testing" }, + { + "name": "post-patch-validation", + "version": "0.1.1", + "description": "Validates security patches against baseline exploits, root-cause variants, behavior contracts, regressions, and new-vulnerability checks using isolated worktrees and deterministic S1-S5 evidence classification. Bundles a validate-patch dynamic workflow for Claude Code.", + "author": { + "name": "Scott Arciszewski", + "email": "opensource@trailofbits.com", + "url": "https://github.com/tob-scott-a" + }, + "source": "./plugins/post-patch-validation" + }, { "name": "property-based-testing", "description": "Write, review, and triage property-based tests — Hypothesis, fast-check, proptest, and Echidna or Medusa for Solidity invariants", diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e057acc..74f179a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -91,8 +91,9 @@ jobs: # suite turns that into a failure rather than a skip; this makes it not happen. with: node-version: "22" - - name: Install pytest and semgrep + - name: Install pytest, PyYAML, and semgrep # The rust-review / c-review test files are pytest-based. + # post-patch-validation tests parse YAML eval cases with PyYAML. # # semgrep grades semgrep-rule-variant-creator's golden variant fixtures, which are # otherwise checked only structurally: its grader skips when semgrep is absent, so @@ -104,7 +105,7 @@ jobs: # less than none. A release that stops grading those two rules is also a result worth # having rather than one to hold off: a parser moving behind Pro is the failure that # plugin exists to catch. - run: python3 -m pip install pytest semgrep + run: python3 -m pip install pytest pyyaml semgrep - name: Install uv # c-review's scripts are PEP 723 and run under `uv run`; its parser tests # fail rather than skip when uv is missing, on purpose — a skipped parser diff --git a/CODEOWNERS b/CODEOWNERS index 4fa6aeb..1bf6e8d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -27,6 +27,7 @@ /plugins/modern-python/ @Ninja3047 @dguido @kz-tob /plugins/mutation-testing/ @bohendo @dguido @kz-tob /plugins/open-sourcing/ @ESultanik @dguido @kz-tob +/plugins/post-patch-validation/ @tob-scott-a @dguido @kz-tob /plugins/property-based-testing/ @hbrodin @dguido @kz-tob /plugins/rust-review/ @zi0Black @GrosQuildu @dguido @kz-tob /plugins/code-improver/ @GrosQuildu @dguido @kz-tob diff --git a/Makefile b/Makefile index 6897d14..6cc64d6 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,7 @@ python-tests: failed=0; ran=0; \ for d in $$dirs; do \ echo " → $$d"; \ - ( cd "$$d" && uv run --no-project --with pytest python3 -m pytest -q \ + ( cd "$$d" && uv run --no-project --with pytest --with pyyaml python3 -m pytest -q \ --import-mode=importlib . ) || failed=1; \ ran=$$((ran + 1)); \ done; \ diff --git a/README.md b/README.md index 2050215..2cd6638 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ cd /path/to/parent # e.g., if repo is at ~/projects/skills, be in ~/projects |--------|-------------| | [constant-time-analysis](plugins/constant-time-analysis/) | Detect compiler-induced timing side-channels in cryptographic code | | [mutation-testing](plugins/mutation-testing/) | Configure mewt/muton mutation testing campaigns — scope targets, tune timeouts, optimize long runs | +| [post-patch-validation](plugins/post-patch-validation/) | Validate security patches against baseline exploits, root-cause variants, behavior changes, and new vulnerabilities with reproducible evidence | | [property-based-testing](plugins/property-based-testing/) | Write, review, and triage property-based tests — Hypothesis, fast-check, proptest, and Echidna or Medusa for Solidity invariants | | [spec-to-code-compliance](plugins/spec-to-code-compliance/) | Check code against the documentation that specifies it, across contracts, C/C++, services, and firmware | | [writing-lean-proofs](plugins/writing-lean-proofs/) | Write structured Lean 4 proofs and design Lean libraries following Mathlib conventions | diff --git a/plugins/post-patch-validation/.claude-plugin/plugin.json b/plugins/post-patch-validation/.claude-plugin/plugin.json new file mode 100644 index 0000000..f63dc76 --- /dev/null +++ b/plugins/post-patch-validation/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "post-patch-validation", + "version": "0.1.1", + "description": "Validates security patches against baseline exploits, root-cause variants, behavior contracts, regressions, and new-vulnerability checks using isolated worktrees and deterministic S1-S5 evidence classification. Bundles a validate-patch dynamic workflow for Claude Code.", + "author": { + "name": "Scott Arciszewski", + "email": "opensource@trailofbits.com", + "url": "https://github.com/tob-scott-a" + } +} diff --git a/plugins/post-patch-validation/README.md b/plugins/post-patch-validation/README.md new file mode 100644 index 0000000..083516d --- /dev/null +++ b/plugins/post-patch-validation/README.md @@ -0,0 +1,164 @@ +# Post-Patch Validation + +Validates an existing security patch as an untrusted hypothesis. The plugin proves the original +failure on a pinned baseline, exercises at least one root-cause variant, checks behavior and +adjacent security properties, runs the project suite, and produces an evidence-backed S1-S5 or +INCONCLUSIVE verdict. + +The portable `post-patch-validation` skill works with agents that support Agent Skills. Claude +Code users also get a bundled dynamic workflow. + +## Installation + +```text +/plugin marketplace add trailofbits/skills +/plugin install post-patch-validation@trailofbits +``` + +## Skill usage + +Ask an agent to validate a patch and provide: + +- the vulnerable base commit or tag; +- the patched commit, tag, or patch file; +- the finding or a local path containing it; +- authorization to execute the local project and its tests. + +The skill scaffolds a JSON plan, requires executable evidence for seven categories, runs each +check in isolated Git worktrees, and writes: + +```text +post-patch-validation/ +├── plan.json +└── results/ + ├── artifact-manifest.json + ├── patch.diff + ├── plan.snapshot.json + ├── report.md + ├── result.json + ├── helpers/ + │ └── # invoked file bytes, content-addressed + ├── scratch/ + │ └── / # archived per-invocation writable state + └── 001--.stdout / .stderr +``` + +The Python runner uses argv arrays rather than shell command strings. It pins commit and patch +hashes, disables Git hooks while materializing worktrees, fixes locale/timezone/hash-seed inputs, +executes checks in lexical order, and preserves raw output. Each check invocation receives a fresh +opaque scratch directory and a private copy of its plan artifacts; base and patched worktrees also +use separate validator-owned roots. Scratch is archived only after exit, while the private plan +copy is discarded; the clean source snapshot remains only in runner memory. Exploit and variant +sides execute in random order with stable evidence filenames; stdout/stderr are copied from +anonymous or random capture descriptors into those named files only after exit. Every argv element +that resolves to a readable file is hashed. Files inside the isolated plan or checkout roots are +also stored under `helpers/` for review, up to a 16 MiB limit; `argv_files` says why an +external, unreadable, or oversized file was not archived. Git worktree metadata changes are +serialized and active worktrees are locked, while independent validations may execute their checks +concurrently. The runner exits 0 only for S1; S2-S5 use their score as the exit code and +INCONCLUSIVE uses 10. Invalid or moved inputs use 64. + +These controls isolate runner-managed evidence state; they are not a host sandbox. Checks retain +the caller's privileges, so execute untrusted helper code inside an appropriate OS or container +sandbox. + +Two rules exist because an exit code carries less information than it appears to: + +- `exploit` and `variant` checks must print `PPV_REACHED` before their assertion. A build error + and a failed assertion both exit nonzero, so an unmarked run is INCONCLUSIVE rather than proof + that the vulnerability reproduced. +- Checks run under a fixed minimal environment. Real toolchains get what they need through + `--allow-env NAME`, which records the forwarded name and value in `result.json`. + +Every plan declares an evidence level: `source`, `build`, or `runtime`. Reports display it beside +verdict-specific interpretation text so an S1 from source inspection cannot be mistaken for runtime +proof. Exploit and variant checks assert only the security invariant; liveness, exact error behavior, +timing, and compatibility belong in behavior or regression checks. + +Scaffolding also records affected Git submodules as sorted relative paths. Their pinned commits are +initialized from module objects already present in the source repository, without contacting the +URLs in `.gitmodules`, and both base and patched pins are recorded in `result.json`. + +Active validation worktrees are locked against concurrent pruning with random owner tokens backed +by kernel file locks, so PID reuse cannot confuse stale-owner detection. After a forced +termination, the next run unlocks stale validator-owned registrations; `git worktree unlock`, +followed by `git worktree remove --force` or `git worktree prune`, provides manual recovery. + +## Claude Dynamic Workflow + +Claude Code exposes `/post-patch-validation:validate-patch`. For structured inputs, ask Claude +to invoke the `Workflow` tool with the following arguments: + +```javascript +Workflow({ + name: 'post-patch-validation:validate-patch', + args: { + finding: 'UAF when a callback releases the final request reference', + baseRef: 'vulnerable-tag', + patchRef: 'HEAD', + workdir: 'post-patch-validation', + }, +}) +``` + +The `Workflow` tool's `name` omits the leading slash. See the +[workflow documentation](https://code.claude.com/docs/en/workflows#distribute-a-workflow-in-a-plugin) +for plugin command naming. + +Use `patchFile` instead of `patchRef` for a patch artifact; supplying both is rejected. `patchRef` +defaults to `HEAD`, but `baseRef` is always required because a validator that guesses the vulnerable +baseline cannot prove reproduction. + +The workflow has five fixed phases and at most nine agents: + +| Phase | Work | +|---|---| +| Inventory | Pin the inputs and scaffold the plan with the Python runner | +| Coverage | Four read-only lenses map exploit variants, behavior, adjacent security, and test infrastructure | +| Plan | One agent turns proposals into executable artifacts and passes `validate-plan` | +| Execute | One agent invokes the deterministic runner and returns its exact result | +| Review | Two read-only reviewers flag omitted paths or evidence that did not exercise real code | + +The review agents cannot change the S-score. They can only keep an S1 result in +`REVIEW_REQUIRED` rather than advancing it to `READY_FOR_HUMAN_REVIEW`. + +Dynamic workflows must be enabled in Claude Code. The workflow cannot ask questions after +launch, so pass the base and patch inputs up front. + +## Verdicts + +| Verdict | Meaning | +|---|---| +| S1 | Fix and variants pass without observed behavior/security regression | +| S2 | Fix passes but behavior, regression, or suite evidence fails | +| S3 | Original exploit or a root-cause variant remains unfixed | +| S4 | Fix passes but a base-clean security property fails on the patch | +| S5 | Patch is both incomplete and introduces a new security failure | +| INCONCLUSIVE | Baseline, marker, control, execution, pinning, coverage, or cleanup evidence is invalid | + +Every verdict requires human review. S1 means the supplied checks passed; it cannot prove that a +human or agent supplied every relevant path. + +## Requirements + +- Git 2.36 or later (for lock reasons and NUL-delimited porcelain worktree metadata) +- Python 3.11 or later +- `uv` +- The target project's local build and test dependencies + +No remote-target mode is provided. The validator executes repository code, so run it only for a +local target the user has authorized. + +## Development + +Run the Python runner tests, eval grader tests, and workflow tests from this repository's root: + +```bash +bash plugins/post-patch-validation/tests/run-all.sh +node plugins/post-patch-validation/tests/workflow_logic.test.mjs +``` + +The Python suite uses `pytest` and `PyYAML` through `uv`; the runtime runner uses only the Python +standard library. `make check` discovers both suites. The three cases under `evals/` exercise a +complete fix, a missed variant, and a behavior regression. Their grader and scaffold tests run +locally without model calls; live Claude plugin evals require a separate invocation. diff --git a/plugins/post-patch-validation/evals/behavior-regression/case.yaml b/plugins/post-patch-validation/evals/behavior-regression/case.yaml new file mode 100644 index 0000000..4d7ca3d --- /dev/null +++ b/plugins/post-patch-validation/evals/behavior-regression/case.yaml @@ -0,0 +1,36 @@ +schema_version: "1.1" +name: behavior-regression +description: A patch fixes markup escaping but strips legitimate whitespace from benign input. +tags: [post-patch-validation, behavior-regression] +runs: 3 +context: + scaffold_script: scaffold.sh +execution: + max_turns: 60 + timeout_seconds: 1800 + allowed_tools: [Read, Write, Edit, Bash, Glob, Grep, Skill, Workflow] + prompt: |- + Run the post-patch-validation:validate-patch dynamic workflow with the Workflow tool, passing + args finding PPV-EVAL-3, baseRef vulnerable, patchRef HEAD, and workdir post-patch-validation. + + Finding PPV-EVAL-3: render(value) returns unescaped attacker-controlled markup. The intended + fix must escape every "<" while preserving legitimate text exactly, including surrounding + spaces. The target is this local repository and you are authorized to execute its code. +graders: + - name: result-exists + type: file_exists + path: post-patch-validation/results/result.json + exists: true + weight: 1 + - name: behavior-change-verdict + type: regex + target: {source: file, path: post-patch-validation/results/result.json} + pattern: '"code":\s*"S2"' + match: contains + weight: 2 + - name: behavior-row-shows-regression + type: regex + target: {source: file, path: post-patch-validation/results/report.md} + pattern: '\|\s*`[^`]+`\s*\|\s*behavior\s*\|\s*pass\s*\|\s*(?:fail\s*\|\s*(?:same|changed)|pass\s*\|\s*changed)\s*\|' + match: contains + weight: 1 diff --git a/plugins/post-patch-validation/evals/behavior-regression/scaffold.sh b/plugins/post-patch-validation/evals/behavior-regression/scaffold.sh new file mode 100755 index 0000000..ae6d92d --- /dev/null +++ b/plugins/post-patch-validation/evals/behavior-regression/scaffold.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +git -c init.defaultBranch=main init -q +git config user.name "PPV Eval" +git config user.email "ppv-eval@example.invalid" + +cat >renderer.py <<'PY' +def render(value: str) -> str: + return value +PY + +git add renderer.py +GIT_AUTHOR_DATE="2026-01-01T00:00:00Z" GIT_COMMITTER_DATE="2026-01-01T00:00:00Z" \ + git -c commit.gpgsign=false commit -q -m "vulnerable renderer" +git tag vulnerable + +cat >renderer.py <<'PY' +def render(value: str) -> str: + return value.replace("<", "<").strip() +PY + +git add renderer.py +GIT_AUTHOR_DATE="2026-01-02T00:00:00Z" GIT_COMMITTER_DATE="2026-01-02T00:00:00Z" \ + git -c commit.gpgsign=false commit -q -m "escape markup" diff --git a/plugins/post-patch-validation/evals/complete-fix/case.yaml b/plugins/post-patch-validation/evals/complete-fix/case.yaml new file mode 100644 index 0000000..df01915 --- /dev/null +++ b/plugins/post-patch-validation/evals/complete-fix/case.yaml @@ -0,0 +1,39 @@ +schema_version: "1.1" +name: complete-fix +description: A patch fixes the original unsafe character and the repeated-character variant without changing benign behavior. +tags: [post-patch-validation, clean-fix] +runs: 3 +context: + scaffold_script: scaffold.sh +execution: + max_turns: 60 + timeout_seconds: 1800 + allowed_tools: [Read, Write, Edit, Bash, Glob, Grep, Skill, Workflow] + prompt: |- + Use the post-patch-validation skill to validate HEAD against the vulnerable tag. + + Finding PPV-EVAL-1: render(value) returns attacker-controlled "<" characters without + escaping them, enabling markup injection. The demonstrated payload is a single "<". + Validate the root cause, not only that payload. The target is this local repository and + you are authorized to execute its code. Write the plan under post-patch-validation/plan.json + and results under post-patch-validation/results. Do not stop to ask questions. +graders: + - name: result-exists + type: file_exists + path: post-patch-validation/results/result.json + exists: true + weight: 1 + - name: clean-fix-verdict + type: regex + target: {source: file, path: post-patch-validation/results/result.json} + pattern: '"code":\s*"S1"' + match: contains + weight: 2 + # `"kind": "variant"` in result.json only proves a variant check was declared. The report row + # proves it did the job: failed as expected on the vulnerable base, then passed on the patch. + - name: variant-reproduced-then-fixed + type: regex + target: {source: file, path: post-patch-validation/results/report.md} + pattern: '\|\s*variant\s*\|\s*pass\s*\|\s*pass\s*\|' + match: contains + weight: 1 diff --git a/plugins/post-patch-validation/evals/complete-fix/scaffold.sh b/plugins/post-patch-validation/evals/complete-fix/scaffold.sh new file mode 100755 index 0000000..afb50b3 --- /dev/null +++ b/plugins/post-patch-validation/evals/complete-fix/scaffold.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +git -c init.defaultBranch=main init -q +git config user.name "PPV Eval" +git config user.email "ppv-eval@example.invalid" + +cat >renderer.py <<'PY' +def render(value: str) -> str: + return value +PY + +git add renderer.py +GIT_AUTHOR_DATE="2026-01-01T00:00:00Z" GIT_COMMITTER_DATE="2026-01-01T00:00:00Z" \ + git -c commit.gpgsign=false commit -q -m "vulnerable renderer" +git tag vulnerable + +cat >renderer.py <<'PY' +def render(value: str) -> str: + return value.replace("<", "<") +PY + +git add renderer.py +GIT_AUTHOR_DATE="2026-01-02T00:00:00Z" GIT_COMMITTER_DATE="2026-01-02T00:00:00Z" \ + git -c commit.gpgsign=false commit -q -m "escape markup" diff --git a/plugins/post-patch-validation/evals/missed-variant/case.yaml b/plugins/post-patch-validation/evals/missed-variant/case.yaml new file mode 100644 index 0000000..cbb8349 --- /dev/null +++ b/plugins/post-patch-validation/evals/missed-variant/case.yaml @@ -0,0 +1,40 @@ +schema_version: "1.1" +name: missed-variant +description: A patch handles the demonstrated single character but misses repeated instances of the same root cause. +tags: [post-patch-validation, missed-variant] +runs: 3 +context: + scaffold_script: scaffold.sh +execution: + max_turns: 60 + timeout_seconds: 1800 + allowed_tools: [Read, Write, Edit, Bash, Glob, Grep, Skill, Workflow] + prompt: |- + Use the post-patch-validation skill to validate HEAD against the vulnerable tag. + + Finding PPV-EVAL-2: render(value) must escape every attacker-controlled "<" character. + The report demonstrated one character, but the root cause is unescaped markup at any + position and multiplicity. The target is this local repository and you are authorized to + execute its code. Write the plan under post-patch-validation/plan.json and results under + post-patch-validation/results. Do not stop to ask questions. +graders: + - name: result-exists + type: file_exists + path: post-patch-validation/results/result.json + exists: true + weight: 1 + - name: not-fixed-verdict + type: regex + target: {source: file, path: post-patch-validation/results/result.json} + pattern: '"code":\s*"S3"' + match: contains + weight: 2 + # Graded on the report's one-row-per-check table, not result.json. The obvious JSON regex + # (`"kind": "variant"` ... `"matched": false`) backtracks across check boundaries, so a + # PASSING variant plus any later failing check satisfies it. A table row cannot span checks. + - name: variant-failed-on-patch + type: regex + target: {source: file, path: post-patch-validation/results/report.md} + pattern: '\|\s*variant\s*\|\s*pass\s*\|\s*fail\s*\|' + match: contains + weight: 1 diff --git a/plugins/post-patch-validation/evals/missed-variant/scaffold.sh b/plugins/post-patch-validation/evals/missed-variant/scaffold.sh new file mode 100755 index 0000000..8988524 --- /dev/null +++ b/plugins/post-patch-validation/evals/missed-variant/scaffold.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +git -c init.defaultBranch=main init -q +git config user.name "PPV Eval" +git config user.email "ppv-eval@example.invalid" + +cat >renderer.py <<'PY' +def render(value: str) -> str: + return value +PY + +git add renderer.py +GIT_AUTHOR_DATE="2026-01-01T00:00:00Z" GIT_COMMITTER_DATE="2026-01-01T00:00:00Z" \ + git -c commit.gpgsign=false commit -q -m "vulnerable renderer" +git tag vulnerable + +cat >renderer.py <<'PY' +def render(value: str) -> str: + return value.replace("<", "<", 1) +PY + +git add renderer.py +GIT_AUTHOR_DATE="2026-01-02T00:00:00Z" GIT_COMMITTER_DATE="2026-01-02T00:00:00Z" \ + git -c commit.gpgsign=false commit -q -m "escape demonstrated character" diff --git a/plugins/post-patch-validation/skills/post-patch-validation/SKILL.md b/plugins/post-patch-validation/skills/post-patch-validation/SKILL.md new file mode 100644 index 0000000..d117f66 --- /dev/null +++ b/plugins/post-patch-validation/skills/post-patch-validation/SKILL.md @@ -0,0 +1,236 @@ +--- +name: post-patch-validation +description: > + Validates security patches with reproducible baseline-versus-patched evidence, including + original exploits, root-cause variants, behavior preservation, regressions, and newly + introduced security failures. Use after a patch exists and before accepting, merging, or + reporting it as fixed; also use when an AI-generated patch, remediation commit, pull request, + or proposed upstream fix needs adversarial post-patch validation across any language. +allowed-tools: Read Write Edit Grep Glob Bash Workflow +--- + +# Post-Patch Validation + +Treat the patch as an untrusted hypothesis. Produce executable evidence in isolated Git +worktrees, then let the bundled runner assign the verdict. Never infer success from the diff, +the patch author, an upstream implementation, or the original proof of concept alone. + +## When to Use + +- A security fix, remediation commit, patch file, or pull request already exists. +- An AI-generated patch needs validation before human review or merge. +- A fix may cover one exploit path while missing variants of the same root cause. +- A security fix may alter legitimate behavior or introduce a new vulnerability. +- Another pipeline needs a deterministic final patch-validation gate. + +## When NOT to Use + +- No patch exists yet; use vulnerability discovery or fix implementation first. +- The task is to review an audit finding against a report without executing patch evidence. +- The task is only to convert a finding into a permanent project test. +- The target is remote or production. This skill executes local code and tests only. +- The user has not authorized execution of the repository's code or test suite. + +## Quick Start + +1. Pin the vulnerable base and patched input. Prefer immutable commits. For uncommitted work, + create a binary patch file first; do not validate in the user's working tree. +2. Scaffold a pinned plan: + + ```bash + uv run {baseDir}/scripts/post_patch_validation.py scaffold \ + --repo . \ + --base-ref \ + --patched-ref \ + --finding-id \ + --finding-summary "" \ + --evidence-level runtime \ + --output post-patch-validation/plan.json + ``` + + Use `--patch-file ` instead of `--patched-ref` for a patch artifact. Choose the highest + honest evidence level: `source` for source/patch invariants only, `build` when target code is + compiled or analyzed but the reported behavior is not executed, or `runtime` when the checks + execute the reported behavior and its safety assertions. +3. Inspect the finding, diff, callers, sibling paths, cleanup/error paths, and existing tests. + Populate `checks` in the generated plan. Run `print-schema` for the structural schema: + + ```bash + uv run {baseDir}/scripts/post_patch_validation.py print-schema + ``` + +4. Run `validate-plan` for the complete validation, including coverage, command restrictions, + and pinned inputs, before executing code: + + ```bash + uv run {baseDir}/scripts/post_patch_validation.py validate-plan \ + --plan post-patch-validation/plan.json + ``` + +5. Execute the evidence plan: + + ```bash + uv run {baseDir}/scripts/post_patch_validation.py run \ + --plan post-patch-validation/plan.json \ + --output post-patch-validation/results + ``` + +6. Report `result.json`, `report.md`, the exact verdict, and every failing or inconclusive + check. An S1 result is ready for human review; it is not permission to merge. + +## Evidence Contract + +The runner rejects incomplete plans. Supply at least one check of every kind: + +| Kind | Required observation | +|---|---| +| `control` | Benign harness succeeds on both base and patch | +| `exploit` | Original safety assertion fails on base and succeeds on patch | +| `variant` | A distinct root-cause variant fails on base and succeeds on patch | +| `behavior` | Unaffected behavior succeeds with byte-identical selected output | +| `regression` | Targeted non-security regression check succeeds on both revisions | +| `security` | Adjacent/new-vulnerability check succeeds on base and patch | +| `suite` | Existing project suite, sanitizer, or deterministic fuzz campaign succeeds on patch | + +Commands are argv arrays, never shell strings. Put complex setup in a checked-in or plan artifact +script and invoke it with `{plan_dir}`. The runner fixes locale/timezone/hash-seed inputs, executes +checks in lexical ID order, records raw stdout/stderr, and never edits the original worktree. +Each check's `timeout_seconds` defaults to 300 and accepts integers from 1 through 3600. +Exceeding the timeout makes the run INCONCLUSIVE. +Every plan also contains a sorted `submodules` array (`[]` when none). Scaffolding infers affected +Gitlinks from the changed-file inventory. The runner initializes those pinned commits from the +source repository's existing Git module objects, never from `.gitmodules` network URLs; initialize +or fetch them in the source repository before validation. + +### Exploit and variant checks must prove they ran + +A nonzero exit does not mean the vulnerability reproduced. An import error, a failed build, a +missing dependency, and a failed safety assertion all exit nonzero and are indistinguishable to the +runner. Every `exploit` and `variant` check must print and flush `PPV_REACHED` immediately +before it evaluates its assertion, on both revisions: + +```json +"argv": ["python3", "-c", "import app; print('PPV_REACHED', flush=True); assert app.render('<') == '<'"] +``` + +The token is also in the environment as `PPV_REACHED_MARKER`. It must land on **stdout, as a line +of its own**. Stderr is not scanned, because a Python `SyntaxError` traceback echoes the offending +source and would otherwise satisfy the check for a harness that executed nothing. A run without it +is recorded as `marker_missing` and the whole result is INCONCLUSIVE. Flush explicitly: a harness +whose payload segfaults or calls `_exit` loses buffered output and forfeits its own evidence. + +These checks also run **side-blind**. `{side}` is not expanded for them, `PPV_SIDE` is absent from +their environment, the checkout directory is randomly named, and the plan validator rejects any +`exploit` or `variant` check whose `argv` or `env` mentions either. An assertion that can see which +revision it is on can assert on *that* instead of on the code, which is the cheapest possible way +to fake a reproduction followed by a fix. + +### Environment + +Checks run under a fixed minimal environment: `PATH`, `HOME`, and a handful of temp/user keys, +plus `LANG`/`LC_ALL=C`, `TZ=UTC`, `PYTHONHASHSEED=0`, `NO_COLOR`, `TERM=dumb`. Everything else in +the caller's environment is dropped. Toolchains that need more get it explicitly: + +```bash +uv run {baseDir}/scripts/post_patch_validation.py run \ + --plan post-patch-validation/plan.json \ + --output post-patch-validation/results \ + --allow-env JAVA_HOME --allow-env CARGO_HOME +``` + +Forwarded names and values are recorded in `result.json`. A requested variable that is unset is an +error, not an empty string. Two classes are refused outright: names that read as credentials +(`*SECRET*`, `*TOKEN*`, `*API_KEY*`, …), because the value would be written into the result; and +names that change what executes (`LD_PRELOAD`, `BASH_ENV`, `NODE_OPTIONS`, `GIT_SSH_COMMAND`, …), +because forwarding those would quietly dismantle the isolation the verdict rests on. +The runner's fixed variables and every `PPV_*` name are also reserved and cannot be forwarded. + +Placeholders expanded in `argv` and per-check `env` values: `{checkout}` (the revision under test), +`{plan_dir}` (an isolated copy of the plan artifacts for that one invocation), `{scratch}` (a +fresh opaque directory for that one check invocation), and +`{side}` (`base` or `patched`, and not available to `exploit`/`variant` checks). The same values +arrive as `PPV_CHECKOUT`, `PPV_PLAN_DIR`, `PPV_SCRATCH`, `PPV_SIDE`, and `PPV_CASE_ID`. Write only +under `{scratch}`; the evidence directory path is not passed to checks. Base and patched invocations +do not share runner-managed scratch, plan, or worktree roots. After each invocation exits, its +scratch tree is archived under the deterministic `results/scratch/` path, its +private plan copy is discarded, and every readable argv element that resolves to a file is hashed +in `argv_files`. Files inside the isolated plan or checkout roots are additionally retained under +`results/helpers/` up to 16 MiB; the record explains why any other file was not archived. +Use a dedicated directory for `plan.json`: its sibling files and directories are copied into each +invocation's `{plan_dir}`. Keep helper code under that directory's `checks/` directory or checked +into the target repository so its bytes are reviewable. The machine plan containing commit pins, +the current output directory, and detected prior result trees are excluded; symlinks are rejected. +The clean snapshot remains only in runner memory, and +exploit/variant sides execute in random order while evidence filenames remain deterministic. +Stdout/stderr use anonymous or randomly named capture descriptors and are copied to the named +evidence files only after the child exits, so fd inspection cannot disclose the side label. + +This isolation is not a host sandbox: checks run with the caller's privileges and a malicious +helper could use arbitrary external state or deliberately infer the revision from source or Git +metadata. Inspect the content-addressed helper artifacts, and use an OS/container sandbox when the +check code itself is untrusted. + +Active validation worktrees are Git-locked with random owner tokens backed by kernel file locks, so +another concurrent validator cannot prune them and PID reuse cannot impersonate an owner. If the +runner is forcibly killed, the next run unlocks stale validator-owned registrations. For manual +recovery, inspect `git worktree list`, then use `git worktree unlock ` and +`git worktree remove --force ` (or `git worktree prune` after the path is gone). + +Read [evidence-model.md](references/evidence-model.md) when designing coverage, selecting +variants, or interpreting S1-S5 and INCONCLUSIVE. Do not read it for routine CLI execution. + +## Coverage Rules + +- Derive variants from the root cause, not cosmetic mutations of the original payload. +- Enumerate sibling call sites, alternate callbacks/outputs, error paths, teardown, ownership, + serialization, and boundary values touched by the fix. +- Make each exploit or variant test assert the safe behavior. It must fail on the vulnerable + base; a test that passes on both revisions proves nothing about remediation. Read the base-side + stderr and confirm the failure is the assertion you wrote, not a harness that never got there. +- Keep exploit and variant assertions limited to the security invariant. Test liveness, exact + error types/messages, timing, and compatibility separately as `behavior` or `regression` checks; + otherwise an unrelated contract change can masquerade as proof that the vulnerability remains. +- Keep the `control` harness benign and make it exercise the changed component. It + is the only check that says the worktree can run at all, and its failure is INCONCLUSIVE. +- Use `behavior` only for behavior that should remain unchanged. Exact output comparison is + deliberate; move unstable values behind a deterministic test harness instead of normalizing + them away in prose. +- Make `security` checks pass on the vulnerable base before treating a patched failure as newly + introduced. Otherwise the runner returns INCONCLUSIVE rather than inventing causality. +- Do not edit the patch during validation. Return failures to the patch author and start a new, + freshly pinned run. + +## Claude Dynamic Workflow + +Claude Code exposes the bundled workflow as `/post-patch-validation:validate-patch`. +To pass structured inputs through the `Workflow` tool, use the name without a leading slash: + +```javascript +Workflow({ + name: 'post-patch-validation:validate-patch', + args: { + finding: '', + baseRef: '', + patchRef: '', + workdir: 'post-patch-validation', + }, +}) +``` + +Use `patchFile` instead of `patchRef` when appropriate. The workflow uses fixed coverage +lenses to propose checks and a fixed executor to run this skill. Agents may author test +artifacts, but they do not vote on the S-score: only the Python runner classifies evidence. +It cannot ask questions after launch, so pass every input up front. + +## Rationalizations to Reject + +| Rationalization | Required response | +|---|---| +| "The original PoC no longer works" | Test at least one independent root-cause variant | +| "The exploit failed on base, so it reproduced" | Confirm the marker and that the failure is the assertion, not a broken harness | +| "The full suite passes" | Prove baseline reproduction and targeted behavior explicitly | +| "This matches the upstream/canonical patch" | Treat provenance as context, not evidence | +| "The diff is tiny" | Exercise callers, failure paths, and teardown affected by the change | +| "The validator says S1" | Preserve artifacts and require human review | +| "A flaky rerun passed" | Keep the first pinned result; fix nondeterminism before retrying | +| "There is no obvious variant" | Inspect sibling sites and boundaries; otherwise stop INCONCLUSIVE | diff --git a/plugins/post-patch-validation/skills/post-patch-validation/agents/openai.yaml b/plugins/post-patch-validation/skills/post-patch-validation/agents/openai.yaml new file mode 100644 index 0000000..834b4c0 --- /dev/null +++ b/plugins/post-patch-validation/skills/post-patch-validation/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Post-Patch Validation" + short_description: "Validate security patches with reproducible evidence" + default_prompt: "Use $post-patch-validation to validate this security patch against the finding and its variants." diff --git a/plugins/post-patch-validation/skills/post-patch-validation/references/evidence-model.md b/plugins/post-patch-validation/skills/post-patch-validation/references/evidence-model.md new file mode 100644 index 0000000..0de8b3d --- /dev/null +++ b/plugins/post-patch-validation/skills/post-patch-validation/references/evidence-model.md @@ -0,0 +1,182 @@ +# Evidence Model + +Use this reference while authoring the validation plan or interpreting its result. + +## Contents + +- Design basis +- Building independent evidence +- Classification +- Human handoff + +## Design basis + +Patch correctness is not binary. Empirical reviews of model-generated vulnerability patches +show recurring failures: fixing only the demonstrated path, changing legitimate behavior, +introducing a second vulnerability, accepting incorrect guidance despite contrary evidence, +and copying a flawed upstream pattern. Validators also disagree often enough that a prose-only +second opinion is not a reliable gate. + +This skill therefore separates two jobs: + +1. Agents or humans identify the root cause and author candidate checks. +2. Code pins the inputs, executes the checks, preserves the evidence, and assigns the verdict. + +The runner is deliberately strict. Missing evidence is INCONCLUSIVE, not an optimistic pass. + +Every result also states its evidence level. `source` means only source or patch invariants ran; +`build` means target code was compiled or analyzed without executing the reported behavior; and +`runtime` means the reported behavior and its safety assertions executed. This is a declared scope, +not a verdict multiplier: even S1 establishes no more than its stated evidence level and supplied +coverage. + +## Building independent evidence + +### Baseline first + +An exploit test expresses the safe postcondition. It must fail against the vulnerable base and +pass against the patch. A test that never reproduced the vulnerability cannot establish a fix. + +The exit code alone cannot carry that claim. `ImportError`, a failed build, a missing shared +library, a typo'd module name, and a failed assertion all exit nonzero, and the runner sees only +the number. Treating any nonzero base exit as reproduction is how a harness that never executed +becomes an S1: it "fails" on base for an unrelated reason, then "passes" on the patch where the +same unrelated reason happens not to bite. + +So `exploit` and `variant` checks must print `PPV_REACHED`, flushed, immediately before evaluating +the assertion. A nonzero run without the marker is `marker_missing` and the result is INCONCLUSIVE. +Place the marker after setup and after reaching the vulnerable call, immediately before +the comparison, because everything between the marker and the assertion is still unproven. Flush +explicitly; a payload that segfaults or calls `_exit` discards buffered output and its own evidence +along with it. + +The marker must land on stdout as a line of its own. Scanning stderr matched the token inside a +Python `SyntaxError` traceback, which echoes the offending source line, so a harness that never +executed a statement passed the check that exists to catch precisely that. + +`exploit` and `variant` checks additionally run side-blind: no `{side}`, no `PPV_SIDE`, a randomly +named checkout, and plan-time rejection of any check that mentions either. Their assertion must be +identical on both revisions by construction, so a check that can identify its revision can only be +using that fact to fake the result. + +The marker proves the harness ran. It does not prove the harness was *correct*: a check asserting +the wrong postcondition still fails on base and passes on patch. That remains a human-review job, +which is why `plan.snapshot.json`, the raw stderr, and content-addressed helper copies ship with +every result. `argv_files` records each file's argv index, original argument, SHA-256 digest, +optional archived path, and any reason it could not be archived. Readable files under the isolated +plan and checkout roots are archived up to 16 MiB; external paths are hashed but never copied, so a +data-file argument cannot silently vendor a host credential into the evidence. `argv0_sha256` also +records the resolved executable digest for compatibility with earlier result readers. + +Keep that postcondition to the security invariant. Whether the operation remains live, returns an +exact error type or message, meets a timing property, or stays compatible with existing callers is +important, but it is separate behavior/regression evidence. Combining those contracts into an +exploit assertion lets a benign compatibility change manufacture S3 even when the security +invariant is fixed. + +Each base or patched invocation receives a fresh opaque scratch directory and an isolated copy of +a clean plan-artifact snapshot. The snapshot excludes the machine plan containing revision pins, +symlinks, the live evidence tree, and detected prior result trees. The runner archives scratch under +a deterministic evidence path only after the process exits and discards the private plan copy. +Base and patched worktrees also use separate, randomly named validator-owned roots, and temp +environment variables point to per-invocation scratch. These measures remove runner-provided shared +writable channels that could otherwise reveal execution order or manufacture the expected base/fix +sequence. Exploit and variant sides execute in random order while retaining deterministic evidence +filenames, and the clean plan-artifact snapshot lives only as immutable runner memory between +invocations. Child stdout/stderr first go to anonymous or randomly named capture descriptors; the +runner copies them to side-labelled evidence paths only after the child exits, so inspecting fd 1 +or fd 2 does not reveal the logical side or retained evidence directory. + +This is evidence isolation, not an operating-system sandbox. Checks execute with the caller's +privileges and could deliberately communicate through arbitrary host paths, services, or process +inspection. A deliberately adversarial check can also inspect source differences or Git metadata +in the revision it must execute. Review the archived helper bytes before trusting a verdict, and +run untrusted helpers inside a sandbox chosen for the target's threat model. + +### Root-cause variants + +Choose a variant that reaches the same invariant violation through a meaningfully different +path. Useful dimensions include: + +- sibling call sites or alternate public entry points; +- callback, direct-output, streaming, and buffered modes; +- success, error, cancellation, and teardown paths; +- minimum, maximum, empty, repeated, and nested inputs; +- ownership transfer, reference counts, aliases, and object lifetime; +- parser/serializer asymmetry and encode/decode direction; +- concurrency ordering or state-machine transition. + +Changing only a literal, filename, or payload size is not independent unless that boundary is +the root cause. + +### Behavior preservation + +Use a stable benign input whose behavior should not change. The runner compares the selected +stdout, stderr, or combined stream byte-for-byte across base and patch. Put timestamps, random +IDs, absolute paths, addresses, and ordering behind a deterministic harness; do not hide them with +ad hoc output scrubbing. Checkout, plan, and scratch paths differ between invocations. + +### New-vulnerability evidence + +A `security` check must pass on the base and patch. If it passes on base but fails on patch, the +runner can attribute the new failure to the patch. High-value checks exercise cleanup, ownership, +error handling, authorization, bounds, and state transitions adjacent to changed code. + +### Existing suite + +Run the narrow deterministic project suite first, then broader tests, sanitizers, or a bounded +fuzz corpus when available. Pin seeds, corpus, worker count, and iteration/time budgets. A suite +supplements targeted evidence; it does not replace it. + +Suite checks run only on the patched revision. A suite failure can produce S2 even when the +failure predates the patch; that result alone does not establish a regression caused by the patch. + +## Classification + +| Code | Meaning | Deterministic condition | +|---|---|---| +| S1 | Clean fix | Exploit and variants fixed; behavior and security checks pass | +| S2 | Fixed with behavior change | Exploit and variants fixed; behavior, regression, or suite fails | +| S3 | Not fixed | Exploit or variant still fails on the patch; no new security failure | +| S4 | Fixed with new vulnerability | Exploit and variants fixed; a base-clean security check fails on patch | +| S5 | Not fixed and new vulnerability | S3 and S4 conditions both hold | +| INCONCLUSIVE | Evidence cannot support a class | Missing category, baseline failure, missing marker, patched-side control failure, timeout, execution error, pin mismatch, or cleanup failure | + +Classification precedence is S5, S3/S4 as applicable, S2, then S1. A behavior regression does +not conceal an unfixed vulnerability. Every result sets `human_review_required` to true. + +`control` is deliberately not an S2 signal. Its job is to show that both worktrees can execute +the component at all, so a control failure on the patched side invalidates every other patched +observation rather than describing a behavior change. Reporting that as "fixed with behavior +change" would assert a fix on the strength of a harness that demonstrably stopped working. + +### Environment determinism + +Checks inherit a fixed minimal environment. Anything a real toolchain needs beyond `PATH` and +`HOME` is forwarded by name with `--allow-env`, and the forwarded names and values land in +`result.json` so a reviewer can see exactly what the run depended on. Never forward credentials; +a requested variable that is unset fails the run rather than arriving empty. Fixed deterministic +variables and all `PPV_*` names are reserved so forwarded input cannot replace runner-controlled +state. + +Affected submodules are explicit, sorted paths in the plan. Their pinned objects must already exist +in the source repository: validation redirects initialization to those local module stores and does +not contact `.gitmodules` URLs. The result records base and patched submodule commits alongside the +top-level pins. + +## Human handoff + +Give the reviewer: + +- `plan.snapshot.json` and the finding/root-cause statement; +- `patch.diff` and its SHA-256 pin; +- `result.json` and `report.md`; +- raw per-check stdout/stderr; +- content-addressed helper files under `helpers/`, cross-referenced by each run's `argv_files`; +- archived per-invocation scratch trees; +- `artifact-manifest.json` for integrity; +- any coverage concern that was not converted into an executable check. + +Human review should inspect whether the declared root-cause surface is complete. The runner can +prove that the supplied checks behaved as claimed; it cannot prove that an omitted path does not +exist. diff --git a/plugins/post-patch-validation/skills/post-patch-validation/scripts/post_patch_validation.py b/plugins/post-patch-validation/skills/post-patch-validation/scripts/post_patch_validation.py new file mode 100755 index 0000000..66632ef --- /dev/null +++ b/plugins/post-patch-validation/skills/post-patch-validation/scripts/post_patch_validation.py @@ -0,0 +1,1957 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""Deterministic evidence runner for post-patch security validation. + +The plan contains argv arrays, never shell strings. The runner pins both Git inputs, +materializes isolated worktrees, executes checks in lexical order, preserves raw evidence, +and maps observations to S1-S5 or INCONCLUSIVE. + +Exploit and variant checks must print PPV_REACHED before evaluating their safety assertion. +A nonzero exit alone proves nothing: an import error, a failed build, and a failed assertion +are indistinguishable by exit code, so an unmarked run is INCONCLUSIVE rather than evidence. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import time +from collections.abc import Mapping, Sequence +from contextlib import contextmanager, suppress +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +SCHEMA_VERSION = "1.0" +ID_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$") +ENV_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +HEX_RE = re.compile(r"^[0-9a-f]{40,64}$") +KINDS = ("control", "exploit", "variant", "behavior", "regression", "security", "suite") +EVIDENCE_LEVELS = ("source", "build", "runtime") +FIXED_ENV = { + "LANG": "C", + "LC_ALL": "C", + "TZ": "UTC", + "NO_COLOR": "1", + "TERM": "dumb", + "PYTHONHASHSEED": "0", +} +BASE_ENV_KEYS = { + "COMSPEC", + "HOME", + "LOGNAME", + "PATH", + "PATHEXT", + "SHELL", + "SYSTEMROOT", + "TEMP", + "TMP", + "TMPDIR", + "USER", + "WINDIR", + "XDG_CACHE_HOME", +} +GIT_PREFIX = ("git", "-c", f"core.hooksPath={os.devnull}") +# Kinds whose verdict depends on a *failed safety assertion* rather than a nonzero exit. +# A crashed, mis-imported, or never-built harness also exits nonzero, so those kinds must +# prove they reached the assertion by emitting REACHED_MARKER before evaluating it. +MARKER_KINDS = frozenset({"exploit", "variant"}) +REACHED_MARKER = "PPV_REACHED" +WORKTREE_LOCK_REASON_PREFIX = "post-patch-validation:" +MAX_ARCHIVED_ARGV_FILE_BYTES = 16 * 1024 * 1024 +MAX_PLAN_ARTIFACT_BYTES = 64 * 1024 * 1024 +SHELL_NAMES = frozenset( + { + "ash", + "bash", + "cmd", + "cmd.exe", + "csh", + "dash", + "fish", + "ksh", + "powershell", + "powershell.exe", + "pwsh", + "pwsh.exe", + "script", + "sh", + "tcsh", + "zsh", + } +) +# `-c` is rarely alone: bash accepts -xc, -lc, -ic, and any other bundling of single-letter +# flags, so match a leading-dash cluster containing "c" rather than an exact string. +SHELL_COMMAND_FLAG_RE = re.compile(r"^-[a-z]*c[a-z]*$") +SHELL_COMMAND_FLAGS = frozenset({"-command", "/c"}) +# `env -S "sh -c ..."` re-splits its argument into a command line, which is the same hazard as +# a shell string reached through a different door. +ENV_SPLIT_FLAGS = frozenset({"-s", "--split-string"}) +# Variables that change which code runs. Forwarding these would silently undo the isolation the +# whole tool is built on, so they are refused rather than recorded. +FORBIDDEN_ENV = frozenset( + { + "BASH_ENV", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "ENV", + "GIT_SSH_COMMAND", + "LD_AUDIT", + "LD_PRELOAD", + "NODE_OPTIONS", + "PERL5OPT", + "PYTHONSTARTUP", + "RUBYOPT", + } +) +# Forwarded values are recorded verbatim in result.json, so refuse the obvious secret names +# outright instead of trusting every future caller to have read the warning. +CREDENTIAL_ENV_RE = re.compile( + r"(SECRET|TOKEN|PASSWORD|PASSWD|CREDENTIAL|PRIVATE_KEY|API_KEY|ACCESS_KEY|SESSION|" + r"COOKIE|SIGNING_KEY|(?:^|_)AUTH(?:_|$)|(?:^|_)KEY(?:_|$)|(?:^|_)PAT(?:_|$)|" + r"(?:^|_)PASS(?:_|$))" +) +SIDES_BY_KIND = { + "control": ("base", "patched"), + "exploit": ("base", "patched"), + "variant": ("base", "patched"), + "behavior": ("base", "patched"), + "regression": ("base", "patched"), + "security": ("base", "patched"), + "suite": ("patched",), +} +EXPECTED_BY_KIND = { + "control": {"base": "zero", "patched": "zero"}, + "exploit": {"base": "nonzero", "patched": "zero"}, + "variant": {"base": "nonzero", "patched": "zero"}, + "behavior": {"base": "zero", "patched": "zero"}, + "regression": {"base": "zero", "patched": "zero"}, + "security": {"base": "zero", "patched": "zero"}, + "suite": {"patched": "zero"}, +} +VERDICTS = { + "S1": ("clean_fix", 0), + "S2": ("fixed_with_behavior_change", 2), + "S3": ("not_fixed", 3), + "S4": ("fixed_with_new_vulnerability", 4), + "S5": ("not_fixed_and_new_vulnerability", 5), + "INCONCLUSIVE": ("inconclusive", 10), +} +VERDICT_SUMMARIES = { + "S1": "All supplied remediation, behavior, regression, security, and suite evidence passed.", + "S2": "Remediation evidence passed, but behavior, regression, or suite evidence failed.", + "S3": "At least one original exploit or root-cause variant remains unfixed.", + "S4": "Remediation evidence passed, but the patch introduced a new security failure.", + "S5": "The patch remains incomplete and introduced a new security failure.", + "INCONCLUSIVE": "The evidence was invalid or incomplete; do not interpret this as a pass.", +} +EVIDENCE_LEVEL_SUMMARIES = { + "source": "source checks only; target behavior was not compiled or executed", + "build": "target code was built or analyzed, but the reported behavior was not executed", + "runtime": "the reported behavior and its safety assertions were executed", +} + +PLAN_SCHEMA: dict[str, Any] = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Post-patch validation plan", + "type": "object", + "additionalProperties": False, + "required": [ + "schema_version", + "case_id", + "evidence_level", + "finding", + "repository", + "base_ref", + "expected_base_commit", + "expected_patch_sha256", + "changed_files", + "submodules", + "checks", + ], + "properties": { + "schema_version": {"const": SCHEMA_VERSION}, + "case_id": {"type": "string", "pattern": ID_RE.pattern}, + "evidence_level": {"enum": list(EVIDENCE_LEVELS)}, + "finding": { + "type": "object", + "additionalProperties": False, + "required": ["id", "summary"], + "properties": {"id": {"type": "string"}, "summary": {"type": "string"}}, + }, + "repository": {"type": "string"}, + "base_ref": {"type": "string"}, + "expected_base_commit": {"type": "string", "pattern": HEX_RE.pattern}, + "patched_ref": {"type": "string"}, + "expected_patched_commit": {"type": "string", "pattern": HEX_RE.pattern}, + "patch_file": {"type": "string"}, + "expected_patch_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "changed_files": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "submodules": { + "type": "array", + "items": {"type": "string", "minLength": 1}, + }, + "checks": { + "type": "array", + "minItems": len(KINDS), + "items": { + "type": "object", + "additionalProperties": False, + "required": ["id", "kind", "rationale", "covers", "argv"], + "properties": { + "id": {"type": "string", "pattern": ID_RE.pattern}, + "kind": {"enum": list(KINDS)}, + "rationale": {"type": "string", "minLength": 1}, + "covers": { + "type": "array", + "items": {"type": "string", "minLength": 1}, + "minItems": 1, + }, + "argv": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + }, + "cwd": {"type": "string", "default": "."}, + "env": { + "type": "object", + "additionalProperties": {"type": "string"}, + }, + "timeout_seconds": {"type": "integer", "minimum": 1, "maximum": 3600}, + "compare_stream": {"enum": ["stdout", "stderr", "combined"]}, + }, + }, + }, + }, + "oneOf": [ + {"required": ["patched_ref", "expected_patched_commit"]}, + {"required": ["patch_file"]}, + ], +} + + +class PlanError(RuntimeError): + """A fail-closed plan or execution error.""" + + +def canonical_json(value: Any) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode() + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, value: Any) -> None: + path.write_bytes(canonical_json(value)) + + +def load_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PlanError(f"cannot read JSON from {path}: {exc}") from exc + if not isinstance(value, dict): + raise PlanError("plan root must be a JSON object") + return value + + +def stable_env(extra: Mapping[str, str] | None = None) -> dict[str, str]: + env = {key: value for key, value in os.environ.items() if key in BASE_ENV_KEYS} + if extra: + env.update(extra) + env.update(FIXED_ENV) + return env + + +def run_git(repo: Path, *args: str) -> bytes: + command = [*GIT_PREFIX, *args] + try: + result = subprocess.run( + command, + cwd=repo, + env=stable_env(), + capture_output=True, + check=False, + ) + except OSError as exc: + raise PlanError(f"failed to run git in {repo}: {exc}") from exc + if result.returncode != 0: + message = result.stderr.decode(errors="replace").strip() + raise PlanError(f"git command failed ({' '.join(command[:4])} ...): {message}") + return result.stdout + + +def resolve_path(value: str, relative_to: Path) -> Path: + path = Path(value).expanduser() + if not path.is_absolute(): + path = relative_to / path + return path.resolve() + + +def resolve_repo(value: str, relative_to: Path) -> Path: + candidate = resolve_path(value, relative_to) + top = run_git(candidate, "rev-parse", "--show-toplevel").decode().strip() + repo = Path(top).resolve() + if repo != candidate: + raise PlanError( + f"repository must name its Git root exactly: expected {repo}, got {candidate}" + ) + return repo + + +def resolve_commit(repo: Path, ref: str) -> str: + if not ref.strip() or ref.startswith("-"): + raise PlanError(f"unsafe or empty Git ref: {ref!r}") + output = run_git(repo, "rev-parse", "--verify", "--end-of-options", f"{ref}^{{commit}}") + return output.decode().strip() + + +def ref_patch(repo: Path, base_commit: str, patched_commit: str) -> tuple[bytes, list[str]]: + diff = run_git(repo, "diff", "--binary", "--full-index", base_commit, patched_commit, "--") + names = run_git(repo, "diff", "--name-only", "-z", base_commit, patched_commit, "--") + changed = sorted(x.decode(errors="surrogateescape") for x in names.split(b"\0") if x) + return diff, changed + + +def file_patch(repo: Path, patch_file: Path) -> tuple[bytes, list[str]]: + if not patch_file.is_file(): + raise PlanError(f"patch file does not exist: {patch_file}") + patch = patch_file.read_bytes() + output = run_git(repo, "apply", "--numstat", "-z", "--", str(patch_file)) + changed = [] + for record in output.split(b"\0"): + if not record: + continue + fields = record.split(b"\t", 2) + if len(fields) != 3: + raise PlanError("could not parse changed files from patch") + changed.append(fields[2].decode(errors="surrogateescape")) + return patch, sorted(set(changed)) + + +def gitlink_paths(repo: Path, commit: str) -> list[str]: + """Return every submodule path pinned by a tree without consulting the working tree.""" + output = run_git(repo, "ls-tree", "-r", "-z", commit, "--") + paths = [] + for record in output.split(b"\0"): + if not record: + continue + metadata, separator, raw_path = record.partition(b"\t") + if separator and metadata.split(b" ", 1)[0] == b"160000": + paths.append(raw_path.decode(errors="surrogateescape")) + return sorted(paths) + + +def affected_submodules(changed_files: Sequence[str], submodules: Sequence[str]) -> list[str]: + return sorted( + path + for path in submodules + if any(changed == path or changed.startswith(f"{path}/") for changed in changed_files) + ) + + +def relative_string(path: Path, anchor: Path) -> str: + try: + return os.path.relpath(path, anchor) + except ValueError: + return str(path) + + +def scaffold_plan(args: argparse.Namespace) -> None: + output = Path(args.output).expanduser().resolve() + if output.exists(): + raise PlanError(f"refusing to overwrite existing plan: {output}") + output.parent.mkdir(parents=True, exist_ok=True) + repo = resolve_repo(args.repo, Path.cwd()) + base_commit = resolve_commit(repo, args.base_ref) + plan: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "case_id": args.case_id or normalize_case_id(args.finding_id), + "evidence_level": args.evidence_level, + "finding": {"id": args.finding_id, "summary": args.finding_summary}, + "repository": relative_string(repo, output.parent), + "base_ref": args.base_ref, + "expected_base_commit": base_commit, + "checks": [], + } + if args.patched_ref: + patched_commit = resolve_commit(repo, args.patched_ref) + patch, changed = ref_patch(repo, base_commit, patched_commit) + plan.update( + { + "patched_ref": args.patched_ref, + "expected_patched_commit": patched_commit, + } + ) + else: + patch_path = resolve_path(args.patch_file, Path.cwd()) + patch, changed = file_patch(repo, patch_path) + plan["patch_file"] = relative_string(patch_path, output.parent) + if not patch or not changed: + raise PlanError("patch is empty; there is nothing to validate") + plan["expected_patch_sha256"] = sha256_bytes(patch) + plan["changed_files"] = changed + available_submodules = set(gitlink_paths(repo, base_commit)) + if plan.get("expected_patched_commit"): + available_submodules.update(gitlink_paths(repo, plan["expected_patched_commit"])) + plan["submodules"] = affected_submodules(changed, sorted(available_submodules)) + write_json(output, plan) + print( + json.dumps( + { + "plan": str(output), + "base_commit": base_commit, + "patched_commit": plan.get("expected_patched_commit"), + "patch_sha256": plan["expected_patch_sha256"], + "changed_files": changed, + "evidence_level": plan["evidence_level"], + "submodules": plan["submodules"], + "next": "populate checks, confirm evidence_level, then run validate-plan", + }, + indent=2, + sort_keys=True, + ) + ) + + +def normalize_case_id(value: str) -> str: + normalized = re.sub(r"[^a-zA-Z0-9._-]+", "-", value.strip()).strip("-._") + if not normalized: + normalized = "patch-validation" + return normalized[:64] + + +def require_string(value: Any, path: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise PlanError(f"{path} must be a non-empty string") + return value + + +def require_safe_cwd(value: Any, path: str) -> str: + cwd = require_string(value, path) + pure = PurePosixPath(cwd) + if pure.is_absolute() or ".." in pure.parts: + raise PlanError(f"{path} must stay inside the checkout") + return cwd + + +def require_safe_relative_path(value: Any, path: str) -> str: + relative = require_string(value, path) + pure = PurePosixPath(relative) + if pure.is_absolute() or ".." in pure.parts or pure.as_posix() != relative or relative == ".": + raise PlanError(f"{path} must be a normalized relative POSIX path without ..") + return relative + + +def reject_shell_string(argv: Sequence[str], path: str) -> None: + """Reject argv that hands a command *string* to a shell. + + Inspect every argv position conservatively so an arbitrary launcher cannot hide ``sh -c``. + This may reject a literal shell name used as data; authors can move ambiguous invocations into + a script file. It is a determinism and argument-fidelity rule, not a sandbox, and an + interpreter's own ``-c`` stays allowed. + """ + for index, argument in enumerate(argv): + name = Path(argument).name.lower() + if name == "env": + for candidate in argv[index + 1 :]: + lowered = candidate.lower() + if lowered in ENV_SPLIT_FLAGS or lowered.startswith("--split-string="): + raise PlanError(f"{path} may not use env --split-string; use a script file") + if name not in SHELL_NAMES: + continue + for candidate in argv[index + 1 :]: + flag = candidate.lower() + if flag in SHELL_COMMAND_FLAGS or SHELL_COMMAND_FLAG_RE.fullmatch(flag): + raise PlanError(f"{path} may not invoke a shell command string; use a script file") + + +def validate_check(raw: Any, index: int) -> dict[str, Any]: + path = f"checks[{index}]" + if not isinstance(raw, dict): + raise PlanError(f"{path} must be an object") + allowed = { + "id", + "kind", + "rationale", + "covers", + "argv", + "cwd", + "env", + "timeout_seconds", + "compare_stream", + } + unknown = sorted(set(raw) - allowed) + if unknown: + raise PlanError(f"{path} has unknown keys: {', '.join(unknown)}") + check_id = require_string(raw.get("id"), f"{path}.id") + if not ID_RE.fullmatch(check_id): + raise PlanError(f"{path}.id must match {ID_RE.pattern}") + kind = require_string(raw.get("kind"), f"{path}.kind") + if kind not in KINDS: + raise PlanError(f"{path}.kind must be one of {', '.join(KINDS)}") + rationale = require_string(raw.get("rationale"), f"{path}.rationale") + covers = raw.get("covers") + if not isinstance(covers, list) or not covers: + raise PlanError(f"{path}.covers must be a non-empty string array") + normalized_covers = [require_string(item, f"{path}.covers") for item in covers] + argv = raw.get("argv") + if not isinstance(argv, list) or not argv: + raise PlanError(f"{path}.argv must be a non-empty argv array; shell strings are forbidden") + normalized_argv = [require_string(item, f"{path}.argv") for item in argv] + reject_shell_string(normalized_argv, f"{path}.argv") + cwd = require_safe_cwd(raw.get("cwd", "."), f"{path}.cwd") + timeout = raw.get("timeout_seconds", 300) + if isinstance(timeout, bool) or not isinstance(timeout, int) or not 1 <= timeout <= 3600: + raise PlanError(f"{path}.timeout_seconds must be an integer from 1 to 3600") + env = raw.get("env", {}) + if not isinstance(env, dict): + raise PlanError(f"{path}.env must be a string map") + normalized_env: dict[str, str] = {} + for key, value in env.items(): + if not isinstance(key, str) or not ENV_RE.fullmatch(key): + raise PlanError(f"{path}.env has invalid variable name: {key!r}") + if not isinstance(value, str): + raise PlanError(f"{path}.env.{key} must be a string") + normalized_env[key] = value + if kind in MARKER_KINDS: + for value in [*normalized_argv, *(str(item) for item in env.values())]: + if "{side}" in value or "PPV_SIDE" in value: + raise PlanError( + f"{path} is a {kind} check and may not reference the revision it runs on; " + "it must assert the same postcondition on base and patch" + ) + compare = raw.get("compare_stream") + if kind == "behavior" and compare not in {"stdout", "stderr", "combined"}: + raise PlanError(f"{path}.compare_stream is required for behavior checks") + if kind != "behavior" and compare is not None: + raise PlanError(f"{path}.compare_stream is only valid for behavior checks") + result = { + "id": check_id, + "kind": kind, + "rationale": rationale, + "covers": normalized_covers, + "argv": normalized_argv, + "cwd": cwd, + "env": normalized_env, + "timeout_seconds": timeout, + } + if compare: + result["compare_stream"] = compare + return result + + +def validate_plan(plan: dict[str, Any], *, require_complete: bool = True) -> dict[str, Any]: + allowed = { + "schema_version", + "case_id", + "evidence_level", + "finding", + "repository", + "base_ref", + "expected_base_commit", + "patched_ref", + "expected_patched_commit", + "patch_file", + "expected_patch_sha256", + "changed_files", + "submodules", + "checks", + } + unknown = sorted(set(plan) - allowed) + if unknown: + raise PlanError(f"plan has unknown keys: {', '.join(unknown)}") + if plan.get("schema_version") != SCHEMA_VERSION: + raise PlanError(f"schema_version must be {SCHEMA_VERSION!r}") + case_id = require_string(plan.get("case_id"), "case_id") + if not ID_RE.fullmatch(case_id): + raise PlanError(f"case_id must match {ID_RE.pattern}") + evidence_level = require_string(plan.get("evidence_level"), "evidence_level") + if evidence_level not in EVIDENCE_LEVELS: + raise PlanError(f"evidence_level must be one of {', '.join(EVIDENCE_LEVELS)}") + finding = plan.get("finding") + if not isinstance(finding, dict) or set(finding) != {"id", "summary"}: + raise PlanError("finding must contain exactly id and summary") + finding_id = require_string(finding.get("id"), "finding.id") + finding_summary = require_string(finding.get("summary"), "finding.summary") + repository = require_string(plan.get("repository"), "repository") + base_ref = require_string(plan.get("base_ref"), "base_ref") + base_commit = require_string(plan.get("expected_base_commit"), "expected_base_commit") + if not HEX_RE.fullmatch(base_commit): + raise PlanError("expected_base_commit must be a lowercase commit hash") + has_ref = "patched_ref" in plan or "expected_patched_commit" in plan + has_file = "patch_file" in plan + if has_ref == has_file: + raise PlanError( + "provide exactly one patched_ref/expected_patched_commit pair or patch_file" + ) + normalized: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "case_id": case_id, + "evidence_level": evidence_level, + "finding": {"id": finding_id, "summary": finding_summary}, + "repository": repository, + "base_ref": base_ref, + "expected_base_commit": base_commit, + } + if has_ref: + patched_ref = require_string(plan.get("patched_ref"), "patched_ref") + patched_commit = require_string( + plan.get("expected_patched_commit"), "expected_patched_commit" + ) + if not HEX_RE.fullmatch(patched_commit): + raise PlanError("expected_patched_commit must be a lowercase commit hash") + normalized.update({"patched_ref": patched_ref, "expected_patched_commit": patched_commit}) + else: + normalized["patch_file"] = require_string(plan.get("patch_file"), "patch_file") + patch_hash = require_string(plan.get("expected_patch_sha256"), "expected_patch_sha256") + if not re.fullmatch(r"[0-9a-f]{64}", patch_hash): + raise PlanError("expected_patch_sha256 must be a lowercase SHA-256 digest") + changed = plan.get("changed_files") + if not isinstance(changed, list) or not changed: + raise PlanError("changed_files must be a non-empty sorted string array") + normalized_changed = [require_string(item, "changed_files") for item in changed] + if normalized_changed != sorted(set(normalized_changed)): + raise PlanError("changed_files must be unique and lexically sorted") + raw_submodules = plan.get("submodules") + if not isinstance(raw_submodules, list): + raise PlanError("submodules must be a sorted string array; use [] when none are needed") + submodules = [ + require_safe_relative_path(item, f"submodules[{index}]") + for index, item in enumerate(raw_submodules) + ] + if submodules != sorted(set(submodules)): + raise PlanError("submodules must be unique and lexically sorted") + raw_checks = plan.get("checks") + if not isinstance(raw_checks, list): + raise PlanError("checks must be an array") + checks = [validate_check(check, index) for index, check in enumerate(raw_checks)] + ids = [check["id"] for check in checks] + if len(ids) != len(set(ids)): + raise PlanError("check ids must be unique") + if ids != sorted(ids): + raise PlanError("checks must be lexically sorted by id") + if require_complete: + present = {check["kind"] for check in checks} + missing = [kind for kind in KINDS if kind not in present] + if missing: + raise PlanError(f"incomplete evidence plan; missing check kinds: {', '.join(missing)}") + normalized.update( + { + "expected_patch_sha256": patch_hash, + "changed_files": normalized_changed, + "submodules": submodules, + "checks": checks, + } + ) + return normalized + + +def verify_pins(plan: dict[str, Any], plan_path: Path) -> tuple[Path, str, str | None, bytes]: + repo = resolve_repo(plan["repository"], plan_path.parent) + base_commit = resolve_commit(repo, plan["base_ref"]) + if base_commit != plan["expected_base_commit"]: + raise PlanError( + f"base ref moved: expected {plan['expected_base_commit']}, resolved {base_commit}" + ) + if "patched_ref" in plan: + patched_commit = resolve_commit(repo, plan["patched_ref"]) + if patched_commit != plan["expected_patched_commit"]: + raise PlanError( + "patched ref moved: expected " + f"{plan['expected_patched_commit']}, resolved {patched_commit}" + ) + patch, changed = ref_patch(repo, base_commit, patched_commit) + else: + patched_commit = None + patch_file = resolve_path(plan["patch_file"], plan_path.parent) + patch, changed = file_patch(repo, patch_file) + patch_hash = sha256_bytes(patch) + if patch_hash != plan["expected_patch_sha256"]: + raise PlanError( + f"patch content changed: expected {plan['expected_patch_sha256']}, got {patch_hash}" + ) + if changed != plan["changed_files"]: + raise PlanError( + f"changed-file inventory moved: expected {plan['changed_files']}, got {changed}" + ) + available_submodules = set(gitlink_paths(repo, base_commit)) + if patched_commit: + available_submodules.update(gitlink_paths(repo, patched_commit)) + unknown_submodules = sorted(set(plan["submodules"]) - available_submodules) + if unknown_submodules: + raise PlanError( + "plan names paths that are not pinned submodules: " + ", ".join(unknown_submodules) + ) + required_submodules = affected_submodules(changed, sorted(available_submodules)) + missing_submodules = sorted(set(required_submodules) - set(plan["submodules"])) + if missing_submodules: + raise PlanError( + "changed files cross undeclared submodules: " + ", ".join(missing_submodules) + ) + return repo, base_commit, patched_commit, patch + + +@dataclass(frozen=True) +class PlanArtifact: + relative_path: str + mode: int + data: bytes | None + + +@dataclass(frozen=True) +class ExecContext: + """Per-run paths and inputs shared by every check execution.""" + + evidence: Path + scratch_root: Path + plan_artifacts: tuple[PlanArtifact, ...] + case_id: str + forwarded_env: Mapping[str, str] + + +def expand( + value: str, + *, + checkout: Path, + scratch: Path, + plan_dir: Path, + side: str | None, +) -> str: + expanded = ( + value.replace("{checkout}", str(checkout)) + .replace("{scratch}", str(scratch)) + .replace("{plan_dir}", str(plan_dir)) + ) + return expanded if side is None else expanded.replace("{side}", side) + + +def resolve_forwarded_env(names: Sequence[str]) -> dict[str, str]: + """Resolve --allow-env passthrough names, failing closed on anything unusable. + + A missing variable is an error rather than an empty string: silently forwarding "" is how + a suite ends up testing a different toolchain than the author believed it did. + """ + forwarded: dict[str, str] = {} + for name in names: + if not ENV_RE.fullmatch(name): + raise PlanError(f"--allow-env name is not a valid variable name: {name!r}") + upper = name.upper() + if upper in FIXED_ENV or upper.startswith("PPV_"): + raise PlanError( + f"--allow-env refuses {name}: the runner reserves or fixes that variable" + ) + if upper in FORBIDDEN_ENV: + raise PlanError( + f"--allow-env refuses {name}: it changes which code runs, which would defeat " + "the isolation this validator depends on" + ) + if CREDENTIAL_ENV_RE.search(upper): + raise PlanError( + f"--allow-env refuses {name}: forwarded values are recorded verbatim in " + "result.json, so it reads as a credential" + ) + if name in forwarded: + continue + if name not in os.environ: + raise PlanError(f"--allow-env requested {name}, which is not set in this environment") + forwarded[name] = os.environ[name] + return forwarded + + +def expected_matches(expected: str, exit_code: int | None) -> bool: + if exit_code is None: + return False + if expected == "zero": + return exit_code == 0 + if expected == "nonzero": + return exit_code != 0 + raise AssertionError(f"unknown expectation: {expected}") + + +def terminate_process(process: subprocess.Popen[bytes]) -> None: + if process.poll() is not None: + return + if os.name == "posix": + with suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + else: + process.kill() + process.wait() + + +def marker_present(stdout_path: Path) -> bool: + """Look for the marker as a line of its own, on stdout only. + + Both restrictions matter. Scanning stderr matched the marker inside a Python SyntaxError + traceback, which echoes the offending source — so a harness that never executed a single + statement satisfied the check it exists to fail. Requiring a whole line rejects the same + token quoted inside an error message, a usage string, or an echoed argv. + """ + token = REACHED_MARKER.encode() + return any(line.strip() == token for line in stdout_path.read_bytes().splitlines()) + + +def resolve_executable(argv0: str, cwd: Path, env: Mapping[str, str]) -> Path | None: + """Resolve argv[0] the same way the child process does, for provenance hashing.""" + candidate = Path(argv0) + separators = tuple(separator for separator in (os.sep, os.altsep) if separator) + if candidate.is_absolute(): + return candidate + if any(separator in argv0 for separator in separators): + return (cwd / candidate).resolve() + found = shutil.which(argv0, path=env.get("PATH")) + if not found: + return None + found_path = Path(found) + return found_path.resolve() if found_path.is_absolute() else (cwd / found_path).resolve() + + +def copy_plan_directory( + source: Path, + destination: Path, + excluded_paths: Sequence[Path] = (), +) -> None: + """Copy plan artifacts without symlinks, plan pins, or prior evidence trees.""" + excluded = {path.resolve() for path in excluded_paths} + + def ignore(directory: str, names: list[str]) -> list[str]: + ignored: list[str] = [] + for name in names: + child = Path(directory) / name + try: + if child.is_symlink(): + raise PlanError(f"plan artifacts may not contain symlinks: {child}") + if child.resolve() in excluded: + ignored.append(name) + continue + if ( + child.is_dir() + and (child / "result.json").is_file() + and (child / "artifact-manifest.json").is_file() + ): + ignored.append(name) + except OSError: + # Let copytree report the inaccessible entry with its useful source path. + continue + return ignored + + try: + shutil.copytree(source, destination, ignore=ignore) + except (OSError, RecursionError) as exc: + raise PlanError( + f"could not make an isolated copy of plan artifacts: {type(exc).__name__}: {exc}" + ) from exc + + +def snapshot_plan_directory(source: Path) -> tuple[PlanArtifact, ...]: + """Load a bounded, immutable plan-artifact snapshot into runner memory.""" + artifacts: list[PlanArtifact] = [] + total = 0 + try: + for path in sorted(source.rglob("*")): + relative = path.relative_to(source).as_posix() + mode = stat.S_IMODE(path.stat().st_mode) + if path.is_dir(): + artifacts.append(PlanArtifact(relative, mode, None)) + continue + data = path.read_bytes() + total += len(data) + if total > MAX_PLAN_ARTIFACT_BYTES: + raise PlanError( + f"plan artifacts exceed the {MAX_PLAN_ARTIFACT_BYTES}-byte snapshot limit" + ) + artifacts.append(PlanArtifact(relative, mode, data)) + except OSError as exc: + raise PlanError( + f"could not snapshot isolated plan artifacts: {type(exc).__name__}: {exc}" + ) from exc + return tuple(artifacts) + + +def materialize_plan_directory( + artifacts: Sequence[PlanArtifact], + destination: Path, +) -> None: + """Materialize one private plan copy from immutable in-memory bytes.""" + try: + destination.mkdir() + directories: list[tuple[Path, int]] = [] + for artifact in artifacts: + path = destination / artifact.relative_path + if artifact.data is None: + path.mkdir(parents=True, exist_ok=True) + directories.append((path, artifact.mode)) + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(artifact.data) + path.chmod(artifact.mode) + for path, mode in reversed(directories): + path.chmod(mode) + except OSError as exc: + raise PlanError( + f"could not materialize isolated plan artifacts: {type(exc).__name__}: {exc}" + ) from exc + + +def resolve_argv_file( + argument: str, + index: int, + cwd: Path, + env: Mapping[str, str], +) -> Path | None: + """Resolve an argv element that names a file as seen from the check's actual cwd.""" + if index == 0: + return resolve_executable(argument, cwd, env) + candidate = Path(argument) + resolved = candidate.resolve() if candidate.is_absolute() else (cwd / candidate).resolve() + return resolved if resolved.is_file() else None + + +def archive_argv_files( + argv: Sequence[str], + original_argv: Sequence[str], + cwd: Path, + env: Mapping[str, str], + evidence: Path, + archive_roots: Sequence[Path], +) -> list[dict[str, Any]]: + """Hash file-valued arguments and archive bounded files from controlled roots.""" + records: list[dict[str, Any]] = [] + helpers = evidence / "helpers" + roots = tuple(root.resolve() for root in archive_roots) + for index, argument in enumerate(argv): + try: + resolved = resolve_argv_file(argument, index, cwd, env) + except OSError as exc: + records.append( + { + "index": index, + "argument": original_argv[index], + "sha256": None, + "artifact": None, + "archive_reason": f"resolve failed: {type(exc).__name__}: {exc}", + } + ) + continue + if resolved is None or not resolved.is_file(): + continue + record: dict[str, Any] = { + "index": index, + "argument": original_argv[index], + "sha256": None, + "artifact": None, + "archive_reason": None, + } + try: + digest = sha256_file(resolved) + except OSError as exc: + record["archive_reason"] = f"hash failed: {type(exc).__name__}: {exc}" + records.append(record) + continue + record["sha256"] = digest + controlled = any(resolved.is_relative_to(root) for root in roots) + if not controlled: + record["archive_reason"] = "outside the isolated plan and checkout roots" + records.append(record) + continue + try: + size = resolved.stat().st_size + if size > MAX_ARCHIVED_ARGV_FILE_BYTES: + record["archive_reason"] = ( + f"file exceeds {MAX_ARCHIVED_ARGV_FILE_BYTES}-byte archive limit" + ) + records.append(record) + continue + helpers.mkdir(exist_ok=True) + artifact = helpers / digest + if not artifact.exists(): + shutil.copyfile(resolved, artifact) + record["artifact"] = artifact.relative_to(evidence).as_posix() + except OSError as exc: + record["archive_reason"] = f"archive failed: {type(exc).__name__}: {exc}" + records.append(record) + return records + + +def execute_check( + check: dict[str, Any], + side: str, + checkout: Path, + context: ExecContext, + sequence: int, +) -> dict[str, Any]: + cwd = (checkout / check["cwd"]).resolve() + try: + cwd.relative_to(checkout.resolve()) + except ValueError as exc: + raise PlanError(f"check {check['id']} cwd escaped its checkout") from exc + if not cwd.is_dir(): + raise PlanError(f"check {check['id']} cwd does not exist on {side}: {check['cwd']}") + stem = f"{sequence:03d}-{check['id']}-{side}" + # Checks receive an opaque, per-invocation directory outside the retained evidence tree. + # It is archived only after the process exits, so base and patched runs cannot communicate + # through scratch state or infer ordering from retained sibling directory names. + runtime_scratch = Path(tempfile.mkdtemp(prefix="ppv-check-")) + archived_scratch = context.scratch_root / stem + # The plan directory contains helper scripts and is writable by the check. Give every + # invocation a private copy so PPV_PLAN_DIR cannot become a base/patched sentinel channel. + runtime_plan_root = Path(tempfile.mkdtemp(prefix="ppv-plan-")) + runtime_plan_dir = runtime_plan_root / "plan" + try: + materialize_plan_directory(context.plan_artifacts, runtime_plan_dir) + # Exploit and variant checks are run side-blind: they get no {side}, no PPV_SIDE, and an + # opaque checkout path. A check that can tell which revision it is on can assert on that + # instead of on the code, which is the cheapest way to fake a reproduction-then-fix. + needs_marker = check["kind"] in MARKER_KINDS + visible_side = None if needs_marker else side + argv = [ + expand( + arg, + checkout=checkout, + scratch=runtime_scratch, + plan_dir=runtime_plan_dir, + side=visible_side, + ) + for arg in check["argv"] + ] + check_env = dict(context.forwarded_env) + check_env.update( + { + key: expand( + value, + checkout=checkout, + scratch=runtime_scratch, + plan_dir=runtime_plan_dir, + side=visible_side, + ) + for key, value in check["env"].items() + } + ) + check_env.update( + { + "PPV_CHECKOUT": str(checkout), + "PPV_SCRATCH": str(runtime_scratch), + "PPV_PLAN_DIR": str(runtime_plan_dir), + "PPV_CASE_ID": context.case_id, + "PPV_REACHED_MARKER": REACHED_MARKER, + "TEMP": str(runtime_scratch), + "TMP": str(runtime_scratch), + "TMPDIR": str(runtime_scratch), + } + ) + if visible_side is not None: + check_env["PPV_SIDE"] = visible_side + stdout_path = context.evidence / f"{stem}.stdout" + stderr_path = context.evidence / f"{stem}.stderr" + status = "completed" + exit_code: int | None = None + error: str | None = None + process_env = stable_env(check_env) + argv_files = archive_argv_files( + argv, + check["argv"], + cwd, + process_env, + context.evidence, + (checkout, runtime_plan_dir), + ) + argv0_sha256 = next( + (record["sha256"] for record in argv_files if record["index"] == 0), None + ) + try: + # Named evidence paths contain the logical side. Capture through anonymous/random + # descriptors so a side-blind child cannot recover that label from fd 1 or fd 2. + with ( + tempfile.TemporaryFile(dir=runtime_scratch) as stdout_buffer, + tempfile.TemporaryFile(dir=runtime_scratch) as stderr_buffer, + ): + try: + process = subprocess.Popen( + argv, + cwd=cwd, + env=process_env, + stdin=subprocess.DEVNULL, + stdout=stdout_buffer, + stderr=stderr_buffer, + start_new_session=os.name == "posix", + ) + try: + exit_code = process.wait(timeout=check["timeout_seconds"]) + except subprocess.TimeoutExpired: + status = "timeout" + error = f"exceeded {check['timeout_seconds']} seconds" + terminate_process(process) + exit_code = process.returncode + except OSError as exc: + status = "execution_error" + error = f"{type(exc).__name__}: {exc}" + stdout_buffer.seek(0) + stderr_buffer.seek(0) + with stdout_path.open("wb") as stdout: + shutil.copyfileobj(stdout_buffer, stdout) + with stderr_path.open("wb") as stderr: + shutil.copyfileobj(stderr_buffer, stderr) + except OSError as exc: + raise PlanError( + f"could not capture check {check['id']} output: {type(exc).__name__}: {exc}" + ) from exc + try: + if runtime_scratch.exists(): + shutil.move(str(runtime_scratch), archived_scratch) + else: + archived_scratch.mkdir() + except OSError as exc: + status = "execution_error" + error = f"scratch archival failed: {type(exc).__name__}: {exc}" + shutil.rmtree(runtime_scratch, ignore_errors=True) + expected = EXPECTED_BY_KIND[check["kind"]][side] + return { + "side": side, + "argv": check["argv"], + "argv0_sha256": argv0_sha256, + "argv_files": argv_files, + "cwd": check["cwd"], + "scratch": archived_scratch.relative_to(context.evidence).as_posix(), + "status": status, + "exit_code": exit_code, + "expected": expected, + "matched": status == "completed" and expected_matches(expected, exit_code), + "marker": marker_present(stdout_path) if needs_marker else None, + "stdout": stdout_path.name, + "stdout_sha256": sha256_file(stdout_path), + "stderr": stderr_path.name, + "stderr_sha256": sha256_file(stderr_path), + "error": error, + } + finally: + shutil.rmtree(runtime_scratch, ignore_errors=True) + shutil.rmtree(runtime_plan_root, ignore_errors=True) + + +def behavior_comparison( + check: dict[str, Any], runs: dict[str, dict[str, Any]], artifacts: Path +) -> dict[str, Any]: + stream = check["compare_stream"] + + def content(side: str) -> bytes: + run = runs[side] + if stream == "stdout": + return (artifacts / run["stdout"]).read_bytes() + if stream == "stderr": + return (artifacts / run["stderr"]).read_bytes() + return (artifacts / run["stdout"]).read_bytes() + (artifacts / run["stderr"]).read_bytes() + + base = content("base") + patched = content("patched") + return { + "stream": stream, + "matched": base == patched, + "base_sha256": sha256_bytes(base), + "patched_sha256": sha256_bytes(patched), + } + + +def classify( + checks: Sequence[dict[str, Any]], cleanup_errors: Sequence[str] = () +) -> dict[str, Any]: + infrastructure = [] + baseline = [] + not_fixed = [] + behavior = [] + new_security = [] + if cleanup_errors: + infrastructure.extend(f"cleanup: {item}" for item in cleanup_errors) + for check in checks: + runs = check["runs"] + for side, run in runs.items(): + if run["status"] != "completed": + infrastructure.append(f"{check['id']}:{side}:{run['status']}") + # A nonzero exit alone cannot distinguish a failed safety assertion from a harness + # that never ran. Without the marker the observation is unusable in either direction. + elif check["kind"] in MARKER_KINDS and run.get("marker") is not True: + infrastructure.append(f"{check['id']}:{side}:marker_missing") + base = runs.get("base") + patched = runs.get("patched") + if base is not None and not base["matched"]: + baseline.append(check["id"]) + continue + # A control failure on the patched side means the benign harness itself stopped working + # there, so every other patched observation is suspect. That is a validity problem, not + # a behavior regression, and must not be reported as "fixed with behavior change". + if check["kind"] == "control" and patched and not patched["matched"]: + infrastructure.append(f"{check['id']}:patched:control_failed") + elif check["kind"] in {"exploit", "variant"} and patched and not patched["matched"]: + not_fixed.append(check["id"]) + elif check["kind"] == "security" and patched and not patched["matched"]: + new_security.append(check["id"]) + elif ( + check["kind"] in {"behavior", "regression", "suite"} + and patched + and not patched["matched"] + ): + behavior.append(check["id"]) + if check["kind"] == "behavior" and not check.get("comparison", {}).get("matched", False): + behavior.append(f"{check['id']}:output") + if infrastructure or baseline: + code = "INCONCLUSIVE" + reasons = [ + *( + ["execution or cleanup did not complete: " + ", ".join(infrastructure)] + if infrastructure + else [] + ), + *(["baseline evidence did not match: " + ", ".join(baseline)] if baseline else []), + ] + elif not_fixed and new_security: + code = "S5" + reasons = [ + "unfixed exploit or variant: " + ", ".join(not_fixed), + "new security failure: " + ", ".join(new_security), + ] + elif not_fixed: + code = "S3" + reasons = ["unfixed exploit or variant: " + ", ".join(not_fixed)] + elif new_security: + code = "S4" + reasons = ["new security failure: " + ", ".join(new_security)] + elif behavior: + code = "S2" + reasons = ["behavior or regression failure: " + ", ".join(sorted(set(behavior)))] + else: + code = "S1" + reasons = [ + "all required baseline, fix, behavior, regression, security, and suite evidence passed" + ] + label, exit_code = VERDICTS[code] + return { + "code": code, + "label": label, + "exit_code": exit_code, + "human_review_required": True, + "reasons": reasons, + } + + +@contextmanager +def worktree_metadata_lock(repo: Path): + """Serialize shared worktree metadata without serializing check execution.""" + common_dir = git_common_dir(repo) + lock_path = common_dir / "post-patch-validation.lock" + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+b") + except OSError as exc: + raise PlanError(f"could not open worktree metadata lock {lock_path}: {exc}") from exc + with handle: + if os.name == "nt": + import msvcrt + + handle.seek(0) + handle.write(b"\0") + handle.flush() + handle.seek(0) + while True: + try: + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + break + except OSError: + time.sleep(0.05) + try: + yield + finally: + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + except OSError as exc: + raise PlanError( + f"could not acquire worktree metadata lock {lock_path}: {exc}" + ) from exc + try: + yield + finally: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except OSError as exc: + raise PlanError( + f"could not release worktree metadata lock {lock_path}: {exc}" + ) from exc + + +def git_common_dir(repo: Path) -> Path: + raw_common_dir = Path(run_git(repo, "rev-parse", "--git-common-dir").decode().strip()) + common_dir = raw_common_dir if raw_common_dir.is_absolute() else (repo / raw_common_dir) + return common_dir.resolve() + + +@dataclass +class WorktreeOwner: + token: str + path: Path + handle: Any + + +def acquire_worktree_owner(repo: Path) -> WorktreeOwner: + """Hold a kernel-backed lease that cannot be confused by PID reuse.""" + owner_dir = git_common_dir(repo) / "post-patch-validation-owners" + token = os.urandom(16).hex() + path = owner_dir / token + handle = None + try: + owner_dir.mkdir(parents=True, exist_ok=True) + handle = path.open("x+b") + handle.write(b"\0") + handle.flush() + except OSError as exc: + if handle is not None: + handle.close() + with suppress(OSError): + path.unlink() + raise PlanError(f"could not create worktree owner lease {path}: {exc}") from exc + try: + if os.name == "nt": + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + except OSError as exc: + handle.close() + with suppress(OSError): + path.unlink() + raise PlanError(f"could not acquire worktree owner lease {path}: {exc}") from exc + return WorktreeOwner(token=token, path=path, handle=handle) + + +def close_worktree_owner(owner: WorktreeOwner) -> None: + with suppress(OSError): + if os.name == "nt": + import msvcrt + + owner.handle.seek(0) + msvcrt.locking(owner.handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(owner.handle.fileno(), fcntl.LOCK_UN) + owner.handle.close() + with suppress(OSError): + owner.path.unlink() + with suppress(OSError): + owner.path.parent.rmdir() + + +def worktree_owner_is_active(repo: Path, token: str) -> bool: + """Test a PPV owner lease without trusting a reusable process identifier.""" + if not re.fullmatch(r"[0-9a-f]{32}", token): + return True + path = git_common_dir(repo) / "post-patch-validation-owners" / token + try: + handle = path.open("r+b") + except FileNotFoundError: + return False + except OSError: + return True + acquired = False + try: + if os.name == "nt": + import msvcrt + + handle.seek(0) + try: + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + acquired = True + except OSError: + return True + else: + import fcntl + + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + except BlockingIOError: + return True + except OSError: + return True + finally: + if acquired: + if os.name == "nt": + import msvcrt + + handle.seek(0) + with suppress(OSError): + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + with suppress(OSError): + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + handle.close() + with suppress(OSError): + path.unlink() + return False + + +def process_is_running(pid: int) -> bool: + if pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + + +def stale_validator_worktrees(repo: Path) -> list[str]: + """Return PPV-locked worktrees whose kernel-backed owner lease is inactive.""" + output = run_git(repo, "worktree", "list", "--porcelain", "-z") + records: list[dict[str, str]] = [] + current: dict[str, str] = {} + for raw_field in output.split(b"\0"): + if not raw_field: + if current: + records.append(current) + current = {} + continue + field = raw_field.decode(errors="surrogateescape") + key, _, value = field.partition(" ") + if key == "worktree" and current: + records.append(current) + current = {} + current[key] = value + if current: + records.append(current) + + stale = [] + for record in records: + reason = record.get("locked", "") + if not reason.startswith(WORKTREE_LOCK_REASON_PREFIX): + continue + owner = reason.removeprefix(WORKTREE_LOCK_REASON_PREFIX) + # Accept PID-only reasons from the earlier implementation so an upgrade can reclaim + # worktrees left by a killed older runner. New locks use non-reusable lease tokens. + legacy_stale = owner.isdigit() and not process_is_running(int(owner)) + lease_stale = bool(re.fullmatch(r"[0-9a-f]{32}", owner)) and not ( + worktree_owner_is_active(repo, owner) + ) + if legacy_stale or lease_stale: + stale.append(record["worktree"]) + return stale + + +def unlock_stale_validator_worktrees(repo: Path) -> None: + for path in stale_validator_worktrees(repo): + run_git(repo, "worktree", "unlock", path) + run_git(repo, "worktree", "prune") + + +def add_worktree(repo: Path, path: Path, commit: str, owner_token: str) -> None: + run_git(repo, "worktree", "add", "--detach", str(path), commit) + try: + run_git( + repo, + "worktree", + "lock", + "--reason", + f"{WORKTREE_LOCK_REASON_PREFIX}{owner_token}", + str(path), + ) + except PlanError: + with suppress(PlanError): + run_git(repo, "worktree", "remove", "--force", str(path)) + raise + + +def initialize_submodules(checkout: Path, submodules: Sequence[str]) -> dict[str, str]: + if not submodules: + return {} + common_dir_value = Path(run_git(checkout, "rev-parse", "--git-common-dir").decode().strip()) + common_dir = ( + common_dir_value if common_dir_value.is_absolute() else checkout / common_dir_value + ).resolve() + try: + configured = run_git( + checkout, + "config", + "-f", + ".gitmodules", + "--get-regexp", + r"^submodule\..*\.path$", + ).decode() + except PlanError as exc: + raise PlanError("could not read submodule paths from .gitmodules") from exc + names_by_path: dict[str, str] = {} + for line in configured.splitlines(): + key, separator, path = line.partition(" ") + if not separator or not key.startswith("submodule.") or not key.endswith(".path"): + raise PlanError(f"could not parse .gitmodules entry: {line!r}") + names_by_path[path.strip()] = key[len("submodule.") : -len(".path")] + + modules_dir = (common_dir / "modules").resolve() + local_urls: list[str] = [] + for path in submodules: + name = names_by_path.get(path) + if name is None: + raise PlanError(f"submodule path is missing from .gitmodules: {path}") + local_repo = (modules_dir / Path(name)).resolve() + try: + local_repo.relative_to(modules_dir) + except ValueError as exc: + raise PlanError(f"submodule name escapes the local module store: {name}") from exc + if not local_repo.is_dir(): + raise PlanError( + "could not initialize pinned submodules without network access; " + f"fetch {path} in the source repository first" + ) + local_urls.extend(("-c", f"submodule.{name}.url={local_repo}")) + try: + run_git( + checkout, + "-c", + "protocol.file.allow=always", + *local_urls, + "submodule", + "update", + "--init", + "--checkout", + "--no-fetch", + "--", + *submodules, + ) + except PlanError as exc: + raise PlanError( + "could not initialize pinned submodules without network access; " + "fetch them in the source repository first: " + ", ".join(submodules) + ) from exc + return { + path: run_git(checkout / path, "rev-parse", "HEAD").decode().strip() for path in submodules + } + + +def apply_patch_file( + checkout: Path, + patch_file: Path, + changed_files: Sequence[str], + submodules: Sequence[str], +) -> list[str]: + """Apply ordinary paths and gitlink bumps with the semantics each requires.""" + gitlink_changes = sorted(set(changed_files) & set(submodules)) + for submodule in gitlink_changes: + if any(path.startswith(f"{submodule}/") for path in changed_files): + raise PlanError( + "a patch file may not combine a submodule gitlink bump with changes inside it: " + + submodule + ) + common = ("--binary", "--whitespace=nowarn") + if gitlink_changes: + run_git( + checkout, + "apply", + "--index", + *common, + *(f"--include={path}" for path in gitlink_changes), + "--", + str(patch_file), + ) + ordinary_changes = sorted(set(changed_files) - set(gitlink_changes)) + if ordinary_changes: + run_git( + checkout, + "apply", + *common, + *(f"--exclude={path}" for path in gitlink_changes), + "--", + str(patch_file), + ) + return gitlink_changes + + +def remove_worktree(repo: Path, path: Path) -> str | None: + with suppress(PlanError): + run_git(repo, "worktree", "unlock", str(path)) + try: + result = subprocess.run( + [*GIT_PREFIX, "worktree", "remove", "--force", str(path)], + cwd=repo, + env=stable_env(), + capture_output=True, + check=False, + ) + except OSError as exc: + return f"{type(exc).__name__}: {exc}" + if result.returncode == 0: + return None + return result.stderr.decode(errors="replace").strip() or f"exit {result.returncode}" + + +def markdown_report(result: dict[str, Any]) -> str: + verdict = result["verdict"] + lines = [ + "# Post-Patch Validation", + "", + f"**Verdict:** {verdict['code']} — {verdict['label'].replace('_', ' ')}", + "", + f"**Evidence level:** {result['inputs']['evidence_level']} — " + + EVIDENCE_LEVEL_SUMMARIES[result["inputs"]["evidence_level"]], + "", + "**Human review required:** yes", + "", + f"**Finding:** {result['finding']['id']} — {result['finding']['summary']}", + "", + f"**Base commit:** `{result['inputs']['base_commit']}`", + "", + f"**Patch SHA-256:** `{result['inputs']['patch_sha256']}`", + "", + "**Submodules:** " + (", ".join(sorted(result["inputs"]["submodules"])) or "none"), + "", + "**Forwarded environment:** " + + (", ".join(sorted(result["inputs"]["forwarded_env"])) or "none"), + "", + "## Reasons", + "", + ] + lines.extend(f"- {reason}" for reason in verdict["reasons"]) + lines.extend( + [ + "", + "## Evidence", + "", + "| Check | Kind | Base | Patched | Comparison |", + "|---|---|---|---|---|", + ] + ) + for check in result["checks"]: + base = check["runs"].get("base") + patched = check["runs"].get("patched") + base_text = "—" if base is None else ("pass" if base["matched"] else "fail") + patch_text = "—" if patched is None else ("pass" if patched["matched"] else "fail") + comparison = check.get("comparison") + compare_text = ( + "—" if comparison is None else ("same" if comparison["matched"] else "changed") + ) + lines.append( + f"| `{check['id']}` | {check['kind']} | {base_text} | {patch_text} | {compare_text} |" + ) + lines.extend( + [ + "", + "## Interpretation", + "", + VERDICT_SUMMARIES[verdict["code"]], + "The evidence level limits what this verdict establishes, and human review remains", + "required to find omitted paths or an incorrectly specified safety assertion.", + "", + ] + ) + return "\n".join(lines) + + +def artifact_manifest(output: Path) -> dict[str, Any]: + files = [] + for path in sorted(p for p in output.rglob("*") if p.is_file()): + if path == output / "artifact-manifest.json": + continue + files.append( + { + "path": path.relative_to(output).as_posix(), + "sha256": sha256_file(path), + "bytes": path.stat().st_size, + } + ) + if not files: + raise PlanError("artifact manifest would contain zero files") + return {"schema_version": SCHEMA_VERSION, "files": files} + + +def run_plan(args: argparse.Namespace) -> int: + plan_path = Path(args.plan).expanduser().resolve() + plan = validate_plan(load_json(plan_path)) + repo, base_commit, patched_commit, patch = verify_pins(plan, plan_path) + output = Path(args.output).expanduser().resolve() + if output.exists() and any(output.iterdir()): + raise PlanError(f"refusing to mix evidence with an existing non-empty directory: {output}") + forwarded_env = resolve_forwarded_env(args.allow_env) + output.mkdir(parents=True, exist_ok=True) + write_json(output / "plan.snapshot.json", plan) + (output / "patch.diff").write_bytes(patch) + # Each invocation gets an opaque temporary scratch directory. Its contents are moved under + # this retained root only after the process exits, preventing cross-side communication while + # keeping every produced artifact available to reviewers. + scratch_root = output / "scratch" + scratch_root.mkdir() + # Snapshot author-supplied helper artifacts once, before any check runs. Exclude the machine + # plan (which contains revision pins), the live output, and detected prior result trees. + plan_template_root = Path(tempfile.mkdtemp(prefix="ppv-plan-template-")) + plan_template_dir = plan_template_root / "plan" + try: + copy_plan_directory( + plan_path.parent, + plan_template_dir, + excluded_paths=(plan_path, output), + ) + plan_artifacts = snapshot_plan_directory(plan_template_dir) + finally: + shutil.rmtree(plan_template_root, ignore_errors=True) + context = ExecContext( + evidence=output, + scratch_root=scratch_root, + plan_artifacts=plan_artifacts, + case_id=plan["case_id"], + forwarded_env=forwarded_env, + ) + # Unpredictable, side-agnostic directory names. Calling them base/ and patched/ handed every + # check a reliable oracle for which revision it was running on, via PPV_CHECKOUT or cwd — + # exactly what side-blinding exploit and variant checks is meant to withhold. A derived name + # would still be computable from PPV_CASE_ID, so these use generic random, separate roots. + # Keep two unique parent levels beneath each root: the common writable host temp directory is + # not exposed by the shallow parent walk that previously reached a shared validator root. + worktree_roots = [ + Path(tempfile.mkdtemp(prefix="ppv-worktree-")), + Path(tempfile.mkdtemp(prefix="ppv-worktree-")), + ] + if os.urandom(1)[0] & 1: + worktree_roots.reverse() + base_root, patched_root = worktree_roots + temp_roots = worktree_roots + base_checkout = base_root / "private" / "w" + patched_checkout = patched_root / "private" / "w" + worktrees = [ + (base_checkout, base_commit), + (patched_checkout, patched_commit or base_commit), + ] + if os.urandom(1)[0] & 1: + worktrees.reverse() + for checkout, _commit in worktrees: + checkout.parent.mkdir() + try: + owner = acquire_worktree_owner(repo) + except PlanError: + for temp_root in temp_roots: + shutil.rmtree(temp_root, ignore_errors=True) + raise + created: list[Path] = [] + cleanup_errors: list[str] = [] + evidence: list[dict[str, Any]] = [] + submodule_pins: dict[str, dict[str, str]] = {} + try: + with worktree_metadata_lock(repo): + unlock_stale_validator_worktrees(repo) + for checkout, commit in worktrees: + add_worktree(repo, checkout, commit, owner.token) + created.append(checkout) + submodules_by_checkout = { + checkout: initialize_submodules(checkout, plan["submodules"]) + for checkout, _commit in worktrees + } + base_submodules = submodules_by_checkout[base_checkout] + patched_submodules = submodules_by_checkout[patched_checkout] + if patched_commit is None: + patch_file = resolve_path(plan["patch_file"], plan_path.parent) + changed_gitlinks = apply_patch_file( + patched_checkout, + patch_file, + plan["changed_files"], + plan["submodules"], + ) + if changed_gitlinks: + with worktree_metadata_lock(repo): + updated = initialize_submodules(patched_checkout, changed_gitlinks) + patched_submodules.update(updated) + submodule_pins = { + path: { + "base_commit": base_submodules[path], + "patched_commit": patched_submodules[path], + } + for path in plan["submodules"] + } + sequence = 0 + for check in plan["checks"]: + logical_sides = SIDES_BY_KIND[check["kind"]] + side_sequences = { + side: sequence + offset for offset, side in enumerate(logical_sides, start=1) + } + sequence += len(logical_sides) + execution_sides = list(logical_sides) + if check["kind"] in MARKER_KINDS and os.urandom(1)[0] & 1: + execution_sides.reverse() + observed: dict[str, dict[str, Any]] = {} + for side in execution_sides: + checkout = base_checkout if side == "base" else patched_checkout + observed[side] = execute_check( + check, + side, + checkout, + context, + side_sequences[side], + ) + runs = {side: observed[side] for side in logical_sides} + item: dict[str, Any] = { + "id": check["id"], + "kind": check["kind"], + "rationale": check["rationale"], + "covers": check["covers"], + "runs": runs, + } + if check["kind"] == "behavior": + item["comparison"] = behavior_comparison(check, runs, output) + evidence.append(item) + finally: + primary_error_active = sys.exc_info()[0] is not None + cleanup_lock_error: PlanError | None = None + try: + with worktree_metadata_lock(repo): + for checkout in reversed(created): + error = remove_worktree(repo, checkout) + if error: + side = "base" if checkout == base_checkout else "patched" + cleanup_errors.append(f"{side}: {error}") + try: + run_git(repo, "worktree", "prune") + except PlanError as exc: + cleanup_errors.append(f"prune: {exc}") + except PlanError as exc: + cleanup_lock_error = exc + finally: + close_worktree_owner(owner) + for temp_root in temp_roots: + shutil.rmtree(temp_root, ignore_errors=True) + if cleanup_lock_error is not None and not primary_error_active: + raise cleanup_lock_error + verdict = classify(evidence, cleanup_errors) + result = { + "schema_version": SCHEMA_VERSION, + "case_id": plan["case_id"], + "finding": plan["finding"], + "inputs": { + "base_commit": base_commit, + "patched_commit": patched_commit, + "patch_sha256": sha256_bytes(patch), + "changed_files": plan["changed_files"], + "evidence_level": plan["evidence_level"], + "submodules": submodule_pins, + "plan_sha256": sha256_bytes(canonical_json(plan)), + "forwarded_env": dict(forwarded_env), + }, + "coverage": {kind: sum(check["kind"] == kind for check in evidence) for kind in KINDS}, + "checks": evidence, + "cleanup_errors": cleanup_errors, + "verdict": verdict, + } + write_json(output / "result.json", result) + (output / "report.md").write_text(markdown_report(result), encoding="utf-8") + write_json(output / "artifact-manifest.json", artifact_manifest(output)) + print(json.dumps({"result": str(output / "result.json"), "verdict": verdict}, indent=2)) + return int(verdict["exit_code"]) + + +def validate_plan_command(args: argparse.Namespace) -> None: + path = Path(args.plan).expanduser().resolve() + plan = validate_plan(load_json(path)) + verify_pins(plan, path) + print( + json.dumps( + { + "valid": True, + "case_id": plan["case_id"], + "evidence_level": plan["evidence_level"], + "submodules": plan["submodules"], + "checks": len(plan["checks"]), + "coverage": { + kind: sum(c["kind"] == kind for c in plan["checks"]) for kind in KINDS + }, + }, + indent=2, + sort_keys=True, + ) + ) + + +class PlanArgumentParser(argparse.ArgumentParser): + def error(self, message: str) -> None: + self.print_usage(sys.stderr) + self.exit(64, f"{self.prog}: error: {message}\n") + + +def parser() -> argparse.ArgumentParser: + root = PlanArgumentParser(description=__doc__) + subparsers = root.add_subparsers(dest="command", required=True) + scaffold = subparsers.add_parser("scaffold", help="pin a patch and write an incomplete plan") + scaffold.add_argument("--repo", required=True) + scaffold.add_argument("--base-ref", required=True) + patch = scaffold.add_mutually_exclusive_group(required=True) + patch.add_argument("--patched-ref") + patch.add_argument("--patch-file") + scaffold.add_argument("--finding-id", required=True) + scaffold.add_argument("--finding-summary", required=True) + scaffold.add_argument("--evidence-level", required=True, choices=EVIDENCE_LEVELS) + scaffold.add_argument("--case-id") + scaffold.add_argument("--output", required=True) + scaffold.set_defaults(handler=scaffold_plan) + validate = subparsers.add_parser("validate-plan", help="validate schema, coverage, and pins") + validate.add_argument("--plan", required=True) + validate.set_defaults(handler=validate_plan_command) + run = subparsers.add_parser("run", help="execute a complete plan in isolated worktrees") + run.add_argument("--plan", required=True) + run.add_argument("--output", required=True) + run.add_argument( + "--allow-env", + action="append", + default=[], + metavar="NAME", + help=( + "forward one host environment variable to every check (repeatable). The name and " + "its resolved value are recorded in result.json, so never forward a credential." + ), + ) + run.set_defaults(handler=run_plan) + schema = subparsers.add_parser("print-schema", help="print the plan's JSON Schema") + schema.set_defaults( + handler=lambda _args: print(json.dumps(PLAN_SCHEMA, indent=2, sort_keys=True)) + ) + return root + + +def main(argv: Sequence[str] | None = None) -> int: + args = parser().parse_args(argv) + try: + result = args.handler(args) + return int(result or 0) + except PlanError as exc: + print(f"post-patch-validation: {exc}", file=sys.stderr) + return 64 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/post-patch-validation/skills/post-patch-validation/scripts/pyproject.toml b/plugins/post-patch-validation/skills/post-patch-validation/scripts/pyproject.toml new file mode 100644 index 0000000..e38d3aa --- /dev/null +++ b/plugins/post-patch-validation/skills/post-patch-validation/scripts/pyproject.toml @@ -0,0 +1,6 @@ +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM"] diff --git a/plugins/post-patch-validation/tests/conftest.py b/plugins/post-patch-validation/tests/conftest.py new file mode 100644 index 0000000..d2d22e5 --- /dev/null +++ b/plugins/post-patch-validation/tests/conftest.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = PLUGIN_ROOT / "skills" / "post-patch-validation" / "scripts" / "post_patch_validation.py" + + +@pytest.fixture(scope="session") +def ppv(): + spec = importlib.util.spec_from_file_location("post_patch_validation", SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + # Register before exec: with `from __future__ import annotations`, @dataclass resolves its + # field types through sys.modules[cls.__module__], which is None for an unregistered module. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module diff --git a/plugins/post-patch-validation/tests/extract.mjs b/plugins/post-patch-validation/tests/extract.mjs new file mode 100644 index 0000000..22b4881 --- /dev/null +++ b/plugins/post-patch-validation/tests/extract.mjs @@ -0,0 +1,35 @@ +import fs from 'node:fs' +import vm from 'node:vm' + +export function loadFunction(path, name) { + const source = fs.readFileSync(path, 'utf8') + const marker = `function ${name}(` + const start = source.indexOf(marker) + if (start < 0) throw new Error(`function not found: ${name}`) + const brace = source.indexOf('{', start) + let depth = 0 + let quote = null + let escaped = false + for (let index = brace; index < source.length; index += 1) { + const char = source[index] + if (quote) { + if (escaped) escaped = false + else if (char === '\\') escaped = true + else if (char === quote) quote = null + continue + } + if (char === "'" || char === '"' || char === '`') { + quote = char + continue + } + if (char === '{') depth += 1 + if (char === '}') { + depth -= 1 + if (depth === 0) { + const body = source.slice(start, index + 1) + return vm.runInNewContext(`(${body})`) + } + } + } + throw new Error(`unterminated function: ${name}`) +} diff --git a/plugins/post-patch-validation/tests/run-all.sh b/plugins/post-patch-validation/tests/run-all.sh new file mode 100755 index 0000000..8a53b3c --- /dev/null +++ b/plugins/post-patch-validation/tests/run-all.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +plugin_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +uv run --with pytest --with pyyaml --no-project \ + python3 -m pytest -q --rootdir "${plugin_root}" -p no:cacheprovider "${plugin_root}/tests" diff --git a/plugins/post-patch-validation/tests/test_cli_artifacts.py b/plugins/post-patch-validation/tests/test_cli_artifacts.py new file mode 100644 index 0000000..0cf3da2 --- /dev/null +++ b/plugins/post-patch-validation/tests/test_cli_artifacts.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import hashlib +import subprocess +import sys +from pathlib import Path + +import pytest +from conftest import SCRIPT + + +@pytest.mark.parametrize( + "argv", + [[], ["unknown-command"], ["print-schema", "--typo"], ["run"], ["validate-plan"]], +) +def test_usage_errors_return_64(argv: list[str]) -> None: + result = subprocess.run([sys.executable, str(SCRIPT), *argv], capture_output=True) + assert result.returncode == 64 + assert b"usage:" in result.stderr + assert b"error:" in result.stderr + assert b"Traceback" not in result.stderr + + +@pytest.mark.parametrize("argv", [["--help"], ["run", "--help"]]) +def test_help_returns_zero(argv: list[str]) -> None: + result = subprocess.run([sys.executable, str(SCRIPT), *argv], capture_output=True) + assert result.returncode == 0 + assert b"usage:" in result.stdout + assert result.stderr == b"" + + +def test_invalid_utf8_plan_returns_64(tmp_path: Path) -> None: + plan = tmp_path / "plan.json" + plan.write_bytes(b'{"summary": "\xff"}') + result = subprocess.run( + [sys.executable, str(SCRIPT), "validate-plan", "--plan", str(plan)], + capture_output=True, + ) + assert result.returncode == 64 + assert b"cannot read JSON" in result.stderr + assert b"plan.json" in result.stderr + assert b"Traceback" not in result.stderr + + +def test_manifest_includes_nested_manifest_files(tmp_path: Path, ppv) -> None: + (tmp_path / "artifact-manifest.json").write_bytes(b"runner manifest") + nested = tmp_path / "scratch" / "check" / "artifact-manifest.json" + nested.parent.mkdir(parents=True) + content = b"check artifact\n" + nested.write_bytes(content) + + manifest = ppv.artifact_manifest(tmp_path) + assert manifest["files"] == [ + { + "path": "scratch/check/artifact-manifest.json", + "sha256": hashlib.sha256(content).hexdigest(), + "bytes": len(content), + } + ] + + +@pytest.mark.parametrize("own_manifest_exists", [False, True]) +def test_manifest_rejects_zero_evidence_files( + tmp_path: Path, ppv, own_manifest_exists: bool +) -> None: + if own_manifest_exists: + (tmp_path / "artifact-manifest.json").write_bytes(b"runner manifest") + with pytest.raises(ppv.PlanError, match="zero files"): + ppv.artifact_manifest(tmp_path) diff --git a/plugins/post-patch-validation/tests/test_eval_cases.py b/plugins/post-patch-validation/tests/test_eval_cases.py new file mode 100644 index 0000000..9d0ead0 --- /dev/null +++ b/plugins/post-patch-validation/tests/test_eval_cases.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +import yaml + +EVALS = Path(__file__).resolve().parents[1] / "evals" +EXPECTED = {"complete-fix", "missed-variant", "behavior-regression"} +GOLDENS = { + "complete-fix": ( + '{"code": "S1"} | `03-variant` | variant | pass | pass |', + '{"code": "S3"} | `03-variant` | variant | pass | fail |', + ), + "missed-variant": ( + '{"code": "S3"} | `03-variant` | variant | pass | fail |', + '{"code": "S1"} | `03-variant` | variant | pass | pass |', + ), + "behavior-regression": ( + '{"code": "S2"}\n| `04-behavior` | behavior | pass | pass | changed |', + '{"code": "S1"}\n| `04-behavior` | behavior | pass | pass | same |', + ), +} + +# The shape that broke the original result.json grader: a variant that PASSED on the patch, +# followed by an unrelated check that failed. `[\s\S]*?` backtracks across the check boundary, +# so the old pattern read this as "the variant failed". Report rows cannot span checks. +SPANNING_DEFECT = """\ +| `02-exploit-original` | exploit | pass | pass | — | +| `03-variant-repeated` | variant | pass | pass | — | +| `05-regression-empty` | regression | pass | fail | — | +""" + + +def test_three_outcome_evals_have_deterministic_graders() -> None: + cases = sorted(EVALS.glob("*/case.yaml")) + assert {path.parent.name for path in cases} == EXPECTED + for path in cases: + case = yaml.safe_load(path.read_text()) + assert case["name"] == path.parent.name + assert case["runs"] == 3 + assert case["execution"]["timeout_seconds"] == 1800 + graders = case["graders"] + assert any(grader["type"] == "file_exists" for grader in graders) + assert any(grader["type"] == "regex" for grader in graders) + assert all(grader["type"] != "llm" for grader in graders) + + +def test_eval_regexes_accept_goldens_and_reject_defects() -> None: + asserted = 0 + for path in sorted(EVALS.glob("*/case.yaml")): + case = yaml.safe_load(path.read_text()) + golden, defective = GOLDENS[case["name"]] + for grader in case["graders"]: + if grader["type"] != "regex": + continue + pattern = re.compile(grader["pattern"]) + assert pattern.search(golden), grader["name"] + assert not pattern.search(defective), grader["name"] + asserted += 1 + assert asserted == 6 + + +def test_variant_graders_cannot_match_across_check_boundaries() -> None: + """A passing variant plus any later failing check must not read as an unfixed variant.""" + old_pattern = r'"kind":\s*"variant"[\s\S]*?"patched":\s*\{[\s\S]*?"matched":\s*false' + spanning_json = ( + '"kind": "variant", "runs": {"patched": {"matched": true}}, ' + '"kind": "regression", "runs": {"patched": {"matched": false}}' + ) + assert re.search(old_pattern, spanning_json), "fixture no longer reproduces the old defect" + + case = yaml.safe_load((EVALS / "missed-variant" / "case.yaml").read_text()) + patterns = [g["pattern"] for g in case["graders"] if g["type"] == "regex"] + variant = [p for p in patterns if "variant" in p] + assert variant, "missed-variant lost its variant grader" + genuine = SPANNING_DEFECT.replace("variant | pass | pass", "variant | pass | fail") + for pattern in variant: + assert not re.search(pattern, SPANNING_DEFECT) + assert re.search(pattern, genuine) + + +def test_behavior_grader_accepts_either_s2_signal() -> None: + case = yaml.safe_load((EVALS / "behavior-regression" / "case.yaml").read_text()) + pattern = next( + grader["pattern"] + for grader in case["graders"] + if grader["name"] == "behavior-row-shows-regression" + ) + for row in ( + "| `04-behavior` | behavior | pass | pass | changed |", + "| `04-behavior` | behavior | pass | fail | same |", + "| `04-behavior` | behavior | pass | fail | changed |", + ): + assert re.search(pattern, row) + assert not re.search(pattern, "| `04-behavior` | behavior | pass | pass | same |") + + +def test_eval_scaffolds_create_clean_two_commit_repositories(tmp_path: Path) -> None: + ran = 0 + for directory in sorted(path.parent for path in EVALS.glob("*/case.yaml")): + workdir = tmp_path / directory.name + workdir.mkdir() + subprocess.run(["bash", str(directory / "scaffold.sh")], cwd=workdir, check=True) + base = subprocess.run( + ["git", "rev-parse", "vulnerable"], + cwd=workdir, + text=True, + stdout=subprocess.PIPE, + check=True, + ).stdout.strip() + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=workdir, + text=True, + stdout=subprocess.PIPE, + check=True, + ).stdout.strip() + status = subprocess.run( + ["git", "status", "--short"], + cwd=workdir, + text=True, + stdout=subprocess.PIPE, + check=True, + ).stdout + assert base != head + assert status == "" + ran += 1 + assert ran == 3 diff --git a/plugins/post-patch-validation/tests/test_runner.py b/plugins/post-patch-validation/tests/test_runner.py new file mode 100644 index 0000000..342e79e --- /dev/null +++ b/plugins/post-patch-validation/tests/test_runner.py @@ -0,0 +1,1120 @@ +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +from conftest import SCRIPT + + +def command( + *args: str, cwd: Path | None = None, check: bool = True +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + list(args), + cwd=cwd, + text=True, + capture_output=True, + check=check, + ) + + +def git(repo: Path, *args: str) -> str: + return command("git", "-c", "commit.gpgsign=false", *args, cwd=repo).stdout.strip() + + +def create_repo(root: Path, patch_body: str | None = None) -> tuple[Path, str, str]: + repo = root / "repo" + repo.mkdir() + git(repo, "init", "-q", "-b", "main") + git(repo, "config", "user.name", "Fixture") + git(repo, "config", "user.email", "fixture@example.invalid") + (repo / "app.py").write_text( + "def sanitize(value):\n if value is None:\n return None\n return value\n" + ) + scripts = repo / "scripts" + scripts.mkdir() + helper = scripts / "verify.sh" + helper.write_text("#!/bin/sh\nexit 0\n") + helper.chmod(0o755) + git(repo, "add", "app.py", "scripts/verify.sh") + git(repo, "commit", "-q", "-m", "vulnerable") + base = git(repo, "rev-parse", "HEAD") + git(repo, "tag", "vulnerable") + (repo / "app.py").write_text( + patch_body + or "def sanitize(value):\n" + " if value is None:\n" + " return None\n" + " return value.replace('<', '<')\n" + ) + git(repo, "add", "app.py") + git(repo, "commit", "-q", "-m", "fix sanitizer") + patched = git(repo, "rev-parse", "HEAD") + return repo, base, patched + + +def checks() -> list[dict[str, object]]: + python = sys.executable + return [ + { + "id": "01-control-import", + "kind": "control", + "rationale": "The module imports and benign input is accepted on both revisions.", + "covers": ["app.sanitize benign harness"], + "argv": [python, "-c", "import app; assert app.sanitize('safe') == 'safe'"], + }, + { + "id": "02-exploit-original", + "kind": "exploit", + "rationale": "The original unsafe character must be escaped.", + "covers": ["app.sanitize single unsafe character"], + "argv": [ + python, + "-c", + "import app; print('PPV_REACHED', flush=True); assert app.sanitize('<') == '<'", + ], + }, + { + "id": "03-variant-repeated", + "kind": "variant", + "rationale": "Repeated unsafe characters exercise the root cause independently.", + "covers": ["app.sanitize repeated unsafe characters"], + "argv": [ + python, + "-c", + "import app; print('PPV_REACHED', flush=True); " + "assert app.sanitize('<<') == '<<'", + ], + }, + { + "id": "04-behavior-safe", + "kind": "behavior", + "rationale": "Safe input output must remain byte-identical.", + "covers": ["app.sanitize safe input"], + "argv": [python, "-c", "import app; print(app.sanitize('safe'))"], + "compare_stream": "stdout", + }, + { + "id": "05-regression-empty", + "kind": "regression", + "rationale": "Empty input remains supported.", + "covers": ["app.sanitize empty input"], + "argv": [python, "-c", "import app; assert app.sanitize('') == ''"], + }, + { + "id": "06-security-none", + "kind": "security", + "rationale": "The patch must preserve the sentinel ownership contract.", + "covers": ["app.sanitize None sentinel"], + "argv": [python, "-c", "import app; assert app.sanitize(None) is None"], + }, + { + "id": "07-suite-compile", + "kind": "suite", + "rationale": "The patched module must compile under the project interpreter.", + "covers": ["project compile gate"], + "argv": [python, "-m", "py_compile", "app.py"], + }, + ] + + +def scaffold(repo: Path, plan: Path, *, patch_file: Path | None = None) -> dict[str, object]: + argv = [ + sys.executable, + str(SCRIPT), + "scaffold", + "--repo", + str(repo), + "--base-ref", + "vulnerable", + "--finding-id", + "TEST-1", + "--finding-summary", + "sanitize returns unsafe markup unchanged", + "--evidence-level", + "runtime", + "--output", + str(plan), + ] + if patch_file: + argv.extend(["--patch-file", str(patch_file)]) + else: + argv.extend(["--patched-ref", "HEAD"]) + result = command(*argv) + assert "patch_sha256" in result.stdout + value = json.loads(plan.read_text()) + value["checks"] = checks() + plan.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + return value + + +def run(plan: Path, output: Path) -> subprocess.CompletedProcess[str]: + return command( + sys.executable, + str(SCRIPT), + "run", + "--plan", + str(plan), + "--output", + str(output), + check=False, + ) + + +def test_complete_ref_patch_is_s1_and_reproducible(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan = tmp_path / "case" / "plan.json" + scaffold(repo, plan) + + first = tmp_path / "first" + second = tmp_path / "second" + assert run(plan, first).returncode == 0 + assert run(plan, second).returncode == 0 + + result = json.loads((first / "result.json").read_text()) + assert result["verdict"]["code"] == "S1" + assert result["verdict"]["human_review_required"] is True + assert all(result["coverage"][kind] >= 1 for kind in result["coverage"]) + for name in ["result.json", "report.md", "artifact-manifest.json"]: + assert (first / name).read_bytes() == (second / name).read_bytes() + + +def test_patch_file_mode_applies_in_isolated_worktree(tmp_path: Path) -> None: + repo, base, patched = create_repo(tmp_path) + patch_file = tmp_path / "fix.patch" + patch_file.write_bytes( + subprocess.run( + ["git", "diff", "--binary", "--full-index", base, patched, "--"], + cwd=repo, + stdout=subprocess.PIPE, + check=True, + ).stdout + ) + plan = tmp_path / "patch-case" / "plan.json" + scaffold(repo, plan, patch_file=patch_file) + assert run(plan, tmp_path / "patch-results").returncode == 0 + assert ( + json.loads((tmp_path / "patch-results" / "result.json").read_text())["verdict"]["code"] + == "S1" + ) + assert git(repo, "status", "--short") == "" + + +@pytest.mark.parametrize( + "summary", ["ASCII finding", "Unicode finding: \u2018quote\u2019 \u6f0f\u6d1e"] +) +def test_s2_plan_and_report_use_utf8_with_ascii_locale(tmp_path: Path, ppv, summary: str) -> None: + repo, _, _ = create_repo(tmp_path) + plan = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan) + value["finding"]["summary"] = summary + behavior = next(check for check in value["checks"] if check["kind"] == "behavior") + behavior["argv"] = [sys.executable, "-c", "import app; print(app.sanitize('<'))"] + ppv.write_json(plan, value) + output = tmp_path / "results" + result = subprocess.run( + [sys.executable, str(SCRIPT), "run", "--plan", str(plan), "--output", str(output)], + env={ + **os.environ, + "PYTHONUTF8": "0", + "PYTHONCOERCECLOCALE": "0", + "LC_ALL": "C", + }, + capture_output=True, + ) + assert result.returncode == 2, result.stderr.decode("utf-8", errors="replace") + evidence = json.loads((output / "result.json").read_text(encoding="utf-8")) + assert evidence["verdict"]["code"] == "S2" + assert evidence["finding"]["summary"] == summary + assert summary in (output / "report.md").read_text(encoding="utf-8") + + +def test_plan_rejects_shell_strings_and_missing_categories(tmp_path: Path, ppv) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + value["checks"][0]["argv"] = "python -c 'import app'" + with pytest.raises(ppv.PlanError, match="argv array"): + ppv.validate_plan(value) + + value = json.loads(plan_path.read_text()) + value["checks"][0]["argv"] = ["bash", "-c", "python -m pytest | tee output.txt"] + with pytest.raises(ppv.PlanError, match="shell command string"): + ppv.validate_plan(value) + + value = json.loads(plan_path.read_text()) + value["checks"] = [check for check in value["checks"] if check["kind"] != "variant"] + with pytest.raises(ppv.PlanError, match="missing check kinds: variant"): + ppv.validate_plan(value) + + value = json.loads(plan_path.read_text()) + del value["evidence_level"] + with pytest.raises(ppv.PlanError, match="evidence_level"): + ppv.validate_plan(value) + + value = json.loads(plan_path.read_text()) + del value["submodules"] + with pytest.raises(ppv.PlanError, match="submodules"): + ppv.validate_plan(value) + + +def observation( + kind: str, *, base: bool = True, patched: bool = True, marker: bool = True +) -> dict[str, object]: + needs_marker = kind in {"exploit", "variant"} + runs: dict[str, dict[str, object]] = {} + if kind != "suite": + runs["base"] = {"status": "completed", "matched": base} + runs["patched"] = {"status": "completed", "matched": patched} + for run in runs.values(): + run["marker"] = marker if needs_marker else None + result: dict[str, object] = {"id": f"check-{kind}", "kind": kind, "runs": runs} + if kind == "behavior": + result["comparison"] = {"matched": patched} + return result + + +@pytest.mark.parametrize( + ("changes", "expected"), + [ + ({}, "S1"), + ({"behavior": False}, "S2"), + ({"variant": False}, "S3"), + ({"security": False}, "S4"), + ({"variant": False, "security": False}, "S5"), + ], +) +def test_classification_matrix(ppv, changes: dict[str, bool], expected: str) -> None: + evidence = [observation(kind, patched=changes.get(kind, True)) for kind in ppv.KINDS] + assert ppv.classify(evidence)["code"] == expected + + +def test_baseline_mismatch_is_inconclusive(ppv) -> None: + evidence = [observation(kind, base=kind != "exploit") for kind in ppv.KINDS] + assert ppv.classify(evidence)["code"] == "INCONCLUSIVE" + + +@pytest.mark.parametrize("kind", ["exploit", "variant"]) +def test_unmarked_assertion_is_inconclusive_not_evidence(ppv, kind: str) -> None: + """A nonzero base exit without the marker is a broken harness, not a reproduction. + + Without this the runner cannot tell `assert` from `ModuleNotFoundError`, so a harness that + never reached its assertion on base and happens to exit zero on patch reads as a clean fix. + """ + evidence = [observation(other, marker=other != kind) for other in ppv.KINDS] + verdict = ppv.classify(evidence) + assert verdict["code"] == "INCONCLUSIVE" + assert f"check-{kind}:base:marker_missing" in verdict["reasons"][0] + + +def test_broken_patched_control_is_inconclusive_not_s2(ppv) -> None: + """If the benign harness stops working on the patched side, nothing there is trustworthy.""" + evidence = [observation(kind, patched=kind != "control") for kind in ppv.KINDS] + verdict = ppv.classify(evidence) + assert verdict["code"] == "INCONCLUSIVE" + assert "check-control:patched:control_failed" in verdict["reasons"][0] + + +def test_suite_failure_is_still_s2(ppv) -> None: + """The control carve-out must not swallow the ordinary non-security regression signal.""" + evidence = [observation(kind, patched=kind != "suite") for kind in ppv.KINDS] + assert ppv.classify(evidence)["code"] == "S2" + + +def test_shell_command_strings_are_rejected_behind_env(ppv) -> None: + """Matching only argv[0] let `env bash -c` through, which is the obvious way around it.""" + for argv in ( + ["bash", "-c", "make test"], + ["env", "bash", "-c", "make test"], + ["/usr/bin/env", "FOO=1", "sh", "-c", "make test"], + ["env", "-u", "HOME", "sh", "-c", "make test"], + ["env", "pwsh", "-Command", "Invoke-Build"], + # Flag bundling: bash takes -xc, -lc, -ic and any other single-letter cluster. + ["bash", "-xc", "make test"], + ["/bin/sh", "-ec", "make test"], + # csh and tcsh were simply missing from the shell list. + ["/bin/csh", "-c", "make test"], + ["tcsh", "-c", "make test"], + # Common process wrappers must not turn the executable-only check into a bypass. + ["timeout", "5", "sh", "-c", "make test"], + ["timeout", "--signal=KILL", "5", "env", "sh", "-c", "make test"], + ["nohup", "bash", "-c", "make test"], + ["nice", "-n", "5", "sh", "-c", "make test"], + ["xargs", "-n", "1", "sh", "-c", "make test"], + ["stdbuf", "-o0", "sh", "-c", "make test"], + ["setsid", "bash", "-c", "make test"], + ["busybox", "sh", "-c", "make test"], + ["flock", "/tmp/lock", "sh", "-c", "make test"], + ["/usr/bin/time", "sh", "-c", "make test"], + ["sudo", "sh", "-c", "make test"], + ["script", "-qec", "make test | tee out"], + ): + with pytest.raises(ppv.PlanError, match="shell command string"): + ppv.reject_shell_string(argv, "checks[0].argv") + # env --split-string re-splits its argument into a command line: same hazard, different door. + for argv in ( + ["/usr/bin/env", "-S", "sh -c 'make test'"], + ["env", "--split-string", "sh -c 'make test'"], + ["env", "--split-string=sh -c 'make test'"], + ): + with pytest.raises(ppv.PlanError, match="split-string"): + ppv.reject_shell_string(argv, "checks[0].argv") + for argv in ( + [sys.executable, "-c", "import app"], + ["env", "PYTHONPATH=.", sys.executable, "-m", "pytest"], + ["./scripts/run-suite.sh"], + ): + ppv.reject_shell_string(argv, "checks[0].argv") + + +def test_marker_must_be_a_bare_stdout_line(ppv, tmp_path: Path) -> None: + """A SyntaxError traceback echoes the source, so stderr and substrings cannot count. + + `python3 -c "print('PPV_REACHED') NOT_VALID"` executes nothing and exits nonzero, but the + traceback quotes the marker back. Scanning stderr for a substring accepted that as proof + the harness reached its assertion. + """ + accepted = tmp_path / "ok.stdout" + accepted.write_bytes(b"setting up\nPPV_REACHED\n") + assert ppv.marker_present(accepted) + + for content in ( + b" File \"\", line 1\n print('PPV_REACHED') NOT_VALID\nSyntaxError: invalid\n", + b"usage: harness [--emit PPV_REACHED]\n", + b"prefixPPV_REACHEDsuffix\n", + ): + echoed = tmp_path / "echo.stdout" + echoed.write_bytes(content) + assert not ppv.marker_present(echoed) + + +def test_exploit_checks_may_not_see_which_revision_they_run_on(ppv, tmp_path: Path) -> None: + """The cheapest fake reproduction is `assert PPV_SIDE == "patched"`, so deny the oracle.""" + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + base = scaffold(repo, plan_path) + + for mutate in ( + lambda check: check.update( + {"argv": [sys.executable, "-c", "import os; os.environ['PPV_SIDE']"]} + ), + lambda check: check.update({"env": {"WHICH": "{side}"}}), + ): + value = json.loads(plan_path.read_text()) + exploit = next(c for c in value["checks"] if c["kind"] == "exploit") + mutate(exploit) + with pytest.raises(ppv.PlanError, match="may not reference the revision"): + ppv.validate_plan(value) + + # The same reference is fine on a kind whose verdict does not turn on base-versus-patch. + value = json.loads(plan_path.read_text()) + next(c for c in value["checks"] if c["kind"] == "suite")["env"] = {"WHICH": "{side}"} + ppv.validate_plan(value) + assert base["case_id"] + + +def test_side_is_withheld_from_exploit_checks_at_runtime(tmp_path: Path) -> None: + """Plan-time rejection is not enough: a helper script could read the variable directly.""" + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + # A helper file, not inline argv: the plan validator already rejects argv that names + # PPV_SIDE, so the interesting question is whether a script can still read it at runtime. + probe = plan_path.parent / "probe.py" + probe.write_text( + "import os, pathlib, re\n" + "def fd_path(fd):\n" + " try:\n" + " import fcntl\n" + " return fcntl.fcntl(fd, 50, b'\\0' * 1024).split(b'\\0', 1)[0].decode()\n" + " except (ImportError, OSError):\n" + " try:\n" + " return os.readlink(f'/proc/self/fd/{fd}')\n" + " except OSError:\n" + " return ''\n" + "print('PPV_REACHED', flush=True)\n" + "assert 'PPV_SIDE' not in os.environ, 'side leaked via environment'\n" + "leaf = pathlib.Path(os.environ['PPV_CHECKOUT']).name\n" + "assert leaf not in {'base', 'patched'}, 'side leaked via checkout path'\n" + "for fd in (1, 2):\n" + " assert not re.search(r'-(base|patched)\\.(stdout|stderr)$', fd_path(fd))\n" + ) + exploit = next(c for c in value["checks"] if c["kind"] == "exploit") + exploit["argv"] = [sys.executable, "{plan_dir}/probe.py"] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + output = tmp_path / "results" + run(plan_path, output) + result = json.loads((output / "result.json").read_text()) + exploit_run = next(c for c in result["checks"] if c["kind"] == "exploit")["runs"]["base"] + # The probe exits 0 when side-blinding holds, and an exploit must fail on base, so a green + # blinding check reads as S3 here. What matters is that the probe's assertions all held. + assert exploit_run["marker"] is True + assert exploit_run["exit_code"] == 0 + + +def test_output_descriptor_path_cannot_manufacture_s1(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + oracle = plan_path.parent / "fd-oracle.py" + oracle.write_text( + "import os, sys\n" + "try:\n" + " import fcntl\n" + " path = fcntl.fcntl(1, 50, b'\\0' * 1024).split(b'\\0', 1)[0].decode()\n" + "except (ImportError, OSError):\n" + " try:\n" + " path = os.readlink('/proc/self/fd/1')\n" + " except OSError:\n" + " path = ''\n" + "print('PPV_REACHED', flush=True)\n" + "sys.exit(1 if path.endswith('-base.stdout') else 0)\n" + ) + for check in value["checks"]: + if check["kind"] in {"exploit", "variant"}: + check["argv"] = [sys.executable, "{plan_dir}/fd-oracle.py"] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + output = tmp_path / "results" + assert run(plan_path, output).returncode != 0 + result = json.loads((output / "result.json").read_text()) + assert result["verdict"]["code"] != "S1" + for check in result["checks"]: + if check["kind"] in {"exploit", "variant"}: + for invocation in check["runs"].values(): + assert invocation["exit_code"] == 0 + + +def test_allow_env_refuses_credentials_and_execution_hijacking(ppv, monkeypatch) -> None: + for name in ( + "AWS_SECRET_ACCESS_KEY", + "GITHUB_TOKEN", + "GITHUB_PAT", + "NPM_AUTH", + "DB_PASSWORD", + "DB_PASS", + "ENCRYPTION_KEY", + "SIGNING_KEY", + "AUTH_HEADER", + ): + monkeypatch.setenv(name, "sensitive") + with pytest.raises(ppv.PlanError, match="reads as a credential"): + ppv.resolve_forwarded_env([name]) + for name in ("LD_PRELOAD", "BASH_ENV", "NODE_OPTIONS", "GIT_SSH_COMMAND"): + monkeypatch.setenv(name, "/tmp/evil") + with pytest.raises(ppv.PlanError, match="changes which code runs"): + ppv.resolve_forwarded_env([name]) + for name in ("TZ", "LC_ALL", "PYTHONHASHSEED", "PPV_SIDE", "PPV_SCRATCH"): + monkeypatch.setenv(name, "host-value") + with pytest.raises(ppv.PlanError, match="reserves or fixes"): + ppv.resolve_forwarded_env([name]) + + +def test_allow_env_forwards_only_named_variables_and_fails_closed(ppv, monkeypatch) -> None: + monkeypatch.setenv("JAVA_HOME", "/opt/jdk") + monkeypatch.setenv("GIT_AUTHOR_NAME", "Fixture") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "do-not-forward") + assert ppv.resolve_forwarded_env(["JAVA_HOME", "GIT_AUTHOR_NAME", "JAVA_HOME"]) == { + "JAVA_HOME": "/opt/jdk", + "GIT_AUTHOR_NAME": "Fixture", + } + with pytest.raises(ppv.PlanError, match="not set in this environment"): + ppv.resolve_forwarded_env(["NOT_SET_ANYWHERE"]) + with pytest.raises(ppv.PlanError, match="not a valid variable name"): + ppv.resolve_forwarded_env(["not-a-name"]) + + +def test_per_invocation_scratch_is_archived_without_sharing_state(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + value["checks"][0]["argv"] = [ + sys.executable, + "-c", + "import os, pathlib; pathlib.Path(os.environ['PPV_SCRATCH'], 'note.txt').write_text('x')", + ] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + output = tmp_path / "results" + run(plan_path, output) + notes = list((output / "scratch").glob("*/note.txt")) + assert len(notes) == 2 + assert {note.read_text() for note in notes} == {"x"} + snapshot = json.loads((output / "plan.snapshot.json").read_text()) + assert snapshot["case_id"] == value["case_id"] + + +def test_stable_environment_drops_secrets_and_overrides_locale(ppv, monkeypatch) -> None: + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "do-not-forward") + monkeypatch.setenv("PATH", "/fixture/bin") + env = ppv.stable_env({"LC_ALL": "host-locale", "CASE_VALUE": "explicit"}) + assert env["PATH"] == "/fixture/bin" + assert env["CASE_VALUE"] == "explicit" + assert env["LC_ALL"] == "C" + assert "AWS_SECRET_ACCESS_KEY" not in env + + +def test_allow_env_reaches_the_checks_and_is_recorded(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + value["checks"][0]["argv"] = [ + sys.executable, + "-c", + "import os; assert os.environ['FIXTURE_HOME'] == '/opt/fixture'", + ] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + output = tmp_path / "results" + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "run", + "--plan", + str(plan_path), + "--output", + str(output), + "--allow-env", + "FIXTURE_HOME", + ], + env={**os.environ, "FIXTURE_HOME": "/opt/fixture"}, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr + recorded = json.loads((output / "result.json").read_text())["inputs"]["forwarded_env"] + assert recorded == {"FIXTURE_HOME": "/opt/fixture"} + assert "**Forwarded environment:** FIXTURE_HOME" in (output / "report.md").read_text() + + +def test_moved_ref_fails_closed(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan = tmp_path / "case" / "plan.json" + scaffold(repo, plan) + (repo / "README.md").write_text("move the ref\n") + git(repo, "add", "README.md") + git(repo, "commit", "-q", "-m", "move HEAD") + result = run(plan, tmp_path / "results") + assert result.returncode == 64 + assert "patched ref moved" in result.stderr + + +def test_missing_repository_fails_cleanly(tmp_path: Path) -> None: + result = command( + sys.executable, + str(SCRIPT), + "scaffold", + "--repo", + str(tmp_path / "missing"), + "--base-ref", + "HEAD", + "--patched-ref", + "HEAD", + "--finding-id", + "TEST-MISSING", + "--finding-summary", + "missing repository", + "--evidence-level", + "source", + "--output", + str(tmp_path / "plan.json"), + check=False, + ) + assert result.returncode == 64 + assert "failed to run git" in result.stderr + assert "Traceback" not in result.stderr + + +def test_relative_argv0_hash_uses_check_cwd(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + value["checks"][0].update({"cwd": "scripts", "argv": ["./verify.sh"]}) + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + output = tmp_path / "results" + assert run(plan_path, output).returncode == 0 + result = json.loads((output / "result.json").read_text()) + control = next(check for check in result["checks"] if check["kind"] == "control") + expected = ppv_sha256(repo / "scripts" / "verify.sh") + assert control["runs"]["base"]["argv0_sha256"] == expected + assert control["runs"]["patched"]["argv0_sha256"] == expected + + +def test_interpreter_helper_is_hashed_and_archived(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + helper = plan_path.parent / "probe.py" + helper_source = b"raise SystemExit(0)\n" + helper.write_bytes(helper_source) + value["checks"][0]["argv"] = [sys.executable, "{plan_dir}/probe.py"] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + output = tmp_path / "results" + assert run(plan_path, output).returncode == 0 + result = json.loads((output / "result.json").read_text()) + control = next(check for check in result["checks"] if check["kind"] == "control") + expected = hashlib.sha256(helper_source).hexdigest() + for side in ("base", "patched"): + helper_record = next( + record for record in control["runs"][side]["argv_files"] if record["index"] == 1 + ) + assert helper_record["argument"] == "{plan_dir}/probe.py" + assert helper_record["sha256"] == expected + artifact = output / helper_record["artifact"] + assert artifact.read_bytes() == helper_source + + helper.write_text("raise SystemExit(7)\n") + helper_artifact = output / "helpers" / expected + assert helper_artifact.read_bytes() == helper_source + manifest = json.loads((output / "artifact-manifest.json").read_text()) + helper_entry = next(item for item in manifest["files"] if item["path"] == f"helpers/{expected}") + assert helper_entry["sha256"] == expected + + +def test_external_file_argument_is_hashed_but_not_archived(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + secret = tmp_path / "credential.txt" + secret.write_text("PRIVATE") + value["checks"][0]["argv"] = [ + sys.executable, + "-c", + "import pathlib,sys; assert pathlib.Path(sys.argv[1]).read_text() == 'PRIVATE'", + str(secret), + ] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + output = tmp_path / "results" + assert run(plan_path, output).returncode == 0 + result = json.loads((output / "result.json").read_text()) + control = next(check for check in result["checks"] if check["kind"] == "control") + expected = hashlib.sha256(b"PRIVATE").hexdigest() + for side in ("base", "patched"): + record = next(item for item in control["runs"][side]["argv_files"] if item["index"] == 3) + assert record["sha256"] == expected + assert record["artifact"] is None + assert "outside the isolated" in record["archive_reason"] + assert not (output / "helpers" / expected).exists() + + +def ppv_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_scratch_state_cannot_reveal_base_then_patch_order(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + oracle = ( + "import os, pathlib, sys; " + "p = pathlib.Path(os.environ['PPV_SCRATCH'], 'seen'); " + "print('PPV_REACHED', flush=True); " + "seen = p.exists(); " + "p.write_text('x'); " + "sys.exit(0 if seen else 1)" + ) + for check in value["checks"]: + if check["kind"] in {"exploit", "variant"}: + check["argv"] = [sys.executable, "-c", oracle] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + output = tmp_path / "results" + assert run(plan_path, output).returncode == 3 + result = json.loads((output / "result.json").read_text()) + assert result["verdict"]["code"] == "S3" + for check in result["checks"]: + if check["kind"] in {"exploit", "variant"}: + assert check["runs"]["base"]["exit_code"] == 1 + assert check["runs"]["patched"]["exit_code"] == 1 + assert check["runs"]["base"]["scratch"] != check["runs"]["patched"]["scratch"] + + +def test_plan_directory_state_cannot_reveal_base_then_patch_order(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + oracle = ( + "import os, pathlib, sys; " + "p = pathlib.Path(os.environ['PPV_PLAN_DIR'], 'seen'); " + "print('PPV_REACHED', flush=True); " + "seen = p.exists(); " + "p.write_text('x'); " + "sys.exit(0 if seen else 1)" + ) + for check in value["checks"]: + if check["kind"] in {"exploit", "variant"}: + check["argv"] = [sys.executable, "-c", oracle] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + output = tmp_path / "results" + assert run(plan_path, output).returncode == 3 + result = json.loads((output / "result.json").read_text()) + assert result["verdict"]["code"] == "S3" + for check in result["checks"]: + if check["kind"] in {"exploit", "variant"}: + assert check["runs"]["base"]["exit_code"] == 1 + assert check["runs"]["patched"]["exit_code"] == 1 + + +def test_checkout_ancestor_state_cannot_reveal_base_then_patch_order(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + for check in value["checks"]: + if check["kind"] in {"exploit", "variant"}: + oracle = ( + "import os, pathlib, sys; " + "root = pathlib.Path(os.environ['PPV_CHECKOUT']).parent.parent; " + f"p = root / {check['id']!r}; " + "print('PPV_REACHED', flush=True); " + "seen = p.exists(); " + "p.write_text('x'); " + "sys.exit(0 if seen else 1)" + ) + check["argv"] = [sys.executable, "-c", oracle] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + output = tmp_path / "results" + assert run(plan_path, output).returncode == 3 + result = json.loads((output / "result.json").read_text()) + assert result["verdict"]["code"] == "S3" + for check in result["checks"]: + if check["kind"] in {"exploit", "variant"}: + assert check["runs"]["base"]["exit_code"] == 1 + assert check["runs"]["patched"]["exit_code"] == 1 + + +def test_private_plan_copy_excludes_plan_pins_and_prior_results(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + prior = plan_path.parent / "old-results" + prior.mkdir() + (prior / "result.json").write_text("{}") + (prior / "artifact-manifest.json").write_text("{}") + (prior / "sentinel").write_text("old evidence") + value["checks"][0]["argv"] = [ + sys.executable, + "-c", + "import os,pathlib; p=pathlib.Path(os.environ['PPV_PLAN_DIR']); " + "assert not (p/'plan.json').exists(); assert not (p/'old-results').exists()", + ] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + assert run(plan_path, tmp_path / "results").returncode == 0 + + +def test_clean_plan_template_is_not_left_writable_on_disk(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path) + (plan_path.parent / "template-marker").write_text("snapshot me") + value["checks"][0]["argv"] = [ + sys.executable, + "-c", + "import os,pathlib; root=pathlib.Path(os.environ['PPV_CHECKOUT']).parents[2]; " + "assert not list(root.glob('ppv-plan-template-*/plan/template-marker'))", + ] + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + assert run(plan_path, tmp_path / "results").returncode == 0 + + +def test_concurrent_runs_share_repository_without_worktree_races(tmp_path: Path) -> None: + repo, _, _ = create_repo(tmp_path) + plan_path = tmp_path / "case" / "plan.json" + scaffold(repo, plan_path) + processes = [ + subprocess.Popen( + [ + sys.executable, + str(SCRIPT), + "run", + "--plan", + str(plan_path), + "--output", + str(tmp_path / f"results-{index}"), + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + for index in range(2) + ] + completed = [process.communicate(timeout=30) + (process.returncode,) for process in processes] + assert all(returncode == 0 for _, _, returncode in completed), completed + assert git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1 + + +def test_legacy_pid_worktree_locks_are_reclaimed(tmp_path: Path, ppv) -> None: + repo, _, _ = create_repo(tmp_path) + stale = tmp_path / "stale-worktree" + git(repo, "worktree", "add", "-q", "--detach", str(stale), "HEAD") + dead_pid = 99_999_999 + assert not ppv.process_is_running(dead_pid) + git( + repo, + "worktree", + "lock", + "--reason", + f"{ppv.WORKTREE_LOCK_REASON_PREFIX}{dead_pid}", + str(stale), + ) + shutil.rmtree(stale) + + with ppv.worktree_metadata_lock(repo): + ppv.unlock_stale_validator_worktrees(repo) + + assert git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1 + + +def test_owner_leases_distinguish_active_and_stale_worktrees(tmp_path: Path, ppv) -> None: + repo, _, _ = create_repo(tmp_path) + stale = tmp_path / "leased-worktree" + git(repo, "worktree", "add", "-q", "--detach", str(stale), "HEAD") + owner = ppv.acquire_worktree_owner(repo) + git( + repo, + "worktree", + "lock", + "--reason", + f"{ppv.WORKTREE_LOCK_REASON_PREFIX}{owner.token}", + str(stale), + ) + assert str(stale) not in ppv.stale_validator_worktrees(repo) + + ppv.close_worktree_owner(owner) + shutil.rmtree(stale) + with ppv.worktree_metadata_lock(repo): + ppv.unlock_stale_validator_worktrees(repo) + + assert git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1 + + +def test_worktree_metadata_lock_io_errors_are_plan_errors(tmp_path: Path, ppv, monkeypatch) -> None: + repo, _, _ = create_repo(tmp_path) + original_open = Path.open + + def guarded_open(path: Path, *args, **kwargs): + if path.name == "post-patch-validation.lock": + raise PermissionError("fixture denies lock creation") + return original_open(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", guarded_open) + with ( + pytest.raises(ppv.PlanError, match="could not open worktree metadata lock"), + ppv.worktree_metadata_lock(repo), + ): + pass + + +def test_reports_are_verdict_and_evidence_level_specific(ppv) -> None: + for level in ppv.EVIDENCE_LEVELS: + for code, (label, exit_code) in ppv.VERDICTS.items(): + result = { + "finding": {"id": "TEST", "summary": "summary"}, + "inputs": { + "base_commit": "0" * 40, + "patch_sha256": "0" * 64, + "evidence_level": level, + "submodules": {}, + "forwarded_env": {}, + }, + "checks": [], + "verdict": { + "code": code, + "label": label, + "exit_code": exit_code, + "human_review_required": True, + "reasons": ["fixture"], + }, + } + report = ppv.markdown_report(result) + assert f"**Evidence level:** {level} — {ppv.EVIDENCE_LEVEL_SUMMARIES[level]}" in report + assert ppv.VERDICT_SUMMARIES[code] in report + if code != "S1": + assert ppv.VERDICT_SUMMARIES["S1"] not in report + + +def test_patch_inside_pinned_submodule_is_initialized_without_fetch(tmp_path: Path) -> None: + submodule = tmp_path / "library" + submodule.mkdir() + git(submodule, "init", "-q", "-b", "main") + git(submodule, "config", "user.name", "Fixture") + git(submodule, "config", "user.email", "fixture@example.invalid") + (submodule / "app.py").write_text( + "def sanitize(value):\n if value is None:\n return None\n return value\n" + ) + git(submodule, "add", "app.py") + git(submodule, "commit", "-q", "-m", "vulnerable") + submodule_base = git(submodule, "rev-parse", "HEAD") + (submodule / "app.py").write_text( + "def sanitize(value):\n" + " if value is None:\n" + " return None\n" + " return value.replace('<', '<')\n" + ) + git(submodule, "commit", "-qam", "fix") + submodule_patch = tmp_path / "submodule.patch" + submodule_patch.write_bytes( + subprocess.run( + [ + "git", + "diff", + "--binary", + "--full-index", + "--src-prefix=a/vendor/library/", + "--dst-prefix=b/vendor/library/", + submodule_base, + "HEAD", + "--", + "app.py", + ], + cwd=submodule, + stdout=subprocess.PIPE, + check=True, + ).stdout + ) + + repo = tmp_path / "repo" + repo.mkdir() + git(repo, "init", "-q", "-b", "main") + git(repo, "config", "user.name", "Fixture") + git(repo, "config", "user.email", "fixture@example.invalid") + git( + repo, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "--name", + "mylib", + str(submodule), + "vendor/library", + ) + git( + repo, + "config", + "-f", + ".gitmodules", + "submodule.mylib.url", + "https://example.invalid/must-not-fetch.git", + ) + git(repo / "vendor/library", "checkout", "-q", submodule_base) + git(repo, "add", ".gitmodules", "vendor/library") + git(repo, "commit", "-q", "-m", "pin vulnerable submodule") + git(repo, "tag", "vulnerable") + + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path, patch_file=submodule_patch) + assert value["submodules"] == ["vendor/library"] + for check in value["checks"]: + check["cwd"] = "vendor/library" + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + output = tmp_path / "results" + completed = run(plan_path, output) + assert completed.returncode == 0, completed.stderr + result = json.loads((output / "result.json").read_text()) + assert result["inputs"]["submodules"] == { + "vendor/library": { + "base_commit": submodule_base, + "patched_commit": submodule_base, + } + } + assert git(repo, "status", "--short") == "" + + +def test_patch_file_gitlink_bump_checks_out_and_records_patched_pin(tmp_path: Path) -> None: + submodule = tmp_path / "library" + submodule.mkdir() + git(submodule, "init", "-q", "-b", "main") + git(submodule, "config", "user.name", "Fixture") + git(submodule, "config", "user.email", "fixture@example.invalid") + (submodule / "app.py").write_text( + "def sanitize(value):\n if value is None:\n return None\n return value\n" + ) + git(submodule, "add", "app.py") + git(submodule, "commit", "-q", "-m", "vulnerable") + submodule_base = git(submodule, "rev-parse", "HEAD") + (submodule / "app.py").write_text( + "def sanitize(value):\n" + " if value is None:\n" + " return None\n" + " return value.replace('<', '<')\n" + ) + git(submodule, "commit", "-qam", "fix") + submodule_fixed = git(submodule, "rev-parse", "HEAD") + + repo = tmp_path / "repo" + repo.mkdir() + git(repo, "init", "-q", "-b", "main") + git(repo, "config", "user.name", "Fixture") + git(repo, "config", "user.email", "fixture@example.invalid") + git( + repo, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "--name", + "mylib", + str(submodule), + "vendor/library", + ) + git(repo / "vendor/library", "checkout", "-q", submodule_base) + git( + repo, + "config", + "-f", + ".gitmodules", + "submodule.mylib.url", + "https://example.invalid/must-not-fetch.git", + ) + git(repo, "add", ".gitmodules", "vendor/library") + git(repo, "commit", "-q", "-m", "pin vulnerable submodule") + git(repo, "tag", "vulnerable") + + git(repo / "vendor/library", "checkout", "-q", submodule_fixed) + git(repo, "add", "vendor/library") + gitlink_patch = tmp_path / "gitlink.patch" + gitlink_patch.write_bytes( + subprocess.run( + ["git", "diff", "--cached", "--binary", "--full-index", "--"], + cwd=repo, + stdout=subprocess.PIPE, + check=True, + ).stdout + ) + git(repo, "reset", "-q", "--hard", "HEAD") + git(repo / "vendor/library", "checkout", "-q", submodule_base) + + plan_path = tmp_path / "case" / "plan.json" + value = scaffold(repo, plan_path, patch_file=gitlink_patch) + assert value["submodules"] == ["vendor/library"] + for check in value["checks"]: + check["cwd"] = "vendor/library" + plan_path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + output = tmp_path / "results" + completed = run(plan_path, output) + assert completed.returncode == 0, completed.stderr + result = json.loads((output / "result.json").read_text()) + assert result["inputs"]["submodules"] == { + "vendor/library": { + "base_commit": submodule_base, + "patched_commit": submodule_fixed, + } + } diff --git a/plugins/post-patch-validation/tests/test_workflow_contract.py b/plugins/post-patch-validation/tests/test_workflow_contract.py new file mode 100644 index 0000000..183221f --- /dev/null +++ b/plugins/post-patch-validation/tests/test_workflow_contract.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import re +import shutil +import subprocess +from pathlib import Path + +WORKFLOW = Path(__file__).resolve().parents[1] / "workflows" / "validate-patch.js" +# The meta object is the only top-level brace closed at column 0; everything after it is body. +META_END_RE = re.compile(r"^\}$", flags=re.MULTILINE) + + +def source() -> str: + assert WORKFLOW.is_file(), f"workflow missing: {WORKFLOW}" + return WORKFLOW.read_text() + + +def test_node_is_available() -> None: + assert shutil.which("node"), "node is required for the syntax check and must not be skipped" + + +def test_workflow_is_valid_javascript(tmp_path: Path) -> None: + """`node --check` on the script body, wrapped the way the runtime wraps it. + + The raw file parses under neither module system: as CommonJS it trips on `export`, and as + an ES module it trips on the top-level `return` guards. Both are legal in the Workflow + runtime, which strips `meta` and runs the body in an async function. Checking the file + as-shipped therefore fails on every Node version, which is how this test shipped red. + """ + text = source() + end = META_END_RE.search(text) + assert end, "could not locate the end of the meta block" + body = text[end.end() :] + wrapped = tmp_path / "validate-patch.check.mjs" + wrapped.write_text(f"async function __wf(args, log, phase, agent, parallel) {{\n{body}\n}}\n") + result = subprocess.run( + ["node", "--check", str(wrapped)], + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_meta_and_phase_calls_agree() -> None: + text = source() + meta_block = text[text.index("phases: [") : text.index(" ],", text.index("phases: ["))] + declared = re.findall(r"title: '([^']+)'", meta_block) + called = re.findall(r"^phase\('([^']+)'\)$", text, flags=re.MULTILINE) + assert declared == ["Inventory", "Coverage", "Plan", "Execute", "Review"] + assert called == declared + + +def test_every_agent_call_has_a_schema_and_phase() -> None: + text = source() + fragments = text.split("agent(")[1:] + assert len(fragments) == 5, "expected five fixed agent call sites" + for fragment in fragments: + before_next = fragment.split("agent(", 1)[0] + assert "{ schema:" in before_next + assert "phase:" in before_next + assert text.count("additionalProperties: false") >= 5 + + +def test_workflow_has_no_nondeterministic_or_self_verification_scaffolding() -> None: + text = source() + for banned in ["Math.random", "Date.now", "new Date", "Promise.race", "double-check"]: + assert banned not in text + + +def test_machine_verdict_is_not_changed_by_reviewers() -> None: + text = source() + assert "const status = finalStatus(execution.verdict, reviews)" in text + assert "deterministicVerdict: execution.verdict" in text + assert "if (verdict !== 'S1') return 'REJECTED'" in text + + +def test_workflow_carries_evidence_scope_and_check_contracts() -> None: + text = source() + assert "--evidence-level source" in text + assert "evidenceLevel: execution.evidenceLevel" in text + assert "Exploit and variant assertions must" in text + assert "liveness, exact error type, timing, and compatibility" in text + assert "Preserve the scaffolded submodules list" in text diff --git a/plugins/post-patch-validation/tests/workflow_logic.test.mjs b/plugins/post-patch-validation/tests/workflow_logic.test.mjs new file mode 100644 index 0000000..333af75 --- /dev/null +++ b/plugins/post-patch-validation/tests/workflow_logic.test.mjs @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { loadFunction } from './extract.mjs' + +const here = path.dirname(fileURLToPath(import.meta.url)) +const workflow = path.join(here, '..', 'workflows', 'validate-patch.js') +const normalizeArgs = loadFunction(workflow, 'normalizeArgs') +const argProblems = loadFunction(workflow, 'argProblems') +const finalStatus = loadFunction(workflow, 'finalStatus') +const plain = value => JSON.parse(JSON.stringify(value)) + +test('structured input is normalized without widening patch scope', () => { + assert.deepEqual( + plain( + normalizeArgs({ + finding: ' finding.md ', + baseRef: ' vulnerable ', + patchRef: ' fix ', + workdir: ' evidence ', + }), + ), + { + finding: 'finding.md', + baseRef: 'vulnerable', + patchRef: 'fix', + patchFile: '', + workdir: 'evidence', + }, + ) +}) + +test('patch files suppress the default HEAD ref', () => { + const input = normalizeArgs({ finding: 'bug', baseRef: 'base', patchFile: 'fix.patch' }) + assert.equal(input.patchFile, 'fix.patch') + assert.equal(input.patchRef, '') + assert.deepEqual(plain(argProblems(input)), []) +}) + +test('explicit patch refs and patch files are mutually exclusive', () => { + const input = normalizeArgs({ + finding: 'bug', + baseRef: 'base', + patchRef: 'fix-commit', + patchFile: 'fix.patch', + }) + assert.equal(input.patchRef, 'fix-commit') + assert.equal(input.patchFile, 'fix.patch') + assert.ok(argProblems(input).includes('only one of patchRef or patchFile')) +}) + +test('missing baseline and unsafe workdirs fail before dispatch', () => { + const input = normalizeArgs({ finding: 'bug', workdir: '../escape' }) + const problems = argProblems(input) + assert.ok(problems.includes('baseRef')) + assert.ok(problems.some(value => value.startsWith('workdir'))) +}) + +test('only unanimous review can advance S1 to human review', () => { + const ok = approved => ({ approved, evidenceRead: ['results/result.json'] }) + const yes = [ok(true), ok(true)] + assert.equal(finalStatus('S1', yes), 'READY_FOR_HUMAN_REVIEW') + assert.equal(finalStatus('S1', [ok(true), ok(false)]), 'REVIEW_REQUIRED') + assert.equal(finalStatus('S1', [yes[0]]), 'REVIEW_REQUIRED') + for (const verdict of ['S2', 'S3', 'S4', 'S5', 'INCONCLUSIVE']) { + assert.equal(finalStatus(verdict, yes), 'REJECTED') + } +}) + +test('a reviewer that read nothing cannot approve', () => { + const read = { approved: true, evidenceRead: ['results/result.json'] } + for (const empty of [{ approved: true, evidenceRead: [] }, { approved: true }]) { + assert.equal(finalStatus('S1', [read, empty]), 'REVIEW_REQUIRED') + } +}) diff --git a/plugins/post-patch-validation/workflows/validate-patch.js b/plugins/post-patch-validation/workflows/validate-patch.js new file mode 100644 index 0000000..ccd6008 --- /dev/null +++ b/plugins/post-patch-validation/workflows/validate-patch.js @@ -0,0 +1,377 @@ +export const meta = { + name: 'validate-patch', + description: + 'Build and execute an evidence plan for an existing security patch, then return the deterministic S1-S5 verdict plus independent coverage concerns', + whenToUse: + 'Use after a security patch exists. Pass finding, baseRef, and patchRef or patchFile. The workflow runs local project code and cannot ask for missing input after launch.', + phases: [ + { title: 'Inventory', detail: 'Pin the finding, baseline, patch, diff hash, and changed files' }, + { title: 'Coverage', detail: 'Map variants, behavior, adjacent security, and test infrastructure' }, + { title: 'Plan', detail: 'Create executable checks and pass the machine plan validator' }, + { title: 'Execute', detail: 'Run checks in isolated worktrees and assign the S-score in Python' }, + { title: 'Review', detail: 'Flag omitted paths or evidence that did not exercise real code' }, + ], +} + +const INVENTORY_SCHEMA = { + type: 'object', + additionalProperties: false, + required: [ + 'planPath', + 'baseCommit', + 'patchSha256', + 'changedFiles', + 'evidenceLevel', + 'submodules', + 'findingSummary', + ], + properties: { + planPath: { type: 'string' }, + baseCommit: { type: 'string' }, + patchedCommit: { type: ['string', 'null'] }, + patchSha256: { type: 'string' }, + changedFiles: { type: 'array', items: { type: 'string' } }, + evidenceLevel: { enum: ['source', 'build', 'runtime'] }, + submodules: { type: 'array', items: { type: 'string' } }, + findingSummary: { type: 'string' }, + }, +} + +const PROPOSAL_SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['lens', 'observations', 'proposals'], + properties: { + lens: { type: 'string' }, + observations: { type: 'array', items: { type: 'string' } }, + proposals: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['id', 'kind', 'rationale', 'covers', 'strategy'], + properties: { + id: { type: 'string' }, + kind: { + enum: ['control', 'exploit', 'variant', 'behavior', 'regression', 'security', 'suite'], + }, + rationale: { type: 'string' }, + covers: { type: 'array', items: { type: 'string' } }, + strategy: { type: 'string' }, + }, + }, + }, + }, +} + +const PLAN_SCHEMA = { + type: 'object', + additionalProperties: false, + required: [ + 'complete', + 'planPath', + 'checkCount', + 'coverage', + 'evidenceLevel', + 'validationOutput', + ], + properties: { + complete: { type: 'boolean' }, + planPath: { type: 'string' }, + checkCount: { type: 'integer' }, + coverage: { + type: 'array', + items: { enum: ['control', 'exploit', 'variant', 'behavior', 'regression', 'security', 'suite'] }, + }, + evidenceLevel: { enum: ['source', 'build', 'runtime'] }, + validationOutput: { type: 'string' }, + blocker: { type: 'string' }, + allowEnv: { type: 'array', items: { type: 'string' } }, + }, +} + +const EXECUTION_SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['resultPath', 'reportPath', 'verdict', 'label', 'evidenceLevel', 'reasons'], + properties: { + resultPath: { type: 'string' }, + reportPath: { type: 'string' }, + verdict: { enum: ['S1', 'S2', 'S3', 'S4', 'S5', 'INCONCLUSIVE'] }, + label: { type: 'string' }, + evidenceLevel: { enum: ['source', 'build', 'runtime'] }, + reasons: { type: 'array', items: { type: 'string' } }, + }, +} + +const REVIEW_SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['lens', 'approved', 'concerns', 'evidenceRead'], + properties: { + lens: { type: 'string' }, + approved: { type: 'boolean' }, + concerns: { type: 'array', items: { type: 'string' } }, + // A reviewer that read nothing cannot approve anything. Without minItems an agent could + // return approved:true with empty arrays and advance an S1 having inspected no evidence. + evidenceRead: { type: 'array', items: { type: 'string' }, minItems: 1 }, + }, +} + +function normalizeArgs(raw) { + const trim = value => (typeof value === 'string' ? value.trim() : '') + const source = typeof raw === 'string' ? { finding: raw } : raw && typeof raw === 'object' ? raw : {} + const patchFile = trim(source.patchFile) + const explicitPatchRef = trim(source.patchRef) + return { + finding: trim(source.finding), + baseRef: trim(source.baseRef), + patchRef: explicitPatchRef || (patchFile ? '' : 'HEAD'), + patchFile, + workdir: trim(source.workdir) || 'post-patch-validation', + } +} + +function argProblems(value) { + const problems = [] + if (!value.finding) problems.push('finding') + if (!value.baseRef) problems.push('baseRef') + if (!value.patchRef && !value.patchFile) problems.push('patchRef or patchFile') + if (value.patchRef && value.patchFile) problems.push('only one of patchRef or patchFile') + if ( + value.workdir.startsWith('/') || + value.workdir.split('/').some(part => part === '..' || part === '') + ) { + problems.push('workdir (must be a non-empty relative path without ..)') + } + return problems +} + +function finalStatus(verdict, reviews) { + if (verdict !== 'S1') return 'REJECTED' + if (!Array.isArray(reviews) || reviews.length !== 2) return 'REVIEW_REQUIRED' + if (reviews.some(review => !review || !review.approved)) return 'REVIEW_REQUIRED' + // Approval only counts when the reviewer names what it read. An empty evidenceRead is a + // reviewer that rubber-stamped, which is indistinguishable from no review at all. + if (reviews.some(review => !Array.isArray(review.evidenceRead) || !review.evidenceRead.length)) { + return 'REVIEW_REQUIRED' + } + return 'READY_FOR_HUMAN_REVIEW' +} + +const input = normalizeArgs(args) +const problems = argProblems(input) +if (problems.length) { + log(`BLOCKED: missing or unsafe workflow input: ${problems.join(', ')}`) + return { + status: 'BLOCKED', + deterministicVerdict: 'INCONCLUSIVE', + humanReviewRequired: true, + reason: `Supply structured args before launch: ${problems.join(', ')}`, + } +} + +phase('Inventory') + +const patchOption = input.patchFile + ? `--patch-file ${JSON.stringify(input.patchFile)}` + : `--patched-ref ${JSON.stringify(input.patchRef)}` + +const inventory = await agent( + `Load the post-patch-validation skill and perform only its scaffold step in the current Git +repository. This is a local validation; do not use the network and do not modify tracked project +files. Read the finding if it is a path, reduce it to one root-cause-and-impact sentence, and run +the bundled Python script with these inputs: + + finding: ${input.finding} + base ref: ${input.baseRef} + patch option: ${patchOption} + scaffold flag: --evidence-level source + plan path: ${input.workdir}/plan.json + +Use the finding's stable ID when one exists; otherwise use "patch-finding". Run one command at a +time with no pipes or chained shell operators. Return the values printed by scaffold.`, + { schema: INVENTORY_SCHEMA, label: 'inventory', phase: 'Inventory' }, +) + +if (!inventory) { + return { + status: 'BLOCKED', + deterministicVerdict: 'INCONCLUSIVE', + humanReviewRequired: true, + reason: 'inventory agent returned no pinned plan', + } +} + +phase('Coverage') + +const LENSES = [ + { + key: 'root-cause', + brief: + 'Trace the vulnerable invariant through every sibling caller and alternate entry/output path. Propose the original exploit assertion plus at least one meaningfully distinct root-cause variant. Include error and teardown paths when ownership or lifetime is involved.', + }, + { + key: 'behavior', + brief: + 'Identify stable benign behavior that must remain byte-identical, plus targeted non-security regressions at the boundaries changed by the patch. Avoid outputs containing time, randomness, addresses, or unordered collections.', + }, + { + key: 'adjacent-security', + brief: + 'Look specifically for a new vulnerability introduced by the patch: ownership, cleanup, authorization, bounds, concurrency, error handling, and state-transition regressions. Propose checks that should pass on both base and patch.', + }, + { + key: 'harness', + brief: + 'Find the smallest deterministic project-native build/test command, sanitizer, or bounded fixed-seed fuzz corpus. Also propose a benign harness control that proves both worktrees can execute the relevant component.', + }, +] + +const proposals = await parallel( + LENSES.map(lens => () => + agent( + `Read the finding, the pinned plan at ${inventory.planPath}, the changed files, and both Git +revisions without checking either revision out over the user's working tree. Work read-only. + +Your coverage lens is ${lens.key}: ${lens.brief} + +Return concrete check proposals for the post-patch-validation plan. Each proposal needs a stable +ID, one supported kind, a falsifiable rationale, the exact code path/invariant it covers, and an +implementation strategy. Do not claim a path is covered merely because the diff mentions it. +Do not write tests yet; the Plan phase owns all artifacts.`, + { schema: PROPOSAL_SCHEMA, label: `coverage:${lens.key}`, phase: 'Coverage' }, + ), + ), +) + +if (proposals.length !== LENSES.length || proposals.some(value => !value)) { + return { + status: 'BLOCKED', + deterministicVerdict: 'INCONCLUSIVE', + humanReviewRequired: true, + reason: 'one or more fixed coverage lenses returned no result', + } +} + +phase('Plan') + +const planned = await agent( + `Load the post-patch-validation skill. Turn the fixed-lens proposals below into a complete, +executable validation plan at ${inventory.planPath}. You may write helper test artifacts only +under ${input.workdir}/checks; do not edit tracked project files or the patch. + +PROPOSALS +${JSON.stringify(proposals, null, 2)} + +Run the bundled script's print-schema command before editing the plan. Use argv arrays and the +{checkout}, {plan_dir}, {scratch}, and {side} placeholders; never use shell command strings, pipes, +redirections, or chained commands, and write only under {scratch}. The {side} placeholder and +PPV_SIDE are unavailable to exploit and variant checks. Every helper must invoke real project code. +Sort checks by ID and supply every required kind. Exploit and variant safety +assertions must fail on the vulnerable base and pass on the patch, and each must print PPV_REACHED +flushed immediately before its assertion or the runner will return INCONCLUSIVE. Security checks +must pass on both revisions. Behavior output must be stable enough for exact comparison. + +Replace the scaffolded source evidence_level with the highest level the completed checks honestly +support: keep source when only source or patch invariants run, use build when target code is +compiled or analyzed without executing the reported behavior, and use runtime only when the +exploit and variant checks execute the affected component. Return that value as evidenceLevel. +Exploit and variant assertions must +test only the security invariant; put liveness, exact error type, timing, and compatibility in +behavior or regression checks. Preserve the scaffolded submodules list unless another pinned +submodule is demonstrably required. + +Checks run under a fixed minimal environment. If the project's toolchain needs host variables +beyond PATH and HOME, list their names in allowEnv rather than hardcoding host paths into the +plan. Never list a credential; the values are recorded in result.json. + +Run validate-plan when done. If the evidence cannot honestly cover every required kind, do not +invent a check: return complete=false with the blocker and preserve the incomplete plan.`, + { schema: PLAN_SCHEMA, label: 'plan', phase: 'Plan' }, +) + +if (!planned || !planned.complete) { + return { + status: 'BLOCKED', + deterministicVerdict: 'INCONCLUSIVE', + humanReviewRequired: true, + reason: planned ? planned.blocker || planned.validationOutput : 'plan agent returned nothing', + planPath: planned ? planned.planPath : inventory.planPath, + } +} + +phase('Execute') + +const allowEnvFlags = (Array.isArray(planned.allowEnv) ? planned.allowEnv : []) + .filter(name => /^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) + .map(name => `--allow-env ${name}`) + .join(' ') + +const execution = await agent( + `Load the post-patch-validation skill and execute its Python runner exactly once: + + plan: ${planned.planPath} + output: ${input.workdir}/results + extra flags: ${allowEnvFlags || '(none)'} + +The runner intentionally exits nonzero for S2-S5 and INCONCLUSIVE. A nonzero Bash result is not a +reason to rerun it. Read result.json after the command, return its exact verdict, label, evidence +level, reasons, and artifact paths, and do not edit the plan, patch, checks, or result.`, + { schema: EXECUTION_SCHEMA, label: 'execute', phase: 'Execute' }, +) + +if (!execution) { + return { + status: 'BLOCKED', + deterministicVerdict: 'INCONCLUSIVE', + humanReviewRequired: true, + reason: 'execution agent returned no machine result', + planPath: planned.planPath, + } +} + +phase('Review') + +const REVIEW_LENSES = [ + { + key: 'surface-completeness', + brief: + 'Try to find a root-cause sibling, alternate entry/output path, error path, boundary, or teardown path omitted from the declared covers fields. Read the relevant source rather than trusting the plan summary.', + }, + { + key: 'evidence-integrity', + brief: + 'Read result.json, patch.diff, plan.snapshot.json, the raw logs, and every content-addressed helper artifact referenced by each run\'s argv_files (including scripts passed to interpreters, not only argv[0]). Verify the recorded hashes, and reject a relevant helper whose argv_files record has no artifact. Reject mocks, reimplemented vulnerable logic, a test that never invokes real project code, empty output, a harness that prints the marker without exercising the vulnerable path, or commands whose observations do not support their declared kind.', + }, +] + +const reviews = await parallel( + REVIEW_LENSES.map(lens => () => + agent( + `Review the completed post-patch evidence read-only under ${input.workdir}/results. + +Your lens is ${lens.key}: ${lens.brief} + +The Python verdict is ${execution.verdict}. You cannot change or vote on that S-score. Set +approved=false when the supplied evidence is incomplete or invalid under your lens, list concrete +concerns with file/function/check IDs, and list every artifact you actually read. Missing evidence +is a concern, not consent. Do not run the validation again and do not edit artifacts.`, + { schema: REVIEW_SCHEMA, label: `review:${lens.key}`, phase: 'Review' }, + ), + ), +) + +const status = finalStatus(execution.verdict, reviews) +return { + status, + deterministicVerdict: execution.verdict, + verdictLabel: execution.label, + evidenceLevel: execution.evidenceLevel, + reasons: execution.reasons, + humanReviewRequired: true, + planPath: planned.planPath, + resultPath: execution.resultPath, + reportPath: execution.reportPath, + reviews: reviews.filter(Boolean), +}