fix(evals): bind model plan to exact subject

Bind every external model execution packet to the candidate commit and
tree resolved from the repository at plan generation time.

- retain the pinned retained-v1 commit and tree
- expose exact candidate commit and tree in the non-evidence plan
- regress the subject bindings alongside all 24 task packets

This does not create model-run evidence or weaken signature requirements.
External runner and evaluator evidence remains mandatory.

Verified:
- model gate tests: 16 passed
- full suite: 558 passed, 24 skipped
- ruff, compileall, shell syntax, pip check, and diff checks: passed
- release audit: 354 tracked files passed
- dependency audit: 3 profiles, 16 not affected, 0 unhandled
- added diff: 20 lines, 0 em dash or private path hits
- remaining: remote CI, model evidence, and signed independent reviews

Co-Authored-By: GPT-5 <noreply@openai.com>
This commit is contained in:
Agrici Daniel
2026-08-26 17:27:21 +03:00
parent ecc0fb0374
commit 119a9fcb67
2 changed files with 20 additions and 1 deletions
+10 -1
View File
@@ -829,6 +829,15 @@ def build_plan(contract_path: Path, root: Path) -> dict[str, Any]:
contract = _load_contract(contract_path)
_load_schemas(root, contract)
suite = _load_suite(root, contract)
candidate_commit, candidate_tree = _git_identity(root, "HEAD")
subjects = {
"candidate": {
**contract["subjects"]["candidate"],
"git_commit": candidate_commit,
"git_tree": candidate_tree,
},
"retained_v1": contract["subjects"]["retained_v1"],
}
return {
"schema_version": "2.0.0",
"artifact_class": "external_model_execution_plan",
@@ -837,7 +846,7 @@ def build_plan(contract_path: Path, root: Path) -> dict[str, Any]:
"schemas": contract["schemas"],
"suite": contract["suite"],
"result_path": contract["result_path"],
"subjects": contract["subjects"],
"subjects": subjects,
"runtime_requirements": contract["runtime"],
"evaluation_requirements": contract["evaluation"],
"authentication_requirements": contract["authentication_policy"],
+10
View File
@@ -261,8 +261,18 @@ def _assess_with_auth(
def test_execution_plan_is_pinned_and_explicitly_not_run_evidence(repo_root):
plan = build_plan(repo_root / "evals" / "model-eval-contract.json", repo_root)
candidate_commit = subprocess.check_output(
["git", "rev-parse", "HEAD^{commit}"], cwd=repo_root, text=True
).strip()
candidate_tree = subprocess.check_output(
["git", "rev-parse", "HEAD^{tree}"], cwd=repo_root, text=True
).strip()
assert plan["artifact_class"] == "external_model_execution_plan"
assert plan["is_model_run_evidence"] is False
assert plan["subjects"]["candidate"]["git_commit"] == candidate_commit
assert plan["subjects"]["candidate"]["git_tree"] == candidate_tree
assert plan["subjects"]["retained_v1"]["git_commit"] == BASELINE_COMMIT
assert plan["subjects"]["retained_v1"]["git_tree"] == BASELINE_TREE
assert len(plan["task_packets"]) == 24
assert plan["suite"]["sha256"] == _sha(
(repo_root / "evals" / "v2-behavior-evals.json").read_text(encoding="utf-8")