mirror of
https://github.com/AgriciDaniel/claude-ads.git
synced 2026-09-14 20:36:30 +08:00
fix(evals): bind model plan to exact subject
Bind every external model execution packet to the candidate commit and tree resolved from the repository at plan generation time. - retain the pinned retained-v1 commit and tree - expose exact candidate commit and tree in the non-evidence plan - regress the subject bindings alongside all 24 task packets This does not create model-run evidence or weaken signature requirements. External runner and evaluator evidence remains mandatory. Verified: - model gate tests: 16 passed - full suite: 558 passed, 24 skipped - ruff, compileall, shell syntax, pip check, and diff checks: passed - release audit: 354 tracked files passed - dependency audit: 3 profiles, 16 not affected, 0 unhandled - added diff: 20 lines, 0 em dash or private path hits - remaining: remote CI, model evidence, and signed independent reviews Co-Authored-By: GPT-5 <noreply@openai.com>
This commit is contained in:
@@ -829,6 +829,15 @@ def build_plan(contract_path: Path, root: Path) -> dict[str, Any]:
|
||||
contract = _load_contract(contract_path)
|
||||
_load_schemas(root, contract)
|
||||
suite = _load_suite(root, contract)
|
||||
candidate_commit, candidate_tree = _git_identity(root, "HEAD")
|
||||
subjects = {
|
||||
"candidate": {
|
||||
**contract["subjects"]["candidate"],
|
||||
"git_commit": candidate_commit,
|
||||
"git_tree": candidate_tree,
|
||||
},
|
||||
"retained_v1": contract["subjects"]["retained_v1"],
|
||||
}
|
||||
return {
|
||||
"schema_version": "2.0.0",
|
||||
"artifact_class": "external_model_execution_plan",
|
||||
@@ -837,7 +846,7 @@ def build_plan(contract_path: Path, root: Path) -> dict[str, Any]:
|
||||
"schemas": contract["schemas"],
|
||||
"suite": contract["suite"],
|
||||
"result_path": contract["result_path"],
|
||||
"subjects": contract["subjects"],
|
||||
"subjects": subjects,
|
||||
"runtime_requirements": contract["runtime"],
|
||||
"evaluation_requirements": contract["evaluation"],
|
||||
"authentication_requirements": contract["authentication_policy"],
|
||||
|
||||
@@ -261,8 +261,18 @@ def _assess_with_auth(
|
||||
|
||||
def test_execution_plan_is_pinned_and_explicitly_not_run_evidence(repo_root):
|
||||
plan = build_plan(repo_root / "evals" / "model-eval-contract.json", repo_root)
|
||||
candidate_commit = subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD^{commit}"], cwd=repo_root, text=True
|
||||
).strip()
|
||||
candidate_tree = subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD^{tree}"], cwd=repo_root, text=True
|
||||
).strip()
|
||||
assert plan["artifact_class"] == "external_model_execution_plan"
|
||||
assert plan["is_model_run_evidence"] is False
|
||||
assert plan["subjects"]["candidate"]["git_commit"] == candidate_commit
|
||||
assert plan["subjects"]["candidate"]["git_tree"] == candidate_tree
|
||||
assert plan["subjects"]["retained_v1"]["git_commit"] == BASELINE_COMMIT
|
||||
assert plan["subjects"]["retained_v1"]["git_tree"] == BASELINE_TREE
|
||||
assert len(plan["task_packets"]) == 24
|
||||
assert plan["suite"]["sha256"] == _sha(
|
||||
(repo_root / "evals" / "v2-behavior-evals.json").read_text(encoding="utf-8")
|
||||
|
||||
Reference in New Issue
Block a user