mirror of
https://github.com/Imbad0202/academic-research-skills.git
synced 2026-09-14 13:51:17 +08:00
Edward Cheng-I Wu
68c8594c26
Refactor the Kong A1 Commitment Ledger from three index-aligned parallel lists (commitment_extracted objects + top-level fulfillment_status[] + unfulfilled_rationale[]) to a nested-object shape: fulfillment_status and unfulfilled_rationale now nest inside each commitment_extracted object. Length mismatch / index desync becomes structurally impossible, closing the fragility where a dropped Markdown <br> or numbering error silently mispaired a status with the wrong commitment and produced a false COMMITMENT_GAP. REPLACE, not coexist: the parallel-list shape is removed entirely. No executable consumer, lint, or fixture carried it (the calibration harness is unshipped; the seed is a non-runnable seed), so coexistence would only preserve the failure mode. Lifecycle fields are absent at extraction time and appended per-object during revision execution; the old "" placeholder for fulfilled commitments is dropped (omitted, not empty-string). The equal-length invariant is retired (now structurally impossible); a legacy-normalization note instructs verifying equal length before zipping any pre-refactor top-level arrays onto the nested objects (refusing to auto-zip a desynced ledger). Synced across all six ledger surfaces: handoff_schemas.md Schema 11 (incl. the residual_action coherence prose, reworded from index notation to object-field notation), revision_coach_agent Step 3.5, re_review_mode_protocol Commitment Ledger Verification, revision_tracking_template.md (three fragile <br>-separated columns collapsed into one per-commitment nested YAML ledger), worked example, and the 12-case calibration seed. author_fulfillment_claim deferred (not required for the structural fix). New scripts/check_268_nested_commitment_ledger.py (N1-N5 + N3b) + 18 mutation tests, wired into spec-consistency.yml + the pytest manifest. Advisory semantics unchanged. Spec: docs/design/2026-05-31-ars-268-schema11-nested-commitment-ledger-spec.md Closes #268. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
195 lines
8.1 KiB
Python
195 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Mutation tests for check_268_nested_commitment_ledger.
|
|
|
|
Each invariant is exercised against a deliberately-mutated in-memory fixture to
|
|
confirm it FAILs — guarding against a trivial accept-all regression
|
|
(feedback_schema_mutation_test_for_constraints). The clean fixture must PASS.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import unittest
|
|
|
|
import scripts.check_268_nested_commitment_ledger as lint
|
|
|
|
|
|
CLEAN_SEED = {
|
|
"cases": [
|
|
{
|
|
"case_id": "F1",
|
|
"expected_commitments": [
|
|
{
|
|
"commitment_text": "add ablation on CIFAR-100",
|
|
"commitment_type": "add_experiment",
|
|
"required_evidence_type": "new_table",
|
|
"fulfillment_status": "fulfilled",
|
|
}
|
|
],
|
|
"expected_commitment_gap": False,
|
|
},
|
|
{
|
|
"case_id": "P2",
|
|
"expected_commitments": [
|
|
{
|
|
"commitment_text": "add ImageNet experiments",
|
|
"commitment_type": "add_experiment",
|
|
"required_evidence_type": "new_table",
|
|
"fulfillment_status": "not-fulfilled",
|
|
"unfulfilled_rationale": "deferred — compute budget; acknowledged in §6.",
|
|
},
|
|
{
|
|
"commitment_text": "add CIFAR-100 experiments",
|
|
"commitment_type": "add_experiment",
|
|
"required_evidence_type": "new_table",
|
|
"fulfillment_status": "fulfilled",
|
|
},
|
|
],
|
|
"expected_commitment_gap": False,
|
|
},
|
|
{
|
|
"case_id": "N1",
|
|
"expected_commitments": [
|
|
{
|
|
"commitment_text": "add error bars across 5 seeds",
|
|
"commitment_type": "add_experiment",
|
|
"required_evidence_type": "new_figure",
|
|
"fulfillment_status": "not-fulfilled",
|
|
# unfulfilled_rationale omitted → COMMITMENT_GAP trigger (valid)
|
|
}
|
|
],
|
|
"expected_commitment_gap": True,
|
|
},
|
|
]
|
|
}
|
|
|
|
CLEAN_PROSE = "A commitment object with fulfillment_status fulfilled carries no rationale."
|
|
|
|
|
|
class TestCleanPasses(unittest.TestCase):
|
|
def test_clean_seed_passes(self):
|
|
self.assertEqual(lint.check_seed(copy.deepcopy(CLEAN_SEED)), [])
|
|
|
|
def test_clean_prose_passes(self):
|
|
self.assertEqual(lint.check_index_notation("x", CLEAN_PROSE), [])
|
|
|
|
|
|
class TestRealFiles(unittest.TestCase):
|
|
def test_real_files_pass(self):
|
|
# The shipped artifacts must satisfy the lint end-to-end (hits disk).
|
|
self.assertEqual(lint.main(), 0)
|
|
|
|
|
|
class TestN1MissingField(unittest.TestCase):
|
|
def test_missing_commitment_type_fails(self):
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
del seed["cases"][0]["expected_commitments"][0]["commitment_type"]
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N1" in e and "commitment_type" in e for e in errs), errs)
|
|
|
|
def test_non_mapping_commitment_fails(self):
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
seed["cases"][0]["expected_commitments"][0] = "not a mapping"
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(
|
|
any("N1" in e and "commitment entry is not a mapping" in e for e in errs), errs
|
|
)
|
|
|
|
|
|
class TestN2ReintroducedParallelList(unittest.TestCase):
|
|
def test_parallel_status_list_fails(self):
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
seed["cases"][0]["expected_fulfillment_status"] = ["fulfilled"]
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N2" in e and "expected_fulfillment_status" in e for e in errs), errs)
|
|
|
|
def test_parallel_rationale_list_fails(self):
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
seed["cases"][0]["expected_unfulfilled_rationale"] = [""]
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N2" in e and "expected_unfulfilled_rationale" in e for e in errs), errs)
|
|
|
|
|
|
class TestN3LifecycleCoherence(unittest.TestCase):
|
|
def test_fulfilled_with_rationale_fails(self):
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
seed["cases"][0]["expected_commitments"][0]["unfulfilled_rationale"] = "should not be here"
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N3" in e and "fulfilled commitment carries" in e for e in errs), errs)
|
|
|
|
def test_nonfulfilled_with_empty_rationale_fails(self):
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
# P2 first commitment is not-fulfilled with rationale; blank it out.
|
|
seed["cases"][1]["expected_commitments"][0]["unfulfilled_rationale"] = " "
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N3" in e and "empty" in e for e in errs), errs)
|
|
|
|
def test_nonfulfilled_with_null_rationale_fails(self):
|
|
# A bare `unfulfilled_rationale:` (YAML null) must not read as populated
|
|
# via the str(None) == "None" trap — it is a present-but-blank key.
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
seed["cases"][1]["expected_commitments"][0]["unfulfilled_rationale"] = None
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N3" in e and "empty" in e for e in errs), errs)
|
|
|
|
def test_bad_status_enum_fails(self):
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
seed["cases"][0]["expected_commitments"][0]["fulfillment_status"] = "kinda-done"
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N3" in e and "not in enum" in e for e in errs), errs)
|
|
|
|
def test_rationale_without_status_fails(self):
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
com = seed["cases"][0]["expected_commitments"][0]
|
|
del com["fulfillment_status"]
|
|
com["unfulfilled_rationale"] = "orphan rationale"
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N3" in e and "without fulfillment_status" in e for e in errs), errs)
|
|
|
|
|
|
class TestN3bGapOracleCoherence(unittest.TestCase):
|
|
def test_false_gap_on_silent_omission_fails(self):
|
|
# N1 (not-fulfilled, no rationale) genuinely SHOULD gap; flipping its
|
|
# oracle to False must be caught.
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
n1 = next(c for c in seed["cases"] if c["case_id"] == "N1")
|
|
n1["expected_commitment_gap"] = False
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N3b" in e and "N1" in e for e in errs), errs)
|
|
|
|
def test_true_gap_on_fully_fulfilled_fails(self):
|
|
# F1 is fully fulfilled (no gap); claiming a gap must be caught.
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
f1 = next(c for c in seed["cases"] if c["case_id"] == "F1")
|
|
f1["expected_commitment_gap"] = True
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N3b" in e and "F1" in e for e in errs), errs)
|
|
|
|
def test_quoted_boolean_gap_flag_fails(self):
|
|
# A quoted "false" on a genuine-gap case would coerce truthy under bool();
|
|
# the oracle must reject non-boolean expected_commitment_gap outright.
|
|
seed = copy.deepcopy(CLEAN_SEED)
|
|
n1 = next(c for c in seed["cases"] if c["case_id"] == "N1")
|
|
n1["expected_commitment_gap"] = "false"
|
|
errs = lint.check_seed(seed)
|
|
self.assertTrue(any("N3b" in e and "must be a boolean" in e for e in errs), errs)
|
|
|
|
|
|
class TestN4N5IndexNotation(unittest.TestCase):
|
|
def test_fulfillment_status_index_fails(self):
|
|
errs = lint.check_index_notation("x", "where fulfillment_status[i] is empty")
|
|
self.assertTrue(any("index notation" in e for e in errs), errs)
|
|
|
|
def test_unfulfilled_rationale_index_fails(self):
|
|
errs = lint.check_index_notation("x", "the unfulfilled_rationale[0] placeholder")
|
|
self.assertTrue(any("index notation" in e for e in errs), errs)
|
|
|
|
def test_legacy_array_description_passes(self):
|
|
# The schema's legacy-normalization note references `fulfillment_status: [...]`
|
|
# (array literal, not a subscript) — that must NOT trip the index regex.
|
|
errs = lint.check_index_notation("x", "old top-level `fulfillment_status: [...]` arrays")
|
|
self.assertEqual(errs, [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|