imbad0202__academic-research-skills/scripts/test_check_215_field_norm.py

"""Unit tests for check_215_field_norm.py (#215).

The lint reads the three real reviewer surfaces. These tests (a) assert the shipped
files pass, and (b) monkey-patch the file reader to inject mutated content so every
assertion is shown to be non-vacuous (RED when the #215 block/marker is removed or
de-scoped). Mirrors the falsifiability discipline of check_v3_9_2_phase_boundary.
"""
from __future__ import annotations

import re
import unittest

from scripts import check_215_field_norm as cfn


DOMAIN = "academic-paper-reviewer/agents/domain_reviewer_agent.md"
DA = "academic-paper-reviewer/agents/devils_advocate_reviewer_agent.md"
CAL = "academic-paper-reviewer/references/calibration_mode_protocol.md"


def _real_reads() -> dict[str, str]:
    return {p: cfn._read(p) for p in (DOMAIN, DA, CAL)}


class TestShipped(unittest.TestCase):
    def test_shipped_surfaces_pass(self) -> None:
        self.assertEqual(cfn.check(), [], msg="shipped reviewer surfaces should pass #215 lint")


class TestMutations(unittest.TestCase):
    """Each test patches cfn._read to serve a mutated copy of ONE surface and asserts the
    lint goes RED with a message naming that surface."""

    def setUp(self) -> None:
        self._orig_read = cfn._read
        self._files = _real_reads()

    def tearDown(self) -> None:
        cfn._read = self._orig_read

    def _patch(self, path: str, mutate) -> list[str]:
        files = dict(self._files)
        files[path] = mutate(files[path])
        cfn._read = lambda p: files[p]
        return cfn.check()

    def test_domain_step5_removed_fails(self) -> None:
        errors = self._patch(
            DOMAIN, lambda t: t.replace("### Step 5: Field-Norm Severity Discipline (#215)", "### Step 5: Removed")
        )
        self.assertTrue(
            any("domain_reviewer_agent.md" in e and "Step 5" in e for e in errors),
            msg=f"expected domain Step 5 error: {errors!r}",
        )

    def test_domain_broadened_evidence_removed_fails(self) -> None:
        errors = self._patch(
            DOMAIN, lambda t: t.replace("not limited to a literature citation", "limited to a literature citation")
        )
        self.assertTrue(
            any("broadened-evidence" in e for e in errors),
            msg=f"expected broadened-evidence error: {errors!r}",
        )

    def test_domain_must_not_scoped_to_block(self) -> None:
        """Removing 'MUST NOT' from the Step 5 block must fail even though MUST NOT may exist
        elsewhere in the file — proving the check is block-scoped, not file-wide."""
        errors = self._patch(
            DOMAIN,
            lambda t: self._strip_in_block(
                t, "### Step 5: Field-Norm Severity Discipline (#215)", "MUST NOT", "must-not"
            ),
        )
        self.assertTrue(
            any("domain_reviewer_agent.md Step 5" in e and "MUST NOT" in e for e in errors),
            msg=f"expected block-scoped MUST NOT error: {errors!r}",
        )

    def test_da_dimension9_removed_fails(self) -> None:
        errors = self._patch(
            DA, lambda t: t.replace("### 9. Field-Norm Severity Calibration (#215)", "### 9. Removed")
        )
        self.assertTrue(
            any("devils_advocate_reviewer_agent.md" in e and "dimension" in e for e in errors),
            msg=f"expected DA dimension 9 error: {errors!r}",
        )

    def test_da_field_removed_fails(self) -> None:
        errors = self._patch(
            DA, lambda t: t.replace("field_norm_boundary", "xxx").replace("evidence_crossing_rationale", "yyy")
        )
        self.assertTrue(
            any("field_norm_boundary" in e for e in errors)
            and any("evidence_crossing_rationale" in e for e in errors),
            msg=f"expected DA missing-field errors: {errors!r}",
        )

    def test_calibration_phase35_removed_fails(self) -> None:
        errors = self._patch(
            CAL, lambda t: t.replace("### Phase 3.5: Severity-miscalibration measurement (#215)", "### Phase 3.5: Removed")
        )
        self.assertTrue(
            any("calibration_mode_protocol.md" in e and "Phase 3.5" in e for e in errors),
            msg=f"expected calibration Phase 3.5 error: {errors!r}",
        )

    def test_calibration_anti_circularity_removed_fails(self) -> None:
        """Removing the anti-circularity anchor (the gold-set pointer) from Phase 3.5 must fail —
        this is the codex P1 fix that the classifier rates grounding, not norm-correctness."""
        errors = self._patch(
            CAL,
            lambda t: self._strip_in_block(
                t,
                "### Phase 3.5: Severity-miscalibration measurement (#215)",
                "evals/gold/field_norm_severity",
                "evals/gold/REMOVED",
            ),
        )
        self.assertTrue(
            any("calibration_mode_protocol.md Phase 3.5" in e and "anti-circularity" in e for e in errors),
            msg=f"expected anti-circularity error: {errors!r}",
        )

    def test_domain_grounding_clause_removed_fails(self) -> None:
        """codex P2: a bare `MUST` check passes on `MUST NOT` alone. Deleting the load-bearing
        positive grounding clause must still fail — assert the specific clause, not a modal verb."""
        errors = self._patch(
            DOMAIN,
            lambda t: self._strip_in_block(
                t,
                "### Step 5: Field-Norm Severity Discipline (#215)",
                "ground the norm in an external",
                "do something with the norm",
            ),
        )
        self.assertTrue(
            any("domain_reviewer_agent.md Step 5" in e and "ground the norm in an external" in e for e in errors),
            msg=f"expected grounding-clause error: {errors!r}",
        )

    def test_da_output_format_column_removed_fails(self) -> None:
        """codex P1: deleting the CRITICAL/MAJOR table columns from the Output Format block must
        fail even though the snake_case field names remain in the gating prose elsewhere — proves
        the column check is scoped to the output block, not file-wide."""
        errors = self._patch(
            DA,
            lambda t: self._strip_in_block(
                t, "## Output Format", "Field-Norm Boundary", "Removed-Column",
            ),
        )
        self.assertTrue(
            any("Output Format" in e and "Field-Norm Boundary" in e for e in errors),
            msg=f"expected scoped output-format column error: {errors!r}",
        )

    def test_da_single_table_column_removed_fails(self) -> None:
        """codex re-review P1: if ONLY the CRITICAL table loses its columns while MAJOR keeps
        them, a whole-block check would find the names in MAJOR and false-pass. Each severity
        subsection must be checked separately. Mutate only the CRITICAL table header row."""
        def drop_critical_columns(t: str) -> str:
            # Replace the column names on the CRITICAL table's header row only (first occurrence
            # after '#### CRITICAL'), leaving the MAJOR table intact.
            idx = t.index("#### CRITICAL")
            head = t[:idx]
            tail = t[idx:]
            tail = tail.replace(
                "| # | Dimension | Issue Description | Evidence Anchor | Confidence | Field-Norm Boundary | Evidence-Crossing Rationale |",
                "| # | Dimension | Issue Description | Evidence Anchor | Confidence |",
                1,
            )
            return head + tail

        errors = self._patch(DA, drop_critical_columns)
        self.assertTrue(
            any("CRITICAL table" in e and "Field-Norm Boundary" in e for e in errors),
            msg=f"expected single-table (CRITICAL) column error: {errors!r}",
        )

    def test_domain_must_weakened_to_should_fails(self) -> None:
        """codex re-review P1: weakening 'MUST** ground' to 'SHOULD** ground' must fail — the
        load-bearing invariant is the positive MUST requirement, not just the grounding phrase."""
        errors = self._patch(
            DOMAIN,
            lambda t: self._strip_in_block(
                t,
                "### Step 5: Field-Norm Severity Discipline (#215)",
                "**MUST** ground the norm",
                "**SHOULD** ground the norm",
            ),
        )
        self.assertTrue(
            any("Step 5" in e and "MUST** ground the norm in an external" in e for e in errors),
            msg=f"expected weakened-modal error: {errors!r}",
        )

    def test_calibration_risk_definitions_removed_fails(self) -> None:
        """codex P2: the intro line already contains 'low / med / high', so a bare-word check
        passes after the three definition bullets are deleted. Removing a definition marker
        (**`high`**) must fail."""
        errors = self._patch(
            CAL,
            lambda t: self._strip_in_block(
                t,
                "### Phase 3.5: Severity-miscalibration measurement (#215)",
                "**`high`**",
                "**removed**",
            ),
        )
        self.assertTrue(
            any("Phase 3.5" in e and "'high' risk-level definition" in e for e in errors),
            msg=f"expected risk-definition error: {errors!r}",
        )

    def test_block_not_truncated_by_code_fence(self) -> None:
        """The Output Format block embeds a ```markdown sample report whose code fence contains
        '## Devil's Advocate Review' etc. _block() must NOT treat those fenced ## lines as the
        next header — otherwise the block truncates early and drops the CRITICAL table below it.
        This asserts the real columns (below the fence) are inside the captured block."""
        block = cfn._block(self._files[DA], r"^## Output Format")
        self.assertIsNotNone(block)
        self.assertIn("Field-Norm Boundary", block, "fence-aware _block must reach the CRITICAL table")
        self.assertIn("#### CRITICAL", block)

    @staticmethod
    def _strip_in_block(text: str, header: str, needle: str, replacement: str) -> str:
        """Replace `needle` with `replacement` ONLY inside the block starting at `header`
        (up to the next ##/### header), leaving any same-string occurrences elsewhere intact.
        Lets a test prove block-scoping rather than file-wide presence. Reuses the lint's own
        `_block()` so the test and the production scope logic cannot drift apart."""
        block = cfn._block(text, re.escape(header))
        if block is None:
            return text
        start = text.index(block)
        return text[:start] + block.replace(needle, replacement) + text[start + len(block):]


if __name__ == "__main__":
    unittest.main()