imbad0202__academic-research-skills/scripts/check_215_field_norm.py

#!/usr/bin/env python3
"""Static lint for #215 field-norm severity coverage across the three reviewer surfaces.

Issue: #215 (field-norm severity calibration, Kim et al. 2026 arXiv:2605.20668v1 W1/§F.3.4).

Enforces that the three #215 changes are present AND scoped to their own block, so a
bare keyword appearing elsewhere in a file cannot make the lint pass while the actual
load-bearing instruction is missing (falsifiability discipline, per
feedback_lint_passes_but_prompt_silent.md — same pattern as check_v3_9_2_phase_boundary).

Three surfaces:

1. domain_reviewer_agent.md — a `### Step 5: Field-Norm Severity Discipline (#215)` block
   that contains the hard rule (MUST ground the norm in an external source, MUST NOT
   assert from model knowledge), the broadened evidence definition (not just a literature
   citation), and the `[FIELD-NORM UNVERIFIED]` down-rate label.

2. devils_advocate_reviewer_agent.md — a 9th challenge dimension
   `### 9. Field-Norm Severity Calibration (#215)` AND the two required CRITICAL/MAJOR
   fields `field_norm_boundary` + `evidence_crossing_rationale`.

3. calibration_mode_protocol.md — a `### Phase 3.5: Severity-miscalibration measurement (#215)`
   block carrying the low/med/high risk classification and the anti-circularity grounding
   discipline (classify grounding, NOT norm-correctness — do not repeat the W1 failure).

Exit 0 = clean, 1 = any failure.
"""
from __future__ import annotations

import re
import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent


def _read(rel: str) -> str:
    return (REPO_ROOT / rel).read_text(encoding="utf-8")


def _block(text: str, header_re: str) -> str | None:
    """Return the markdown block from a header matching header_re up to the next header of
    the same-or-higher level (## or ###), or end of file. None if the header is absent.

    Scopes keyword checks to the block so a keyword elsewhere in the file does not count.
    A ``##``/``###`` line INSIDE a ``` fenced code block is NOT a real header (e.g. an
    Output Format section embeds a sample report whose code fence contains ``## ...`` lines);
    treating it as one would truncate the block early and drop content below it.
    """
    m = re.search(header_re, text, re.M)
    if not m:
        return None
    start = m.start()
    rest = text[m.end():]
    in_fence = False
    offset = 0
    for line in rest.splitlines(keepends=True):
        if line.lstrip().startswith("```"):
            in_fence = not in_fence
        elif not in_fence and re.match(r"\#{2,3} ", line):
            # A real header line (column 0, outside any fence) ends the block.
            return text[start : m.end() + offset]
        offset += len(line)
    return text[start:]


def check() -> list[str]:
    errors: list[str] = []

    # --- Surface 1: domain_reviewer_agent.md Step 5 ---
    dr = _read("academic-paper-reviewer/agents/domain_reviewer_agent.md")
    step5 = _block(dr, r"^### Step 5: Field-Norm Severity Discipline \(#215\)")
    if step5 is None:
        errors.append("domain_reviewer_agent.md: missing '### Step 5: Field-Norm Severity Discipline (#215)' block")
    else:
        # The down-rate prohibition AND the load-bearing positive MUST-ground clause must
        # both survive. The positive clause binds the modal to the grounding requirement
        # ("MUST** ground the norm in an external"), so weakening MUST→SHOULD is caught — a
        # bare "ground the norm" substring would still pass under SHOULD (codex re-review P1).
        for clause in ("MUST NOT", "[FIELD-NORM UNVERIFIED]", "MUST** ground the norm in an external"):
            if clause not in step5:
                errors.append(f"domain_reviewer_agent.md Step 5: missing required clause {clause!r}")
        # codex P1: evidence is NOT limited to a literature citation.
        if "not limited to a literature citation" not in step5:
            errors.append(
                "domain_reviewer_agent.md Step 5: missing the broadened-evidence rule "
                "('not limited to a literature citation')"
            )

    # --- Surface 2: devils_advocate_reviewer_agent.md dimension 9 + CRITICAL fields ---
    da = _read("academic-paper-reviewer/agents/devils_advocate_reviewer_agent.md")
    dim9 = _block(da, r"^### 9\. Field-Norm Severity Calibration \(#215\)")
    if dim9 is None:
        errors.append("devils_advocate_reviewer_agent.md: missing '### 9. Field-Norm Severity Calibration (#215)' dimension")
    # The two required fields must land in the OUTPUT FORMAT block — that is what makes the
    # rule reach the actual review output. A file-wide check would pass on the prose mention
    # in the gating section even if the output-format columns were deleted (codex P1).
    output_fmt = _block(da, r"^## Output Format")
    if output_fmt is None:
        errors.append("devils_advocate_reviewer_agent.md: missing '## Output Format' block")
    else:
        # The CRITICAL/MAJOR tables carry the fields as human-readable column HEADERS
        # (Title Case), which is what actually reaches the review output. The snake_case
        # names live only in the gating prose. Scope to EACH severity subsection separately:
        # if only one table loses its columns, a whole-block check still finds the names in
        # the other table and false-passes (codex re-review P1). These #### headers sit inside
        # the ```markdown sample, so slice between #### markers rather than using _block.
        for severity in ("CRITICAL", "MAJOR"):
            sub = re.search(rf"^#### {severity}\n(.*?)(?=^#### |\Z)", output_fmt, re.M | re.S)
            if sub is None:
                errors.append(f"devils_advocate_reviewer_agent.md Output Format: missing '#### {severity}' table")
                continue
            for column in ("Field-Norm Boundary", "Evidence-Crossing Rationale"):
                if column not in sub.group(1):
                    errors.append(
                        f"devils_advocate_reviewer_agent.md Output Format {severity} table: "
                        f"missing required column {column!r}"
                    )
    # The snake_case field NAMES + their grounding definition live in the CRITICAL-finding
    # gating block; check them there (scoped) so a definition deleted from that block is
    # caught independently of the output-format columns.
    crit_block = _block(da, r"^### What Constitutes a CRITICAL Finding")
    if crit_block is None:
        errors.append("devils_advocate_reviewer_agent.md: missing '### What Constitutes a CRITICAL Finding' block")
    else:
        for field in ("field_norm_boundary", "evidence_crossing_rationale"):
            if field not in crit_block:
                errors.append(
                    f"devils_advocate_reviewer_agent.md CRITICAL-finding block: missing field "
                    f"definition {field!r}"
                )
        if "[FIELD-NORM UNVERIFIED]" not in crit_block:
            errors.append(
                "devils_advocate_reviewer_agent.md CRITICAL-finding block: missing "
                "'[FIELD-NORM UNVERIFIED]' down-rate label"
            )

    # --- Surface 3: calibration_mode_protocol.md Phase 3.5 ---
    cal = _read("academic-paper-reviewer/references/calibration_mode_protocol.md")
    phase35 = _block(cal, r"^### Phase 3\.5: Severity-miscalibration measurement \(#215\)")
    if phase35 is None:
        errors.append("calibration_mode_protocol.md: missing '### Phase 3.5: Severity-miscalibration measurement (#215)' block")
    else:
        # Require the actual risk-level DEFINITIONS, not bare words. The intro line already
        # contains "low / med / high", so a substring check passes even if all three
        # definition bullets are deleted (codex P2). Each level is defined as **`level`** — …
        for level in ("low", "med", "high"):
            if f"**`{level}`**" not in phase35:
                errors.append(
                    f"calibration_mode_protocol.md Phase 3.5: missing the {level!r} risk-level definition"
                )
        # codex P1: classify GROUNDING, not norm-correctness — do not repeat the failure.
        if "MUST NOT" not in phase35 or "evals/gold/field_norm_severity" not in phase35:
            errors.append(
                "calibration_mode_protocol.md Phase 3.5: missing the anti-circularity grounding "
                "discipline (MUST NOT guess norm-correctness; anchor to evals/gold/field_norm_severity)"
            )

    return errors


def main() -> int:
    errors = check()
    if errors:
        for e in errors:
            print(f"ERROR: {e}", file=sys.stderr)
        return 1
    print("215_field_norm: all three reviewer surfaces carry their scoped #215 blocks")
    return 0


if __name__ == "__main__":
    sys.exit(main())