imbad0202__academic-research-skills/scripts/test_check_setup_cross_model_parity.py

"""Unit tests for check_setup_cross_model_parity.py (#491 fold-in)."""
import unittest
from pathlib import Path

from tests.test_helpers import load_module_from_path, run_script

SCRIPT = Path(__file__).resolve().parent / "check_setup_cross_model_parity.py"

EN_OK = 'export ARS_CROSS_MODEL="gpt-5.5"\n# or: export ARS_CROSS_MODEL="gemini-3.1-pro-preview"\n'
ZH_OK = EN_OK
CANONICAL_OK = (
    "| Model | API ID | Provider |\n"
    "|-------|--------|----------|\n"
    "| GPT-5.5 | `gpt-5.5` | OpenAI |\n"
    "| Gemini 3.1 Pro | `gemini-3.1-pro-preview` | Google |\n"
    "\n"
    "(`gpt-5.4` / `gpt-5.4-pro` remain accepted for existing setups.)\n"
)


def _load_module():
    return load_module_from_path("check_setup_cross_model_parity", SCRIPT)


class SetupCrossModelParityTests(unittest.TestCase):

    def test_repo_baseline_passes(self) -> None:
        """The committed SETUP + canonical doc state must pass."""
        result = run_script(SCRIPT)
        self.assertEqual(result.returncode, 0, msg=f"stderr: {result.stderr}")
        self.assertIn("PASSED", result.stdout)

    def test_clean_fixture_passes(self) -> None:
        module = _load_module()
        self.assertEqual(module.check(EN_OK, ZH_OK, CANONICAL_OK), [])

    def test_en_zh_drift_fails(self) -> None:
        """One-sided edit (the B4-F02 shape: en updated, zh-TW forgotten)."""
        module = _load_module()
        zh_stale = 'export ARS_CROSS_MODEL="gpt-5.4-pro"\n'
        errors = module.check(EN_OK, zh_stale, CANONICAL_OK)
        self.assertTrue(any("drift" in e for e in errors), msg=f"errors: {errors}")

    def test_legacy_id_in_prose_note_does_not_count(self) -> None:
        """codex P1 regression (PR #492): `gpt-5.4-pro` is backticked in the
        canonical doc's legacy-accepted NOTE but absent from the model tables —
        a SETUP example reverting to it must FAIL, not pass on the note."""
        module = _load_module()
        stale = 'export ARS_CROSS_MODEL="gpt-5.4-pro"\n'
        errors = module.check(stale, stale, CANONICAL_OK)
        self.assertTrue(
            any("gpt-5.4-pro" in e and "not in" in e for e in errors),
            msg=f"errors: {errors}",
        )

    def test_canonical_table_parse_fails_closed(self) -> None:
        """No 'API ID' table in the canonical doc = parser stale = error."""
        module = _load_module()
        errors = module.check(EN_OK, ZH_OK, "prose only, `gpt-5.5` backticked")
        self.assertTrue(
            any("parser went stale" in e for e in errors), msg=f"errors: {errors}"
        )

    def test_compat_table_column_counts(self) -> None:
        """Ids in the compat table's 'Example API ID(s)' column are members."""
        module = _load_module()
        canonical = CANONICAL_OK + (
            "\n| Provider | Example API ID(s) | Endpoint |\n"
            "|----------|-------------------|----------|\n"
            "| DeepSeek | `deepseek-v4-pro` | https://api.deepseek.com/v1 |\n"
        )
        ids = module.canonical_model_ids(canonical)
        self.assertIn("deepseek-v4-pro", ids)
        self.assertNotIn("gpt-5.4-pro", ids)

    def test_wildcard_prefix_tokens_are_not_ids(self) -> None:
        """codex P2 regression (PR #492): the compat table's "any
        non-`gpt-*`/`gemini-*` id" prose sits in an API ID column — globs are
        prefix patterns, not model ids, and a SETUP example of `gpt-*` must
        fail, not pass on the leaked token."""
        module = _load_module()
        canonical = CANONICAL_OK + (
            "\n| Provider | Example API ID(s) | Endpoint |\n"
            "|----------|-------------------|----------|\n"
            "| Any OpenAI-compatible | any non-`gpt-*`/`gemini-*` id | any |\n"
        )
        self.assertNotIn("gpt-*", module.canonical_model_ids(canonical))
        glob_setup = 'export ARS_CROSS_MODEL="gpt-*"\n'
        errors = module.check(glob_setup, glob_setup, canonical)
        self.assertTrue(
            any("gpt-*" in e and "not in" in e for e in errors),
            msg=f"errors: {errors}",
        )

    def test_unknown_model_fails(self) -> None:
        """Example naming a model outside the canonical lineup."""
        module = _load_module()
        bad = 'export ARS_CROSS_MODEL="gpt-9.9-imaginary"\n'
        errors = module.check(bad, bad, CANONICAL_OK)
        self.assertTrue(
            any("gpt-9.9-imaginary" in e and "canonical" in e for e in errors),
            msg=f"errors: {errors}",
        )

    def test_zero_examples_fails_closed(self) -> None:
        """Regex-went-stale / block-removed must be an error, not a pass."""
        module = _load_module()
        errors = module.check("no examples here", ZH_OK, CANONICAL_OK)
        self.assertTrue(
            any("Fail-closed" in e for e in errors), msg=f"errors: {errors}"
        )

    def test_commented_example_lines_are_extracted(self) -> None:
        """`# or:` alternates count — they are user-pasteable examples too."""
        module = _load_module()
        ids = module.extract_ids(EN_OK)
        self.assertEqual(ids, ["gpt-5.5", "gemini-3.1-pro-preview"])


if __name__ == "__main__":
    unittest.main()