mirror of
https://github.com/Imbad0202/academic-research-skills.git
synced 2026-09-14 13:51:17 +08:00
Edward Cheng-I Wu
e0f0c7b54a
The three agents/ files were relative symlinks into deep-research/agents/ (v3.7.0 Phase 2.1); on Windows checkouts without core.symlinks and in zip-download installs they materialise as one-line text files, silently breaking the three plugin agents (#413, external audit). Materialized as real byte-identical copies. New CI lint scripts/check_agents_mirror_sync.py takes over the single-source guarantee (hard-pinned mirror roster: set equality + regular-file-never-symlink + byte-equality; symlink check runs before byte-equality). The two symlink-dependent lints adapt: version-consistency invariant 8 excludes the mirror dir from the agent count; write-scope I5 maps direct children of root agents/ by name to their deep-research sources (codex P2: nested agents/sub/agents/ files do NOT remap — fail-open guard, pinned by a negative test). skills/ directory symlinks deliberately unchanged. 10 new mirror-sync tests, 3 mutations killed, 6 tests adapted/added across the two existing suites; lint + pytest companion wired into CI. Dual-track review: codex (1 P2, adopted) + security review (0 findings). Closes #413 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
252 lines
12 KiB
Python
252 lines
12 KiB
Python
"""Mutation tests for check_v3_10_134_write_scope.py (the fail-open guard lint).
|
|
|
|
feedback_schema_mutation_test_for_constraints: after a lint passes on the real repo,
|
|
inject deliberately-broken state and assert the lint FAILS. A lint that passes on both
|
|
the clean repo AND a mutated repo is vacuous (trivially accept-all). Each test below
|
|
mutates one invariant's input and asserts a matching error surfaces.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
import check_v3_10_134_write_scope as lint # noqa: E402
|
|
|
|
|
|
class CleanRepoTest(unittest.TestCase):
|
|
def test_clean_repo_passes(self):
|
|
# Baseline: the real repo state must pass (0 errors).
|
|
self.assertEqual(lint.run_checks(), [])
|
|
|
|
|
|
class MutationTest(unittest.TestCase):
|
|
"""Each mutation must make run_checks() report at least one error."""
|
|
|
|
def setUp(self):
|
|
# snapshot the real loaders to restore after each mutation
|
|
self._real_keys = lint.load_manifest_keys
|
|
self._real_manifest = lint.load_manifest
|
|
self._real_name = lint.read_frontmatter_name
|
|
self._real_a = list(lint.BUCKET_A_AGENT_FILES)
|
|
self._real_bcd = list(lint.BUCKET_BCD_AGENT_FILES)
|
|
|
|
def tearDown(self):
|
|
lint.load_manifest_keys = self._real_keys
|
|
lint.load_manifest = self._real_manifest
|
|
lint.read_frontmatter_name = self._real_name
|
|
lint.BUCKET_A_AGENT_FILES = self._real_a
|
|
lint.BUCKET_BCD_AGENT_FILES = self._real_bcd
|
|
|
|
def _assert_fails(self, needle=None):
|
|
errs = lint.run_checks()
|
|
self.assertTrue(errs, "mutation should have produced at least one error")
|
|
if needle:
|
|
self.assertTrue(any(needle in e for e in errs),
|
|
f"expected an error mentioning {needle!r}; got {errs}")
|
|
|
|
def test_I1_roster_size_drift_fails(self):
|
|
# Drop one Bucket A agent from the roster -> size != 23.
|
|
lint.BUCKET_A_AGENT_FILES = self._real_a[:-1]
|
|
self._assert_fails("I1")
|
|
|
|
def test_I2_manifest_missing_key_fails(self):
|
|
# A real agent on disk has no manifest entry -> fail-open risk.
|
|
real = self._real_keys()
|
|
dropped = sorted(real)[0]
|
|
lint.load_manifest_keys = lambda: real - {dropped}
|
|
self._assert_fails("I2")
|
|
|
|
def test_I2_manifest_typo_key_fails(self):
|
|
# A manifest key that matches no on-disk name (rename/typo).
|
|
real = self._real_keys()
|
|
lint.load_manifest_keys = lambda: (real - {sorted(real)[0]}) | {"bibliografy_agent_typo"}
|
|
self._assert_fails("I2")
|
|
|
|
def test_I2_agent_renamed_on_disk_fails(self):
|
|
# An agent file's frontmatter name drifts away from its manifest key.
|
|
def fake_name(rel):
|
|
if rel.endswith("bibliography_agent.md"):
|
|
return "renamed_bibliography_agent"
|
|
return self._real_name(rel)
|
|
lint.read_frontmatter_name = fake_name
|
|
self._assert_fails("I2")
|
|
|
|
def test_I3_bcd_leak_into_manifest_fails(self):
|
|
# A Bucket B agent's name (report_compiler_agent) appears as a manifest key.
|
|
real = self._real_keys()
|
|
lint.load_manifest_keys = lambda: real | {"report_compiler_agent"}
|
|
errs = lint.run_checks()
|
|
self.assertTrue(any("I3" in e for e in errs),
|
|
f"expected an I3 leak error; got {errs}")
|
|
|
|
def test_I4_empty_globs_fails(self):
|
|
real = self._real_manifest()
|
|
import copy
|
|
mutated = copy.deepcopy(real)
|
|
first = sorted(mutated["agents"])[0]
|
|
mutated["agents"][first]["allowed_write_globs"] = []
|
|
lint.load_manifest = lambda: mutated
|
|
self._assert_fails("I4")
|
|
|
|
def test_I4_wrong_bucket_fails(self):
|
|
real = self._real_manifest()
|
|
import copy
|
|
mutated = copy.deepcopy(real)
|
|
first = sorted(mutated["agents"])[0]
|
|
mutated["agents"][first]["bucket"] = "B"
|
|
lint.load_manifest = lambda: mutated
|
|
self._assert_fails("I4")
|
|
|
|
def test_I5_undeclared_agent_on_disk_fails(self):
|
|
# A real agent file dropped from BOTH rosters must be caught by the filesystem
|
|
# exhaustiveness glob (NON-vacuous guard): the hook would fail OPEN for it.
|
|
lint.BUCKET_A_AGENT_FILES = self._real_a[:-1] # drop one Bucket A file from roster
|
|
# (it still exists on disk, so I5's filesystem glob must flag it as undeclared)
|
|
errs = lint.run_checks()
|
|
self.assertTrue(any("I5" in e for e in errs),
|
|
f"expected an I5 undeclared-agent error; got {errs}")
|
|
|
|
def test_I5_stale_roster_entry_fails(self):
|
|
# A roster entry pointing at a non-existent file is a stale entry.
|
|
lint.BUCKET_A_AGENT_FILES = self._real_a + ["deep-research/agents/ghost_agent.md"]
|
|
errs = lint.run_checks()
|
|
self.assertTrue(any("I5" in e for e in errs),
|
|
f"expected an I5 stale-entry error; got {errs}")
|
|
|
|
|
|
class I5DepthAndSymlinkTest(unittest.TestCase):
|
|
"""I5 must (a) catch an agent dir nested DEEPER than one level, and
|
|
(b) NOT false-flag the plugin-root `agents/` mirror dir — real
|
|
byte-identical copies since #413 (symlinks before that; both file kinds
|
|
map back to the rostered deep-research source). Runs run_checks()
|
|
against a synthetic REPO_ROOT so the real repo is untouched."""
|
|
|
|
def setUp(self):
|
|
self._real_root = lint.REPO_ROOT
|
|
self._real_a = list(lint.BUCKET_A_AGENT_FILES)
|
|
self._real_bcd = list(lint.BUCKET_BCD_AGENT_FILES)
|
|
self._real_keys = lint.load_manifest_keys
|
|
self._real_manifest = lint.load_manifest
|
|
self._real_name = lint.read_frontmatter_name
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self._tmp.name).resolve()
|
|
lint.REPO_ROOT = self.root
|
|
|
|
def tearDown(self):
|
|
lint.REPO_ROOT = self._real_root
|
|
lint.BUCKET_A_AGENT_FILES = self._real_a
|
|
lint.BUCKET_BCD_AGENT_FILES = self._real_bcd
|
|
lint.load_manifest_keys = self._real_keys
|
|
lint.load_manifest = self._real_manifest
|
|
lint.read_frontmatter_name = self._real_name
|
|
self._tmp.cleanup()
|
|
|
|
def _write_agent(self, rel, name):
|
|
p = self.root / rel
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_text(f"---\nname: {name}\n---\nbody\n", encoding="utf-8")
|
|
return p
|
|
|
|
def _stub_loaders_to(self, a_files, bcd_files, manifest_keys):
|
|
# Point every non-I5 invariant at consistent synthetic data so ONLY I5 can react.
|
|
lint.BUCKET_A_AGENT_FILES = list(a_files)
|
|
lint.BUCKET_BCD_AGENT_FILES = list(bcd_files)
|
|
agents = {k: {"bucket": "A", "phase": "1", "allowed_write_globs": ["phase1_*/**"]}
|
|
for k in manifest_keys}
|
|
lint.load_manifest = lambda: {"agents": agents}
|
|
lint.load_manifest_keys = lambda: set(manifest_keys)
|
|
|
|
def test_root_agents_materialized_copy_not_flagged(self):
|
|
# #413: plugin-root agents/ holds REAL byte-identical copies (relative
|
|
# symlinks broke Windows checkouts / zip installs). I5 maps a root
|
|
# agents/ file to its deep-research/agents/<name> source BY NAME and
|
|
# must not report it as undeclared — byte-equality with that source is
|
|
# check_agents_mirror_sync.py's invariant, not I5's.
|
|
real = self._write_agent("deep-research/agents/x_agent.md", "x_agent")
|
|
agg = self.root / "agents"
|
|
agg.mkdir()
|
|
(agg / "x_agent.md").write_bytes(real.read_bytes())
|
|
self._stub_loaders_to(["deep-research/agents/x_agent.md"], [], ["x_agent"])
|
|
errs = lint.run_checks()
|
|
self.assertFalse(any("I5" in e for e in errs),
|
|
f"root agents/ mirror copy must NOT be I5-undeclared; got {errs}")
|
|
|
|
def test_root_agents_copy_without_rostered_source_is_flagged(self):
|
|
# The name-mapping must not become a blanket allowlist: a file dropped
|
|
# into root agents/ whose name maps to NO rostered deep-research
|
|
# source is still the fail-open case I5 exists to catch.
|
|
self._write_agent("deep-research/agents/x_agent.md", "x_agent")
|
|
self._write_agent("agents/rogue_agent.md", "rogue_agent")
|
|
self._stub_loaders_to(["deep-research/agents/x_agent.md"], [], ["x_agent"])
|
|
errs = lint.run_checks()
|
|
i5 = [e for e in errs if "I5" in e]
|
|
self.assertTrue(any("rogue_agent" in e for e in i5),
|
|
f"unrostered root agents/ file must trigger I5; got {errs}")
|
|
|
|
def test_root_agents_copy_of_non_deep_research_source_fails_closed(self):
|
|
# The by-name mapping points ONLY at deep-research/agents/ — it must
|
|
# not shadow-match a same-named rostered agent living elsewhere. A
|
|
# mirror of e.g. academic-paper/agents/y_agent.md maps to the
|
|
# (unrostered) deep-research path and is flagged: fail-CLOSED, which
|
|
# is the documented lockstep-edit prompt, never a silent pass.
|
|
real = self._write_agent("academic-paper/agents/y_agent.md", "y_agent")
|
|
agg = self.root / "agents"
|
|
agg.mkdir()
|
|
(agg / "y_agent.md").write_bytes(real.read_bytes())
|
|
self._stub_loaders_to(["academic-paper/agents/y_agent.md"], [], ["y_agent"])
|
|
errs = lint.run_checks()
|
|
i5 = [e for e in errs if "I5" in e]
|
|
self.assertTrue(any("agents/y_agent.md" in e for e in i5),
|
|
f"non-deep-research mirror must fail closed; got {errs}")
|
|
|
|
def test_nested_dir_under_root_agents_is_not_remapped(self):
|
|
# codex review (#413 round, P2): the mirror remap applies ONLY to
|
|
# DIRECT children of root agents/. A nested agents/sub/agents/x.md
|
|
# whose NAME collides with a rostered deep-research agent must still
|
|
# be flagged — remapping it would reopen the fail-open case the
|
|
# recursive glob exists to catch.
|
|
self._write_agent("deep-research/agents/x_agent.md", "x_agent")
|
|
self._write_agent("agents/sub/agents/x_agent.md", "rogue")
|
|
self._stub_loaders_to(["deep-research/agents/x_agent.md"], [], ["x_agent"])
|
|
errs = lint.run_checks()
|
|
i5 = [e for e in errs if "I5" in e]
|
|
self.assertTrue(any("agents/sub/agents/x_agent.md" in e for e in i5),
|
|
f"nested file under root agents/ must not be remapped; got {errs}")
|
|
|
|
def test_root_agents_symlink_aggregate_not_flagged(self):
|
|
# Legacy/transition pin (pre-#413 file kind): a symlink in root
|
|
# agents/ maps back the same way and must not be flagged.
|
|
real = self._write_agent("deep-research/agents/x_agent.md", "x_agent")
|
|
agg = self.root / "agents"
|
|
agg.mkdir()
|
|
try:
|
|
os.symlink(real, agg / "x_agent.md")
|
|
except OSError:
|
|
self.skipTest("symlinks unavailable on this platform")
|
|
# roster sizes are checked by I1; bypass that by patching the size expectation is not
|
|
# possible, so just assert no I5 error specifically.
|
|
self._stub_loaders_to(["deep-research/agents/x_agent.md"], [], ["x_agent"])
|
|
errs = lint.run_checks()
|
|
self.assertFalse(any("I5" in e for e in errs),
|
|
f"root agents/ symlink must NOT be I5-undeclared; got {errs}")
|
|
|
|
def test_nested_agents_dir_undeclared_is_caught(self):
|
|
# A genuinely new standalone agent file nested two levels deep, absent from the
|
|
# roster, MUST be flagged — the one-level glob would have silently missed it.
|
|
self._write_agent("deep-research/agents/x_agent.md", "x_agent")
|
|
self._write_agent("skill/sub/agents/sneaky_agent.md", "sneaky_agent") # nested, undeclared
|
|
self._stub_loaders_to(["deep-research/agents/x_agent.md"], [], ["x_agent"])
|
|
errs = lint.run_checks()
|
|
i5 = [e for e in errs if "I5" in e]
|
|
self.assertTrue(i5, f"nested undeclared agent must trigger I5; got {errs}")
|
|
self.assertTrue(any("sneaky_agent" in e for e in i5),
|
|
f"I5 error should name the nested file; got {i5}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|