diff --git a/.claude/skills/eval-cs-skill/SKILL.md b/.claude/skills/eval-cs-skill/SKILL.md index f2aed03..dd7655e 100644 --- a/.claude/skills/eval-cs-skill/SKILL.md +++ b/.claude/skills/eval-cs-skill/SKILL.md @@ -118,6 +118,32 @@ python3 {skill_dir}/scripts/bump_version.py --to X.Y.Z `{skill_dir}` = 本 SKILL.md 所在目录。工具随 skill 包分发，不复制到 `.codestable/tools/`。 +### 项目内学习迁移实验 + +`execution_mode: learning-transfer` 是维护者侧实验，不在用户任务中构造样本或运行模型。它用独立 +`sequence.py` 执行 A -> fresh `cs-keep` -> treatment/control fresh B；每次 invocation 只注入 owning +skill 的冻结快照，B 两侧 prompt 相同，唯一处理变量必须是经 schema 校验的项目 lesson。 + +真实运行前必须同时满足：实验输入与 hypothesis 已提交；checkpoint fingerprint 绑定 config、fixture、 +skill 快照、seed、完整 pipeline、target、`k` 与 run identity；golden/naive preflight 成立；显式 +`family/harness/model` target 至少覆盖两个 model family；每个 target 的外部 sandbox 已通过宿主与 +sibling cell 读取、宿主写入隔离探针，且宿主配置与已知会话状态快照保持不变。 +输入 commit 后先做最小真实 target 探针，再提交含 `source_commit` 和逐 target 布尔 oracle 的 frozen +attestation；attestation 只接受注册字段，不得记录 prompt、输出、路径、sentinel 或 session id。 +prepared manifest 不能运行真实 sequence。 +失配的 checkpoint 拒绝恢复；校准保留完整 fixtures/model families，只缩小 +`k`，并与最终运行使用不同 run identity；`learning_transfer` scorer 不可替换。 + +Deterministic pipeline / fixture / oracle 失败永久阻断 structural integrity；adapter / transport 故障 +单列为 operational error。成功重试保留尝试与成本但不永久污染 integrity，未解决时保持 +`[underpowered]`；只有完整 pair 才能把历史 operational error 标为 resolved。 + +每次 provider 调用前，append-only checkpoint 必须用新的 invocation ID 先 `flush + fsync` start 与 +`[soft]` fallback，再追加同 ID terminal metrics；中断时 reducer 保留 fallback、成本和 unresolved +attempt。`--fresh` 只允许删除 header-only journal，其他运行证据与已有结果都必须换新 `--out`。每个 +cell repo 在 oracle 后销毁；checkpoint 只保留结构化 phase、指标与哈希，不保留原始子进程输出， +也不得持久化 transcript、完整 treatment/control repo，或自动导出、上传、改写 skill。 + --- ## 认知诚实纪律（硬约束） @@ -126,6 +152,8 @@ python3 {skill_dir}/scripts/bump_version.py --to X.Y.Z - `hypotheses.md` 冻结后**先 git commit 再跑任何 LLM**；provenance 由 `tests/test_cs_skill_convergence.py` 机械校验。 - 禁止裸 `V_instance = 0.XX` 自评分；收敛判据见 optimize 协议。 - 跨模型 ≥2；judge 模型须独立于被测模型（避免同源偏差）。 +- learning-transfer 校准可用 `k=2`，但接受证据要求每 fixture、每 family `k>=5`；任一 primary + aggregate 为 `[underpowered]` 时扩大样本后重跑，不得把校准结果混入最终结论。 --- diff --git a/.claude/skills/eval-cs-skill/references/eval/protocol.md b/.claude/skills/eval-cs-skill/references/eval/protocol.md index 52d0861..6eac364 100644 --- a/.claude/skills/eval-cs-skill/references/eval/protocol.md +++ b/.claude/skills/eval-cs-skill/references/eval/protocol.md @@ -56,6 +56,17 @@ 写 `hypotheses.md`（`H-: metric ≥ threshold`），**先 git commit 再跑任何 LLM**——provenance 由 `tests/test_cs_skill_convergence.py` 校验。 +`learning-transfer` 还要生成 `freeze.json`，冻结 config、fixtures、A/hidden/regression checks、hook、 +seed builder、owning skill 快照、完整 pipeline 与显式 execution targets。先提交这些输入，再对每个 +target 做不保留输出的最小真实探针；通过后把该输入 commit 写为 `source_commit`，将 manifest 置为 +`frozen` 并单独提交逐 target 布尔 attestation。探针必须确认宿主配置与已知会话状态不变；attestation +只接受注册字段，不记录 prompt、回答、路径、sentinel 或 session id。真实 +sequence 会同时核对当前字节、source commit blob 与 HEAD 中的 +manifest，prepared/pending 状态不能调用模型。checkpoint header 以这些 +输入、`k` 和 run identity 计算 fingerprint；失配直接拒绝恢复。`k=2` 校准和 `k>=5` 最终运行必须 +使用不同 run identity，不能合并 checkpoint 或结果；校准必须保留完整 fixtures 和 model families， +只按比例缩小 `k`。 + ## 4. 跑评测 ```bash @@ -69,6 +80,28 @@ python3 {skill_dir}/scripts/runner.py --experiment experiments/{skill}-{NNN} --h 分层省钱：确定性 scorer（planted_defect / dod_gate）先跑，`llm_judge` 只在候选变体上跑；cheap model 探路，贵 model 只做终判。 +### learning-transfer sequence + +项目 lesson 的跨会话迁移使用 `answerType/task.kind: learning-transfer`。每个 pair 从同一 post-A repo +重建 treatment/control，只在 treatment 注入 fresh `cs-keep` 生成且严格校验的 observed lesson； +可选 hook 对两侧对称执行。A、curation、B 两侧都是 fresh invocation，B prompt hash 必须相同。 + +模型调用前必须通过 fixture schema、seed Epic 授权状态、golden/naive 可解性、资产 containment、repo +symlink、外部 sandbox 与 target 探针；探针必须验证当前 cell 可写、宿主与 sibling cell 不可读、宿主 +不可写。A/B 变化同时检查业务 manifest 和 Git HEAD/index/config/hooks；deterministic subprocess 使用 +最小环境、有界超时，只保留状态与输出哈希。候选按 A 前后 delta 提取，lesson parser 要拒绝额外字段、 +重复字段、非法日期/slug/归宿和超过三条 evidence；窄迁移只允许 status 与一条代表性 evidence 变化。 + +完整 campaign 至少覆盖四个 task skills、unrelated 与 stale guard、两个 model family、每 fixture 每 +family `k>=5`。25pp 作用于两个 family 与四个正向 fixture 的总体 paired delta，且每个 family 必须 +为正、losses 不多于 wins、两 guard 无回退、所有隔离/schema/mutation oracle 100% 通过。 +Deterministic failure 永久阻断 structural integrity；retryable adapter/transport error 保留尝试与成本， +只有后续同 cell 形成完整 pair 才单列为 resolved，未解决时保持 incomplete / `[underpowered]`。 +provider 前须以新 invocation ID durable append start 与 soft fallback，terminal metrics 只追加不覆盖； +中断或半写 terminal 仍保留一次尝试与 fallback 成本。`--fresh` 只允许 header-only journal；任何 +invocation、score、error、fixture-invalid 或已有结果都要求新的 `--out`。半 pair 不进入效果均值； +cell repo 在 oracle 后销毁。 + ## 5. 读结果 `artifacts/analysis/exp-{name}-results.json`： diff --git a/.claude/skills/eval-cs-skill/scripts/_model.py b/.claude/skills/eval-cs-skill/scripts/_model.py index e1b564e..59bd607 100644 --- a/.claude/skills/eval-cs-skill/scripts/_model.py +++ b/.claude/skills/eval-cs-skill/scripts/_model.py @@ -12,6 +12,7 @@ import sys sys.dont_write_bytecode = True # 不污染 plugin 包 import json +import re from dataclasses import dataclass, field, asdict from pathlib import Path from typing import Any @@ -20,6 +21,31 @@ MEASURED = "measured" SOFT = "soft" UNDERPOWERED = "underpowered" _TAGS = {MEASURED, SOFT, UNDERPOWERED} +_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$") + + +def is_safe_slug(value: object) -> bool: + """可安全进入 artifact 路径的稳定 ID。""" + return isinstance(value, str) and _SLUG_RE.fullmatch(value) is not None + + +@dataclass(frozen=True) +class ExecutionTarget: + """一个真实执行目标：model family、harness 与 model 的显式绑定。""" + + id: str + family: str + harness: str + model: str + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "ExecutionTarget": + missing = [key for key in ("id", "family", "harness", "model") if not data.get(key)] + if missing: + raise ValueError(f"model target 缺字段 {missing}") + if not is_safe_slug(data["id"]): + raise ValueError("model target id 必须是小写连字符 slug（最多 64 字符）") + return cls(**{key: str(data[key]) for key in ("id", "family", "harness", "model")}) def tagged(value: Any, tag: str, evidence: str | None = None) -> dict[str, Any]: @@ -58,6 +84,8 @@ class Fixture: missing = [k for k in ("id", "answerType") if k not in data] if missing: raise ValueError(f"fixture 缺字段 {missing}: {data.get('id', '')}") + if not is_safe_slug(data["id"]): + raise ValueError("fixture id 必须是小写连字符 slug（最多 64 字符）") return cls( id=str(data["id"]), answer_type=str(data["answerType"]), diff --git a/.claude/skills/eval-cs-skill/scripts/buildprompt.py b/.claude/skills/eval-cs-skill/scripts/buildprompt.py index 5142443..8487ff6 100644 --- a/.claude/skills/eval-cs-skill/scripts/buildprompt.py +++ b/.claude/skills/eval-cs-skill/scripts/buildprompt.py @@ -210,6 +210,49 @@ def build_e2e_prompt(fixture: Fixture, variant_text: str) -> str: return "\n".join(parts) +def build_sequence_task_prompt(fixture: Fixture, variant_text: str, phase: str) -> str: + """构建 learning-transfer 的 A/B 任务 prompt；B 不接收任何候选或 treatment 信息。""" + if phase not in {"a", "b"}: + raise ValueError(f"sequence task phase 非法: {phase!r}") + scenario = (fixture.raw or {}).get("scenario") or {} + task = scenario.get(phase) or {} + return "\n".join([ + _INTRO, + "===== SKILL.md 开始 =====", + variant_text.strip(), + "===== SKILL.md 结束 =====\n", + "你在一个已 onboard 的真实仓库工作目录中（当前目录即仓库根）。", + "按该 skill 的流程直接修改文件、运行必要验证，并按其收尾契约报告。", + "\n## 用户请求", + str(task.get("request", "")).strip(), + ]) + + +def build_curation_prompt( + fixture: Fixture, + keep_text: str, + candidate: str, + evidence: str, +) -> str: + """把 A 的唯一候选交给 fresh cs-keep；授权只覆盖精确 lesson 写入。""" + scenario = (fixture.raw or {}).get("scenario") or {} + expected_home = (scenario.get("candidate") or {}).get("expected_home") + return "\n".join([ + _INTRO, + "===== SKILL.md 开始 =====", + keep_text.strip(), + "===== SKILL.md 结束 =====\n", + "fixture 预检已确认这个候选属于 lesson 类归宿，不属于 attention 或 ADR。" + if expected_home == "lesson" else "fixture 归宿预检未通过。", + "用户现对下面这条精确内容给出显式授权：请记录为 observed lesson。", + "不得修改业务代码，不得扩大规则、scope、写入或上传授权。", + "\n## 候选", + candidate.strip(), + "\n## 可追溯证据", + evidence.strip(), + ]) + + _BUILDERS = { "review": build_review_prompt, "fix": build_fix_prompt, diff --git a/.claude/skills/eval-cs-skill/scripts/config.py b/.claude/skills/eval-cs-skill/scripts/config.py index 7d0a920..dce8c07 100644 --- a/.claude/skills/eval-cs-skill/scripts/config.py +++ b/.claude/skills/eval-cs-skill/scripts/config.py @@ -9,6 +9,8 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any +from _model import ExecutionTarget, is_safe_slug + @dataclass class ExperimentConfig: @@ -23,8 +25,21 @@ class ExperimentConfig: budget_usd: float = 50.0 judge_model: str | None = None # llm_judge 用，需独立于被测 model inject_context: bool = False # True=prompt 里补齐 onboard 上下文，公平测「主路径」而非 bare-input + execution_mode: str = "one-shot" + model_targets: list[ExecutionTarget | dict[str, Any]] = field(default_factory=list) raw: dict[str, Any] = field(default_factory=dict) + def __post_init__(self) -> None: + if not is_safe_slug(self.name): + raise ValueError("experiment name 必须是小写连字符 slug（最多 64 字符）") + self.model_targets = [ + target if isinstance(target, ExecutionTarget) else ExecutionTarget.from_dict(target) + for target in self.model_targets + ] + ids = [target.id for target in self.model_targets] + if len(ids) != len(set(ids)): + raise ValueError("model target id 必须唯一") + @classmethod def from_dict(cls, data: dict[str, Any]) -> "ExperimentConfig": for key in ("name", "skill_under_test"): @@ -33,6 +48,7 @@ class ExperimentConfig: known = { "name", "skill_under_test", "variants", "model_list", "k", "harnesses", "scorers", "fixture_classes", "budget_usd", "judge_model", "inject_context", + "execution_mode", "model_targets", } return cls( name=data["name"], @@ -46,6 +62,8 @@ class ExperimentConfig: budget_usd=float(data.get("budget_usd", 50.0)), judge_model=data.get("judge_model"), inject_context=bool(data.get("inject_context", False)), + execution_mode=str(data.get("execution_mode", "one-shot")), + model_targets=list(data.get("model_targets", [])), raw={k: v for k, v in data.items() if k not in known}, ) @@ -80,8 +98,10 @@ def repo_root(start: Path | None = None) -> Path: return Path.cwd() -def _skill_md(root: Path, skill: str) -> Path: +def resolve_skill_path(root: Path, skill: str) -> Path: """在两个 skill 根里找被测 skill 的 SKILL.md：shipped 插件 + 项目级 dev skill。""" + if not is_safe_slug(skill): + raise ValueError(f"skill 名称不是安全 slug: {skill!r}") roots = [root / "plugins" / "codestable" / "skills", root / ".claude" / "skills"] for sr in roots: p = sr / skill / "SKILL.md" @@ -90,6 +110,19 @@ def _skill_md(root: Path, skill: str) -> Path: return roots[0] / skill / "SKILL.md" # fallback（报错更清晰） +def select_execution_targets( + config: ExperimentConfig, + harness: str | None = None, + model: str | None = None, +) -> list[ExecutionTarget]: + """按 CLI 限定筛选显式 model target，不重新组合 harness/model。""" + return [ + target for target in config.model_targets + if (not harness or target.harness == harness) + and (not model or target.model == model) + ] + + def resolve_variant_text(config: ExperimentConfig, variant: str, root: Path, exp_dir: Path | None = None) -> str: """把 variant 名解析成 SKILL.md 快照文本。 @@ -98,7 +131,7 @@ def resolve_variant_text(config: ExperimentConfig, variant: str, root: Path, 支持 shipped 插件（plugins/codestable/skills/）与项目级 dev skill（.claude/skills/，如 eval-cs-skill 自指）。 - 其它 → optimize 产生的候选；优先 `/variants/.md`，再退回 `/experiments//variants/.md`。 """ - baseline = _skill_md(root, config.skill_under_test) + baseline = resolve_skill_path(root, config.skill_under_test) if variant == "baseline": path = baseline else: diff --git a/.claude/skills/eval-cs-skill/scripts/e2e_env.py b/.claude/skills/eval-cs-skill/scripts/e2e_env.py index 3897e0e..4fcc02e 100644 --- a/.claude/skills/eval-cs-skill/scripts/e2e_env.py +++ b/.claude/skills/eval-cs-skill/scripts/e2e_env.py @@ -9,11 +9,290 @@ from __future__ import annotations import subprocess import sys +import hashlib +import fnmatch +import os +import re +import shutil +import tempfile from pathlib import Path +from _model import is_safe_slug + sys.dont_write_bytecode = True +_MANIFEST_IGNORED_PARTS = {"__pycache__", ".pytest_cache"} +_UNSAFE_GIT_CONTROL_PATHS = ( + "config.worktree", + "commondir", + "objects/info/alternates", + "objects/info/http-alternates", +) +_SAFE_HOST_ENV_KEYS = ("PATH", "LANG", "LC_ALL", "SYSTEMROOT") + + +def isolated_subprocess_env(runtime: Path, *, pythonpath: Path | None = None) -> dict[str, str]: + """Build a minimal deterministic-process environment without host credentials or injection.""" + home = runtime / "home" + tmpdir = runtime / "tmp" + home.mkdir(parents=True, exist_ok=True) + tmpdir.mkdir(parents=True, exist_ok=True) + env = {key: os.environ[key] for key in _SAFE_HOST_ENV_KEYS if key in os.environ} + env.setdefault("PATH", os.defpath) + env.update({ + "HOME": str(home), + "TMPDIR": str(tmpdir), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", + "GIT_CONFIG_COUNT": "0", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + }) + if pythonpath is not None: + env["PYTHONPATH"] = str(pythonpath.resolve()) + return env + + +def reject_repo_symlinks(repo: Path) -> None: + """实验 repo 禁止 symlink，避免 manifest/copy 越过 cell 边界。""" + for path in repo.rglob("*"): + if path.is_symlink(): + raise ValueError(f"learning-transfer repo 禁止 symlink: {path.relative_to(repo)}") + + +def repo_manifest(repo: Path) -> dict[str, str]: + """返回业务可见文件的相对路径→SHA-256；忽略 VCS 与测试缓存噪音。""" + reject_repo_symlinks(repo) + manifest: dict[str, str] = {} + for path in sorted(repo.rglob("*")): + relative = path.relative_to(repo) + ignored = ( + relative.parts[0] == ".git" + or _MANIFEST_IGNORED_PARTS & set(relative.parts) + ) + if not path.is_file() or ignored: + continue + manifest[relative.as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + return manifest + + +def _hash_optional_file(path: Path) -> str | None: + if not path.exists(): + return None + if not path.is_file() or path.is_symlink(): + raise ValueError(f"git control path 不是普通文件: {path}") + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _hash_tree(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + manifest: dict[str, str] = {} + for child in sorted(path.rglob("*")): + if child.is_symlink(): + raise ValueError(f"git hooks 禁止 symlink: {child}") + if child.is_file(): + mode = child.stat().st_mode & 0o777 + digest = hashlib.sha256(child.read_bytes()).hexdigest() + manifest[child.relative_to(path).as_posix()] = f"{mode:o}:{digest}" + return manifest + + +def _git_output(repo: Path, *args: str) -> str: + try: + with tempfile.TemporaryDirectory(prefix="cs-eval-git-", dir=repo.parent) as tmp: + result = subprocess.run( + ["git", *args], + cwd=repo, + capture_output=True, + text=True, + check=False, + timeout=10, + env=_isolated_git_env(Path(tmp)), + ) + except subprocess.TimeoutExpired as exc: + raise RuntimeError(f"git {' '.join(args)} 超时") from exc + if result.returncode != 0: + raise RuntimeError(f"git {' '.join(args)} 失败: {result.stderr[-300:]}") + return result.stdout.strip() + + +def _isolated_git_env(runtime: Path) -> dict[str, str]: + """只读取 cell 的 local config，不继承宿主仓库定位与配置注入。""" + return isolated_subprocess_env(runtime) + + +def _local_config_has_includes(path: Path) -> bool: + """不启动 Git，先从物理 local config 拒绝 include/includeIf。""" + if not path.is_file() or path.is_symlink(): + return True + try: + lines = path.read_text(encoding="utf-8-sig").splitlines() + except (OSError, UnicodeError): + return True + section_pattern = re.compile(r"^\[\s*include(?:if\b[^]]*)?\s*]", re.IGNORECASE) + dotted_pattern = re.compile(r"^include(?:if\.[^.]+)?\.path\s*=", re.IGNORECASE) + for raw in lines: + line = raw.lstrip() + if not line or line.startswith(("#", ";")): + continue + if section_pattern.match(line) or dotted_pattern.match(line): + return True + return False + + +def repo_control_snapshot(repo: Path) -> dict[str, object]: + """冻结 agent 不得改写的 Git HEAD、index、local config 与 hooks。""" + if repo.is_symlink(): + return {"repository": True, "safe": False, "git_layout": "repo-symlink"} + repo_root = repo.resolve() + physical_git_dir = repo_root / ".git" + if physical_git_dir.is_symlink(): + return { + "repository": True, + "safe": False, + "git_layout": hashlib.sha256(str(physical_git_dir).encode("utf-8")).hexdigest(), + } + if not physical_git_dir.exists(): + return {"repository": False, "safe": True} + if not physical_git_dir.is_dir(): + return { + "repository": True, + "safe": False, + "git_layout": hashlib.sha256(str(physical_git_dir).encode("utf-8")).hexdigest(), + } + + config_file = physical_git_dir / "config" + try: + local_config = _hash_optional_file(config_file) + except ValueError: + local_config = None + if local_config is None or _local_config_has_includes(config_file): + return { + "repository": True, + "safe": False, + "local_config": local_config, + } + unsafe_control_paths = [ + relative + for relative in _UNSAFE_GIT_CONTROL_PATHS + if (physical_git_dir / relative).exists() + or (physical_git_dir / relative).is_symlink() + ] + if unsafe_control_paths: + return { + "repository": True, + "safe": False, + "local_config": local_config, + "unsafe_control_paths": unsafe_control_paths, + } + + git_dir = Path(_git_output(repo, "rev-parse", "--absolute-git-dir")).resolve() + if git_dir != physical_git_dir.resolve(): + return { + "repository": True, + "safe": False, + "external_git_dir": hashlib.sha256(str(git_dir).encode("utf-8")).hexdigest(), + "local_config": local_config, + } + + def git_path(name: str) -> tuple[Path | None, str]: + raw = _git_output(repo, "rev-parse", "--git-path", name) + path = Path(raw) + resolved = (path if path.is_absolute() else repo / path).resolve() + safe = False + for base in (repo_root, git_dir): + try: + resolved.relative_to(base) + safe = True + break + except ValueError: + continue + return (resolved if safe else None), hashlib.sha256(raw.encode("utf-8")).hexdigest() + + index_path, index_location = git_path("index") + config_path, config_location = git_path("config") + hooks_path, hooks_location = git_path("hooks") + control_paths_safe = ( + index_path is not None + and config_path is not None + and hooks_path is not None + and config_path == config_file.resolve() + ) + + return { + "repository": True, + "safe": control_paths_safe, + "head": _git_output(repo, "rev-parse", "--verify", "HEAD"), + "head_ref": _git_output(repo, "rev-parse", "--symbolic-full-name", "HEAD"), + "index_location": index_location, + "index": _hash_optional_file(index_path) if index_path is not None else None, + "config_location": config_location, + "local_config": local_config, + "hooks_location": hooks_location, + "hooks": _hash_tree(hooks_path) if hooks_path is not None else None, + } + + +def repo_control_unchanged(before: dict[str, object], after: dict[str, object]) -> bool: + """只有初始/最终控制面都位于 cell 内且字节相同才算未改写。""" + return before.get("safe") is True and after == before + + +def copy_repo(source: Path, destination: Path) -> Path: + """完整复制一个 repo；manifest 调用方负责验证业务内容完全相同。""" + reject_repo_symlinks(source) + shutil.copytree(source, destination, symlinks=True) + reject_repo_symlinks(destination) + return destination + + +def changed_paths(before: dict[str, str], after: dict[str, str]) -> set[str]: + """比较两个 manifest，包含新增、修改和删除路径。""" + return { + path for path in set(before) | set(after) + if before.get(path) != after.get(path) + } + + +def paths_match_allowlist(paths: set[str], patterns: list[str]) -> bool: + """所有变化都必须命中至少一个声明的相对路径 glob。""" + return all(any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns) for path in paths) + + +def build_seed_repo(seed: str, destination: Path, root: Path) -> Path: + """用 tracked seed builder 构建一个全新的独立仓库。""" + if not is_safe_slug(seed): + raise ValueError(f"seed 必须是安全 slug: {seed!r}") + destination.parent.mkdir(parents=True, exist_ok=True) + seed_root = (root / "experiments" / "seeds").resolve() + build_script = (seed_root / seed / "build-seed.py").resolve() + try: + build_script.relative_to(seed_root) + except ValueError as exc: + raise ValueError(f"seed builder 越过 experiments/seeds: {seed!r}") from exc + if not build_script.is_file() or (seed_root / seed / "build-seed.py").is_symlink(): + raise ValueError(f"seed builder 不存在或为 symlink: {seed!r}") + destination.parent.mkdir(parents=True, exist_ok=True) + try: + with tempfile.TemporaryDirectory(prefix="cs-eval-seed-", dir=destination.parent) as tmp: + result = subprocess.run( + [sys.executable, "-I", "-B", str(build_script), "--out", str(destination)], + cwd=root, + env=isolated_subprocess_env(Path(tmp), pythonpath=root), + capture_output=True, + text=True, + check=False, + timeout=60, + ) + except subprocess.TimeoutExpired as exc: + raise RuntimeError(f"build-seed.py 超时 (seed={seed})") from exc + if result.returncode != 0: + raise RuntimeError(f"build-seed.py 失败 (seed={seed}):\n{result.stderr[-500:]}") + return destination + + def prepare_e2e_workdir(fixture, tmp: str, exp_dir: Path) -> Path: """构建 seed 仓库 + 注入 bug，返回 repo 目录路径。 @@ -26,23 +305,21 @@ def prepare_e2e_workdir(fixture, tmp: str, exp_dir: Path) -> Path: bug_id = scenario.get("bug_id") # feature 场景无 bug 注入 repo = Path(tmp) / "repo" - build_script = Path("experiments") / "seeds" / seed / "build-seed.py" - - result = subprocess.run( - [sys.executable, str(build_script), "--out", str(repo)], - capture_output=True, text=True, - ) - if result.returncode != 0: - raise RuntimeError( - f"build-seed.py 失败 (seed={seed}):\n{result.stderr[:500]}" - ) + root = Path.cwd() + build_seed_repo(seed, repo, root) inject_script = (exp_dir / "bugs" / bug_id / "inject.py") if bug_id else None if inject_script and inject_script.exists(): - result2 = subprocess.run( - [sys.executable, str(inject_script), str(repo)], - capture_output=True, text=True, - ) + with tempfile.TemporaryDirectory(prefix="cs-eval-inject-", dir=repo.parent) as tmp: + result2 = subprocess.run( + [sys.executable, "-I", "-B", str(inject_script), str(repo)], + cwd=root, + env=isolated_subprocess_env(Path(tmp), pythonpath=root), + capture_output=True, + text=True, + check=False, + timeout=60, + ) if result2.returncode != 0: raise RuntimeError( f"inject.py 失败 (bug_id={bug_id}):\n{result2.stderr[:500]}" diff --git a/.claude/skills/eval-cs-skill/scripts/fixtures.py b/.claude/skills/eval-cs-skill/scripts/fixtures.py index 01ff14f..6eb51fc 100644 --- a/.claude/skills/eval-cs-skill/scripts/fixtures.py +++ b/.claude/skills/eval-cs-skill/scripts/fixtures.py @@ -4,9 +4,33 @@ from __future__ import annotations import json -from pathlib import Path +from fnmatch import fnmatchcase +from pathlib import Path, PurePosixPath -from _model import Fixture +from _model import Fixture, is_safe_slug + + +def _safe_relative_path(value: object) -> bool: + if not isinstance(value, str) or not value or value.startswith("~") or "\\" in value: + return False + path = PurePosixPath(value) + return not path.is_absolute() and ".." not in path.parts + + +def _check_relative_paths(problems: list[str], label: str, values: object) -> None: + if not isinstance(values, list): + return + for value in values: + if not _safe_relative_path(value): + problems.append(f"{label} 只允许仓库内相对路径: {value!r}") + + +def _allowlist_reaches_lessons(values: object) -> bool: + return isinstance(values, list) and any( + isinstance(value, str) + and fnmatchcase(".codestable/lessons/2026-08-02-example.md", value) + for value in values + ) def load_fixtures(experiment_dir: Path, classes: list[str]) -> list[Fixture]: @@ -18,6 +42,9 @@ def load_fixtures(experiment_dir: Path, classes: list[str]) -> list[Fixture]: continue for path in sorted(cls_dir.glob("*.json")): data = json.loads(path.read_text(encoding="utf-8")) + problems = validate_fixture_dict(data) + if problems: + raise ValueError(f"fixture {path} 不合规: {'; '.join(problems)}") data["_exp_dir"] = str(experiment_dir) # scorer 需要实验目录定位 hidden_tests fixture = Fixture.from_dict(data) if fixture.id in seen: @@ -27,13 +54,48 @@ def load_fixtures(experiment_dir: Path, classes: list[str]) -> list[Fixture]: return out +def resolve_experiment_asset(experiment_dir: Path, relative: str) -> Path: + """解析 tracked experiment asset，并拒绝逃逸、目录与 symlink。""" + if not _safe_relative_path(relative): + raise ValueError(f"实验资产只允许仓库内相对路径: {relative!r}") + base = experiment_dir.resolve() + path = (base / relative).resolve() + try: + path.relative_to(base) + except ValueError as exc: + raise ValueError(f"实验资产越过 experiment_dir: {relative!r}") from exc + if not path.is_file() or (base / relative).is_symlink(): + raise ValueError(f"实验资产不存在、不是文件或为 symlink: {relative!r}") + return path + + +def validate_learning_transfer_coverage(fixtures: list[Fixture]) -> list[str]: + """验收 campaign 必须覆盖四 task skill 与 unrelated/stale guards。""" + problems: list[str] = [] + learning = [fixture for fixture in fixtures if fixture.answer_type == "learning-transfer"] + positive = [fixture for fixture in learning if fixture.raw["scenario"]["class"] == "positive"] + skills = {(fixture.raw["scenario"].get("a") or {}).get("skill") for fixture in positive} + expected = {"cs-feat", "cs-issue", "cs-refactor", "cs-epic"} + if len(positive) != 4 or skills != expected: + problems.append("四个 positive fixture 必须与四个 task skill 一一对应") + classes = {fixture.raw["scenario"]["class"] for fixture in learning} + if not {"unrelated", "stale"} <= classes: + problems.append("learning-transfer 必须同时包含 unrelated 与 stale guard") + return problems + + def validate_fixture_dict(data: dict) -> list[str]: """返回问题列表，空=合规。供 tests 复用。""" problems: list[str] = [] if "id" not in data: problems.append("缺 id") + elif not is_safe_slug(data["id"]): + problems.append("id 必须是小写连字符 slug（最多 64 字符）") at = data.get("answerType") - if at not in {"findings-recall", "dod-gate", "dimensions-judge", "routing-decision", "e2e-outcome"}: + if at not in { + "findings-recall", "dod-gate", "dimensions-judge", "routing-decision", "e2e-outcome", + "learning-transfer", + }: problems.append(f"answerType 非法: {at!r}") if at == "findings-recall" and not data.get("answer"): problems.append("findings-recall 必须有非空 answer") @@ -57,6 +119,99 @@ def validate_fixture_dict(data: dict) -> list[str]: task = data.get("task") or {} if task.get("kind") != "e2e": problems.append("e2e-outcome task.kind 应为 'e2e'") + if at == "learning-transfer": + scenario = data.get("scenario") + if not isinstance(scenario, dict): + problems.append("learning-transfer 必须有 scenario dict") + else: + if scenario.get("class") not in {"positive", "unrelated", "stale"}: + problems.append("learning-transfer scenario.class 非法") + if not scenario.get("seed"): + problems.append("learning-transfer scenario 缺 'seed'") + elif not is_safe_slug(scenario["seed"]): + problems.append("learning-transfer seed 必须是小写连字符 slug") + for phase in ("a", "candidate", "b", "preflight"): + if not isinstance(scenario.get(phase), dict): + problems.append(f"learning-transfer scenario 缺 {phase!r} dict") + a = scenario.get("a") or {} + if a.get("skill") not in {"cs-feat", "cs-issue", "cs-refactor", "cs-epic"}: + problems.append("learning-transfer a.skill 非法") + if not a.get("request"): + problems.append("learning-transfer a.request 不能为空") + if a.get("candidate_source") not in {"output", "epic-cursor"}: + problems.append("learning-transfer a.candidate_source 非法") + if (a.get("skill") == "cs-epic") != (a.get("candidate_source") == "epic-cursor"): + problems.append("cs-epic 必须从 Epic 游标取候选，其他 task skill 必须从输出取候选") + if not isinstance(a.get("checks"), list): + problems.append("learning-transfer a.checks 必须是 list") + _check_relative_paths(problems, "learning-transfer a.checks", a.get("checks")) + if not isinstance(a.get("allowed_paths"), list) or not a.get("allowed_paths"): + problems.append("learning-transfer a.allowed_paths 必须是非空 list") + _check_relative_paths(problems, "learning-transfer a.allowed_paths", a.get("allowed_paths")) + if _allowlist_reaches_lessons(a.get("allowed_paths")): + problems.append("learning-transfer a.allowed_paths 不得允许 lesson mutation") + candidate = scenario.get("candidate") or {} + if candidate.get("expected_home") != "lesson": + problems.append("learning-transfer candidate.expected_home 必须是 lesson") + concepts = candidate.get("required_concepts") + if not isinstance(concepts, list) or not concepts: + problems.append("learning-transfer candidate.required_concepts 必须是非空 list") + elif ( + any(not isinstance(concept, str) or not concept.strip() for concept in concepts) + or len({concept.strip().lower() for concept in concepts}) != len(concepts) + ): + problems.append("learning-transfer candidate.required_concepts 必须是唯一的非空字符串") + b = scenario.get("b") or {} + if b.get("skill") not in {"cs-feat", "cs-issue", "cs-refactor", "cs-epic"}: + problems.append("learning-transfer b.skill 非法") + if b.get("skill") != a.get("skill"): + problems.append("learning-transfer A/B 必须由同一个 owning skill 执行") + if not b.get("request"): + problems.append("learning-transfer b.request 不能为空") + if not isinstance(b.get("hidden_tests"), list) or not b.get("hidden_tests"): + problems.append("learning-transfer b.hidden_tests 必须是非空 list") + if not isinstance(b.get("regression_tests"), list) or not b.get("regression_tests"): + problems.append("learning-transfer b.regression_tests 必须是非空 list") + if not isinstance(b.get("allowed_paths"), list): + problems.append("learning-transfer b.allowed_paths 必须是 list") + _check_relative_paths(problems, "learning-transfer b.hidden_tests", b.get("hidden_tests")) + _check_relative_paths(problems, "learning-transfer b.regression_tests", b.get("regression_tests")) + _check_relative_paths(problems, "learning-transfer b.allowed_paths", b.get("allowed_paths")) + if _allowlist_reaches_lessons(b.get("allowed_paths")): + problems.append("learning-transfer b.allowed_paths 不得允许额外 lesson mutation") + preflight = scenario.get("preflight") or {} + for hook in ("naive_hook", "golden_hook"): + if not preflight.get(hook): + problems.append(f"learning-transfer preflight 缺 {hook!r}") + elif not _safe_relative_path(preflight[hook]): + problems.append(f"learning-transfer preflight.{hook} 只允许实验内相对路径") + between = scenario.get("between_tasks") + if isinstance(between, dict) and between.get("hook") and not _safe_relative_path(between["hook"]): + problems.append("learning-transfer between_tasks.hook 只允许实验内相对路径") + if scenario.get("class") == "stale" and not ( + isinstance(between, dict) and between.get("hook") + ): + problems.append("learning-transfer stale fixture 必须有 between_tasks.hook") + if isinstance(between, dict) and between.get("hook"): + if not isinstance(between.get("allowed_paths"), list) or not between.get("allowed_paths"): + problems.append("learning-transfer between_tasks.allowed_paths 必须是非空 list") + _check_relative_paths( + problems, + "learning-transfer between_tasks.allowed_paths", + between.get("allowed_paths"), + ) + expect = scenario.get("expect") or {} + transition = expect.get("lesson_transition") + expected_by_class = { + "positive": "observed->validated", + "unrelated": "unchanged-observed", + "stale": "observed->retired", + } + if transition != expected_by_class.get(scenario.get("class")): + problems.append("learning-transfer expect.lesson_transition 与 scenario.class 不一致") + task = data.get("task") or {} + if task.get("kind") != "learning-transfer": + problems.append("learning-transfer task.kind 应为 'learning-transfer'") if "task" not in data: problems.append("缺 task") return problems diff --git a/.claude/skills/eval-cs-skill/scripts/harness/adapter_claude.py b/.claude/skills/eval-cs-skill/scripts/harness/adapter_claude.py index 1b67bc1..cb39ef2 100644 --- a/.claude/skills/eval-cs-skill/scripts/harness/adapter_claude.py +++ b/.claude/skills/eval-cs-skill/scripts/harness/adapter_claude.py @@ -10,10 +10,52 @@ from __future__ import annotations import json import shutil import subprocess +import tempfile import time from pathlib import Path -from .base import HarnessError, HarnessResult, register, whitelisted_env +from .base import ( + CLAUDE_ENV_KEYS, + HarnessError, + HarnessResult, + macos_sandbox_profile, + register, + whitelisted_env, +) + + +_SETTINGS_ENV_KEYS = { + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_MODEL", + "CLAUDE_CODE_OAUTH_TOKEN", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", + "CLAUDE_CODE_ATTRIBUTION_HEADER", +} + + +def _provider_env() -> dict[str, str]: + """复制显式 provider 变量；缺失项可从宿主 Claude settings 白名单补齐。""" + env = whitelisted_env(include=CLAUDE_ENV_KEYS) + settings_path = Path.home() / ".claude/settings.json" + if not settings_path.is_file() or settings_path.is_symlink(): + return env + try: + data = json.loads(settings_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return env + settings_env = data.get("env") if isinstance(data, dict) else None + if not isinstance(settings_env, dict): + return env + for key in _SETTINGS_ENV_KEYS: + value = settings_env.get(key) + if key not in env and isinstance(value, str) and value: + env[key] = value + return env + + +_sandbox_profile = macos_sandbox_profile class ClaudeHarness: @@ -23,24 +65,39 @@ class ClaudeHarness: binary = shutil.which("claude") if not binary: raise HarnessError("找不到 `claude` CLI，无法用 claude-headless harness") + binary_path = Path(binary).resolve() + sandbox = shutil.which("sandbox-exec") + if not sandbox: + raise HarnessError("claude-headless 需要 macOS sandbox-exec 外部文件系统隔离") workdir.mkdir(parents=True, exist_ok=True) - cmd = [binary, "-p", prompt, "--output-format", "json"] - if model: - cmd += ["--model", model] - start = time.monotonic() - try: - completed = subprocess.run( - cmd, - cwd=workdir, - env=whitelisted_env(), - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout_s, - check=False, - ) - except subprocess.TimeoutExpired as exc: - raise HarnessError(f"claude-headless 超时 {timeout_s}s") from exc + with tempfile.TemporaryDirectory(prefix="cs-eval-claude-", dir=workdir.parent) as tmp: + runtime = Path(tmp) + (runtime / "home").mkdir() + (runtime / "tmp").mkdir() + inner = [ + str(binary_path), "-p", prompt, "--output-format", "json", + "--no-session-persistence", "--safe-mode", + "--permission-mode", "bypassPermissions", + ] + if model: + inner += ["--model", model] + cmd = [sandbox, "-p", _sandbox_profile(workdir, runtime, binary_path), *inner] + env = _provider_env() + env.update({"HOME": str(runtime / "home"), "TMPDIR": str(runtime / "tmp")}) + start = time.monotonic() + try: + completed = subprocess.run( + cmd, + cwd=workdir, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout_s, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise HarnessError(f"claude-headless 超时 {timeout_s}s") from exc wall_ms = int((time.monotonic() - start) * 1000) if completed.returncode != 0: raise HarnessError(f"claude 退出码 {completed.returncode}: {completed.stderr[-500:]}") diff --git a/.claude/skills/eval-cs-skill/scripts/harness/adapter_codex.py b/.claude/skills/eval-cs-skill/scripts/harness/adapter_codex.py index 23e9157..8664020 100644 --- a/.claude/skills/eval-cs-skill/scripts/harness/adapter_codex.py +++ b/.claude/skills/eval-cs-skill/scripts/harness/adapter_codex.py @@ -1,18 +1,64 @@ #!/usr/bin/env python3 -"""Codex CLI headless 适配器：`codex exec`。 +"""Codex CLI headless 适配器：`codex exec --json`。 -隔离：独立 workdir + env 白名单。Codex 一般不回传 token usage → 指标走 [soft] 估算。 +隔离：独立 workdir + Seatbelt + ephemeral + env 白名单；JSONL 回收 usage，不保留 session id。 CLI 缺失/超时/非零退出抛 HarnessError。真实运行前请对齐本机 codex 版本的 exec 参数。 """ from __future__ import annotations +import json +import os +import platform import shutil import subprocess +import tempfile import time from pathlib import Path -from .base import HarnessError, HarnessResult, register, whitelisted_env +from .base import ( + CODEX_ENV_KEYS, + HarnessError, + HarnessResult, + macos_sandbox_profile, + register, + whitelisted_env, +) + + +_sandbox_profile = macos_sandbox_profile + + +def _native_codex_runtime(binary: str) -> tuple[Path, tuple[Path, ...]]: + """Resolve the npm launcher to its native binary so Seatbelt need not expose the package tree.""" + resolved = Path(binary).resolve() + if resolved.suffix != ".js": + return resolved, (resolved.parent,) + machine = platform.machine().lower() + targets = { + "arm64": ("codex-darwin-arm64", "aarch64-apple-darwin"), + "aarch64": ("codex-darwin-arm64", "aarch64-apple-darwin"), + "x86_64": ("codex-darwin-x64", "x86_64-apple-darwin"), + "amd64": ("codex-darwin-x64", "x86_64-apple-darwin"), + } + if machine not in targets: + raise HarnessError(f"codex-cli 不支持当前 macOS 架构: {machine}") + package_name, target = targets[machine] + package_root = resolved.parent.parent + candidates = ( + package_root / "node_modules" / "@openai" / package_name / "vendor" / target / "bin" / "codex", + package_root.parent / package_name / "vendor" / target / "bin" / "codex", + package_root / "vendor" / target / "bin" / "codex", + ) + for candidate in candidates: + if candidate.is_file() and not candidate.is_symlink(): + native = candidate.resolve() + return native, (native.parent.parent,) + raise HarnessError("无法从 Codex JavaScript launcher 解析受控 native binary") + + +def _native_codex_binary(binary: str) -> Path: + return _native_codex_runtime(binary)[0] class CodexHarness: @@ -22,25 +68,87 @@ class CodexHarness: binary = shutil.which("codex") if not binary: raise HarnessError("找不到 `codex` CLI，无法用 codex-cli harness") + binary_path, binary_read_roots = _native_codex_runtime(binary) + sandbox = shutil.which("sandbox-exec") + if not sandbox: + raise HarnessError("codex-cli 需要 macOS sandbox-exec 外部文件系统隔离") workdir.mkdir(parents=True, exist_ok=True) - # codex exec：非交互执行；--skip-git-repo-check 便于在 tmp workdir 跑 - cmd = [binary, "exec", "--skip-git-repo-check"] - if model: - cmd += ["--model", model] - cmd.append(prompt) - start = time.monotonic() - try: - completed = subprocess.run( - cmd, cwd=workdir, env=whitelisted_env(), text=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout_s, check=False, + with tempfile.TemporaryDirectory(prefix="cs-eval-codex-", dir=workdir.parent) as tmp: + runtime = Path(tmp) + codex_home = runtime / "codex-home" + home = runtime / "home" + tmpdir = runtime / "tmp" + for path in (codex_home, home, tmpdir): + path.mkdir() + if "OPENAI_API_KEY" not in os.environ: + source_home = Path(os.environ.get("CODEX_HOME", Path.home() / ".codex")) + source_auth = source_home / "auth.json" + if not source_auth.is_file(): + raise HarnessError("codex-cli 需要 OPENAI_API_KEY 或 CODEX_HOME/auth.json") + shutil.copy2(source_auth, codex_home / "auth.json") + inner = [ + str(binary_path), "exec", "--skip-git-repo-check", "--ephemeral", "--json", + "--ignore-user-config", "--ignore-rules", "--sandbox", "workspace-write", + ] + if model: + inner += ["--model", model] + inner.append(prompt) + cmd = [ + sandbox, + "-p", + _sandbox_profile(workdir, runtime, binary_path, binary_read_roots), + *inner, + ] + env = whitelisted_env( + include=CODEX_ENV_KEYS, + extra={ + "HOME": str(home), + "CODEX_HOME": str(codex_home), + "TMPDIR": str(tmpdir), + }, ) - except subprocess.TimeoutExpired as exc: - raise HarnessError(f"codex-cli 超时 {timeout_s}s") from exc + start = time.monotonic() + try: + completed = subprocess.run( + cmd, cwd=workdir, env=env, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout_s, check=False, + ) + except subprocess.TimeoutExpired as exc: + raise HarnessError(f"codex-cli 超时 {timeout_s}s") from exc wall_ms = int((time.monotonic() - start) * 1000) if completed.returncode != 0: raise HarnessError(f"codex 退出码 {completed.returncode}: {completed.stderr[-500:]}") - return HarnessResult(output=completed.stdout, model=model, harness=self.name, - wall_ms=wall_ms, turns=None, usage=None) + output, usage = _parse(completed.stdout) + return HarnessResult(output=output, model=model, harness=self.name, + wall_ms=wall_ms, turns=None, usage=usage) + + +def _parse(stdout: str) -> tuple[str, dict | None]: + """只提取最终 agent message 与 usage，丢弃 thread/session 标识。""" + messages: list[str] = [] + usage = None + parsed_any = False + for raw in stdout.splitlines(): + try: + event = json.loads(raw) + except json.JSONDecodeError: + continue + parsed_any = True + if event.get("type") == "item.completed": + item = event.get("item") or {} + if item.get("type") == "agent_message" and item.get("text"): + messages.append(str(item["text"])) + if event.get("type") == "turn.completed" and isinstance(event.get("usage"), dict): + raw_usage = event["usage"] + usage = { + "input_tokens": raw_usage.get("input_tokens"), + "output_tokens": raw_usage.get("output_tokens"), + "source": "codex-json", + } + output = "\n".join(messages).strip() + if not parsed_any: + output = stdout.strip() + return output, usage register(CodexHarness()) diff --git a/.claude/skills/eval-cs-skill/scripts/harness/base.py b/.claude/skills/eval-cs-skill/scripts/harness/base.py index 0615335..f57a307 100644 --- a/.claude/skills/eval-cs-skill/scripts/harness/base.py +++ b/.claude/skills/eval-cs-skill/scripts/harness/base.py @@ -54,19 +54,87 @@ def available() -> list[str]: return sorted(_REGISTRY) -ENV_WHITELIST = ( - "PATH", "HOME", "LANG", "LC_ALL", "TERM", - "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL", "ANTHROPIC_MODEL", - "OPENAI_API_KEY", "OPENAI_BASE_URL", - "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", - "CLAUDE_CODE_ATTRIBUTION_HEADER", "CODEX_HOME", +COMMON_ENV_KEYS = ( + "PATH", "LANG", "LC_ALL", "TERM", "HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy", "ALL_PROXY", "NO_PROXY", ) +CLAUDE_ENV_KEYS = COMMON_ENV_KEYS + ( + "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL", "ANTHROPIC_MODEL", + "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", + "CLAUDE_CODE_ATTRIBUTION_HEADER", +) +CODEX_ENV_KEYS = COMMON_ENV_KEYS + ("OPENAI_API_KEY", "OPENAI_BASE_URL") +ENV_WHITELIST = CLAUDE_ENV_KEYS + CODEX_ENV_KEYS + ("HOME", "CODEX_HOME") -def whitelisted_env(extra: dict[str, str] | None = None) -> dict[str, str]: +def _sandbox_quote(value: Path | str) -> str: + text = str(value) + if any(character in text for character in ("\x00", "\n", "\r")): + raise ValueError("Seatbelt path 不得包含 NUL 或换行") + return text.replace("\\", "\\\\").replace('"', '\\"') + + +def physical_home() -> Path: + """Return the account home independently of a caller-controlled HOME variable.""" + try: + import os + import pwd + + return Path(pwd.getpwuid(os.getuid()).pw_dir).resolve() + except (ImportError, KeyError, OSError): + return Path.home().resolve() + + +def macos_sandbox_profile( + workdir: Path, + runtime: Path, + binary: Path, + read_roots: tuple[Path, ...] = (), +) -> str: + """Seatbelt profile: protect host home and sibling cells, allow only this cell/runtime.""" + protected_roots = sorted({ + Path.home().resolve(), + physical_home(), + workdir.resolve().parent, + }, key=str) + protected = "\n".join( + f' (subpath "{_sandbox_quote(path)}")' + for path in protected_roots + ) + readable_roots = sorted({ + workdir.resolve(), + runtime.resolve(), + *(path.resolve() for path in read_roots), + }, key=str) + readable = "\n".join( + f' (subpath "{_sandbox_quote(path)}")' + for path in readable_roots + ) + return f'''(version 1) +(allow default) +(deny file-read* + (require-all + (require-any +{protected}) + (require-not + (require-any +{readable} + (literal "{_sandbox_quote(binary.resolve())}"))))) +(deny file-write* + (require-not + (require-any + (subpath "{_sandbox_quote(workdir.resolve())}") + (subpath "{_sandbox_quote(runtime.resolve())}") + (literal "/dev/null")))) +''' + + +def whitelisted_env( + extra: dict[str, str] | None = None, + include: tuple[str, ...] = ENV_WHITELIST, +) -> dict[str, str]: import os - env = {k: os.environ[k] for k in ENV_WHITELIST if k in os.environ} + env = {k: os.environ[k] for k in include if k in os.environ} if extra: env.update(extra) return env diff --git a/.claude/skills/eval-cs-skill/scripts/metrics.py b/.claude/skills/eval-cs-skill/scripts/metrics.py index bc32b54..01a3d9f 100644 --- a/.claude/skills/eval-cs-skill/scripts/metrics.py +++ b/.claude/skills/eval-cs-skill/scripts/metrics.py @@ -50,7 +50,7 @@ def capture(result: HarnessResult, prompt: str | None = None) -> dict[str, Any]: metrics["turns"] = tagged(result.turns, MEASURED) usage = result.usage or {} - real = usage.get("source") == "claude-json" + real = usage.get("source") in {"claude-json", "codex-json"} tag = MEASURED if real else SOFT tin = usage.get("input_tokens") diff --git a/.claude/skills/eval-cs-skill/scripts/probe_targets.py b/.claude/skills/eval-cs-skill/scripts/probe_targets.py new file mode 100644 index 0000000..1d38652 --- /dev/null +++ b/.claude/skills/eval-cs-skill/scripts/probe_targets.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Run minimal real-model filesystem probes before freezing a learning-transfer campaign.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import secrets +import stat +import sys +import tempfile +from pathlib import Path +from typing import Any + +sys.dont_write_bytecode = True + +from config import load_config, repo_root +from fixtures import load_fixtures +from harness import get_harness +from harness.base import physical_home +from sequence import ( + _current_freeze_external_inputs, + _current_freeze_inputs, + _file_hash, + _git_blob_hash, + _isolated_repo_git, + _root_relative, +) + + +_PROBE_ORACLES = ( + "cell_write", + "host_read_blocked", + "sibling_read_blocked", + "host_write_blocked", + "host_config_unchanged", + "runtime_removed", +) +_CONFIG_NAMES = { + "auth.json", + "config.json", + "config.toml", + "settings.json", + "settings.local.json", +} +_CLAUDE_STATE_DIRS = ( + "debug", + "file-history", + "projects", + "session-env", + "sessions", + "shell-snapshots", + "tasks", + "todos", +) +_CODEX_STATE_DIRS = ( + "archived_sessions", + "log", + "rollout", + "rollouts", + "sessions", + "shell_snapshots", +) + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _entry_fingerprint(path: Path) -> str: + info = path.lstat() + kind = stat.S_IFMT(info.st_mode) + parts = [ + str(kind), + str(info.st_mode), + str(info.st_size), + str(info.st_mtime_ns), + str(info.st_ctime_ns), + str(info.st_ino), + ] + if stat.S_ISLNK(info.st_mode): + parts.append(os.readlink(path)) + elif stat.S_ISREG(info.st_mode) and path.name in _CONFIG_NAMES | {".claude.json"}: + parts.append(_sha256(path)) + return ":".join(parts) + + +def _add_tree(candidates: set[Path], root: Path) -> None: + if root.is_symlink(): + candidates.add(root) + return + if not root.is_dir(): + return + candidates.add(root) + for current, directories, files in os.walk(root, followlinks=False): + directories.sort() + files.sort() + base = Path(current) + candidates.update(base / name for name in directories) + candidates.update(base / name for name in files) + + +def _config_snapshot() -> dict[str, str]: + """Snapshot host config plus known Claude/Codex session state without reading transcripts.""" + host_home = physical_home() + candidates = {host_home / ".claude.json"} + roots = { + host_home / ".claude": _CLAUDE_STATE_DIRS, + host_home / ".codex": _CODEX_STATE_DIRS, + Path(os.environ.get("CODEX_HOME", host_home / ".codex")).resolve(): _CODEX_STATE_DIRS, + } + for root, state_directories in roots.items(): + if root.is_dir() and not root.is_symlink(): + candidates.update(path for path in root.iterdir() if path.is_file() or path.is_symlink()) + for name in state_directories: + _add_tree(candidates, root / name) + snapshot: dict[str, str] = {} + for path in sorted(candidates, key=str): + if not path.exists() and not path.is_symlink(): + continue + path_id = hashlib.sha256(str(path).encode()).hexdigest() + snapshot[path_id] = _entry_fingerprint(path) + return snapshot + + +def _source_commit(root: Path) -> str: + result = _isolated_repo_git(root, "rev-parse", "HEAD") + if result.returncode != 0: + raise RuntimeError("source commit unavailable") + return result.stdout.decode("utf-8").strip() + + +def _validate_prepared_source(experiment: Path, root: Path, fixtures: list[Any]) -> str: + manifest_path = experiment / "freeze.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("state") != "prepared-awaiting-commit": + raise ValueError("probe requires prepared-awaiting-commit freeze") + if manifest.get("source_commit") != "pending" or manifest.get("real_llm_runs_started") is not False: + raise ValueError("probe requires pending source with no real LLM runs") + probe = manifest.get("model_target_probe") or {} + if probe != {"status": "pending", "source_commit": "pending", "targets": []}: + raise ValueError("probe attestation must still be pending") + current_inputs = _current_freeze_inputs(experiment, fixtures) + current_external = _current_freeze_external_inputs(root, fixtures) + if manifest.get("inputs") != current_inputs or manifest.get("external_inputs") != current_external: + raise ValueError("prepared freeze hashes do not match current source") + source_commit = _source_commit(root) + experiment_relative = Path(_root_relative(experiment, root)) + for relative, digest in current_inputs.items(): + if _git_blob_hash(root, source_commit, (experiment_relative / relative).as_posix()) != digest: + raise ValueError("experiment inputs must be committed before probe") + for relative, digest in current_external.items(): + if _git_blob_hash(root, source_commit, relative) != digest: + raise ValueError("pipeline inputs must be committed before probe") + manifest_relative = _root_relative(manifest_path, root) + if _git_blob_hash(root, source_commit, manifest_relative) != _file_hash(manifest_path): + raise ValueError("prepared freeze manifest must be committed before probe") + return source_commit + + +def _probe_target(target: Any) -> dict[str, Any]: + harness = get_harness(target.harness) + host_home = physical_home() + token = secrets.token_hex(16) + host_secret = f"host-{secrets.token_hex(16)}" + sibling_secret = f"sibling-{secrets.token_hex(16)}" + host_sentinel = host_home / f".cs-eval-probe-read-{token}" + host_write = host_home / f".cs-eval-probe-write-{token}" + try: + host_sentinel.write_text(host_secret, encoding="utf-8") + before_config = _config_snapshot() + with tempfile.TemporaryDirectory(prefix=f"cs-eval-probe-{target.id}-") as tmp: + probe_root = Path(tmp) + workdir = probe_root / "cell" + sibling = probe_root / "sibling" + workdir.mkdir() + sibling.mkdir() + (sibling / "secret.txt").write_text(sibling_secret, encoding="utf-8") + (workdir / "host-read-link").symlink_to(host_sentinel) + (workdir / "sibling-read-link").symlink_to(sibling / "secret.txt") + (workdir / "host-write-link").symlink_to(host_write) + runtime_before = {path.name for path in probe_root.glob("cs-eval-*")} + prompt = ( + "Execute this filesystem isolation probe exactly. Write the exact token " + f"{token} to inside.txt. Attempt to read host-read-link and sibling-read-link; " + "write each read result to host-read-result.txt and sibling-read-result.txt, or write " + "the exact word BLOCKED when access fails. Attempt to write PROBE through " + "host-write-link, then write the exact word BLOCKED to host-write-result.txt when " + "access fails or WRITTEN when it succeeds. Do not explain; finish after all operations." + ) + result = harness.invoke(prompt, target.model, workdir, timeout_s=180) + host_result = ( + (workdir / "host-read-result.txt").read_text(encoding="utf-8", errors="replace") + if (workdir / "host-read-result.txt").is_file() + else "" + ) + sibling_result = ( + (workdir / "sibling-read-result.txt").read_text(encoding="utf-8", errors="replace") + if (workdir / "sibling-read-result.txt").is_file() + else "" + ) + host_write_result = ( + (workdir / "host-write-result.txt").read_text(encoding="utf-8", errors="replace") + if (workdir / "host-write-result.txt").is_file() + else "" + ) + combined = "\n".join((result.output, host_result, sibling_result, host_write_result)) + evidence = { + "id": target.id, + "family": target.family, + "harness": target.harness, + "model": target.model, + "cell_write": ( + (workdir / "inside.txt").is_file() + and (workdir / "inside.txt").read_text(encoding="utf-8").strip() == token + ), + "host_read_blocked": ( + host_result.strip() == "BLOCKED" and host_secret not in combined + ), + "sibling_read_blocked": ( + sibling_result.strip() == "BLOCKED" and sibling_secret not in combined + ), + "host_write_blocked": ( + host_write_result.strip() == "BLOCKED" and not host_write.exists() + ), + "host_config_unchanged": _config_snapshot() == before_config, + "runtime_removed": ( + {path.name for path in probe_root.glob("cs-eval-*")} == runtime_before + ), + } + failed = [name for name in _PROBE_ORACLES if evidence[name] is not True] + if failed: + raise RuntimeError(f"target probe failed: {','.join(failed)}") + return evidence + finally: + host_sentinel.unlink(missing_ok=True) + host_write.unlink(missing_ok=True) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--experiment", type=Path, required=True) + args = parser.parse_args(argv) + try: + experiment = args.experiment.resolve() + root = repo_root(experiment) + config = load_config(experiment) + fixtures = load_fixtures(experiment, config.fixture_classes) + source_commit = _validate_prepared_source(experiment, root, fixtures) + targets = [_probe_target(target) for target in config.model_targets] + print(json.dumps({ + "status": "passed", + "source_commit": source_commit, + "targets": targets, + }, ensure_ascii=False, sort_keys=True)) + return 0 + except Exception as exc: + print(f"target probe failed: {type(exc).__name__}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.claude/skills/eval-cs-skill/scripts/runner.py b/.claude/skills/eval-cs-skill/scripts/runner.py index 1ff874b..03738fb 100644 --- a/.claude/skills/eval-cs-skill/scripts/runner.py +++ b/.claude/skills/eval-cs-skill/scripts/runner.py @@ -12,12 +12,15 @@ from __future__ import annotations import argparse +import fcntl +import shutil import sys sys.dont_write_bytecode = True # 不污染 plugin 包（check-plugin-package 禁 __pycache__） import json import tempfile +from contextlib import contextmanager from pathlib import Path from statistics import mean from typing import Any @@ -31,9 +34,16 @@ import metrics # noqa: E402 import scorers as scorers_pkg # noqa: E402 from _model import EvalResult, MEASURED, SOFT, UNDERPOWERED, render_tagged, tagged, write_json # noqa: E402 from buildprompt import build_prompt # noqa: E402 -from config import ExperimentConfig, judge_issues, load_config, repo_root, resolve_variant_text # noqa: E402 +from config import ( # noqa: E402 + ExperimentConfig, + judge_issues, + load_config, + repo_root, + resolve_variant_text, + select_execution_targets, +) from e2e_env import prepare_e2e_workdir # noqa: E402 -from fixtures import load_fixtures # noqa: E402 +from fixtures import load_fixtures, validate_learning_transfer_coverage # noqa: E402 def _agg_tag(tags: set[str]) -> str: @@ -84,6 +94,13 @@ def _models_for(config: ExperimentConfig, override: str | None, harness: str) -> def build_matrix(config: ExperimentConfig, args) -> list[tuple[str, str, str]]: variants = [args.variant] if args.variant else config.variants + if config.model_targets: + targets = select_execution_targets(config, args.harness, args.model) + return [ + (variant, target.harness, target.model) + for variant in variants + for target in targets + ] harnesses = [args.harness] if args.harness else config.harnesses cells = [] for variant in variants: @@ -204,6 +221,56 @@ def write_results_md(exp_dir: Path, config: ExperimentConfig, agg: dict, cells: (exp_dir / "results.md").write_text("\n".join(lines) + "\n", encoding="utf-8") +class SequenceOutputBusyError(RuntimeError): + """同一输出已有 learning-transfer 进程持锁。""" + + +def _sequence_run_root(out_path: Path) -> Path: + """不同 --out 使用独立 preflight/cell 空间；同一 out 才共享恢复状态。""" + base = (out_path.parent / "runs").resolve() + if out_path.name in {"", ".", ".."}: + raise ValueError("--out 必须包含安全的文件名") + run_root = (base / out_path.name).resolve() + try: + run_root.relative_to(base) + except ValueError as exc: + raise ValueError("--out 生成的 run root 越过隔离目录") from exc + if run_root == base: + raise ValueError("--out 不能指向 run root 本身") + return run_root + + +def _sequence_checkpoint_path(out_path: Path) -> Path: + return out_path.parent / f"{out_path.name}.partial.jsonl" + + +@contextmanager +def _sequence_output_lock(out_path: Path): + """对单个 --out 加非阻塞进程锁，避免 checkpoint/run root 交叉写。""" + out_path.parent.mkdir(parents=True, exist_ok=True) + lock_path = out_path.parent / f".{out_path.name}.lock" + with lock_path.open("a", encoding="utf-8") as handle: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise SequenceOutputBusyError(f"同一 --out 已有运行: {out_path}") from exc + try: + yield + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +def _clear_sequence_state(run_root: Path, checkpoint_path: Path) -> None: + """`--fresh` 清理该输出专属的 checkpoint、preflight 与 cell 状态。""" + if run_root.is_symlink(): + raise ValueError("拒绝清理 symlink run root") + if run_root.exists() and not run_root.is_dir(): + raise ValueError("拒绝清理非目录 run root") + checkpoint_path.unlink(missing_ok=True) + if run_root.exists(): + shutil.rmtree(run_root) + + def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(description="eval-cs-skill eval runner") p.add_argument("--experiment", required=True) @@ -215,7 +282,7 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--dry-run", action="store_true") p.add_argument("--confirm", action="store_true", help="确认超预算仍执行") p.add_argument("--out") - p.add_argument("--fresh", action="store_true", help="忽略并删除已有 checkpoint，从头跑") + p.add_argument("--fresh", action="store_true", help="删除该输出的 checkpoint 与 run state，从头跑") p.add_argument("--offset", type=int, default=0, help="跳过前 M 个 fixture（分段用）") p.add_argument("--limit", type=int, help="只跑 offset 起 N 个 fixture（分段用，每段配独立 --out）") args = p.parse_args(argv) @@ -223,13 +290,129 @@ def main(argv: list[str] | None = None) -> int: exp_dir = Path(args.experiment).resolve() config = load_config(exp_dir) k = args.k if args.k is not None else config.k + if k < 1: + print("[eval-cs-skill] k 必须至少为 1", file=sys.stderr) + return 2 scorer_names = args.scorer or config.scorers fixtures = load_fixtures(exp_dir, config.fixture_classes) + if config.execution_mode == "learning-transfer": + contract_issues: list[str] = [] + if config.variants != ["baseline"] or args.variant not in {None, "baseline"}: + contract_issues.append("只允许冻结的 baseline 变体") + if config.scorers != ["learning_transfer"] or scorer_names != ["learning_transfer"]: + contract_issues.append("scorer 必须且只能是 learning_transfer") + if contract_issues: + print( + f"[eval-cs-skill] learning-transfer 执行契约不合规: {'; '.join(contract_issues)}", + file=sys.stderr, + ) + return 2 + if config.execution_mode == "learning-transfer" and set(config.fixture_classes) == { + "positive", "unrelated", "stale", + }: + coverage_problems = validate_learning_transfer_coverage(fixtures) + if coverage_problems: + print( + f"[eval-cs-skill] learning-transfer coverage 不合规: {'; '.join(coverage_problems)}", + file=sys.stderr, + ) + return 2 if args.offset or args.limit: fixtures = fixtures[args.offset: (args.offset + args.limit) if args.limit else None] if not fixtures: print(f"[eval-cs-skill] 无 fixtures：{exp_dir}/fixtures/{config.fixture_classes}", file=sys.stderr) return 2 + if config.execution_mode == "learning-transfer": + import sequence + + targets = select_execution_targets(config, args.harness, args.model) + if not targets: + print("[eval-cs-skill] 没有匹配的 model target", file=sys.stderr) + return 2 + est = sequence.dry_run_sequence(config, fixtures, k, repo_root(), targets) + print( + f"[eval-cs-skill] learning-transfer 预估成本 ${est['est_total_usd']} / " + f"预算 ${est['budget_usd']} (invocations={est['invocation_count']}, " + f"hooks={est['hook_runs']})" + ) + if args.dry_run: + if args.out: + dry_run_out = Path(args.out) + dry_run_checkpoint = _sequence_checkpoint_path(dry_run_out) + try: + with _sequence_output_lock(dry_run_out): + if dry_run_out.exists() or dry_run_checkpoint.exists(): + raise ValueError( + "--dry-run 不得覆盖已有结果或 checkpoint；请改用新的 --out" + ) + write_json(dry_run_out, est) + except (SequenceOutputBusyError, ValueError) as exc: + print(f"[eval-cs-skill] learning-transfer 输出不安全: {exc}", file=sys.stderr) + return 2 + return 0 + if est["est_total_usd"] > config.budget_usd and not args.confirm: + print( + f"[eval-cs-skill] 阻断：预估 ${est['est_total_usd']} 超预算 " + f"${config.budget_usd}；加 --confirm 放行。", + file=sys.stderr, + ) + return 3 + real_targets = [ + target for target in targets + if target.harness not in {"mock", "mock-weak"} + ] + freeze_validation = None + if real_targets: + try: + freeze_validation = sequence.validate_freeze_manifest( + experiment_dir=exp_dir, + root=repo_root(), + config=config, + fixtures=fixtures, + k=k, + targets=targets, + ) + except ValueError as exc: + print(f"[eval-cs-skill] 冻结校验失败: {exc}", file=sys.stderr) + return 2 + out_path = Path(args.out) if args.out else ( + exp_dir / "artifacts" / "analysis" / f"exp-{config.name}-results.json" + ) + checkpoint_path = _sequence_checkpoint_path(out_path) + try: + run_root = _sequence_run_root(out_path) + with _sequence_output_lock(out_path): + if out_path.exists() and (args.fresh or not checkpoint_path.exists()): + raise ValueError( + "--out 已包含不可覆盖的运行结果；恢复需保留 checkpoint，重跑请使用新 --out" + ) + if args.fresh: + if sequence.checkpoint_has_irreversible_evidence(checkpoint_path): + raise ValueError( + "--fresh 不得删除 invocation、score、error 或 fixture-invalid 证据；" + "请改用新的 --out" + ) + _clear_sequence_state(run_root, checkpoint_path) + payload = sequence.run_sequence( + config=config, + fixtures=fixtures, + k=k, + experiment_dir=exp_dir, + root=repo_root(), + run_root=run_root, + checkpoint_path=checkpoint_path, + targets=targets, + ) + if freeze_validation is not None: + payload["freeze"] = freeze_validation + write_json(out_path, payload) + if not payload["errors"] and not payload["invalid"]: + checkpoint_path.unlink(missing_ok=True) + except (SequenceOutputBusyError, ValueError) as exc: + print(f"[eval-cs-skill] learning-transfer 运行状态不安全: {exc}", file=sys.stderr) + return 2 + print(f"[eval-cs-skill] 完成 {len(payload['pairs'])} paired runs → {out_path}") + return 0 if not payload["errors"] and not payload["invalid"] else 4 cells = build_matrix(config, args) for issue in judge_issues(config): diff --git a/.claude/skills/eval-cs-skill/scripts/scorers/__init__.py b/.claude/skills/eval-cs-skill/scripts/scorers/__init__.py index e3fab34..3a6239a 100644 --- a/.claude/skills/eval-cs-skill/scripts/scorers/__init__.py +++ b/.claude/skills/eval-cs-skill/scripts/scorers/__init__.py @@ -9,7 +9,10 @@ from .base import applies, available, get_scorer # noqa: F401 from . import planted_defect # noqa: F401 # 后续阶段的 scorer；缺依赖不阻断确定性路径 -for _mod in ("dod_gate", "llm_judge", "recall_judge", "routing_decision", "e2e_outcome"): +for _mod in ( + "dod_gate", "llm_judge", "recall_judge", "routing_decision", "e2e_outcome", + "learning_transfer", +): try: importlib.import_module(f".{_mod}", __name__) except Exception: # noqa: BLE001 diff --git a/.claude/skills/eval-cs-skill/scripts/scorers/learning_transfer.py b/.claude/skills/eval-cs-skill/scripts/scorers/learning_transfer.py new file mode 100644 index 0000000..0354a93 --- /dev/null +++ b/.claude/skills/eval-cs-skill/scripts/scorers/learning_transfer.py @@ -0,0 +1,497 @@ +#!/usr/bin/env python3 +"""learning-transfer paired oracle：lesson schema、任务结果与跨模型 verdict。""" + +from __future__ import annotations + +import re +from collections import Counter +from datetime import date +from pathlib import Path +from statistics import mean +from typing import Any + +import yaml +from yaml.nodes import MappingNode +from yaml.resolver import BaseResolver + +from _model import MEASURED, SOFT, tagged +from .base import register + + +_BODY_FIELDS = ("规则：", "适用 / 不适用：", "证据：", "候选归宿：") +_FRONTMATTER_FIELDS = {"status", "scope", "date"} +_LESSON_STATUSES = {"observed", "validated", "retired"} +_CANDIDATE_HOMES = { + "test", + "checker", + "attention", + "project-doc", + "adr", + "codestable-eval", +} +_LESSON_PATH_RE = re.compile( + r"\.codestable/lessons/" + r"(?P\d{4}-\d{2}-\d{2})-" + r"(?P[a-z0-9-]{1,30})\.md" +) +_CONCRETE_EVIDENCE_RE = re.compile( + r"(?:^|(?<=[\s，,（(]))" + r"(?=[A-Za-z0-9_./-]*[A-Za-z_])" + r"(?:\.{0,2}/)?(?:[A-Za-z0-9_.-]+/)+[A-Za-z0-9_.-]+" + r"|(? dict[Any, Any]: + loader.flatten_mapping(node) + mapping: dict[Any, Any] = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if key in mapping: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + f"duplicate key: {key}", + key_node.start_mark, + ) + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +_UniqueKeyLoader.add_constructor( + BaseResolver.DEFAULT_MAPPING_TAG, + _construct_unique_mapping, +) + + +def score_pair(pair: dict[str, Any]) -> dict[str, Any]: + """把一个完整 paired record 转成不含主观判断的 measured scores。""" + treatment = float(pair.get("treatment_hidden", 0.0)) + control = float(pair.get("control_hidden", 0.0)) + delta = treatment - control + scores = { + name: tagged(1 if pair.get(name) else 0, MEASURED) + for name in _STRUCTURAL_ORACLES + } + scores.update({ + "treatment_hidden": tagged(treatment, MEASURED), + "control_hidden": tagged(control, MEASURED), + "paired_delta": tagged(delta, MEASURED), + "paired_win": tagged(1 if delta > 0 else 0, MEASURED), + "paired_loss": tagged(1 if delta < 0 else 0, MEASURED), + "paired_tie": tagged(1 if delta == 0 else 0, MEASURED), + }) + structural_ok = all(pair.get(name) is True for name in _STRUCTURAL_ORACLES) + return { + "scores": scores, + "evidence": [], + "status": "passed" if structural_ok else "failed", + } + + +@register("learning_transfer", applies_to={"learning-transfer"}) +def score_learning_transfer(_fixture, pair, _config=None, _root=None) -> dict[str, Any]: + """registry adapter；sequence 传入完整 pair，而不是 HarnessResult。""" + return score_pair(pair) + + +def _effect_summary(records: list[dict[str, Any]]) -> dict[str, Any]: + treatment = [float(record.get("treatment_hidden", 0.0)) for record in records] + control = [float(record.get("control_hidden", 0.0)) for record in records] + deltas = [left - right for left, right in zip(treatment, control)] + return { + "pairs": len(records), + "treatment_rate": round(mean(treatment), 4) if treatment else 0.0, + "control_rate": round(mean(control), 4) if control else 0.0, + "paired_delta": round(mean(deltas), 4) if deltas else 0.0, + "wins": sum(delta > 0 for delta in deltas), + "losses": sum(delta < 0 for delta in deltas), + "ties": sum(delta == 0 for delta in deltas), + } + + +def _power_assessment(positive: list[dict[str, Any]], guards: list[dict[str, Any]]) -> dict[str, Any]: + reasons: list[str] = [] + families = sorted({record.get("family") for record in positive if record.get("family")}) + required_skills = {"cs-feat", "cs-issue", "cs-refactor", "cs-epic"} + if len(families) < 2: + reasons.append("需要至少两个 model family") + for family in families: + family_positive = [record for record in positive if record.get("family") == family] + if len(family_positive) < 20: + reasons.append(f"{family} 正向完成 pair 少于 20") + for skill in required_skills: + repeats = { + record.get("k_index") for record in family_positive + if record.get("owning_skill") == skill + } + if len(repeats) < 5: + reasons.append(f"{family}/{skill} 完成 repeat 少于 5") + for guard_class in ("unrelated", "stale"): + repeats = { + record.get("k_index") for record in guards + if record.get("family") == family and record.get("fixture_class") == guard_class + } + if len(repeats) < 5: + reasons.append(f"{family}/{guard_class} guard 完成 repeat 少于 5") + return {"ok": not reasons, "reasons": reasons} + + +def aggregate_pairs( + records: list[dict[str, Any]], + *, + operational_errors: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """按 family 聚合正向迁移，并把 unrelated/stale guard 独立保留。""" + operational_history = operational_errors or [] + completed = [record for record in records if record.get("state") == "completed"] + deterministic_blockers = [ + record + for record in records + if record.get("state") not in {"completed", "retryable-error"} + ] + positive = [record for record in completed if record.get("fixture_class") == "positive"] + guard_records = [ + record for record in completed + if record.get("fixture_class") in {"unrelated", "stale"} + ] + families = { + family: _effect_summary([record for record in positive if record.get("family") == family]) + for family in sorted({record.get("family") for record in positive if record.get("family")}) + } + guards: dict[str, Any] = {} + for guard_class in ("unrelated", "stale"): + selected = [record for record in guard_records if record.get("fixture_class") == guard_class] + guards[guard_class] = { + "pairs": len(selected), + "no_regression": all( + float(record.get("treatment_hidden", 0.0)) >= float(record.get("control_hidden", 0.0)) + and record.get("treatment_regression_ok") is True + and record.get("control_regression_ok") is True + for record in selected + ), + } + if guard_class == "stale": + guards[guard_class]["retired_rate"] = ( + round(mean([1.0 if record.get("stale_retired") else 0.0 for record in selected]), 4) + if selected else 0.0 + ) + attempt_counts = dict(Counter(str(record.get("state", "incomplete")) for record in records)) + structural_total = len(completed) * len(_STRUCTURAL_ORACLES) + structural_passed = sum( + record.get(name) is True + for record in completed + for name in _STRUCTURAL_ORACLES + ) + phase_metrics = [ + phase + for record in completed + for phase in record.get("phase_metrics", []) + ] + cost_items = [ + phase.get("metrics", {}).get("cost_usd") + for phase in phase_metrics + if isinstance(phase.get("metrics", {}).get("cost_usd"), dict) + ] + cost_tag = ( + MEASURED + if len(cost_items) == len(phase_metrics) and all(item.get("tag") == MEASURED for item in cost_items) + else SOFT + ) + return { + "overall": _effect_summary(positive), + "families": families, + "guards": guards, + "power": _power_assessment(positive, guard_records), + "integrity": { + "ok": structural_passed == structural_total and not deterministic_blockers, + "passed": structural_passed, + "total": structural_total, + "blockers": len(deterministic_blockers), + }, + "cost": { + "invocation_count": len(phase_metrics), + "cost_usd": tagged(round(sum(float(item.get("value", 0.0)) for item in cost_items), 6), cost_tag), + }, + "operational_errors": { + "attempts": len(operational_history), + "resolved": sum(error.get("resolved") is True for error in operational_history), + "unresolved": sum(error.get("resolved") is not True for error in operational_history), + }, + "attempt_counts": attempt_counts, + } + + +def transfer_verdict(aggregate: dict[str, Any]) -> dict[str, Any]: + """按预注册复合门槛给出 learning-transfer 最终判定。""" + reasons: list[str] = [] + overall = aggregate.get("overall") or {} + delta = float(overall.get("paired_delta", 0.0)) + if delta < 0.25: + reasons.append("总体 paired delta 低于 25pp") + families = aggregate.get("families") or {} + if len(families) < 2 or any(float(item.get("paired_delta", 0.0)) <= 0 for item in families.values()): + reasons.append("至少一个 model family 未呈正向迁移") + if int(overall.get("losses", 0)) > int(overall.get("wins", 0)): + reasons.append("paired losses 多于 wins") + guards = aggregate.get("guards") or {} + for guard_class in ("unrelated", "stale"): + guard = guards.get(guard_class) or {} + if not guard.get("pairs") or guard.get("no_regression") is not True: + reasons.append(f"{guard_class} guard 回退或缺失") + if float((guards.get("stale") or {}).get("retired_rate", 0.0)) != 1.0: + reasons.append("stale lesson 未 100% 退役") + if (aggregate.get("integrity") or {}).get("ok") is not True: + reasons.append("隔离、schema、mutation 或回归 oracle 未 100% 通过") + operational = aggregate.get("operational_errors") or {} + unresolved_operational = int(operational.get("unresolved", 0)) + powered = (aggregate.get("power") or {}).get("ok") is True and unresolved_operational == 0 + if not powered: + reasons.append("统计功效不足") + if unresolved_operational: + reasons.append("存在未解决 operational error") + accepted = not reasons + direction = "CONFIRMED" if accepted else ("NULL" if delta < 0 else "REJECTED") + return { + "accepted": accepted, + "verdict": { + "direction": direction, + "observed": delta, + "threshold": 0.25, + "confidence": "high" if powered else "underpowered", + }, + "reasons": reasons, + } + + +def _iso_date(value: Any) -> str: + if type(value) is date: + return value.isoformat() + if not isinstance(value, str): + raise ValueError("date must be an ISO date") + try: + parsed = date.fromisoformat(value) + except ValueError as exc: + raise ValueError("date must be an ISO date") from exc + if parsed.isoformat() != value: + raise ValueError("date must be an ISO date") + return value + + +def _evidence_items(value: str) -> list[str]: + items = [item.strip() for item in re.split(r"[；;]", value)] + if not 1 <= len(items) <= 3 or any(not item for item in items): + raise ValueError("evidence must contain 1-3 non-empty items") + return items + + +def _evidence_content(value: str) -> str: + """句末标点不属于 evidence item；追加分项时允许用分隔符替换它。""" + return value[:-1] if value.endswith(("。", ".")) else value + + +def _parse_lesson_text(text: str) -> tuple[dict[str, Any], dict[str, str]]: + lines = text.splitlines() + if not lines or lines[0] != "---": + raise ValueError("missing frontmatter") + try: + closing = lines.index("---", 1) + except ValueError as exc: + raise ValueError("unterminated frontmatter") from exc + + frontmatter = "\n".join(lines[1:closing]) + metadata = yaml.load(frontmatter, Loader=_UniqueKeyLoader) + if not isinstance(metadata, dict): + raise ValueError("frontmatter must be a mapping") + keys = set(metadata) + if keys != _FRONTMATTER_FIELDS: + missing = sorted(_FRONTMATTER_FIELDS - keys) + extra = sorted(str(key) for key in keys - _FRONTMATTER_FIELDS) + raise ValueError(f"frontmatter fields mismatch: missing={missing}, extra={extra}") + status = metadata["status"] + if not isinstance(status, str) or status not in _LESSON_STATUSES: + raise ValueError("invalid status") + scope = metadata["scope"] + if not isinstance(scope, str) or not scope.strip() or scope != scope.strip(): + raise ValueError("scope must be a non-empty string") + metadata["date"] = _iso_date(metadata["date"]) + + body_lines = lines[closing + 1:] + if len(body_lines) != len(_BODY_FIELDS): + raise ValueError("body must contain exactly four field lines") + fields: dict[str, str] = {} + for line in body_lines: + matches = [prefix for prefix in _BODY_FIELDS if line.startswith(prefix)] + if len(matches) != 1: + raise ValueError(f"unexpected body line: {line}") + prefix = matches[0] + if prefix in fields: + raise ValueError(f"duplicate body field: {prefix}") + value = line[len(prefix):] + if not value or value != value.strip(): + raise ValueError(f"invalid body value: {prefix}") + fields[prefix] = value + if set(fields) != set(_BODY_FIELDS): + raise ValueError("body fields mismatch") + if fields["候选归宿："] not in _CANDIDATE_HOMES: + raise ValueError("invalid candidate home") + _evidence_items(fields["证据："]) + return metadata, fields + + +def lesson_status(text: str) -> str | None: + """从 lesson frontmatter 机械读取 lifecycle status。""" + try: + metadata, _ = _parse_lesson_text(text) + except (ValueError, yaml.YAMLError): + return None + status = metadata.get("status") + return str(status) if status is not None else None + + +def validate_lesson_transition(before: str, after: str) -> dict[str, Any]: + """验证 validated/retired 窄迁移不改写规则或 scope。""" + try: + before_meta, before_fields = _parse_lesson_text(before) + after_meta, after_fields = _parse_lesson_text(after) + except (ValueError, yaml.YAMLError) as exc: + return {"ok": False, "errors": [str(exc)]} + errors: list[str] = [] + transition = (before_meta.get("status"), after_meta.get("status")) + if transition not in { + ("observed", "validated"), + ("observed", "retired"), + ("validated", "retired"), + }: + errors.append("status") + if {k: v for k, v in before_meta.items() if k != "status"} != { + k: v for k, v in after_meta.items() if k != "status" + }: + errors.append("scope/date") + for prefix in ("规则：", "适用 / 不适用：", "候选归宿："): + if before_fields.get(prefix) != after_fields.get(prefix): + errors.append(prefix.removesuffix("：")) + before_evidence = before_fields["证据："] + after_evidence = after_fields["证据："] + before_items = _evidence_items(before_evidence) + after_items = _evidence_items(after_evidence) + preserved_prefix = next( + ( + prefix + for prefix in (before_evidence, _evidence_content(before_evidence)) + if after_evidence.startswith(prefix) + and after_evidence[len(prefix):len(prefix) + 1] in "；;" + ), + None, + ) + suffix = after_evidence[len(preserved_prefix):] if preserved_prefix is not None else "" + evidence_ok = ( + len(after_items) == len(before_items) + 1 + and [_evidence_content(item) for item in after_items[:-1]] + == [_evidence_content(item) for item in before_items] + and preserved_prefix is not None + and len(suffix) > 1 + and suffix[0] in "；;" + and _CONCRETE_EVIDENCE_RE.search(after_items[-1]) is not None + ) + if not evidence_ok: + errors.append("证据") + new_evidence = after_items[-1] if len(after_items) > len(before_items) else "" + if transition[1] == "retired" and not any( + marker in new_evidence for marker in ("反证", "替代", "canonical") + ): + errors.append("退役原因") + return {"ok": not errors, "errors": errors} + + +def _lesson_path_error(repo: Path, path: Path, relative: str) -> str | None: + match = _LESSON_PATH_RE.fullmatch(relative) + if match is None: + return "lesson path must match .codestable/lessons/YYYY-MM-DD-{slug}.md" + try: + filename_date = date.fromisoformat(match.group("date")) + except ValueError: + return "lesson filename date must be valid" + if filename_date.isoformat() != match.group("date"): + return "lesson filename date must be valid" + try: + lexical = path.relative_to(repo) + path.resolve().relative_to(repo.resolve()) + except ValueError: + return "lesson path escapes repository" + cursor = repo + for part in lexical.parts: + cursor = cursor / part + if cursor.is_symlink(): + return "lesson path must not contain symlinks" + return None + + +def validate_observed_lesson( + repo: Path, + required_concepts: list[str], + paths: list[str] | None = None, +) -> dict[str, Any]: + """机械验证一次 curation 生成的唯一 observed lesson。""" + repo = repo.resolve() + if paths is not None: + candidates = [(str(relative), Path(str(relative))) for relative in paths] + lesson_paths = [repo / path for _, path in candidates] + else: + lesson_paths = sorted((repo / ".codestable/lessons").glob("*.md")) + candidates = [ + (path.relative_to(repo).as_posix(), path.relative_to(repo)) + for path in lesson_paths + ] + errors: list[str] = [] + if len(lesson_paths) != 1: + return {"ok": False, "path": None, "errors": [f"expected one lesson, got {len(lesson_paths)}"]} + path = lesson_paths[0] + relative_text, relative_path = candidates[0] + if relative_path.is_absolute(): + return {"ok": False, "path": None, "errors": ["lesson path must be relative"]} + path_error = _lesson_path_error(repo, path, relative_text) + if path_error: + return {"ok": False, "path": None, "errors": [path_error]} + try: + text = path.read_text(encoding="utf-8") + metadata, fields = _parse_lesson_text(text) + except (OSError, ValueError, yaml.YAMLError) as exc: + return {"ok": False, "path": None, "errors": [str(exc)]} + if metadata.get("status") != "observed": + errors.append("status must be observed") + filename_date = _LESSON_PATH_RE.fullmatch(relative_text).group("date") + if metadata["date"] != filename_date: + errors.append("frontmatter date must match filename date") + lower = "\n".join(fields.values()).lower() + for concept in required_concepts: + if concept.lower() not in lower: + errors.append(f"missing concept: {concept}") + return { + "ok": not errors, + "path": relative_text, + "errors": errors, + } diff --git a/.claude/skills/eval-cs-skill/scripts/sequence.py b/.claude/skills/eval-cs-skill/scripts/sequence.py new file mode 100644 index 0000000..08b05e3 --- /dev/null +++ b/.claude/skills/eval-cs-skill/scripts/sequence.py @@ -0,0 +1,1881 @@ +#!/usr/bin/env python3 +"""learning-transfer sequence runner：在同源 repo 上执行 A、curation 与 paired B。""" + +from __future__ import annotations + +import json +import hashlib +import importlib.util +import inspect +import os +import re +import secrets +import shutil +import subprocess +import sys +import tempfile +import time + +sys.dont_write_bytecode = True + +from collections import Counter +from dataclasses import asdict +from pathlib import Path +from typing import Any, Callable + +import metrics +import yaml +from _model import ExecutionTarget, Fixture, HarnessResult, MEASURED, SOFT, tagged +from buildprompt import build_curation_prompt, build_sequence_task_prompt +from config import ExperimentConfig, resolve_skill_path +from e2e_env import ( + build_seed_repo, + changed_paths, + copy_repo, + isolated_subprocess_env, + paths_match_allowlist, + repo_control_unchanged, + repo_control_snapshot, + repo_manifest, +) +from fixtures import resolve_experiment_asset +from scorers.learning_transfer import ( + aggregate_pairs, + lesson_status, + transfer_verdict, + validate_lesson_transition, + validate_observed_lesson, +) + + +_CANDIDATE_MARKER = "晶化候选：" +_PHASE_OUTPUT_TOKENS = {"a": 2_000, "curation": 1_000, "b-treatment": 2_000, "b-control": 2_000} +_CHECKPOINT_KIND = "sequence-checkpoint-header" +_CHECKPOINT_SCHEMA_VERSION = 2 +_DETERMINISTIC_TIMEOUT_S = 60 +_PRIMARY_METRIC = { + "name": "overall_paired_hidden_pass_delta", + "threshold": 0.25, + "aggregation": "overall across both model families and four positive fixtures", + "family_guard": "each family delta > 0", +} +_PIPELINE_MODULES = ( + "_model.py", + "buildprompt.py", + "config.py", + "e2e_env.py", + "fixtures.py", + "metrics.py", + "probe_targets.py", + "runner.py", + "sequence.py", + "scorers/__init__.py", + "scorers/base.py", + "scorers/learning_transfer.py", +) +_EXPERIMENT_INPUTS = ( + "config.json", + "freeze.json", + "hypotheses.md", + "_asset_mutations.py", + "fixtures", + "checks", + "hidden", + "regression", + "preflight", + "hooks", + "bugs", + "variants", +) +_PROBE_ORACLES = ( + "cell_write", + "host_read_blocked", + "sibling_read_blocked", + "host_write_blocked", + "host_config_unchanged", + "runtime_removed", +) +_PROBE_FIELDS = frozenset({"status", "source_commit", "targets"}) +_PROBE_TARGET_FIELDS = frozenset({"id", "family", "harness", "model", *_PROBE_ORACLES}) + + +class RetryableSequenceError(RuntimeError): + """模型、adapter 或外部进程故障；保留 checkpoint 后可重试。""" + + def __init__(self, message: str, *, invocation_id: str | None = None) -> None: + super().__init__(message) + self.invocation_id = invocation_id + + +def _contained_child(base: Path, name: str) -> Path: + path = base / name + resolved = path.resolve() + try: + resolved.relative_to(base.resolve()) + except ValueError as exc: + raise ValueError(f"运行路径越过 run_root: {name!r}") from exc + return path + + +def phase_key(target_id: str, fixture_id: str, k_index: int, phase: str) -> str: + """一个 sequence phase 的稳定 checkpoint key。""" + return f"{target_id}|{fixture_id}|{k_index}|{phase}" + + +def append_checkpoint(path: Path, event: dict) -> None: + """追加一个 sequence phase 事件，并立即刷新到磁盘。""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, ensure_ascii=False) + "\n") + handle.flush() + os.fsync(handle.fileno()) + + +def _atomic_write_text(path: Path, payload: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temp_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as handle: + temp_path = Path(handle.name) + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temp_path, path) + finally: + if temp_path is not None: + temp_path.unlink(missing_ok=True) + + +def _repair_checkpoint_tail(path: Path) -> None: + """保留完整 JSONL 事件；崩溃留下的最后半行可安全丢弃。""" + if not path.exists(): + return + text = path.read_text(encoding="utf-8") + if not text or text.endswith("\n"): + return + prefix, _, tail = text.rpartition("\n") + try: + json.loads(tail) + except json.JSONDecodeError: + _atomic_write_text(path, f"{prefix}\n" if prefix else "") + else: + _atomic_write_text(path, text + "\n") + + +def _checkpoint_events(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + events: list[dict[str, Any]] = [] + for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not raw.strip(): + continue + try: + event = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"checkpoint 第 {line_number} 行不是有效 JSON") from exc + if not isinstance(event, dict): + raise ValueError(f"checkpoint 第 {line_number} 行必须是 JSON object") + events.append(event) + return events + + +def _checkpoint_record(event: dict[str, Any], *, error: str) -> dict[str, Any]: + return { + "target_id": event.get("target_id"), + "fixture_id": event.get("fixture_id"), + "k_index": event.get("k_index"), + "state": "retryable-error", + "error": error, + } + + +def _reduce_checkpoint(path: Path) -> dict[str, Any]: + """Reduce the append-only journal once for resume, cost, and fresh eligibility.""" + _repair_checkpoint_tail(path) + events = _checkpoint_events(path) + invocations: dict[str, dict[str, Any]] = {} + legacy_invocations: list[dict[str, Any]] = [] + generic_retryable: list[dict[str, Any]] = [] + pipeline_errors: list[dict[str, Any]] = [] + completed_pairs: list[dict[str, Any]] = [] + fixture_invalid: list[dict[str, Any]] = [] + + for event_index, event in enumerate(events): + phase = event.get("phase") + status = event.get("status") + invocation_id = event.get("invocation_id") + if phase in _PHASE_OUTPUT_TOKENS: + if invocation_id is None: + if isinstance(event.get("metrics"), dict): + legacy_invocations.append({**event, "journal_index": event_index}) + elif not isinstance(invocation_id, str) or not invocation_id: + raise ValueError("checkpoint invocation_id 必须是非空字符串") + elif status == "invocation-started": + if invocation_id in invocations: + raise ValueError("checkpoint invocation_id 重复 start") + if not isinstance(event.get("metrics"), dict): + raise ValueError("checkpoint invocation start 缺 fallback metrics") + invocations[invocation_id] = dict(event) + elif status in {"invocation-complete", "retryable-error"}: + started = invocations.get(invocation_id) + if started is None or started.get("status") != "invocation-started": + raise ValueError("checkpoint invocation terminal 缺 durable start") + identity_fields = ("target_id", "fixture_id", "k_index", "phase") + if any(event.get(key) != started.get(key) for key in identity_fields): + raise ValueError("checkpoint invocation terminal identity 不匹配") + metrics_value = event.get("metrics") + if metrics_value is not None and not isinstance(metrics_value, dict): + raise ValueError("checkpoint invocation terminal metrics 无效") + invocations[invocation_id] = { + **started, + **event, + "metrics": {**started["metrics"], **(metrics_value or {})}, + } + elif invocation_id is not None: + raise ValueError(f"checkpoint invocation status 无效: {status!r}") + + if phase == "score" and isinstance(event.get("pair"), dict): + completed_pairs.append(event["pair"]) + elif phase == "error" and event.get("state") == "retryable-error": + generic_retryable.append(event) + elif phase == "error" and event.get("state") == "pipeline-error": + pipeline_errors.append({ + key: event.get(key) + for key in ("target_id", "fixture_id", "k_index", "state", "error") + }) + elif phase == "fixture-invalid" and isinstance(event.get("preflight"), dict): + fixture_invalid.append({ + "fixture_id": event.get("fixture_id"), + "state": "fixture-invalid", + "preflight": event["preflight"], + }) + + operational_errors: list[dict[str, Any]] = [] + represented_ids: set[str] = set() + for invocation_id, event in invocations.items(): + status = event.get("status") + if status == "invocation-started": + represented_ids.add(invocation_id) + operational_errors.append(_checkpoint_record(event, error="InterruptedInvocation")) + elif status == "retryable-error": + represented_ids.add(invocation_id) + operational_errors.append(_checkpoint_record( + event, + error=str(event.get("error_type") or "HarnessResultError"), + )) + for event in generic_retryable: + invocation_id = event.get("invocation_id") + if isinstance(invocation_id, str) and invocation_id in represented_ids: + continue + operational_errors.append(_checkpoint_record( + event, + error=str(event.get("error") or "RetryableSequenceError"), + )) + + return { + "events": events, + "invocations": [*invocations.values(), *legacy_invocations], + "operational_errors": operational_errors, + "pipeline_errors": pipeline_errors, + "completed_pairs": completed_pairs, + "fixture_invalid": fixture_invalid, + "fresh_eligible": all(event.get("phase") == "header" for event in events), + } + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _canonical_hash(value: Any) -> str: + encoded = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return _sha256_bytes(encoded) + + +def _file_hash(path: Path) -> str: + if not path.is_file() or path.is_symlink(): + raise ValueError(f"checkpoint 输入不存在或不是普通文件: {path}") + return _sha256_bytes(path.read_bytes()) + + +def _seed_input_hash(root: Path, seed: str) -> str: + seed_dir = root / "experiments" / "seeds" / seed + builder = seed_dir / "build-seed.py" + if builder.is_file() and not builder.is_symlink(): + return _canonical_hash(_input_tree_hashes(seed_dir, (".",))) + try: + source = inspect.getsource(build_seed_repo) + except (OSError, TypeError): + code = getattr(build_seed_repo, "__code__", None) + source = repr(( + getattr(build_seed_repo, "__module__", None), + getattr(build_seed_repo, "__qualname__", None), + getattr(code, "co_code", b"").hex(), + getattr(code, "co_consts", ()), + )) + return _canonical_hash({"seed": seed, "injected_builder": source}) + + +def _input_tree_hashes(root: Path, entries: tuple[str, ...]) -> dict[str, str]: + """Hash exact input bytes while excluding generated result directories.""" + hashes: dict[str, str] = {} + for entry in entries: + path = root / entry + if path.is_symlink(): + raise ValueError(f"checkpoint 输入不得为 symlink: {path}") + if path.is_file(): + hashes[path.relative_to(root).as_posix()] = _file_hash(path) + continue + if not path.is_dir(): + continue + for child in sorted(path.rglob("*")): + if child.is_symlink(): + raise ValueError(f"checkpoint 输入不得为 symlink: {child}") + if child.is_file(): + hashes[child.relative_to(root).as_posix()] = _file_hash(child) + return hashes + + +def _pipeline_hashes() -> dict[str, str]: + scripts = Path(__file__).resolve().parent + paths = [scripts / relative for relative in _PIPELINE_MODULES] + paths.extend(sorted((scripts / "harness").glob("*.py"))) + return { + path.relative_to(scripts).as_posix(): _file_hash(path) + for path in paths + } + + +def _fixture_asset_references(fixtures: list[Fixture]) -> set[str]: + references: set[str] = set() + for fixture in fixtures: + scenario = (fixture.raw or {}).get("scenario") or {} + a_spec = scenario.get("a") or {} + b_spec = scenario.get("b") or {} + preflight = scenario.get("preflight") or {} + between = scenario.get("between_tasks") or {} + references.update(str(path) for path in a_spec.get("checks", []) if path) + references.update(str(path) for path in b_spec.get("hidden_tests", []) if path) + references.update(str(path) for path in b_spec.get("regression_tests", []) if path) + references.update( + str(preflight[key]) for key in ("naive_hook", "golden_hook") if preflight.get(key) + ) + if between.get("hook"): + references.add(str(between["hook"])) + return references + + +def _experiment_asset_hashes(experiment_dir: Path, fixtures: list[Fixture]) -> dict[str, str]: + hashes = _input_tree_hashes(experiment_dir, _EXPERIMENT_INPUTS) + for relative in sorted(_fixture_asset_references(fixtures)): + path = experiment_dir / relative + if path.is_symlink(): + raise ValueError(f"checkpoint 输入不得为 symlink: {path}") + hashes.setdefault(relative, _file_hash(path) if path.is_file() else "missing") + return hashes + + +def _root_relative(path: Path, root: Path) -> str: + try: + return path.resolve().relative_to(root.resolve()).as_posix() + except ValueError as exc: + raise ValueError(f"冻结输入不在仓库内: {path}") from exc + + +def _current_freeze_inputs( + experiment_dir: Path, + fixtures: list[Fixture], +) -> dict[str, str]: + inputs = _experiment_asset_hashes(experiment_dir, fixtures) + inputs.pop("freeze.json", None) + return inputs + + +def _current_freeze_external_inputs( + root: Path, + fixtures: list[Fixture], +) -> dict[str, str]: + external: dict[str, str] = {} + scripts = Path(__file__).resolve().parent + for relative, digest in _pipeline_hashes().items(): + external[_root_relative(scripts / relative, root)] = digest + skill_names = { + "cs-keep", + *( + str((scenario.get(phase) or {}).get("skill")) + for fixture in fixtures + for scenario in [(fixture.raw or {}).get("scenario") or {}] + for phase in ("a", "b") + if (scenario.get(phase) or {}).get("skill") + ), + } + for skill in sorted(skill_names): + path = resolve_skill_path(root, skill) + external[_root_relative(path, root)] = _file_hash(path) + seeds = { + str(((fixture.raw or {}).get("scenario") or {}).get("seed")) + for fixture in fixtures + } + for seed in sorted(seeds): + seed_dir = root / "experiments" / "seeds" / seed + for relative, digest in _input_tree_hashes(seed_dir, (".",)).items(): + external[_root_relative(seed_dir / relative, root)] = digest + return dict(sorted(external.items())) + + +def _git_blob_hash(root: Path, revision: str, relative: str) -> str | None: + result = _isolated_repo_git(root, "show", f"{revision}:{relative}") + return _sha256_bytes(result.stdout) if result.returncode == 0 else None + + +def _isolated_repo_git(root: Path, *args: str) -> subprocess.CompletedProcess[bytes]: + """Run provenance Git commands without host config, hooks, trace or credential environment.""" + with tempfile.TemporaryDirectory(prefix="cs-eval-freeze-git-", dir=root.parent) as tmp: + return subprocess.run( + ["git", *args], + cwd=root, + env=isolated_subprocess_env(Path(tmp)), + capture_output=True, + check=False, + timeout=20, + ) + + +def _freeze_run_mode(required: dict[str, Any], actual: dict[str, Any]) -> str: + if actual == required: + return "final" + required_k = int(required.get("k", 0)) + actual_k = int(actual.get("k", 0)) + if not 0 < actual_k < required_k: + raise ValueError("freeze.json required_scale 与本次执行矩阵不一致") + expected = dict(required) + expected["k"] = actual_k + for field in ("pairs", "agent_invocations", "hook_runs"): + total = int(required.get(field, -1)) + if required_k <= 0 or total < 0 or total % required_k: + raise ValueError("freeze.json required_scale 不能等比例校准") + expected[field] = total // required_k * actual_k + if actual != expected: + raise ValueError("校准必须保留完整 fixtures 与 model families，只缩小 k") + return "calibration" + + +def _validate_freeze_metadata(manifest: dict[str, Any]) -> None: + if manifest.get("schema_version") != 1: + raise ValueError("freeze.json schema_version 必须为 1") + if manifest.get("hash_algorithm") != "sha256": + raise ValueError("freeze.json hash_algorithm 必须为 sha256") + if manifest.get("primary_metric") != _PRIMARY_METRIC: + raise ValueError("freeze.json primary_metric 与预注册契约不一致") + + +def _validate_probe_attestation( + manifest: dict[str, Any], + targets: list[ExecutionTarget], + source_commit: str, +) -> None: + probe = manifest.get("model_target_probe") + if not isinstance(probe, dict) or probe.get("status") != "passed": + raise ValueError("freeze.json model_target_probe 必须先通过") + if set(probe) != _PROBE_FIELDS: + raise ValueError("model_target_probe 字段必须严格匹配 attestation schema") + if probe.get("source_commit") != source_commit: + raise ValueError("model_target_probe source_commit 不匹配") + records = probe.get("targets") + if not isinstance(records, list): + raise ValueError("model_target_probe targets 必须是 list") + expected = { + (target.id, target.family, target.harness, target.model) + for target in targets + } + actual: set[tuple[str, str, str, str]] = set() + for record in records: + if not isinstance(record, dict): + raise ValueError("model_target_probe target 必须是 object") + if set(record) != _PROBE_TARGET_FIELDS: + raise ValueError("model_target_probe target 字段必须严格匹配 attestation schema") + identity = tuple(record.get(key) for key in ("id", "family", "harness", "model")) + if len(identity) != 4 or not all(isinstance(value, str) and value for value in identity): + raise ValueError("model_target_probe target identity 无效") + actual.add(identity) + if any(record.get(oracle) is not True for oracle in _PROBE_ORACLES): + raise ValueError("model_target_probe filesystem oracle 未全部通过") + if len(actual) != len(records) or actual != expected: + raise ValueError("model_target_probe targets 与 config 不一致") + + +def validate_freeze_manifest( + *, + experiment_dir: Path, + root: Path, + config: ExperimentConfig, + fixtures: list[Fixture], + k: int, + targets: list[ExecutionTarget], +) -> dict[str, Any]: + """真实模型前验证冻结声明、当前字节与已提交 provenance 三方一致。""" + manifest_path = experiment_dir / "freeze.json" + if not manifest_path.is_file() or manifest_path.is_symlink(): + raise ValueError("真实 learning-transfer 运行缺少普通文件 freeze.json") + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError("freeze.json 不是有效 JSON") from exc + if not isinstance(manifest, dict): + raise ValueError("freeze.json 必须是 JSON object") + if manifest.get("state") != "frozen": + raise ValueError("freeze.json state 必须为 frozen；prepared/pending 禁止真实模型") + _validate_freeze_metadata(manifest) + source_commit = manifest.get("source_commit") + if not isinstance(source_commit, str) or re.fullmatch(r"[0-9a-f]{40}", source_commit) is None: + raise ValueError("freeze.json source_commit 必须是完整 40 位 commit SHA") + _validate_probe_attestation(manifest, targets, source_commit) + if (manifest.get("offline_preflight") or {}).get("status") != "passed": + raise ValueError("freeze.json offline_preflight 必须先通过") + + current_inputs = _current_freeze_inputs(experiment_dir, fixtures) + current_external = _current_freeze_external_inputs(root, fixtures) + if any(digest == "missing" for digest in current_inputs.values()): + raise ValueError("freeze.json 引用的实验资产存在缺失") + if manifest.get("inputs") != current_inputs: + raise ValueError("freeze.json inputs 与当前实验输入字节不一致") + if manifest.get("external_inputs") != current_external: + raise ValueError("freeze.json external_inputs 与当前 pipeline/skill/seed 字节不一致") + if float(manifest.get("budget_usd", -1)) != float(config.budget_usd): + raise ValueError("freeze.json budget_usd 与 config 不一致") + + positive = sum( + ((fixture.raw or {}).get("scenario") or {}).get("class") == "positive" + for fixture in fixtures + ) + guards = sum( + ((fixture.raw or {}).get("scenario") or {}).get("class") in {"unrelated", "stale"} + for fixture in fixtures + ) + pairs = len(targets) * len(fixtures) * k + hook_runs = len(targets) * k * 2 * sum( + bool((((fixture.raw or {}).get("scenario") or {}).get("between_tasks") or {}).get("hook")) + for fixture in fixtures + ) + actual_scale = { + "model_families": len({target.family for target in targets}), + "positive_fixtures": positive, + "guard_fixtures": guards, + "k": k, + "pairs": pairs, + "agent_invocations": pairs * len(_PHASE_OUTPUT_TOKENS), + "hook_runs": hook_runs, + } + run_mode = _freeze_run_mode(manifest.get("required_scale") or {}, actual_scale) + manifest_relative = _root_relative(manifest_path, root) + if _git_blob_hash(root, "HEAD", manifest_relative) != _file_hash(manifest_path): + raise ValueError("freeze.json 的 frozen 版本必须先提交到 HEAD") + ancestry = _isolated_repo_git(root, "merge-base", "--is-ancestor", source_commit, "HEAD") + if ancestry.returncode != 0: + raise ValueError("freeze.json source_commit 不是当前 HEAD 的祖先") + experiment_relative = Path(_root_relative(experiment_dir, root)) + committed_inputs = { + relative: _git_blob_hash(root, source_commit, (experiment_relative / relative).as_posix()) + for relative in current_inputs + } + committed_external = { + relative: _git_blob_hash(root, source_commit, relative) + for relative in current_external + } + if committed_inputs != current_inputs or committed_external != current_external: + raise ValueError("freeze.json source_commit 未包含声明的精确输入字节") + return { + "ok": True, + "source_commit": source_commit, + "run_mode": run_mode, + "inputs": len(current_inputs), + "external_inputs": len(current_external), + } + + +def _checkpoint_header( + *, + config: ExperimentConfig, + fixtures: list[Fixture], + k: int, + experiment_dir: Path, + root: Path, + run_root: Path, + checkpoint_path: Path, + targets: list[ExecutionTarget], +) -> dict[str, Any]: + skill_names = sorted({ + "cs-keep", + *( + str((scenario.get(phase) or {}).get("skill")) + for fixture in fixtures + for scenario in [(fixture.raw or {}).get("scenario") or {}] + for phase in ("a", "b") + if (scenario.get(phase) or {}).get("skill") + ), + }) + seeds = sorted({ + str(((fixture.raw or {}).get("scenario") or {}).get("seed")) + for fixture in fixtures + }) + inputs = { + "config": _canonical_hash(asdict(config)), + "fixtures": { + fixture.id: _canonical_hash(fixture.raw) + for fixture in sorted(fixtures, key=lambda item: item.id) + }, + "skill_snapshots": { + skill: _file_hash(resolve_skill_path(root, skill)) + for skill in skill_names + }, + "pipeline": _pipeline_hashes(), + "experiment_assets": _experiment_asset_hashes(experiment_dir, fixtures), + "seeds": { + seed: _seed_input_hash(root, seed) + for seed in seeds + }, + "targets": _canonical_hash([asdict(target) for target in targets]), + "k": k, + "run_identity": _canonical_hash({ + "experiment_dir": str(experiment_dir.resolve()), + "run_root": str(run_root.resolve()), + "checkpoint": str(checkpoint_path.resolve()), + }), + } + return { + "kind": _CHECKPOINT_KIND, + "phase": "header", + "schema_version": _CHECKPOINT_SCHEMA_VERSION, + "fingerprint": _canonical_hash(inputs), + "inputs": inputs, + } + + +def _ensure_checkpoint_header(path: Path, expected: dict[str, Any]) -> None: + _repair_checkpoint_tail(path) + if not path.exists() or not path.read_text(encoding="utf-8").strip(): + append_checkpoint(path, expected) + return + first = next(line for line in path.read_text(encoding="utf-8").splitlines() if line.strip()) + try: + actual = json.loads(first) + except json.JSONDecodeError as exc: + raise ValueError("checkpoint header 不匹配：首行不是有效 JSON") from exc + if actual != expected: + raise ValueError( + "checkpoint header 不匹配：config、fixture、skill、pipeline、experiment asset、seed、target、k " + "或 run identity 已变化；请使用新的 checkpoint 或 --fresh" + ) + + +def load_completed_pairs(path: Path) -> list[dict]: + """只恢复已经写入 score phase 的完整 pair。""" + return list(_reduce_checkpoint(path)["completed_pairs"]) + + +def load_retryable_errors(path: Path) -> list[dict[str, Any]]: + """恢复历史 operational/half-pair 错误，避免重试成功后幸存者筛选。""" + return list(_reduce_checkpoint(path)["operational_errors"]) + + +def load_pipeline_errors(path: Path) -> list[dict[str, Any]]: + """恢复不可重试的 pipeline 异常；同一冻结输入下不得再次执行。""" + return list(_reduce_checkpoint(path)["pipeline_errors"]) + + +def _checkpoint_actual_cost(path: Path) -> dict[str, Any]: + invocation_events = _reduce_checkpoint(path)["invocations"] + cost_items = [ + event.get("metrics", {}).get("cost_usd") + for event in invocation_events + if isinstance(event.get("metrics", {}).get("cost_usd"), dict) + ] + cost_tag = ( + MEASURED + if len(cost_items) == len(invocation_events) + and all(item.get("tag") == MEASURED for item in cost_items) + else SOFT + ) + return { + "invocation_count": len(invocation_events), + "cost_usd": tagged( + round(sum(float(item.get("value", 0.0)) for item in cost_items), 6), + cost_tag, + ), + } + + +def checkpoint_has_irreversible_evidence(path: Path) -> bool: + """Only a missing/header-only journal may be erased with --fresh.""" + return _reduce_checkpoint(path)["fresh_eligible"] is not True + + +def branch_order(k_index: int) -> tuple[str, str]: + """交替 paired 分支顺序，避免固定先后带来的系统偏差。""" + if k_index % 2 == 0: + return "treatment", "control" + return "control", "treatment" + + +def _lesson_manifest(manifest: dict[str, str]) -> dict[str, str]: + return {path: digest for path, digest in manifest.items() if path.startswith(".codestable/lessons/")} + + +def _without_lessons(manifest: dict[str, str]) -> dict[str, str]: + return {path: digest for path, digest in manifest.items() if not path.startswith(".codestable/lessons/")} + + +def _differs_only_by_path( + treatment: dict[str, str], + control: dict[str, str], + treatment_only_path: str, +) -> bool: + """比较 pair 时只移除新注入 lesson，保留双方原有项目 lessons。""" + reduced = dict(treatment) + reduced.pop(treatment_only_path, None) + return reduced == control + + +def materialize_paired_repos( + post_a: Path, + curation_repo: Path, + lesson_path: str, + treatment_path: Path, + control_path: Path, +) -> tuple[Path, Path]: + """从同一 post-A 基线重建 pair，只把已验证 lesson 注入 treatment。""" + lesson = Path(lesson_path) + if lesson.is_absolute() or ".." in lesson.parts or not lesson_path.startswith(".codestable/lessons/"): + raise ValueError(f"非法 lesson path: {lesson_path!r}") + treatment = copy_repo(post_a, treatment_path) + control = copy_repo(post_a, control_path) + source = curation_repo / lesson + destination = treatment / lesson + if not source.is_file() or source.is_symlink(): + raise ValueError(f"curation lesson 不存在或为 symlink: {lesson_path}") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + return treatment, control + + +def apply_between_tasks_hook( + hook: Path, + treatment: Path, + control: Path, + allowed_paths: list[str], +) -> dict[str, Any]: + """在 paired 两侧运行同一 deterministic hook，并验证 lesson 外结果同源。""" + before_manifests = {"treatment": repo_manifest(treatment), "control": repo_manifest(control)} + before_controls = { + "treatment": repo_control_snapshot(treatment), + "control": repo_control_snapshot(control), + } + before_lessons = {name: _lesson_manifest(manifest) for name, manifest in before_manifests.items()} + results = [_run_repo_script(hook, repo) for repo in (treatment, control)] + treatment_manifest = repo_manifest(treatment) + control_manifest = repo_manifest(control) + control_unchanged = ( + repo_control_unchanged(before_controls["treatment"], repo_control_snapshot(treatment)) + and repo_control_unchanged(before_controls["control"], repo_control_snapshot(control)) + ) + lesson_unchanged = ( + _lesson_manifest(treatment_manifest) == before_lessons["treatment"] + and _lesson_manifest(control_manifest) == before_lessons["control"] + ) + allowed = ( + paths_match_allowlist(changed_paths(before_manifests["treatment"], treatment_manifest), allowed_paths) + and paths_match_allowlist(changed_paths(before_manifests["control"], control_manifest), allowed_paths) + ) + return { + "ok": ( + all(result.returncode == 0 for result in results) + and lesson_unchanged + and control_unchanged + and allowed + and _without_lessons(treatment_manifest) == _without_lessons(control_manifest) + ), + "runs": len(results), + "timed_out": sum(result.returncode == 124 for result in results), + "returncodes": [result.returncode for result in results], + } + + +def _run_repo_script(script: Path, repo: Path) -> subprocess.CompletedProcess[str]: + command = [sys.executable, "-I", "-B", str(script), str(repo)] + try: + with tempfile.TemporaryDirectory(prefix="cs-eval-script-", dir=repo.parent) as tmp: + return subprocess.run( + command, + cwd=repo, + env=isolated_subprocess_env(Path(tmp), pythonpath=repo), + capture_output=True, + text=True, + check=False, + timeout=_DETERMINISTIC_TIMEOUT_S, + ) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess( + args=command, + returncode=124, + stdout="", + stderr=f"deterministic subprocess timeout after {_DETERMINISTIC_TIMEOUT_S}s", + ) + + +def _run_check_files(repo: Path, paths: list[Path]) -> dict[str, Any]: + """执行 A/B deterministic pytest，并把 timeout 记为机械失败。""" + passed = 0 + timed_out = 0 + evidence: list[dict[str, Any]] = [] + pytest_spec = importlib.util.find_spec("pytest") + if pytest_spec is None or pytest_spec.origin is None: + raise RuntimeError("deterministic checks 需要 pytest") + pytest_site = Path(pytest_spec.origin).resolve().parent.parent + bootstrap = ( + "import sys; " + "sys.path[:0] = sys.argv[1:3]; " + "import pytest; " + "raise SystemExit(pytest.main(['-q', '-p', 'no:cacheprovider', sys.argv[3]]))" + ) + with tempfile.TemporaryDirectory(prefix="cs-eval-checks-", dir=repo.parent) as tmp: + env = isolated_subprocess_env(Path(tmp), pythonpath=repo) + for path in paths: + command = [ + sys.executable, + "-I", + "-B", + "-c", + bootstrap, + str(repo.resolve()), + str(pytest_site), + str(path), + ] + try: + result = subprocess.run( + command, + cwd=repo, + env=env, + capture_output=True, + text=True, + check=False, + timeout=_DETERMINISTIC_TIMEOUT_S, + ) + raw_output = result.stdout + result.stderr + ok = result.returncode == 0 + returncode = result.returncode + except subprocess.TimeoutExpired: + timed_out += 1 + raw_output = f"deterministic pytest timeout after {_DETERMINISTIC_TIMEOUT_S}s" + ok = False + returncode = 124 + passed += int(ok) + evidence.append({ + "check_sha256": _file_hash(path), + "passed": ok, + "returncode": returncode, + "output_sha256": _sha256_bytes(raw_output.encode("utf-8", errors="replace")), + }) + total = len(paths) + return { + "passed": passed, + "total": total, + "rate": round(passed / total, 4) if total else 0.0, + "timed_out": timed_out, + "evidence": evidence, + } +def _markdown_frontmatter(path: Path) -> tuple[dict[str, Any], str]: + text = path.read_text(encoding="utf-8") + if not text.startswith("---\n"): + raise ValueError(f"缺少 frontmatter: {path}") + try: + _, raw, body = text.split("---\n", 2) + metadata = yaml.safe_load(raw) or {} + except (ValueError, yaml.YAMLError) as exc: + raise ValueError(f"frontmatter 无效: {path}") from exc + if not isinstance(metadata, dict): + raise ValueError(f"frontmatter 必须是 mapping: {path}") + return metadata, body + + +def _repo_relative_pointer(repo: Path, owner: Path, value: object) -> Path: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"缺少文档指针: {owner}") + path = (owner.parent / value).resolve() + try: + path.relative_to(repo.resolve()) + except ValueError as exc: + raise ValueError(f"文档指针越过 seed repo: {value!r}") from exc + if not path.is_file() or path.is_symlink(): + raise ValueError(f"文档指针不存在或为 symlink: {value!r}") + return path + + +def validate_epic_seed(repo: Path) -> dict[str, Any]: + """验证 Epic fixture 走真实 active 恢复路径，而不是隐式豁免 owner gate。""" + errors: list[str] = [] + cursors = sorted((repo / ".codestable/work").glob("epic-*.md")) + if len(cursors) != 1: + return {"ok": False, "errors": [f"需要恰好一个 Epic 游标，实际 {len(cursors)}"]} + cursor = cursors[0] + try: + cursor_meta, cursor_body = _markdown_frontmatter(cursor) + epic = _repo_relative_pointer(repo, cursor, cursor_meta.get("epic")) + epic_meta, epic_body = _markdown_frontmatter(epic) + except (OSError, ValueError) as exc: + return {"ok": False, "errors": [str(exc)]} + + if epic_meta.get("status") != "active": + errors.append("永久 Epic status 必须为 active") + if cursor_meta.get("phase") != "executing": + errors.append("Epic 游标 phase 必须为 executing") + approved = str(cursor_meta.get("approved_revision") or "") + if approved != _file_hash(epic): + errors.append("approved_revision 与永久 Epic SHA-256 不匹配") + try: + work_pointer = _repo_relative_pointer(repo, epic, epic_meta.get("work")) + if work_pointer != cursor.resolve(): + errors.append("永久 Epic work 指针未指向当前游标") + except (OSError, ValueError) as exc: + errors.append(str(exc)) + + progression = cursor_meta.get("item_progression") + commit = cursor_meta.get("milestone_commit") + publish = cursor_meta.get("remote_publish") + if progression not in {"continuous", "per-item"}: + errors.append("item_progression 非法") + if commit not in {"authorized", "manual"}: + errors.append("milestone_commit 非法") + if publish not in {"each-milestone", "final", "manual"}: + errors.append("remote_publish 非法") + if commit == "manual" and (progression != "per-item" or publish != "manual"): + errors.append("manual milestone 策略组合非法") + if publish == "each-milestone" and commit != "authorized": + errors.append("each-milestone publish 必须搭配 authorized commit") + + item_section = epic_body.split("## 子项契约", 1) + item_text = item_section[1].split("\n## ", 1)[0] if len(item_section) == 2 else "" + item_ids = re.findall(r"^-\s+`([^`]+)`\s*[：:]", item_text, flags=re.MULTILINE) + if len(item_ids) < 2 or len(item_ids) != len(set(item_ids)): + errors.append("永久 Epic 必须含至少两个唯一的已批准子项") + + progress_section = cursor_body.split("## 子项进度", 1) + progress_text = progress_section[1].split("\n## ", 1)[0] if len(progress_section) == 2 else "" + progress_entries = re.findall( + r"^-\s+\[([ xX])\]\s+`?([^`\s]+)`?\s*$", + progress_text, + flags=re.MULTILINE, + ) + progress = {item_id: marker.lower() == "x" for marker, item_id in progress_entries} + if set(progress) != set(item_ids): + errors.append("游标子项进度必须与已批准子项一一对应") + current = str(cursor_meta.get("current_item") or "") + if current not in item_ids: + errors.append("current_item 不属于已批准子项") + elif progress.get(current) is not False: + errors.append("current_item 必须是尚未完成的子项") + if "## 临时决策与证据" not in cursor_body: + errors.append("Epic 游标必须含临时决策与证据区") + return {"ok": not errors, "errors": errors} + + +def preflight_fixture( + fixture: Fixture, + seed_repo: Path, + experiment_dir: Path, + run_root: Path, +) -> dict[str, Any]: + """机械证明 fixture 的 golden 可解且 naive 只在目标不变量上失败。""" + scenario = (fixture.raw or {}).get("scenario") or {} + b_spec = scenario.get("b") or {} + preflight = scenario.get("preflight") or {} + if "cs-epic" in { + (scenario.get("a") or {}).get("skill"), + (scenario.get("b") or {}).get("skill"), + }: + epic_result = validate_epic_seed(seed_repo) + if not epic_result["ok"]: + return { + "ok": False, + "state": "fixture-invalid", + "reason": "; ".join(epic_result["errors"]), + "epic": epic_result, + } + run_root.mkdir(parents=True, exist_ok=True) + repos = { + "naive": copy_repo(seed_repo, run_root / "naive"), + "golden": copy_repo(seed_repo, run_root / "golden"), + } + script_results = { + name: _run_repo_script( + resolve_experiment_asset(experiment_dir, preflight[f"{name}_hook"]), repo, + ) + for name, repo in repos.items() + } + timed_out = [name for name, result in script_results.items() if result.returncode == 124] + failed_hooks = [name for name, result in script_results.items() if result.returncode != 0] + if failed_hooks: + reason = ( + f"preflight hook timeout: {', '.join(timed_out)}" + if timed_out + else f"preflight hook failed: {', '.join(failed_hooks)}" + ) + return { + "ok": False, + "state": "fixture-invalid", + "reason": reason, + "hook_returncodes": { + name: result.returncode for name, result in script_results.items() + }, + } + hidden_paths = [ + resolve_experiment_asset(experiment_dir, path) for path in b_spec.get("hidden_tests", []) + ] + regression_paths = [ + resolve_experiment_asset(experiment_dir, path) for path in b_spec.get("regression_tests", []) + ] + checks = { + name: { + "hidden": _run_check_files(repo, hidden_paths), + "regression": _run_check_files(repo, regression_paths), + } + for name, repo in repos.items() + } + golden_hidden = checks["golden"]["hidden"]["rate"] + naive_hidden = checks["naive"]["hidden"]["rate"] + golden_regression = checks["golden"]["regression"]["rate"] + naive_regression = checks["naive"]["regression"]["rate"] + ok = ( + golden_hidden == 1.0 + and naive_hidden < 1.0 + and golden_regression == 1.0 + and naive_regression == 1.0 + ) + return { + "ok": ok, + "state": "runnable" if ok else "fixture-invalid", + "reason": None if ok else "golden/naive deterministic oracle contract failed", + "golden_hidden": golden_hidden, + "naive_hidden": naive_hidden, + "golden_regression": golden_regression, + "naive_regression": naive_regression, + } + + +def dry_run_sequence( + config: ExperimentConfig, + fixtures: list[Fixture], + k: int, + root: Path, + targets: list[ExecutionTarget] | None = None, +) -> dict[str, Any]: + """估算完整 paired sequence，显式累加 A、curation 与两侧 B。""" + selected_targets = config.model_targets if targets is None else targets + if not selected_targets: + raise ValueError("learning-transfer 需要显式 model_targets") + phase_invocations = {phase: 0 for phase in _PHASE_OUTPUT_TOKENS} + per_target: list[dict[str, Any]] = [] + total = 0.0 + hook_runs = 0 + for target in selected_targets: + target_cost = 0.0 + keep_text = resolve_skill_path(root, "cs-keep").read_text(encoding="utf-8") + for fixture in fixtures: + scenario = (fixture.raw or {}).get("scenario") or {} + a_text = resolve_skill_path(root, (scenario.get("a") or {}).get("skill", "")).read_text(encoding="utf-8") + b_text = resolve_skill_path(root, (scenario.get("b") or {}).get("skill", "")).read_text(encoding="utf-8") + concepts = (scenario.get("candidate") or {}).get("required_concepts") or ["fixture candidate"] + prompts = { + "a": build_sequence_task_prompt(fixture, a_text, "a"), + "curation": build_curation_prompt(fixture, keep_text, "; ".join(concepts), "A checks passed"), + "b-treatment": build_sequence_task_prompt(fixture, b_text, "b"), + "b-control": build_sequence_task_prompt(fixture, b_text, "b"), + } + for phase, prompt in prompts.items(): + phase_invocations[phase] += k + target_cost += metrics.estimate_cost( + prompt, + target.model, + out_tokens=_PHASE_OUTPUT_TOKENS[phase], + ) * k + if (scenario.get("between_tasks") or {}).get("hook"): + hook_runs += 2 * k + total += target_cost + per_target.append({"target_id": target.id, "est_usd": round(target_cost, 4)}) + return { + "est_total_usd": round(total, 2), + "budget_usd": config.budget_usd, + "invocation_count": sum(phase_invocations.values()), + "phase_invocations": phase_invocations, + "hook_runs": hook_runs, + "per_target": per_target, + "fixtures": len(fixtures), + "targets": len(selected_targets), + "k": k, + } + + +def _all_checks_passed(result: dict[str, Any]) -> bool: + return result["passed"] == result["total"] + + +def _pipeline_failure( + *, + target: ExecutionTarget, + fixture: Fixture, + k_index: int, + a_ok: bool, + a_mutation_ok: bool, + a_repo_integrity_ok: bool, + candidate_unique: bool, + reason: str, + phase_metrics: list[dict[str, Any]], + lesson_schema_ok: bool = False, + lesson_only_mutation: bool = False, + curation_repo_integrity_ok: bool | None = None, +) -> dict[str, Any]: + scenario = (fixture.raw or {}).get("scenario") or {} + return { + "state": "pipeline-failed", + "target_id": target.id, + "family": target.family, + "fixture_id": fixture.id, + "fixture_class": scenario.get("class"), + "k_index": k_index, + "a_ok": a_ok, + "a_mutation_ok": a_mutation_ok, + "a_repo_integrity_ok": a_repo_integrity_ok, + "candidate_unique": candidate_unique, + "lesson_schema_ok": lesson_schema_ok, + "lesson_only_mutation": lesson_only_mutation, + "curation_repo_integrity_ok": curation_repo_integrity_ok, + "prompt_equal": False, + "isolation_ok": False, + "lesson_transition_ok": False, + "lesson_expectation_ok": False, + "treatment_mutation_ok": False, + "control_mutation_ok": False, + "treatment_regression_ok": False, + "control_regression_ok": False, + "treatment_hidden": 0.0, + "control_hidden": 0.0, + "phase_metrics": phase_metrics, + "failure_reason": reason, + } + + +def _invoke_phase( + *, + harness: Any, + prompt: str, + target: ExecutionTarget, + workdir: Path, + phase: str, + emit: Callable[[dict[str, Any]], None], +) -> tuple[HarnessResult, dict[str, Any]]: + """Write-ahead one provider attempt, then append its terminal metrics.""" + invocation_id = secrets.token_hex(16) + started = time.monotonic() + fallback = HarnessResult( + output="", + model=target.model, + harness=str(getattr(harness, "name", target.harness)), + wall_ms=0, + usage={"output_tokens": _PHASE_OUTPUT_TOKENS[phase]}, + error="InvocationIncomplete", + ) + fallback_metrics = metrics.capture(fallback, prompt) + emit({ + "phase": phase, + "status": "invocation-started", + "invocation_id": invocation_id, + "metrics": fallback_metrics, + }) + try: + result = harness.invoke(prompt, target.model, workdir, timeout_s=600) + phase_metrics = metrics.capture(result, prompt) + except Exception as exc: + synthetic = HarnessResult( + output="", + model=target.model, + harness=str(getattr(harness, "name", target.harness)), + wall_ms=int((time.monotonic() - started) * 1000), + usage={"output_tokens": _PHASE_OUTPUT_TOKENS[phase]}, + error=type(exc).__name__, + ) + phase_metrics = metrics.capture(synthetic, prompt) + emit({ + "phase": phase, + "status": "retryable-error", + "invocation_id": invocation_id, + "metrics": phase_metrics, + "error_type": type(exc).__name__, + }) + raise RetryableSequenceError( + f"{phase} adapter invocation failed: {type(exc).__name__}", + invocation_id=invocation_id, + ) from exc + if result.error: + emit({ + "phase": phase, + "status": "retryable-error", + "invocation_id": invocation_id, + "metrics": phase_metrics, + "error_type": "HarnessResultError", + }) + raise RetryableSequenceError( + f"{phase} harness returned an error", + invocation_id=invocation_id, + ) + emit({ + "phase": phase, + "status": "invocation-complete", + "invocation_id": invocation_id, + "metrics": phase_metrics, + }) + return result, phase_metrics + + +def run_pair( + *, + fixture: Fixture, + target: ExecutionTarget, + k_index: int, + seed_repo: Path, + experiment_dir: Path, + root: Path, + run_root: Path, + harness: Any, + phase_callback: Callable[[dict[str, Any]], None] | None = None, +) -> dict[str, Any]: + """执行一个完整 paired cell，仅返回可聚合的结构化 oracle。""" + scenario = (fixture.raw or {}).get("scenario") or {} + a_spec = scenario.get("a") or {} + b_spec = scenario.get("b") or {} + candidate_spec = scenario.get("candidate") or {} + run_root.mkdir(parents=True, exist_ok=True) + emit = phase_callback or (lambda _event: None) + + a_repo = copy_repo(seed_repo, run_root / "post-a") + before_a = repo_manifest(a_repo) + before_a_control = repo_control_snapshot(a_repo) + if before_a_control.get("safe") is not True: + return _pipeline_failure( + target=target, + fixture=fixture, + k_index=k_index, + a_ok=False, + a_mutation_ok=False, + a_repo_integrity_ok=False, + candidate_unique=False, + reason="seed Git control plane escapes the cell", + phase_metrics=[], + ) + candidate_before = snapshot_candidates(fixture, a_repo) + a_text = resolve_skill_path(root, a_spec["skill"]).read_text(encoding="utf-8") + a_prompt = build_sequence_task_prompt(fixture, a_text, "a") + a_result, a_metrics = _invoke_phase( + harness=harness, + prompt=a_prompt, + target=target, + workdir=a_repo, + phase="a", + emit=emit, + ) + a_checks = _run_check_files( + a_repo, + [resolve_experiment_asset(experiment_dir, path) for path in a_spec.get("checks", [])], + ) + a_changes = changed_paths(before_a, repo_manifest(a_repo)) + a_repo_integrity_ok = repo_control_unchanged( + before_a_control, + repo_control_snapshot(a_repo), + ) + a_mutation_ok = ( + paths_match_allowlist(a_changes, list(a_spec.get("allowed_paths", []))) + and a_repo_integrity_ok + and not any(path.startswith(".codestable/lessons/") for path in a_changes) + ) + a_ok = not a_result.error and _all_checks_passed(a_checks) and a_mutation_ok + if not a_ok: + return _pipeline_failure( + target=target, + fixture=fixture, + k_index=k_index, + a_ok=False, + a_mutation_ok=a_mutation_ok, + a_repo_integrity_ok=a_repo_integrity_ok, + candidate_unique=False, + reason="A oracle failed", + phase_metrics=[{"phase": "a", "metrics": a_metrics}], + ) + try: + candidate = extract_candidate( + fixture, + a_result.output, + a_repo, + before=candidate_before, + ) + except ValueError as exc: + return _pipeline_failure( + target=target, + fixture=fixture, + k_index=k_index, + a_ok=True, + a_mutation_ok=a_mutation_ok, + a_repo_integrity_ok=a_repo_integrity_ok, + candidate_unique=False, + reason=str(exc), + phase_metrics=[{"phase": "a", "metrics": a_metrics}], + ) + + curation_repo = copy_repo(a_repo, run_root / "curation") + before_curation = repo_manifest(curation_repo) + before_curation_control = repo_control_snapshot(curation_repo) + keep_text = resolve_skill_path(root, "cs-keep").read_text(encoding="utf-8") + curation_prompt = build_curation_prompt( + fixture, + keep_text, + candidate, + ( + f"A checks passed: {a_checks['passed']}/{a_checks['total']}; " + f"changed paths: {', '.join(sorted(a_changes))}" + ), + ) + curation_result, curation_metrics = _invoke_phase( + harness=harness, + prompt=curation_prompt, + target=target, + workdir=curation_repo, + phase="curation", + emit=emit, + ) + after_curation = repo_manifest(curation_repo) + curation_changes = changed_paths(before_curation, after_curation) + curation_repo_integrity_ok = repo_control_unchanged( + before_curation_control, + repo_control_snapshot(curation_repo), + ) + lesson_only_mutation = ( + not curation_result.error + and bool(curation_changes) + and paths_match_allowlist(curation_changes, [".codestable/lessons/**"]) + and curation_repo_integrity_ok + ) + lesson_paths = sorted(path for path in curation_changes if path.startswith(".codestable/lessons/")) + lesson_oracle = validate_observed_lesson( + curation_repo, + list(candidate_spec.get("required_concepts", [])), + lesson_paths, + ) + if not lesson_only_mutation or not lesson_oracle["ok"]: + return _pipeline_failure( + target=target, + fixture=fixture, + k_index=k_index, + a_ok=True, + a_mutation_ok=a_mutation_ok, + a_repo_integrity_ok=a_repo_integrity_ok, + candidate_unique=True, + lesson_schema_ok=lesson_oracle["ok"], + lesson_only_mutation=lesson_only_mutation, + curation_repo_integrity_ok=curation_repo_integrity_ok, + reason="curation mutation/schema failed", + phase_metrics=[ + {"phase": "a", "metrics": a_metrics}, + {"phase": "curation", "metrics": curation_metrics}, + ], + ) + + treatment, control = materialize_paired_repos( + a_repo, + curation_repo, + lesson_paths[0], + run_root / "treatment", + run_root / "control", + ) + treatment_manifest = repo_manifest(treatment) + control_manifest = repo_manifest(control) + post_a_equal = _differs_only_by_path( + treatment_manifest, + control_manifest, + lesson_paths[0], + ) + + hook_result = {"ok": True, "runs": 0} + hook_path = (scenario.get("between_tasks") or {}).get("hook") + if hook_path: + hook_result = apply_between_tasks_hook( + resolve_experiment_asset(experiment_dir, hook_path), + treatment, + control, + list((scenario.get("between_tasks") or {}).get("allowed_paths", [])), + ) + emit({"phase": "hook", "status": "passed" if hook_result["ok"] else "failed", "runs": 2}) + if not hook_result["ok"]: + return _pipeline_failure( + target=target, + fixture=fixture, + k_index=k_index, + a_ok=True, + a_mutation_ok=a_mutation_ok, + a_repo_integrity_ok=a_repo_integrity_ok, + candidate_unique=True, + lesson_schema_ok=lesson_oracle["ok"], + lesson_only_mutation=lesson_only_mutation, + reason="between_tasks hook failed", + phase_metrics=[ + {"phase": "a", "metrics": a_metrics}, + {"phase": "curation", "metrics": curation_metrics}, + ], + ) + + b_text = resolve_skill_path(root, b_spec["skill"]).read_text(encoding="utf-8") + b_prompt = build_sequence_task_prompt(fixture, b_text, "b") + prompt_hash = hashlib.sha256(b_prompt.encode("utf-8")).hexdigest() + before_b = { + "treatment": repo_manifest(treatment), + "control": repo_manifest(control), + } + before_b_control = { + "treatment": repo_control_snapshot(treatment), + "control": repo_control_snapshot(control), + } + before_lesson_text = { + path: (treatment / path).read_text(encoding="utf-8") + for path in lesson_paths + } + repos = {"treatment": treatment, "control": control} + branch_results: dict[str, Any] = {} + branch_metrics: dict[str, dict[str, Any]] = {} + for branch in branch_order(k_index): + branch_results[branch], branch_metrics[branch] = _invoke_phase( + harness=harness, + prompt=b_prompt, + target=target, + workdir=repos[branch], + phase=f"b-{branch}", + emit=emit, + ) + hidden_paths = [ + resolve_experiment_asset(experiment_dir, path) for path in b_spec.get("hidden_tests", []) + ] + regression_paths = [ + resolve_experiment_asset(experiment_dir, path) for path in b_spec.get("regression_tests", []) + ] + branch_checks: dict[str, dict[str, Any]] = {} + branch_regressions: dict[str, dict[str, Any]] = {} + branch_mutation_ok: dict[str, bool] = {} + branch_repo_integrity_ok: dict[str, bool] = {} + lesson_transition_ok = True + for branch, repo in repos.items(): + branch_checks[branch] = _run_check_files(repo, hidden_paths) + branch_regressions[branch] = _run_check_files(repo, regression_paths) + allowed = list(b_spec.get("allowed_paths", [])) + if branch == "treatment": + allowed.extend(lesson_paths) + changes = changed_paths(before_b[branch], repo_manifest(repo)) + changed_lessons = [path for path in changes if path.startswith(".codestable/lessons/")] + branch_repo_integrity_ok[branch] = repo_control_unchanged( + before_b_control[branch], + repo_control_snapshot(repo), + ) + branch_mutation_ok[branch] = ( + paths_match_allowlist(changes, allowed) + and branch_repo_integrity_ok[branch] + and (branch == "treatment" or not changed_lessons) + and (branch != "treatment" or set(changed_lessons) <= set(lesson_paths)) + ) + if branch == "treatment": + changed_lessons = [path for path in lesson_paths if path in changes] + if changed_lessons: + lesson_transition_ok = len(changed_lessons) == 1 and validate_lesson_transition( + before_lesson_text[changed_lessons[0]], + (repo / changed_lessons[0]).read_text(encoding="utf-8"), + )["ok"] + branch_mutation_ok[branch] = branch_mutation_ok[branch] and lesson_transition_ok + + def branch_ok(branch: str) -> bool: + return ( + not branch_results[branch].error + and hook_result["ok"] + and branch_mutation_ok[branch] + and _all_checks_passed(branch_checks[branch]) + and _all_checks_passed(branch_regressions[branch]) + ) + + final_lesson_text = (treatment / lesson_paths[0]).read_text(encoding="utf-8") + final_lesson_status = lesson_status(final_lesson_text) + lesson_unchanged = final_lesson_text == before_lesson_text[lesson_paths[0]] + expected_transition = (scenario.get("expect") or {}).get("lesson_transition") + lesson_expectation_ok = { + "observed->validated": final_lesson_status == "validated" and lesson_transition_ok, + "unchanged-observed": final_lesson_status == "observed" and lesson_unchanged, + "observed->retired": final_lesson_status == "retired" and lesson_transition_ok, + }.get(expected_transition, False) + + return { + "target_id": target.id, + "family": target.family, + "fixture_id": fixture.id, + "fixture_class": scenario.get("class"), + "k_index": k_index, + "a_ok": a_ok, + "a_mutation_ok": a_mutation_ok, + "a_repo_integrity_ok": a_repo_integrity_ok, + "candidate_unique": True, + "post_a_equal": post_a_equal, + "lesson_schema_ok": lesson_oracle["ok"], + "lesson_only_mutation": lesson_only_mutation, + "curation_repo_integrity_ok": curation_repo_integrity_ok, + "prompt_equal": True, + "prompt_hash": prompt_hash, + "hook_ok": hook_result["ok"], + "isolation_ok": post_a_equal and hook_result["ok"], + "lesson_transition_ok": lesson_transition_ok, + "lesson_expectation_ok": lesson_expectation_ok, + "lesson_unchanged": lesson_unchanged, + "lesson_status": final_lesson_status, + "stale_retired": final_lesson_status == "retired" if scenario.get("class") == "stale" else None, + "treatment_ok": branch_ok("treatment"), + "control_ok": branch_ok("control"), + "treatment_hidden": branch_checks["treatment"]["rate"], + "control_hidden": branch_checks["control"]["rate"], + "treatment_mutation_ok": branch_mutation_ok["treatment"], + "control_mutation_ok": branch_mutation_ok["control"], + "treatment_repo_integrity_ok": branch_repo_integrity_ok["treatment"], + "control_repo_integrity_ok": branch_repo_integrity_ok["control"], + "treatment_regression_ok": _all_checks_passed(branch_regressions["treatment"]), + "control_regression_ok": _all_checks_passed(branch_regressions["control"]), + "phase_metrics": [ + {"phase": "a", "metrics": a_metrics}, + {"phase": "curation", "metrics": curation_metrics}, + {"phase": "b-treatment", "metrics": branch_metrics["treatment"]}, + {"phase": "b-control", "metrics": branch_metrics["control"]}, + ], + } + + +def _pair_identity(pair: dict[str, Any]) -> tuple[str, str, int]: + return str(pair["target_id"]), str(pair["fixture_id"]), int(pair["k_index"]) + + +def _read_preflight_cache(path: Path, fingerprint: str) -> dict[str, Any] | None: + try: + cached = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return None + if not isinstance(cached, dict): + return None + result = cached.get("result") + if cached.get("checkpoint_fingerprint") != fingerprint or not isinstance(result, dict): + return None + return result + + +def _write_preflight_cache(path: Path, fingerprint: str, result: dict[str, Any]) -> None: + payload = json.dumps( + {"checkpoint_fingerprint": fingerprint, "result": result}, + ensure_ascii=False, + indent=2, + ) + "\n" + _atomic_write_text(path, payload) + + +def run_sequence( + *, + config: ExperimentConfig, + fixtures: list[Fixture], + k: int, + experiment_dir: Path, + root: Path, + run_root: Path, + checkpoint_path: Path, + targets: list[ExecutionTarget] | None = None, + harness_resolver=None, +) -> dict[str, Any]: + """运行完整 learning-transfer matrix，并从 score checkpoint 幂等恢复。""" + if harness_resolver is None: + import harness as harness_pkg + harness_resolver = harness_pkg.get_harness + selected_targets = config.model_targets if targets is None else targets + if not selected_targets: + raise ValueError("learning-transfer 需要显式 model_targets") + header = _checkpoint_header( + config=config, + fixtures=fixtures, + k=k, + experiment_dir=experiment_dir, + root=root, + run_root=run_root, + checkpoint_path=checkpoint_path, + targets=selected_targets, + ) + _ensure_checkpoint_header(checkpoint_path, header) + restored_journal = _reduce_checkpoint(checkpoint_path) + restored_invalid = { + str(record["fixture_id"]): record + for record in restored_journal["fixture_invalid"] + if record.get("fixture_id") is not None + } + run_root.mkdir(parents=True, exist_ok=True) + preflight_dir = run_root / "preflight" + preflight_dir.mkdir(exist_ok=True) + preflight_results: dict[str, dict[str, Any]] = {} + invalid: list[dict[str, Any]] = [] + runnable: list[Fixture] = [] + for fixture in fixtures: + result_path = _contained_child(preflight_dir, f"{fixture.id}.json") + restored = restored_invalid.get(fixture.id) + preflight_result: dict[str, Any] | None = ( + dict(restored["preflight"]) if restored is not None else None + ) + if preflight_result is None and result_path.exists(): + preflight_result = _read_preflight_cache(result_path, header["fingerprint"]) + if preflight_result is None: + scenario = (fixture.raw or {}).get("scenario") or {} + try: + with tempfile.TemporaryDirectory(prefix=f"{fixture.id}-", dir=preflight_dir) as tmp: + tmp_path = Path(tmp) + seed_repo = build_seed_repo(str(scenario["seed"]), tmp_path / "seed", root) + preflight_result = preflight_fixture( + fixture, + seed_repo, + experiment_dir, + tmp_path / "runs", + ) + except Exception as exc: + preflight_result = { + "ok": False, + "state": "fixture-invalid", + "reason": f"{type(exc).__name__}: {str(exc)[:300]}", + } + if preflight_result.get("ok") is not True: + append_checkpoint(checkpoint_path, { + "fixture_id": fixture.id, + "phase": "fixture-invalid", + "phase_key": f"fixture|{fixture.id}|preflight", + "status": "fixture-invalid", + "preflight": preflight_result, + }) + _write_preflight_cache(result_path, header["fingerprint"], preflight_result) + preflight_results[fixture.id] = preflight_result + if preflight_result.get("ok") is True: + runnable.append(fixture) + else: + invalid.append({ + "fixture_id": fixture.id, + "state": "fixture-invalid", + "preflight": preflight_result, + }) + completed = load_completed_pairs(checkpoint_path) + completed_keys = {_pair_identity(pair) for pair in completed} + pairs = list(completed) + operational_history = load_retryable_errors(checkpoint_path) + pipeline_errors = load_pipeline_errors(checkpoint_path) + terminal_keys = {_pair_identity(error) for error in pipeline_errors} + for target in selected_targets: + harness = harness_resolver(target.harness) + for fixture in runnable: + scenario = (fixture.raw or {}).get("scenario") or {} + for k_index in range(k): + identity = (target.id, fixture.id, k_index) + if identity in completed_keys or identity in terminal_keys: + continue + cell_root = _contained_child(run_root, f"{target.id}__{fixture.id}__{k_index}") + if cell_root.exists(): + shutil.rmtree(cell_root) + def checkpoint_phase(event: dict[str, Any]) -> None: + append_checkpoint(checkpoint_path, { + "target_id": target.id, + "fixture_id": fixture.id, + "k_index": k_index, + **event, + "phase_key": phase_key( + target.id, + fixture.id, + k_index, + str(event.get("phase", "unknown")), + ), + }) + try: + seed_repo = build_seed_repo(str(scenario["seed"]), cell_root / "seed", root) + pair = run_pair( + fixture=fixture, + target=target, + k_index=k_index, + seed_repo=seed_repo, + experiment_dir=experiment_dir, + root=root, + run_root=cell_root / "pair", + harness=harness, + phase_callback=checkpoint_phase, + ) + except RetryableSequenceError as exc: + error = { + "target_id": target.id, + "fixture_id": fixture.id, + "k_index": k_index, + "state": "retryable-error", + "error": type(exc).__name__, + } + if exc.invocation_id is not None: + error["invocation_id"] = exc.invocation_id + operational_history.append(error) + append_checkpoint(checkpoint_path, { + **error, + "phase": "error", + "phase_key": phase_key(target.id, fixture.id, k_index, "error"), + "status": "retryable-error", + }) + continue + except Exception as exc: + error = { + "target_id": target.id, + "fixture_id": fixture.id, + "k_index": k_index, + "state": "pipeline-error", + "error": type(exc).__name__, + } + pipeline_errors.append(error) + terminal_keys.add(identity) + append_checkpoint(checkpoint_path, { + **error, + "phase": "error", + "phase_key": phase_key(target.id, fixture.id, k_index, "error"), + "status": "pipeline-error", + }) + continue + finally: + if cell_root.exists(): + shutil.rmtree(cell_root) + pair.setdefault("state", "completed") + pair.update({ + "target_id": target.id, + "family": target.family, + "model": target.model, + "harness": target.harness, + "fixture_id": fixture.id, + "fixture_class": scenario.get("class"), + "owning_skill": (scenario.get("a") or {}).get("skill"), + "k_index": k_index, + }) + append_checkpoint(checkpoint_path, { + "target_id": target.id, + "fixture_id": fixture.id, + "k_index": k_index, + "phase": "score", + "phase_key": phase_key(target.id, fixture.id, k_index, "score"), + "status": pair["state"], + "pair": pair, + }) + pairs.append(pair) + completed_outcomes = {_pair_identity(pair) for pair in pairs} + operational_errors = [ + {**error, "resolved": _pair_identity(error) in completed_outcomes} + for error in operational_history + ] + unresolved_operational = [ + {key: value for key, value in error.items() if key != "resolved"} + for error in operational_errors + if error["resolved"] is False + ] + errors = [*pipeline_errors, *unresolved_operational] + records = [*pairs, *errors, *invalid] + aggregate = aggregate_pairs(records, operational_errors=operational_errors) + aggregate["cost"] = _checkpoint_actual_cost(checkpoint_path) + return { + "experiment": config.name, + "execution_mode": "learning-transfer", + "k": k, + "pairs": pairs, + "errors": errors, + "operational_errors": operational_errors, + "invalid": invalid, + "preflight": preflight_results, + "aggregate": aggregate, + "verdict": transfer_verdict(aggregate), + } + + +def _candidate_lines(text: str) -> list[str]: + matches: list[str] = [] + for raw in text.splitlines(): + line = raw.strip() + if line.startswith("- "): + line = line[2:].strip() + if line.startswith(_CANDIDATE_MARKER): + candidate = line[len(_CANDIDATE_MARKER):].strip() + if candidate: + matches.append(candidate) + return matches + + +def _epic_candidate_lines(repo: Path) -> list[str]: + cursors = sorted((repo / ".codestable/work").glob("epic-*.md")) + if len(cursors) != 1: + raise ValueError(f"Epic 候选需要恰好一个游标，实际 {len(cursors)} 个") + text = cursors[0].read_text(encoding="utf-8") + sections = text.split("## 临时决策与证据", 1) + if len(sections) != 2: + raise ValueError("Epic 游标缺少临时决策与证据区") + evidence = sections[1].split("\n## ", 1)[0] + return _candidate_lines(evidence) + + +def snapshot_candidates(fixture: Fixture, repo: Path) -> Counter[str]: + """A 前快照持久通道，避免把旧 Epic 候选归因给本轮。""" + scenario = (fixture.raw or {}).get("scenario") or {} + source = (scenario.get("a") or {}).get("candidate_source") + if source == "output": + return Counter() + if source == "epic-cursor": + return Counter(_epic_candidate_lines(repo)) + raise ValueError(f"尚不支持的 candidate_source: {source!r}") + + +def extract_candidate( + fixture: Fixture, + output: str, + repo: Path, + *, + before: Counter[str] | None = None, +) -> str: + """按 fixture 声明提取本轮新增、位置正确且概念完整的唯一候选。""" + scenario = (fixture.raw or {}).get("scenario") or {} + source = (scenario.get("a") or {}).get("candidate_source") + if source == "output": + matches = _candidate_lines(output) + nonempty = [line.strip() for line in output.splitlines() if line.strip()] + first = nonempty[0] if nonempty else "" + if not first.startswith(_CANDIDATE_MARKER): + raise ValueError("普通任务候选必须位于首个非空输出行") + elif source == "epic-cursor": + baseline = before or Counter() + current = Counter(_epic_candidate_lines(repo)) + matches = list((current - baseline).elements()) + if any(baseline[match] for match in matches): + raise ValueError("Epic 候选不得重复既有规则") + else: + raise ValueError(f"尚不支持的 candidate_source: {source!r}") + if len(matches) != 1: + raise ValueError(f"本轮新增候选必须恰好一条，实际 {len(matches)} 条") + candidate = matches[0] + missing = [ + concept for concept in (scenario.get("candidate") or {}).get("required_concepts", []) + if str(concept).lower() not in candidate.lower() + ] + if missing: + raise ValueError(f"候选缺少 required concepts: {missing}") + return candidate diff --git a/.codestable/work/epic-cs-continuous-learning-lifecycle.md b/.codestable/work/epic-cs-continuous-learning-lifecycle.md index 7f20384..bf7e022 100644 --- a/.codestable/work/epic-cs-continuous-learning-lifecycle.md +++ b/.codestable/work/epic-cs-continuous-learning-lifecycle.md @@ -3,7 +3,7 @@ epic: ../epics/cs-continuous-learning-lifecycle.md phase: executing approved_revision: 4b0b8e8e0596e7ee42611864843512ecd9402970bb3682b5605f0dc0f8dcf01a current_item: LEARN-3 -next_action: implement LEARN-3 with tests first +next_action: squash and publish the reviewed source, then run real target probes blocked_by: null item_progression: continuous milestone_commit: authorized @@ -77,3 +77,78 @@ remote_publish: each-milestone `claude-fable-5` / `plan-high`，冻结 staged patch SHA-256 `6a04b044...50bdf`； `0 blocking / 0 important / 2 nit`，结论可合。保留的 nit 仅为英文 validated 术语精度和两套测试 各自保留同名空白归一化 helper，不影响行为、授权或契约真实性。 +- LEARN-3 tests-first：新增 sequence、oracle、harness、fixture/config/cost 契约；实现 paired A / fresh + `cs-keep` / treatment-control B、phase checkpoint、Git/manifest 隔离、成本聚合与 fail-closed freeze gate。 +- LEARN-3 专项审计发现并修复：fixture/skill/seed/pipeline 输入漂移、截断 checkpoint 与 preflight cache、 + 历史 retry integrity、candidate delta、adapter 异常成本、不同 `--out` 隔离、local Git include 预读取及 + Claude 临时 HOME 认证。新增三条回归先得到 `3 failed`，实现后转为 `3 passed`。 +- LEARN-3 离线验证：sequence `37 passed`，oracle/adapter/既有 eval 回归通过；全量 + `252 passed, 1 skipped`，分发 `3 passed, 1 skipped`，plugin package check 与 `git diff --check` 通过。 +- 六个冻结 fixture 逐个重建 seed 后均为 golden hidden/regression `1.0/1.0`、naive `0.0/1.0`；seed + verify `4 passed`。dry-run 为 240 次 agent invocation、20 次 hook、`$3.41 [soft]`，低于 `$50` 预算。 +- model target 离线探针：Claude/Codex CLI 与 adapter flags 均存在，隔离后的 provider auth 均可用； + 探针只执行 help/version/auth status，未发起模型请求。`freeze.json` 仍为 + `prepared-awaiting-commit`，真实 LLM 尚未运行。 +- LEARN-3 frozen input checkpoint：commit + `3022df55b4aca50e38d289558793db3ddd2205ac`，初始 attestation commit `715c831`；两者均未用于模型 + 调用。提交后专项审计补入 Git 隔离修复，freeze gate 因 source bytes 漂移正确阻断，初始 attestation + 随即废弃并等待新 source checkpoint。 +- LEARN-3 checkpoint 后加固：Git snapshot 额外隔离宿主 global/system/env config 注入；同一 `--out` + 使用非阻塞进程锁，`--fresh` 同时清除 checkpoint、preflight 与 cell state；在首个 Git 进程前拒绝 + `config.worktree`、`commondir` 及 object alternates；freeze schema、hash algorithm、aggregation 和 + family guard 均由契约测试锁定。 +- LEARN-3 加固后验证：四份定向套件 `169 passed`、全量 `263 passed, 1 skipped`、分发 + `3 passed, 1 skipped`，plugin package check 与 `git diff --check` 通过；seed verify `4 passed`， + 六 fixture preflight 与 `$3.41 [soft]` dry-run 结果保持不变；未发起真实模型调用。 +- LEARN-3 hardened source checkpoint：commit + `9fb5d0fae63e308cfd7b7375295f65c53a202ef6`；纠正后的 manifest 保持 33 个实验输入与 25 个外部 + 输入，仅替换 source SHA 与加固后的 `e2e_env.py` 哈希。 +- 后续审计发现 UTF-8 BOM 可让 Git 接受 local include 而绕过首进程前检测；现改用 `utf-8-sig` + 解码并以回归测试锁定，sequence `45 passed`。此前 `fe635f1` 未执行协议要求的最小真实 target + probe，因此不作为有效 attestation；候选 manifest 已恢复 `prepared-awaiting-commit` / `pending`。 +- LEARN-3 晶化候选：无。输入冻结与 promotion fail-closed 已由既有反馈管线经验、sequence 契约测试和 + freeze manifest 共同承接，不另建重复 lesson。 +- LEARN-3 diff review round 1：fresh Paseo `814fd85a-2a72-459e-adda-d23512a80a2a`， + `claude-fable-5` / `high`，冻结 base `1d78fdd` 到 index patch SHA-256 `4a047d71...d362f0`； + `0 blocking / 3 important / 6 nit`。important 指向宿主 Claude env 测试泄漏、retry history 与 + structural integrity 混合、Codex 读隔离无真实探针，均在任何真实模型调用前处理。 +- LEARN-3 review 修复：Claude settings 测试清理完整 provider key；operational error 与 deterministic + failure 分流；Claude/Codex 统一采用外层 Seatbelt，Codex 保留内层 `workspace-write`；新增逐 target + 真实模型探针，只允许输出布尔 oracle，不落 prompt、模型回答、sentinel 或 session id。 +- retry/隔离专项审计进一步补齐 append-only write-ahead journal：每次 provider 前用新 invocation ID + `flush + fsync` start 与 soft fallback，terminal 只追加；单 reducer 统一成本、interrupted/retry history、 + fixture-invalid、pipeline/score 与 fresh eligibility。只有完整 pair 才把 operational error 标 resolved， + `--fresh` 仅允许 header-only journal，终态或已有结果必须使用新 `--out`。 +- journal 与输出保全新增回归先得到 `8 failed`，实现后 sequence/oracle `96 passed`；探针显式 + `BLOCKED` marker 与宿主 `~/.claude.json` 快照先得到 `2 failed` 后转绿。四份 eval 定向套件当前 + `203 passed`，`git diff --check` 干净；freeze 机械比对为 33 inputs / 26 external inputs 全部一致， + 仍保持 `prepared-awaiting-commit`、`source_commit: pending`、probe pending、无真实 LLM run。 +- 隔离审计 follow-up 发现并修复 attestation 未拒绝额外字段、宿主状态快照漏掉 session/rollout、首次 + snapshot 失败残留 sentinel：probe/target 字段改为闭集，快照递归覆盖已知持久状态但不读取 transcript， + sentinel 从创建起受 `finally` 清理；相关用例均先红后绿。 +- 六 fixture 复跑时发现 `python -I` 忽略 `PYTHONDONTWRITEBYTECODE` 环境变量并污染冻结资产；seed、 + injector、hook 与 deterministic pytest 现均显式使用 `-B`，五个回归先红后绿，实验目录复跑后无 + `__pycache__`。该错误已机械化，不另建 lesson。 +- LEARN-3 最新验证：四份 eval 定向 `214 passed`，全量 `308 passed, 1 skipped`，seed verify + `4 passed`；六 fixture 继续满足 golden `1.0/1.0`、naive `0.0/1.0`，dry-run 仍为 240 invocation、 + 20 hook、`$3.41 [soft]`。freeze 为 33 inputs / 26 external inputs 全匹配，保持 prepared/pending， + 未执行真实模型。 +- LEARN-3 隔离专项终态复核：内建只读 agent 对最新四个实现文件给出 + `0 blocking / 0 important / 0 nit`；独立确认 snapshot 异常无 sentinel 残留、四条 `-I -B` 路径均由 + 真实 sibling import 测试锁定，freeze external inputs 匹配且无 bytecode 资产。 +- LEARN-3 full diff review round 2：fresh Paseo `cbf5e71d-0d79-4620-8cb9-a9b75be9bb72`， + `claude-fable-5` / `high`，冻结 base `1d78fdd` 到 index patch SHA-256 `955845cd...b70245c`、 + tree `39e5d475...864e2b`；`0 blocking / 1 important / 5 nit`。唯一 important 指出截断 journal tail + 修复仍以非原子整文件重写，二次崩溃窗口可能丢失 append-only 证据。 +- LEARN-3 atomic repair tests-first：故障注入先复现 `os.replace` 失败时的保全要求；实现同目录临时文件、 + `flush + fsync + os.replace` 的 `_atomic_write_text`，checkpoint tail 与 preflight cache 共用该 helper。 + 红态 `1 failed`，修复后聚焦 `5 passed`；四份 eval 定向 `215 passed`、全量 + `309 passed, 1 skipped`、分发 `3 passed, 1 skipped`，package checker、seed verify 与 + `git diff --check` 通过。dry-run 保持 240 invocation / 20 hook / `$3.41 [soft]`，freeze 的 33 inputs + 与 26 external inputs 全匹配，仍为 prepared/pending 且未运行真实模型。 +- LEARN-3 atomic repair 晶化候选：无。该崩溃窗口已由故障注入回归和原子写 helper 机械承接。 +- LEARN-3 final prepared diff review round 3：fresh Paseo + `c95eeb1c-ba20-41b0-88d7-5c69c2927243`，`claude-fable-5` / `high`，冻结 base `1d78fdd` 到 index + patch SHA-256 `7e43904b...c022`、tree `39429368...edf4f`；`0 blocking / 0 important / 4 nit`， + 结论可合。上一轮 atomic tail repair important 经字节级故障注入与全候选复核确认闭合；保留 nit 仅为 + 父目录 fsync、探针错误诊断、锁文件和 kill -9 临时文件清理等可选加固，不影响本轮证据完整性。 diff --git a/docs/adr/003-cs-skill-evaluation-loop.md b/docs/adr/003-cs-skill-evaluation-loop.md index 3beeb46..bf1034b 100644 --- a/docs/adr/003-cs-skill-evaluation-loop.md +++ b/docs/adr/003-cs-skill-evaluation-loop.md @@ -34,6 +34,19 @@ CodeStable 原有 `tests/test_skill_*` 只验证 skill **写得对不对**（路 8. **v2 输入边界**：评测 fixture 是 repo-local 维护者资产，不依赖任何 shipped runtime skill。`promote_feedback_fixture.py` 只保留为 v1 `cs-feedback` candidate 的 legacy-only 导入器，不构成 v2 production feedback 入口。 +9. **项目 lesson 的 paired sequence**：`answerType/task.kind: learning-transfer` 由独立 + `sequence.py` / `learning_transfer` scorer 承载 A -> fresh `cs-keep` -> treatment/control fresh B。 + 它只用于维护者验证项目内跨会话迁移，不在用户使用 CodeStable 时构造实验。post-A 同源、B prompt + equality、严格 lesson schema/窄迁移、Git 与文件 mutation、hidden/regression、stale retirement + 都是机械 oracle。Deterministic failure 永久阻断 structural integrity；adapter/transport error 单列 + operational history，成功重试仍计成本但不永久污染 integrity，未解决则保持 underpowered。调用前 + durable append start 与 soft fallback，terminal 只追加；仅完整 pair 能把历史 operational error 标为 + resolved，`--fresh` 只允许 header-only journal，其他 campaign 必须使用新的输出身份。 +10. **冻结与隔离是接受前提**：真实调用前提交 hypothesis 与完整 campaign 输入，checkpoint fingerprint + 绑定 config、fixtures、skill snapshots、runner/scorer、seed、target、`k` 与 run identity，校准不得 + 混入最终结果。每个 harness 必须通过宿主与 sibling cell 读取、宿主写入隔离探针；deterministic + 子进程只获最小环境且不保留原始输出。cell repo 在 oracle 后销毁，只保留结构化指标和哈希，不保留 + transcript 或完整 treatment/control 仓库。 ## Consequences @@ -42,6 +55,8 @@ CodeStable 原有 `tests/test_skill_*` 只验证 skill **写得对不对**（路 - 冻结的 v1 feedback candidate 仍可显式导入历史 experiment；v2 不承诺 production feedback promotion。 - eval-cs-skill 自身可被同一闭环评测优化（自指）。 - 真实多模型运行需 API/CLI 鉴权并产生成本，受 `--dry-run` + `budget_usd` 护栏约束。 +- paired learning-transfer 的结论要求至少两个 model family、每 fixture 每 family `k>=5`；校准可用 + `k=2` 但只能标探路证据，任何 primary aggregate 仍为 `[underpowered]` 时不得接受。 - **评测效度是头等风险**（首轮真实 campaign 教训）：必须复现被测 skill 实际声明的 context contract，而不是注入统一 onboard runtime；同时用语义 oracle（`recall_judge`）判散文 answer，并让 fixture 内嵌被操作的 subject matter。否则测到的是「skill 在残缺环境下的反应」而非真实能力。核查须分模型看 + 手工读原始输出 + 认 k=1 variance。细则见 `references/eval/protocol.md` 效度三铁律。 ## Rejected alternatives diff --git a/experiments/cs-learning-transfer-001/_asset_mutations.py b/experiments/cs-learning-transfer-001/_asset_mutations.py new file mode 100644 index 0000000..73ed0ac --- /dev/null +++ b/experiments/cs-learning-transfer-001/_asset_mutations.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Deterministic mutations used only by learning-transfer preflight and hooks.""" + +from __future__ import annotations + +from pathlib import Path +from textwrap import dedent + + +def _write(repo: Path, relative: str, content: str) -> None: + path = repo / relative + path.write_text(dedent(content).lstrip(), encoding="utf-8") + + +def apply_stale_reference_policy(repo: Path) -> None: + _write(repo, "dispatchboard/references.py", _references("naive", opaque_v2=True)) + attention = repo / ".codestable/attention.md" + attention_text = attention.read_text(encoding="utf-8") + policy = ( + "- `reference_policy: opaque-v2`：业务引用按存储值精确匹配；不得 trim、casefold " + "或互换 `_` / `-`，返回值保留存储 spelling。\n" + ) + if policy not in attention_text: + attention.write_text(attention_text.rstrip() + "\n" + policy, encoding="utf-8") + _write( + repo, + "tests/test_references.py", + ''' + from dispatchboard.references import find_job, find_route + + + def test_job_reference_uses_opaque_v2_policy(): + assert find_job("JOB-7")["id"] == "JOB-7" + assert find_job(" job-7 ") is None + assert find_job("job-7") is None + assert find_job("JOB_7") is None + + + def test_exact_route_reference_remains_available(): + assert find_route("Route_7")["id"] == "Route_7" + ''', + ) + + +def _patch_body(kind: str) -> str: + if kind == "old": + return """ + if value == "": + continue + if value is None: + result.pop(key, None) + else: + result[key] = value + """ + if kind == "truthy": + return """ + if value: + result[key] = value + """ + raise ValueError(kind) + + +def _patch_function(name: str, kind: str) -> str: + body = "\n".join(f" {line}" for line in dedent(_patch_body(kind)).strip().splitlines()) + return f'''def {name}(record: dict, changes: dict) -> dict: + result = record.copy() + for key, value in changes.items(): +{body} + return result +''' + + +def _patches(mode: str) -> str: + job_kind = "old" + route_kind = "old" if mode == "golden" else "truthy" + return f'''"""Partial-update helpers for dispatchboard records.""" + +{_patch_function("patch_job", job_kind)} + +{_patch_function("patch_route", route_kind)} +''' + + +def _post_a_patch() -> str: + return f'''"""Partial-update helpers for dispatchboard records.""" + +{_patch_function("patch_job", "old")} + +def patch_route(route: dict, changes: dict) -> dict: + raise NotImplementedError("patch_route is not implemented") +''' + + +def _job_sequence() -> str: + return ''' + """Job number allocation.""" + + + def next_job_number(active_numbers: list[int], retired_numbers: list[int]) -> int: + return max([*active_numbers, *retired_numbers], default=0) + 1 + ''' + + +def _sequence(mode: str) -> str: + expression = ( + "max([*active_numbers, *retired_numbers], default=0) + 1" + if mode == "golden" + else "max(active_numbers, default=0) + 1" + ) + return f'''"""Route number allocation.""" + + +def next_route_number(active_numbers: list[int], retired_numbers: list[int]) -> int: + return {expression} +''' + + +def _grouping(mode: str) -> str: + if mode == "naive": + return ''' + """Event grouping helpers.""" + + + def _group_events(events: list[dict]) -> list[tuple[str, list[dict]]]: + groups: dict[str, list[dict]] = {} + for event in events: + groups.setdefault(event["group"], []).append(event) + return list(groups.items()) + + + def group_jobs(events: list[dict]) -> list[tuple[str, list[dict]]]: + return _group_events(events) + + + def group_routes(events: list[dict]) -> list[tuple[str, list[dict]]]: + return _group_events(events) + ''' + return ''' + """Event grouping helpers.""" + + + def _group_events( + events: list[dict], *, deduplicate: bool = False, sort_keys: bool = False + ) -> list[tuple[str, list[dict]]]: + selected = list({event["id"]: event for event in events}.values()) if deduplicate else events + groups: dict[str, list[dict]] = {} + for event in selected: + groups.setdefault(event["group"], []).append(event) + items = list(groups.items()) + return sorted(items) if sort_keys else items + + + def group_jobs(events: list[dict]) -> list[tuple[str, list[dict]]]: + return _group_events(events) + + + def group_routes(events: list[dict]) -> list[tuple[str, list[dict]]]: + return _group_events(events, deduplicate=True, sort_keys=True) + ''' + + +def _display(mode: str) -> str: + expression = ( + "value.strip()" + if mode == "golden" + else 'value.strip().replace("_", "-").casefold()' + ) + return f'''"""Display-only formatting helpers.""" + + +def display_route_label(value: str) -> str: + return {expression} +''' + + +def _references(mode: str, *, opaque_v2: bool = False) -> str: + if opaque_v2: + policy = "opaque-v2" + job_body = "return JOBS.get(reference)" + channel_body = "return CHANNELS.get(reference)" + route_body = ( + "return ROUTES.get(reference)" + if mode == "golden" + else "wanted = canonical_ref(reference)\n" + " return next((item for key, item in ROUTES.items() " + "if canonical_ref(key) == wanted), None)" + ) + else: + policy = "canonical-v1" + job_body = ( + "wanted = canonical_ref(reference)\n" + " return next((item for key, item in JOBS.items() " + "if canonical_ref(key) == wanted), None)" + ) + channel_body = ( + "wanted = canonical_ref(reference)\n" + " return next((item for key, item in CHANNELS.items() " + "if canonical_ref(key) == wanted), None)" + if mode == "golden" + else "return CHANNELS.get(reference)" + ) + route_body = ( + "wanted = canonical_ref(reference)\n" + " return next((item for key, item in ROUTES.items() " + "if canonical_ref(key) == wanted), None)" + ) + return f'''"""Reference lookup policies.""" + +REFERENCE_POLICY = "{policy}" +JOBS = {{"JOB-7": {{"id": "JOB-7", "title": "Pack"}}}} +CHANNELS = {{"Channel-7": {{"id": "Channel-7", "title": "North"}}}} +ROUTES = {{"Route_7": {{"id": "Route_7", "title": "Primary"}}}} + + +def canonical_ref(value: str) -> str: + return value.strip().replace("_", "-").casefold() + + +def find_job(reference: str) -> dict | None: + {job_body} + + +def find_channel(reference: str) -> dict | None: + {channel_body} + + +def find_route(reference: str) -> dict | None: + {route_body} +''' + + +def apply_preflight(repo: Path, scenario: str, mode: str) -> None: + if mode not in {"naive", "golden"}: + raise ValueError(mode) + if scenario == "patch": + _write(repo, "dispatchboard/patches.py", _patches(mode)) + elif scenario == "sequence": + _write(repo, "dispatchboard/job_sequence.py", _job_sequence()) + _write(repo, "dispatchboard/route_sequence.py", _sequence(mode)) + elif scenario == "reference": + _write(repo, "dispatchboard/references.py", _references(mode)) + elif scenario == "epic-sequence": + _write(repo, "dispatchboard/job_sequence.py", _job_sequence()) + _write(repo, "dispatchboard/route_sequence.py", _sequence(mode)) + cursor = repo / ".codestable/work/epic-sequence-rollout.md" + cursor_text = cursor.read_text(encoding="utf-8").replace("- [ ] SEQ-A", "- [x] SEQ-A") + cursor.write_text( + cursor_text.rstrip() + + "\n\n- 晶化候选：实现 sibling allocator 前先盘点全部 persisted history sources，并建立 single snapshot。\n", + encoding="utf-8", + ) + elif scenario == "grouping": + _write(repo, "dispatchboard/grouping.py", _grouping(mode)) + elif scenario == "display": + _write(repo, "dispatchboard/patches.py", _post_a_patch()) + _write(repo, "dispatchboard/display.py", _display(mode)) + elif scenario == "stale-reference-v2": + apply_stale_reference_policy(repo) + _write(repo, "dispatchboard/references.py", _references(mode, opaque_v2=True)) + else: + raise ValueError(scenario) diff --git a/experiments/cs-learning-transfer-001/checks/test_epic_sequence_a.py b/experiments/cs-learning-transfer-001/checks/test_epic_sequence_a.py new file mode 100644 index 0000000..7758282 --- /dev/null +++ b/experiments/cs-learning-transfer-001/checks/test_epic_sequence_a.py @@ -0,0 +1,21 @@ +from hashlib import sha256 +from pathlib import Path +import re + +from dispatchboard.job_sequence import next_job_number + + +def test_epic_current_item_is_complete_and_revision_stays_valid(): + repo = Path.cwd() + epic = repo / ".codestable/epics/sequence-rollout.md" + cursor = repo / ".codestable/work/epic-sequence-rollout.md" + cursor_text = cursor.read_text(encoding="utf-8") + + assert next_job_number([2, 7], [4, 11]) == 12 + assert re.search(r"- \[[xX]\] SEQ-A", cursor_text) + assert "- [ ] SEQ-B" in cursor_text + assert sha256(epic.read_bytes()).hexdigest() in cursor_text + assert cursor_text.count("晶化候选：") == 1 + assert "persisted history sources" in cursor_text + assert "single snapshot" in cursor_text + assert "sibling allocator" in cursor_text diff --git a/experiments/cs-learning-transfer-001/checks/test_grouping_a.py b/experiments/cs-learning-transfer-001/checks/test_grouping_a.py new file mode 100644 index 0000000..6454183 --- /dev/null +++ b/experiments/cs-learning-transfer-001/checks/test_grouping_a.py @@ -0,0 +1,20 @@ +import ast +import inspect + +import dispatchboard.grouping as grouping + + +def test_group_jobs_refactor_preserves_first_seen_order_and_duplicates(): + events = [ + {"id": "e1", "group": "B", "value": 1}, + {"id": "e1", "group": "B", "value": 2}, + {"id": "e2", "group": "A", "value": 3}, + ] + + groups = grouping.group_jobs(events) + + implementation = ast.parse(inspect.getsource(grouping.group_jobs)) + assert not any(isinstance(node, ast.For) for node in ast.walk(implementation)) + assert any(isinstance(node, ast.Call) for node in ast.walk(implementation)) + assert [key for key, _ in groups] == ["B", "A"] + assert [row["value"] for row in groups[0][1]] == [1, 2] diff --git a/experiments/cs-learning-transfer-001/checks/test_patch_a.py b/experiments/cs-learning-transfer-001/checks/test_patch_a.py new file mode 100644 index 0000000..2cb86ea --- /dev/null +++ b/experiments/cs-learning-transfer-001/checks/test_patch_a.py @@ -0,0 +1,17 @@ +from dispatchboard.patches import patch_job + + +def test_patch_job_distinguishes_omit_clear_and_falsey_values(): + original = { + "title": "Pack", + "note": "fragile", + "attempts": 3, + "enabled": True, + } + result = patch_job( + original, + {"title": "", "note": None, "attempts": 0, "enabled": False}, + ) + + assert result == {"title": "Pack", "attempts": 0, "enabled": False} + assert original["note"] == "fragile" diff --git a/experiments/cs-learning-transfer-001/checks/test_reference_job_a.py b/experiments/cs-learning-transfer-001/checks/test_reference_job_a.py new file mode 100644 index 0000000..f0fac06 --- /dev/null +++ b/experiments/cs-learning-transfer-001/checks/test_reference_job_a.py @@ -0,0 +1,8 @@ +from dispatchboard.references import find_job + + +def test_job_reference_normalizes_query_without_rewriting_storage(): + result = find_job(" job_7 ") + + assert result is not None + assert result["id"] == "JOB-7" diff --git a/experiments/cs-learning-transfer-001/config.json b/experiments/cs-learning-transfer-001/config.json new file mode 100644 index 0000000..1d721b6 --- /dev/null +++ b/experiments/cs-learning-transfer-001/config.json @@ -0,0 +1,33 @@ +{ + "name": "cs-learning-transfer-001", + "skill_under_test": "cs-feat", + "execution_mode": "learning-transfer", + "variants": [ + "baseline" + ], + "k": 5, + "scorers": [ + "learning_transfer" + ], + "fixture_classes": [ + "positive", + "unrelated", + "stale" + ], + "budget_usd": 50.0, + "inject_context": false, + "model_targets": [ + { + "id": "claude-haiku", + "family": "claude", + "harness": "claude-headless", + "model": "claude-haiku-4-5" + }, + { + "id": "codex-terra", + "family": "codex", + "harness": "codex-cli", + "model": "gpt-5.6-terra" + } + ] +} diff --git a/experiments/cs-learning-transfer-001/fixtures/positive/lt-epic-sequence.json b/experiments/cs-learning-transfer-001/fixtures/positive/lt-epic-sequence.json new file mode 100644 index 0000000..9db74f3 --- /dev/null +++ b/experiments/cs-learning-transfer-001/fixtures/positive/lt-epic-sequence.json @@ -0,0 +1,54 @@ +{ + "id": "lt-epic-sequence", + "answerType": "learning-transfer", + "task": { + "kind": "learning-transfer" + }, + "scenario": { + "class": "positive", + "seed": "dispatchboard-learning", + "a": { + "skill": "cs-epic", + "request": "fresh reviewer 已复现：Sequence rollout Epic 的 SEQ-A 只读取一处当前记录，遗漏另一处 persisted history source 后会分配已使用编号。恢复 active Epic，执行当前已批准子项 SEQ-A；实现前盘点全部 persisted history sources 并建立 single snapshot。沿用现有批准 revision 与 per-item/manual 策略，不修改拆解或扩大范围；运行必要验证并按游标契约记录进度。收尾的唯一晶化候选只记录可复用方法：实现 sibling allocator 前先盘点全部 persisted history sources，并建立 single snapshot。", + "candidate_source": "epic-cursor", + "checks": [ + "checks/test_epic_sequence_a.py" + ], + "allowed_paths": [ + "dispatchboard/job_sequence.py", + "tests/**", + ".codestable/work/epic-sequence-rollout.md" + ] + }, + "candidate": { + "expected_home": "lesson", + "required_concepts": [ + "persisted history sources", + "single snapshot", + "sibling allocator" + ] + }, + "b": { + "skill": "cs-epic", + "request": "继续仓库中 active 的 Sequence rollout Epic，执行已批准子项 SEQ-B。沿用现有批准 revision 与 per-item/manual 策略，不修改拆解或扩大范围；运行必要验证并按游标契约记录进度。", + "hidden_tests": [ + "hidden/test_route_sequence.py" + ], + "regression_tests": [ + "regression/test_seed_integrity.py" + ], + "allowed_paths": [ + "dispatchboard/route_sequence.py", + "tests/**", + ".codestable/work/epic-sequence-rollout.md" + ] + }, + "preflight": { + "naive_hook": "preflight/epic-sequence-naive.py", + "golden_hook": "preflight/epic-sequence-golden.py" + }, + "expect": { + "lesson_transition": "observed->validated" + } + } +} diff --git a/experiments/cs-learning-transfer-001/fixtures/positive/lt-feat-patch.json b/experiments/cs-learning-transfer-001/fixtures/positive/lt-feat-patch.json new file mode 100644 index 0000000..9ee3666 --- /dev/null +++ b/experiments/cs-learning-transfer-001/fixtures/positive/lt-feat-patch.json @@ -0,0 +1,55 @@ +{ + "id": "lt-feat-patch", + "answerType": "learning-transfer", + "task": { + "kind": "learning-transfer" + }, + "scenario": { + "class": "positive", + "seed": "dispatchboard-learning", + "a": { + "skill": "cs-feat", + "request": "实现 dispatchboard.patches.patch_job。此前用 if value 合并导致 0/False 丢失：partial update 规则是 empty string 表示未提交该字段，None 表示清除已有字段，0 与 False 是必须写入的真实值；返回新 dict，不原地修改输入。运行必要验证。", + "candidate_source": "output", + "checks": [ + "checks/test_patch_a.py" + ], + "allowed_paths": [ + "dispatchboard/patches.py", + "tests/**", + ".codestable/work/**" + ] + }, + "candidate": { + "expected_home": "lesson", + "required_concepts": [ + "empty string", + "None", + "0", + "False" + ] + }, + "b": { + "skill": "cs-feat", + "request": "实现 dispatchboard.patches.patch_route，使 route 的 partial update 与本项目同类更新行为一致；返回新 dict，不原地修改输入，并运行必要验证。", + "hidden_tests": [ + "hidden/test_patch_route.py" + ], + "regression_tests": [ + "regression/test_seed_integrity.py" + ], + "allowed_paths": [ + "dispatchboard/patches.py", + "tests/**", + ".codestable/work/**" + ] + }, + "preflight": { + "naive_hook": "preflight/feat-patch-naive.py", + "golden_hook": "preflight/feat-patch-golden.py" + }, + "expect": { + "lesson_transition": "observed->validated" + } + } +} diff --git a/experiments/cs-learning-transfer-001/fixtures/positive/lt-issue-reference.json b/experiments/cs-learning-transfer-001/fixtures/positive/lt-issue-reference.json new file mode 100644 index 0000000..7796730 --- /dev/null +++ b/experiments/cs-learning-transfer-001/fixtures/positive/lt-issue-reference.json @@ -0,0 +1,55 @@ +{ + "id": "lt-issue-reference", + "answerType": "learning-transfer", + "task": { + "kind": "learning-transfer" + }, + "scenario": { + "class": "positive", + "seed": "dispatchboard-learning", + "a": { + "skill": "cs-issue", + "request": "修复 find_job 无法用 ` job_7 ` 找到已存储 `JOB-7` 的问题。诊断时同时核对 storage 与 query 两侧 normalization，返回对象必须保留存储的 display spelling；不要据此声明全局 canonical policy。运行必要验证，并在收尾时只报告一条可复用的诊断方法候选。", + "candidate_source": "output", + "checks": [ + "checks/test_reference_job_a.py" + ], + "allowed_paths": [ + "dispatchboard/references.py", + "tests/**", + ".codestable/work/**" + ] + }, + "candidate": { + "expected_home": "lesson", + "required_concepts": [ + "storage", + "query", + "normalization", + "display spelling" + ] + }, + "b": { + "skill": "cs-issue", + "request": "修复 find_channel 无法解析与已存储 channel reference 等价的查询形式，同时保留返回对象中原始 id spelling；运行必要验证。", + "hidden_tests": [ + "hidden/test_reference_channel.py" + ], + "regression_tests": [ + "regression/test_seed_integrity.py" + ], + "allowed_paths": [ + "dispatchboard/references.py", + "tests/**", + ".codestable/work/**" + ] + }, + "preflight": { + "naive_hook": "preflight/issue-reference-naive.py", + "golden_hook": "preflight/issue-reference-golden.py" + }, + "expect": { + "lesson_transition": "observed->validated" + } + } +} diff --git a/experiments/cs-learning-transfer-001/fixtures/positive/lt-refactor-events.json b/experiments/cs-learning-transfer-001/fixtures/positive/lt-refactor-events.json new file mode 100644 index 0000000..46d08b1 --- /dev/null +++ b/experiments/cs-learning-transfer-001/fixtures/positive/lt-refactor-events.json @@ -0,0 +1,54 @@ +{ + "id": "lt-refactor-events", + "answerType": "learning-transfer", + "task": { + "kind": "learning-transfer" + }, + "scenario": { + "class": "positive", + "seed": "dispatchboard-learning", + "a": { + "skill": "cs-refactor", + "request": "对 group_jobs 做行为等价重构。上次抽共享 helper 时把 rows 按 id 去重并排序，造成回归；这次提取可复用的私有 helper 前先锁定这个 caller 的 first-seen group order 和 duplicate rows 语义。不得改变公开函数签名；运行必要验证。", + "candidate_source": "output", + "checks": [ + "checks/test_grouping_a.py" + ], + "allowed_paths": [ + "dispatchboard/grouping.py", + "tests/**", + ".codestable/work/**" + ] + }, + "candidate": { + "expected_home": "lesson", + "required_concepts": [ + "first-seen", + "duplicate", + "caller" + ] + }, + "b": { + "skill": "cs-refactor", + "request": "让 group_routes 复用 grouping.py 中的新私有 helper，保持其当前可观察行为完全不变：group key 排序、重复 id 以后出现的 row 为准且只保留一次。运行必要验证。", + "hidden_tests": [ + "hidden/test_group_routes.py" + ], + "regression_tests": [ + "regression/test_seed_integrity.py" + ], + "allowed_paths": [ + "dispatchboard/grouping.py", + "tests/**", + ".codestable/work/**" + ] + }, + "preflight": { + "naive_hook": "preflight/refactor-events-naive.py", + "golden_hook": "preflight/refactor-events-golden.py" + }, + "expect": { + "lesson_transition": "observed->validated" + } + } +} diff --git a/experiments/cs-learning-transfer-001/fixtures/stale/lt-stale-reference-v2.json b/experiments/cs-learning-transfer-001/fixtures/stale/lt-stale-reference-v2.json new file mode 100644 index 0000000..a253888 --- /dev/null +++ b/experiments/cs-learning-transfer-001/fixtures/stale/lt-stale-reference-v2.json @@ -0,0 +1,63 @@ +{ + "id": "lt-stale-reference-v2", + "answerType": "learning-transfer", + "task": { + "kind": "learning-transfer" + }, + "scenario": { + "class": "stale", + "seed": "dispatchboard-learning", + "a": { + "skill": "cs-feat", + "request": "为 find_job 增加对 ` job_7 ` 这类等价 query reference 的查找能力。实现前同时核对 storage 与 query 两侧 normalization，返回对象保留存储的 display spelling；不要声明全局 canonical policy。运行必要验证，并在收尾时只报告一条可复用的实现方法候选。", + "candidate_source": "output", + "checks": [ + "checks/test_reference_job_a.py" + ], + "allowed_paths": [ + "dispatchboard/references.py", + "tests/test_references.py", + ".codestable/work/**" + ] + }, + "candidate": { + "expected_home": "lesson", + "required_concepts": [ + "storage", + "query", + "normalization", + "display spelling" + ] + }, + "between_tasks": { + "hook": "hooks/stale-reference-opaque-v2.py", + "allowed_paths": [ + "dispatchboard/references.py", + ".codestable/attention.md", + "tests/test_references.py" + ] + }, + "b": { + "skill": "cs-feat", + "request": "项目刚升级了 reference policy。按当前仓库事实修正 find_route，使 sibling lookup 与新 policy 一致，同时保留存储的 id spelling；运行必要验证。", + "hidden_tests": [ + "hidden/test_reference_route_v2.py" + ], + "regression_tests": [ + "regression/test_seed_integrity.py" + ], + "allowed_paths": [ + "dispatchboard/references.py", + "tests/**", + ".codestable/work/**" + ] + }, + "preflight": { + "naive_hook": "preflight/stale-reference-v2-naive.py", + "golden_hook": "preflight/stale-reference-v2-golden.py" + }, + "expect": { + "lesson_transition": "observed->retired" + } + } +} diff --git a/experiments/cs-learning-transfer-001/fixtures/unrelated/lt-unrelated-display.json b/experiments/cs-learning-transfer-001/fixtures/unrelated/lt-unrelated-display.json new file mode 100644 index 0000000..ac6fc16 --- /dev/null +++ b/experiments/cs-learning-transfer-001/fixtures/unrelated/lt-unrelated-display.json @@ -0,0 +1,55 @@ +{ + "id": "lt-unrelated-display", + "answerType": "learning-transfer", + "task": { + "kind": "learning-transfer" + }, + "scenario": { + "class": "unrelated", + "seed": "dispatchboard-learning", + "a": { + "skill": "cs-feat", + "request": "实现 dispatchboard.patches.patch_job。此前用 if value 合并导致 0/False 丢失：partial update 规则是 empty string 表示未提交该字段，None 表示清除已有字段，0 与 False 是必须写入的真实值；返回新 dict，不原地修改输入。运行必要验证。", + "candidate_source": "output", + "checks": [ + "checks/test_patch_a.py" + ], + "allowed_paths": [ + "dispatchboard/patches.py", + "tests/**", + ".codestable/work/**" + ] + }, + "candidate": { + "expected_home": "lesson", + "required_concepts": [ + "empty string", + "None", + "0", + "False" + ] + }, + "b": { + "skill": "cs-feat", + "request": "实现 display_route_label：只移除展示字符串两端的空白，大小写、下划线和连字符必须原样保留。运行必要验证。", + "hidden_tests": [ + "hidden/test_display_route_label.py" + ], + "regression_tests": [ + "regression/test_seed_integrity.py" + ], + "allowed_paths": [ + "dispatchboard/display.py", + "tests/**", + ".codestable/work/**" + ] + }, + "preflight": { + "naive_hook": "preflight/unrelated-display-naive.py", + "golden_hook": "preflight/unrelated-display-golden.py" + }, + "expect": { + "lesson_transition": "unchanged-observed" + } + } +} diff --git a/experiments/cs-learning-transfer-001/freeze.json b/experiments/cs-learning-transfer-001/freeze.json new file mode 100644 index 0000000..e18cbbe --- /dev/null +++ b/experiments/cs-learning-transfer-001/freeze.json @@ -0,0 +1,105 @@ +{ + "schema_version": 1, + "state": "prepared-awaiting-commit", + "prepared_on": "2026-08-02", + "hash_algorithm": "sha256", + "source_commit": "pending", + "real_llm_runs_started": false, + "model_target_probe": { + "status": "pending", + "source_commit": "pending", + "targets": [] + }, + "primary_metric": { + "name": "overall_paired_hidden_pass_delta", + "threshold": 0.25, + "aggregation": "overall across both model families and four positive fixtures", + "family_guard": "each family delta > 0" + }, + "budget_usd": 50.0, + "required_scale": { + "model_families": 2, + "positive_fixtures": 4, + "guard_fixtures": 2, + "k": 5, + "pairs": 60, + "agent_invocations": 240, + "hook_runs": 20 + }, + "offline_preflight": { + "status": "passed", + "golden_hidden_rate": 1.0, + "naive_hidden_rate": 0.0, + "golden_regression_rate": 1.0, + "naive_regression_rate": 1.0, + "a_seed_passed": 0, + "a_seed_total": 4, + "a_golden_passed": 4, + "a_golden_total": 4, + "estimated_cost_usd": 3.41, + "estimated_cost_tag": "soft" + }, + "external_inputs": { + ".claude/skills/eval-cs-skill/scripts/_model.py": "5847997e90aba27d998dbd811e9f1507d0979bd89bfb76775da716337d47f24f", + ".claude/skills/eval-cs-skill/scripts/buildprompt.py": "047e64fc119c7e75060fc0bc581a12c52273761c1263d1975e6924986f21ed9f", + ".claude/skills/eval-cs-skill/scripts/config.py": "094923d9b0fb8676cc2910758fd8660a9a5d3fd9637ed2189ddaf83442f3d113", + ".claude/skills/eval-cs-skill/scripts/e2e_env.py": "0728d026c5ebed589d3ae19fe14cb8f5f38d5616501307aaf5624f121c1684e6", + ".claude/skills/eval-cs-skill/scripts/fixtures.py": "cfb5f41817eec3ec28be4322a5959d31afd8f9b90e10fa6a842b270083dd7305", + ".claude/skills/eval-cs-skill/scripts/harness/__init__.py": "c4acec6d69f515784bdab53afbb99ac5c3f7f5e527a62fea290f534f7f021e41", + ".claude/skills/eval-cs-skill/scripts/harness/adapter_api.py": "b8fafd8fe14d8856274880a45c9f470de17624adc6e0a88f99664acaa6f48dfb", + ".claude/skills/eval-cs-skill/scripts/harness/adapter_claude.py": "905dcaee41a8a9b5629489a8734c0a5b968d8e3cc73d6c20173e0a0dbda1c2a8", + ".claude/skills/eval-cs-skill/scripts/harness/adapter_codex.py": "e3fd6425888b30962e28efaac497d847f3cfa72b2673ab44652e55fc0268c6af", + ".claude/skills/eval-cs-skill/scripts/harness/adapter_paseo.py": "15ec2e456b8af9493da2b405d6379136a7872a7aaea0e7c6d8486073468c1a17", + ".claude/skills/eval-cs-skill/scripts/harness/base.py": "85bafd62450ef9e69bd8dc0ac7da8d0a867ff7655f61a96dd580ba890f8be884", + ".claude/skills/eval-cs-skill/scripts/harness/mock.py": "e9de0dad8dc219a27756e5827bdf4ad0dfada9bb17ca461249ff31627de93fae", + ".claude/skills/eval-cs-skill/scripts/harness/mock_weak.py": "b50f2fadf541158791f918cf46dd55f30d13c1eba948bdc2f71026a3750a6234", + ".claude/skills/eval-cs-skill/scripts/metrics.py": "59372930ebf1b55baeea3492055288ce4d9decf3599e919c360a2be731d505df", + ".claude/skills/eval-cs-skill/scripts/probe_targets.py": "9c97a4b2c6afc5a15d8f6971b475d9958e05fd8e2c5787db1395b79d2c7a6039", + ".claude/skills/eval-cs-skill/scripts/runner.py": "a80b55d9ef1ae07fc579a9d56cd832cd08dbb3d33fb5eaea055335e6a10d5a61", + ".claude/skills/eval-cs-skill/scripts/scorers/__init__.py": "917a22c5614078caeb4daf20b5f71687e16838af67a624e30e427c4834a649e0", + ".claude/skills/eval-cs-skill/scripts/scorers/base.py": "dba496f77c139f516d09f7c64a1d2e38db2e5a2b981c76ce165b5d56b4218189", + ".claude/skills/eval-cs-skill/scripts/scorers/learning_transfer.py": "d34ba315c065b880c0043af94a9ca87bd275d7b18e4ee3d8346858d33fdbebce", + ".claude/skills/eval-cs-skill/scripts/sequence.py": "f11637b3474c4161549fe9380c10c734c3d14a18214824707ba2d9f3c34d11df", + "experiments/seeds/dispatchboard-learning/build-seed.py": "4c7f7090abe69aca0b6be2a3568c85a91345374a4879f94da9f6704f028689b1", + "plugins/codestable/skills/cs-epic/SKILL.md": "4f67e0efc6aa446eceaf3e57bf31b4fcc18af7ba87ce927be6b4bbb0d35407aa", + "plugins/codestable/skills/cs-feat/SKILL.md": "54261caf4d0cb8f01b225c3041d41476691efbaa29a5390e7f9887c9cbfb70ff", + "plugins/codestable/skills/cs-issue/SKILL.md": "203ca2710355a56a12236c2d55f57e7264b744be4f13edd633af4f352be1b707", + "plugins/codestable/skills/cs-keep/SKILL.md": "3d2d2cbb0114b963b1847f3d2caaa2d58db193cbf1cf38039d5b52044d3dd6ca", + "plugins/codestable/skills/cs-refactor/SKILL.md": "a225c692ce7776f9305b14cbb7c424656304cbe2b59c0a70aeedb3802e5b01b8" + }, + "inputs": { + "_asset_mutations.py": "fe952ddc35ddc1fd0976627043383f9b4d4ec05fd8202ac5fe963bdd0b904210", + "checks/test_epic_sequence_a.py": "8e4da3a0323f5c5fc933e1ac51da835e082edbed4a6b27a123ba8ab94fbdce05", + "checks/test_grouping_a.py": "1824398cb4bb63393eabd02aed096ebc3d85447d27f7ec7f25d1e311dfa52dcc", + "checks/test_patch_a.py": "1f3aba4d8d0a78037d9463a472406a5d8efe2249724f85256b80e763f7bdfc43", + "checks/test_reference_job_a.py": "fafd9aff6aa8740ee82c08e4694d1b910d393daa843100db19bb2c97c33c757f", + "config.json": "709e23cded9e1a76e4cb4d712445180060050c66e7f3f07922c94ab7fa8bcdce", + "fixtures/positive/lt-epic-sequence.json": "622aba952c16b125c12e5cd278d2079faf2c07bb09a190de4dda34a14b8e9405", + "fixtures/positive/lt-feat-patch.json": "41542cbdcaf7e12ba075695cf1a1cca62379352c07c2bd7a433af11f8cb0c8a8", + "fixtures/positive/lt-issue-reference.json": "3234b15ceee742570e502953e639f14ee5cc69f5b28f51bba749a4b05f35addb", + "fixtures/positive/lt-refactor-events.json": "efebfab2c46ca01608e4c1a6ef3b7e42af328ea587eae39b5e3c466dc3b9b7bb", + "fixtures/stale/lt-stale-reference-v2.json": "8f7b21a8510e1576f70531ceab67bf86a9534ce1e534e8d683acd6243a02b13e", + "fixtures/unrelated/lt-unrelated-display.json": "c4cbc8bd7cc6e96b70853ea611547458ecd743c88058b366d1c445f607ffb40e", + "hidden/test_display_route_label.py": "ac61a72295579ffc9a0b1da6c562ff3afa172b5c6183663833a5c62524453745", + "hidden/test_group_routes.py": "2559076bb914e68e2252d452e43e70f6842c6fa7908fc3ac3abdfd2855cb1c67", + "hidden/test_patch_route.py": "a5edc897899a131e5c4735c12ce066068cb1c69fab07d156e9b133e38f0a7fb7", + "hidden/test_reference_channel.py": "99d3cf5cf66a96ec45de890c1610692266b56d4c4bc149ae7101073ea53be1ab", + "hidden/test_reference_route_v2.py": "74d8cc747446cb7d939658a6e5fe3b322de4ec1fbdd9c075ce37a368a43b56f4", + "hidden/test_route_sequence.py": "f53c2759c5d0eb9b649ba53f01259fa4e3b245a4397d39eb97e26ab34a66d29e", + "hooks/stale-reference-opaque-v2.py": "3fefea656f9317514d64eee4a0e156d69964456a0800e53bb21f3447eda814eb", + "hypotheses.md": "43b2c53a2ab34a77c64019406840c3b7821208817edc5926d8c5a6b598e32fff", + "preflight/epic-sequence-golden.py": "9a0be0b0cdccc94181791d57a6f9c0af126f6b05da22bc0ff5995d9bdb6c89d1", + "preflight/epic-sequence-naive.py": "2b5db110732e6ea4881c2cd5167f9b9a91ca677f421d907a1baf018bbc0612df", + "preflight/feat-patch-golden.py": "3d97700c0311883a0412dfcade5ccb82b9e1580d9c2e46ec8720140b784b3752", + "preflight/feat-patch-naive.py": "f5be0bca9d6540158e7a59a39a9655946864b8198626bf590be504706dcacc69", + "preflight/issue-reference-golden.py": "6be990afa89cce704e6aa0ba16a329a985f90aacca1c2265a342846b3e3023b9", + "preflight/issue-reference-naive.py": "597e0c4f8963788747c295cf0253c367002959b80e2c1899b9c13eb1e4f4ece4", + "preflight/refactor-events-golden.py": "7ddfba5bfd472dbc469758e2aada14e120f71c0f613e89c187d2fb65e0b3153f", + "preflight/refactor-events-naive.py": "1075d745aa2b701d75adb5c15387f1e39acfe39c62b01fd7128ce288e90e12da", + "preflight/stale-reference-v2-golden.py": "54b91c4f4fdefab21a30b5a80c7102fb6613f6aa1c5f5ea806066a0cf308415e", + "preflight/stale-reference-v2-naive.py": "4403c7b33da0b215dc3cbcd334b77507e9a8fe884c732a64b799db8ce2908fa3", + "preflight/unrelated-display-golden.py": "af741c7d1da7d620b51ad32b0d722310aaaccb722aa401e6b07982b9884321ac", + "preflight/unrelated-display-naive.py": "a066548343bf5d8513561beef10c281e4aa362e9c7deb34016e5d0f720413bcf", + "regression/test_seed_integrity.py": "aac084a69f2f7c5da1f6d949ea03d9ab189af5b31abdda13b701652b51c2f043" + } +} diff --git a/experiments/cs-learning-transfer-001/hidden/test_display_route_label.py b/experiments/cs-learning-transfer-001/hidden/test_display_route_label.py new file mode 100644 index 0000000..2a8bab0 --- /dev/null +++ b/experiments/cs-learning-transfer-001/hidden/test_display_route_label.py @@ -0,0 +1,6 @@ +from dispatchboard.display import display_route_label + + +def test_display_label_trims_without_canonicalizing_visible_text(): + assert display_route_label(" Route_Seven-A ") == "Route_Seven-A" + assert display_route_label(" NORTH_wing ") == "NORTH_wing" diff --git a/experiments/cs-learning-transfer-001/hidden/test_group_routes.py b/experiments/cs-learning-transfer-001/hidden/test_group_routes.py new file mode 100644 index 0000000..b931bda --- /dev/null +++ b/experiments/cs-learning-transfer-001/hidden/test_group_routes.py @@ -0,0 +1,17 @@ +from dispatchboard.grouping import group_routes + + +def test_route_grouping_keeps_its_sorted_deduplicated_behavior(): + events = [ + {"id": "r1", "group": "B", "value": 1}, + {"id": "r2", "group": "B", "value": 2}, + {"id": "r1", "group": "A", "value": 3}, + {"id": "r3", "group": "A", "value": 4}, + ] + + groups = group_routes(events) + + assert [key for key, _ in groups] == ["A", "B"] + assert [row["id"] for row in groups[0][1]] == ["r1", "r3"] + assert groups[0][1][0]["value"] == 3 + assert [row["id"] for _, rows in groups for row in rows].count("r1") == 1 diff --git a/experiments/cs-learning-transfer-001/hidden/test_patch_route.py b/experiments/cs-learning-transfer-001/hidden/test_patch_route.py new file mode 100644 index 0000000..8508655 --- /dev/null +++ b/experiments/cs-learning-transfer-001/hidden/test_patch_route.py @@ -0,0 +1,17 @@ +from dispatchboard.patches import patch_route + + +def test_route_patch_uses_project_partial_update_semantics(): + original = { + "label": "North", + "note": "seasonal", + "priority": 5, + "enabled": True, + } + result = patch_route( + original, + {"label": "", "note": None, "priority": 0, "enabled": False}, + ) + + assert result == {"label": "North", "priority": 0, "enabled": False} + assert original["note"] == "seasonal" diff --git a/experiments/cs-learning-transfer-001/hidden/test_reference_channel.py b/experiments/cs-learning-transfer-001/hidden/test_reference_channel.py new file mode 100644 index 0000000..5848c14 --- /dev/null +++ b/experiments/cs-learning-transfer-001/hidden/test_reference_channel.py @@ -0,0 +1,8 @@ +from dispatchboard.references import find_channel + + +def test_channel_reference_reuses_the_diagnosed_lookup_method(): + result = find_channel(" channel_7 ") + + assert result is not None + assert result["id"] == "Channel-7" diff --git a/experiments/cs-learning-transfer-001/hidden/test_reference_route_v2.py b/experiments/cs-learning-transfer-001/hidden/test_reference_route_v2.py new file mode 100644 index 0000000..fe69cba --- /dev/null +++ b/experiments/cs-learning-transfer-001/hidden/test_reference_route_v2.py @@ -0,0 +1,9 @@ +from dispatchboard.references import REFERENCE_POLICY, find_route + + +def test_opaque_v2_route_reference_requires_exact_storage_spelling(): + assert REFERENCE_POLICY == "opaque-v2" + assert find_route("Route_7")["id"] == "Route_7" + assert find_route(" route_7 ") is None + assert find_route("route_7") is None + assert find_route("Route-7") is None diff --git a/experiments/cs-learning-transfer-001/hidden/test_route_sequence.py b/experiments/cs-learning-transfer-001/hidden/test_route_sequence.py new file mode 100644 index 0000000..c42c52a --- /dev/null +++ b/experiments/cs-learning-transfer-001/hidden/test_route_sequence.py @@ -0,0 +1,7 @@ +from dispatchboard.route_sequence import next_route_number + + +def test_route_sequence_never_reuses_active_or_retired_numbers(): + assert next_route_number([3, 5], [6, 13]) == 14 + assert next_route_number([], [9]) == 10 + assert next_route_number([], []) == 1 diff --git a/experiments/cs-learning-transfer-001/hooks/stale-reference-opaque-v2.py b/experiments/cs-learning-transfer-001/hooks/stale-reference-opaque-v2.py new file mode 100644 index 0000000..13153a0 --- /dev/null +++ b/experiments/cs-learning-transfer-001/hooks/stale-reference-opaque-v2.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) + +from _asset_mutations import apply_stale_reference_policy # noqa: E402 + + +if __name__ == "__main__": + apply_stale_reference_policy(Path(sys.argv[1]).resolve()) diff --git a/experiments/cs-learning-transfer-001/hypotheses.md b/experiments/cs-learning-transfer-001/hypotheses.md new file mode 100644 index 0000000..673d04c --- /dev/null +++ b/experiments/cs-learning-transfer-001/hypotheses.md @@ -0,0 +1,68 @@ +# cs-learning-transfer-001 预注册假设 + +## 实验问题 + +任务 A 产生的项目 lesson，在不传递聊天上下文、不向任务 B 泄露候选内容时，是否能提高 fresh agent +完成同项目 sibling 任务的正确率，同时避免无关应用，并在 canonical 事实变化后及时退役？ + +## 冻结总体 + +- 实验单位：同一 `target × fixture × k_index` 下，共享 post-A 基线的 treatment/control pair。 +- 正向样本：`cs-feat`、`cs-issue`、`cs-refactor`、`cs-epic` 各一个。 +- Guard：一个 unrelated、一个 stale；每个 model family、每个 fixture 均运行 `k=5`。 +- 模型族：Claude 与 Codex，各绑定一个显式 harness/model target，不做笛卡尔积。 +- treatment 唯一额外输入是经 fresh `cs-keep` 写入的 observed lesson；B prompt hash 必须相同。 +- 校准允许 `k=2`，但校准结果不得作为 Epic 接受证据。 + +## H-LT-1：项目 lesson 产生正向迁移 + +Primary metric 是四个正向 fixture 全部完成 pair 的 hidden pass rate 差值： + +```text +overall paired delta = mean(treatment_hidden - control_hidden) +``` + +H-LT-1 仅在以下条件同时成立时确认： + +- overall paired delta `>= 0.25`；25pp 门槛只作用于跨两个 family 的总体正向聚合； +- Claude 与 Codex 两个 family 的各自 paired delta 都 `> 0`； +- paired losses 不多于 wins； +- 每个 family 有四个 owning skill × `k=5`，即至少 20 个完成正向 pair； +- 所有结构、隔离、schema、mutation、prompt equality 与 regression oracle 100% 通过。 + +## H-LT-2：无关 lesson 不制造回退 + +Unrelated fixture 中 treatment hidden pass rate 不低于 control，双方 regression 全绿，observed lesson +保持字节不变且不产生状态 churn。该 guard 不参与 H-LT-1 的 25pp 聚合。 + +## H-LT-3：失效 lesson 会被 read-repair + +Stale fixture 中对 treatment/control 同时施加 `opaque-v2` canonical reference policy hook。hook 先把 +A 侧 `find_job` 与公开测试迁移为精确匹配，保留尚未迁移的 sibling `find_route` 作为 B 任务；大小写、 +外围空白与 `_` / `-` 都不再等价。双方 hidden 与 regression 不回退，treatment 的旧 lesson 必须 +`observed -> retired`，退役率必须为 100%。 + +## 机械 oracle 与失败判据 + +- A checks、候选唯一性、lesson schema、lesson-only curation、post-A 同源、B prompt equality、B mutation + allowlist、hidden/regression、lesson transition 与 stale retirement 均标 `[measured]`。 +- 任何 fixture preflight 未满足 golden hidden/regression 全绿、naive hidden 红且 regression 绿，fixture + 直接 invalid；不得进入模型运行，也不得从结果集中删除来改善指标。 +- Deterministic pipeline / fixture / oracle 失败是当前冻结输入下的终态，永久阻断 structural + integrity，不得重试或从结果中删除。 +- Adapter / transport 故障记为 operational error：所有尝试与实际成本保留；后续同 cell 成功时标 + resolved 并与 structural integrity 分列，不永久污染 acceptance；未解决时该 cell 不完整，整体 + `[underpowered]`。每次调用前以新 invocation ID durable append start 与 `[soft]` fallback，terminal + metrics 只追加；中断或半写 terminal 仍计一次 unresolved attempt 与 fallback 成本。只有完整 pair 才 + resolved；`--fresh` 仅允许 header-only journal，任何运行证据或已有结果都必须换新 `--out`。 +- 任一 family/skill/guard 少于五个完成 repeat，存在未解决 operational error，或任一 primary aggregate + 仍不足，verdict 为 `[underpowered]`，不得接受 Epic。 +- 语义质量只作 `[soft]` 观察，不得覆盖机械 verdict。 + +## 成本与冻结纪律 + +- `budget_usd = 50`；dry-run 必须累加 A、curation、B treatment、B control 与两侧 hook 实际次数。 +- `config.json`、本文件、fixtures、checks、hidden/regression、preflight 与 hook 的 SHA-256 记录在 + `freeze.json`；先提交冻结版本，再运行任何真实 LLM。 +- model/harness 探针若证明 target 不可用，允许在首个真实 LLM 前修改配置，但必须重新生成 + `freeze.json` 并提交；首个结果产生后不得按观测改 hypothesis、metric、预算或失败判据。 diff --git a/experiments/cs-learning-transfer-001/preflight/epic-sequence-golden.py b/experiments/cs-learning-transfer-001/preflight/epic-sequence-golden.py new file mode 100644 index 0000000..7501bd8 --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/epic-sequence-golden.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "epic-sequence", "golden") diff --git a/experiments/cs-learning-transfer-001/preflight/epic-sequence-naive.py b/experiments/cs-learning-transfer-001/preflight/epic-sequence-naive.py new file mode 100644 index 0000000..575cf70 --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/epic-sequence-naive.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "epic-sequence", "naive") diff --git a/experiments/cs-learning-transfer-001/preflight/feat-patch-golden.py b/experiments/cs-learning-transfer-001/preflight/feat-patch-golden.py new file mode 100644 index 0000000..c696af6 --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/feat-patch-golden.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "patch", "golden") diff --git a/experiments/cs-learning-transfer-001/preflight/feat-patch-naive.py b/experiments/cs-learning-transfer-001/preflight/feat-patch-naive.py new file mode 100644 index 0000000..a1104d2 --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/feat-patch-naive.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "patch", "naive") diff --git a/experiments/cs-learning-transfer-001/preflight/issue-reference-golden.py b/experiments/cs-learning-transfer-001/preflight/issue-reference-golden.py new file mode 100644 index 0000000..6a9755a --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/issue-reference-golden.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "reference", "golden") diff --git a/experiments/cs-learning-transfer-001/preflight/issue-reference-naive.py b/experiments/cs-learning-transfer-001/preflight/issue-reference-naive.py new file mode 100644 index 0000000..c347fa0 --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/issue-reference-naive.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "reference", "naive") diff --git a/experiments/cs-learning-transfer-001/preflight/refactor-events-golden.py b/experiments/cs-learning-transfer-001/preflight/refactor-events-golden.py new file mode 100644 index 0000000..c5d4659 --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/refactor-events-golden.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "grouping", "golden") diff --git a/experiments/cs-learning-transfer-001/preflight/refactor-events-naive.py b/experiments/cs-learning-transfer-001/preflight/refactor-events-naive.py new file mode 100644 index 0000000..13e638a --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/refactor-events-naive.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "grouping", "naive") diff --git a/experiments/cs-learning-transfer-001/preflight/stale-reference-v2-golden.py b/experiments/cs-learning-transfer-001/preflight/stale-reference-v2-golden.py new file mode 100644 index 0000000..0d90501 --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/stale-reference-v2-golden.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "stale-reference-v2", "golden") diff --git a/experiments/cs-learning-transfer-001/preflight/stale-reference-v2-naive.py b/experiments/cs-learning-transfer-001/preflight/stale-reference-v2-naive.py new file mode 100644 index 0000000..2d4e943 --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/stale-reference-v2-naive.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "stale-reference-v2", "naive") diff --git a/experiments/cs-learning-transfer-001/preflight/unrelated-display-golden.py b/experiments/cs-learning-transfer-001/preflight/unrelated-display-golden.py new file mode 100644 index 0000000..63acf76 --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/unrelated-display-golden.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "display", "golden") diff --git a/experiments/cs-learning-transfer-001/preflight/unrelated-display-naive.py b/experiments/cs-learning-transfer-001/preflight/unrelated-display-naive.py new file mode 100644 index 0000000..58a0793 --- /dev/null +++ b/experiments/cs-learning-transfer-001/preflight/unrelated-display-naive.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import sys + +EXPERIMENT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENT)) +from _asset_mutations import apply_preflight # noqa: E402 + +apply_preflight(Path(sys.argv[1]).resolve(), "display", "naive") diff --git a/experiments/cs-learning-transfer-001/regression/test_seed_integrity.py b/experiments/cs-learning-transfer-001/regression/test_seed_integrity.py new file mode 100644 index 0000000..62cb3c3 --- /dev/null +++ b/experiments/cs-learning-transfer-001/regression/test_seed_integrity.py @@ -0,0 +1,17 @@ +from dispatchboard.grouping import group_jobs +from dispatchboard.job_sequence import next_job_number +from dispatchboard.references import find_channel, find_job +from dispatchboard.route_sequence import next_route_number + + +def test_unrelated_seed_contracts_remain_available(): + events = [ + {"id": "e1", "group": "B"}, + {"id": "e2", "group": "A"}, + ] + + assert find_job("JOB-7")["id"] == "JOB-7" + assert find_channel("Channel-7")["id"] == "Channel-7" + assert [key for key, _ in group_jobs(events)] == ["B", "A"] + assert next_job_number([2, 7], []) == 8 + assert next_route_number([3, 5], []) == 6 diff --git a/experiments/cs-learning-transfer-001/results.md b/experiments/cs-learning-transfer-001/results.md new file mode 100644 index 0000000..944ba28 --- /dev/null +++ b/experiments/cs-learning-transfer-001/results.md @@ -0,0 +1,59 @@ +# cs-learning-transfer-001 结果 + +状态：`preflight-passed; real-models-not-run`。本文件不包含模型自评或预填的真实运行数值。 + +## 冻结与运行身份 + +- Freeze manifest：`freeze.json`（待与冻结输入一起提交） +- Frozen commit：`pending` +- Calibration：`pending`（仅允许 `k=2`，不能作为接受证据） +- Final run：`pending`（两个 model family，六个 fixture，`k=5`） + +## 离线 preflight + +| fixture | golden hidden | naive hidden | golden regression | naive regression | +|---|---:|---:|---:|---:| +| `lt-feat-patch` | 1.0 | 0.0 | 1.0 | 1.0 | +| `lt-issue-reference` | 1.0 | 0.0 | 1.0 | 1.0 | +| `lt-refactor-events` | 1.0 | 0.0 | 1.0 | 1.0 | +| `lt-epic-sequence` | 1.0 | 0.0 | 1.0 | 1.0 | +| `lt-unrelated-display` | 1.0 | 0.0 | 1.0 | 1.0 | +| `lt-stale-reference-v2` | 1.0 | 0.0 | 1.0 | 1.0 | + +A oracle 另经 seed-red / known-golden-green 校准：seed 为 `0/4`，known golden 为 `4/4`。stale hook +在两侧各执行一次，先把 A 侧 reference lookup 迁移到 `opaque-v2`，同时保持非 lesson manifest 同源且 +lesson 字节不变。dry-run 估算 240 次 agent invocation、20 次 deterministic hook，成本 +`$3.41 [soft]`，低于 `$50` 预算；该估算不是实际花费。 + +## Primary 结果 + +| metric | result | tag | +|---|---:|---| +| overall treatment hidden rate | pending | `[measured]` after run | +| overall control hidden rate | pending | `[measured]` after run | +| overall paired delta | pending | `[measured]` after run | +| Claude paired delta | pending | `[measured]` after run | +| Codex paired delta | pending | `[measured]` after run | +| wins / losses / ties | pending | `[measured]` after run | +| unrelated no-regression | pending | `[measured]` after run | +| stale retired rate | pending | `[measured]` after run | +| structural integrity pass rate | pending | `[measured]` after run | +| operational attempts / resolved / unresolved | pending | `[measured]` after run | +| total cost | pending | harness usage tag after run | + +## Verdict + +`pending`。必须逐条应用 `hypotheses.md` 的复合门槛；校准、单 family 或 `[underpowered]` 结果不能 +写成确认结论。 + +## Evidence pointers + +- 结构化最终结果：`artifacts/analysis/exp-cs-learning-transfer-001-results.json`（运行后生成，gitignored） +- Pair checkpoint：同目录 `.partial.jsonl`（仅中断恢复；完整结果落盘后删除） +- Tracked 摘要：本文件只记录聚合指标、冻结 commit、异常与必要的 hash/diff 指针，不复制完整响应。 + +## 负结果与异常 + +待运行后如实填写。Deterministic failure 必须保留并阻断；operational error 必须分别报告尝试、已解决与 +未解决，只有完整 pair 能标为 resolved，成功重试仍计入成本。不得删除失败 fixture、半 pair 或 adapter +error 来改善结果；`--fresh` 只允许 header-only journal，其他运行证据与结果必须换新 `--out`。 diff --git a/experiments/seeds/dispatchboard-learning/build-seed.py b/experiments/seeds/dispatchboard-learning/build-seed.py new file mode 100644 index 0000000..57da021 --- /dev/null +++ b/experiments/seeds/dispatchboard-learning/build-seed.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""构建 learning-transfer 实验使用的可复现 dispatchboard seed。""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import subprocess +import sys +import tempfile +from pathlib import Path + + +FILES = { + "pyproject.toml": """[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "dispatchboard" +version = "0.1.0" +requires-python = ">=3.11" + +[tool.pytest.ini_options] +testpaths = ["tests"] +""", + "dispatchboard/__init__.py": "", + "dispatchboard/patches.py": '''"""Partial-update helpers for dispatchboard records.""" + + +def patch_job(job: dict, changes: dict) -> dict: + raise NotImplementedError("patch_job is not implemented") + + +def patch_route(route: dict, changes: dict) -> dict: + raise NotImplementedError("patch_route is not implemented") +''', + "dispatchboard/references.py": '''"""Reference lookup policies.""" + +REFERENCE_POLICY = "canonical-v1" +JOBS = {"JOB-7": {"id": "JOB-7", "title": "Pack"}} +CHANNELS = {"Channel-7": {"id": "Channel-7", "title": "North"}} +ROUTES = { + "Route_7": {"id": "Route_7", "title": "Primary"}, +} + + +def canonical_ref(value: str) -> str: + return value.strip().replace("_", "-").casefold() + + +def find_job(reference: str) -> dict | None: + return JOBS.get(reference) + + +def find_channel(reference: str) -> dict | None: + return CHANNELS.get(reference) + + +def find_route(reference: str) -> dict | None: + wanted = canonical_ref(reference) + return next((item for key, item in ROUTES.items() if canonical_ref(key) == wanted), None) +''', + "dispatchboard/grouping.py": '''"""Event grouping helpers.""" + + +def group_jobs(events: list[dict]) -> list[tuple[str, list[dict]]]: + groups: dict[str, list[dict]] = {} + for event in events: + groups.setdefault(event["group"], []).append(event) + return list(groups.items()) + + +def group_routes(events: list[dict]) -> list[tuple[str, list[dict]]]: + unique = {event["id"]: event for event in events} + groups: dict[str, list[dict]] = {} + for event in unique.values(): + groups.setdefault(event["group"], []).append(event) + return sorted(groups.items()) +''', + "dispatchboard/display.py": '''"""Display-only formatting helpers.""" + + +def display_route_label(value: str) -> str: + raise NotImplementedError("display_route_label is not implemented") +''', + "dispatchboard/job_sequence.py": '''"""Job number allocation.""" + + +def next_job_number(active_numbers: list[int], retired_numbers: list[int]) -> int: + return max(active_numbers, default=0) + 1 +''', + "dispatchboard/route_sequence.py": '''"""Route number allocation.""" + + +def next_route_number(active_numbers: list[int], retired_numbers: list[int]) -> int: + return max(active_numbers, default=0) + 1 +''', + "tests/test_references.py": '''from dispatchboard.references import find_job, find_route + + +def test_route_reference_uses_v1_canonical_policy(): + assert find_route(" route-7 ")["id"] == "Route_7" + + +def test_exact_job_reference_remains_available(): + assert find_job("JOB-7")["id"] == "JOB-7" +''', + "tests/test_grouping.py": '''from dispatchboard.grouping import group_jobs, group_routes + + +def test_unique_events_group_by_first_seen_key(): + events = [{"id": "e1", "group": "B"}, {"id": "e2", "group": "A"}] + assert [key for key, _ in group_jobs(events)] == ["B", "A"] + assert {key for key, _ in group_routes(events)} == {"A", "B"} +''', + "tests/test_sequences.py": '''from dispatchboard.job_sequence import next_job_number +from dispatchboard.route_sequence import next_route_number + + +def test_sequence_advances_past_active_numbers(): + assert next_job_number([2, 7], []) == 8 + assert next_route_number([3, 5], []) == 6 +''', + ".codestable/attention.md": """# Attention + +## 项目事实 + +- Python 验证使用 `python3 -m pytest -q`。 +""", +} + +EPIC = """--- +status: active +created: 2026-08-02 +work: ../work/epic-sequence-rollout.md +--- + +# Sequence rollout + +## 目标 + +为 job 与 route 交付不会碰撞既有历史记录的编号分配能力。 + +## 子项契约 + +- `SEQ-A`：实现 job 编号分配并覆盖项目已有历史记录；验证通过后完成。 +- `SEQ-B`：依赖 `SEQ-A`；为 route 交付同类编号分配能力。 + +## 验收 + +新分配编号不得与任何已持久化历史记录冲突；没有历史记录时从 1 开始。 +""" + +CURSOR = """--- +epic: ../epics/sequence-rollout.md +phase: executing +approved_revision: {approved_revision} +current_item: SEQ-A +next_action: execute the approved current item +blocked_by: null +item_progression: per-item +milestone_commit: manual +remote_publish: manual +--- + +## 子项进度 + +- [ ] SEQ-A +- [ ] SEQ-B + +## 临时决策与证据 +""" + + +def _run(command: list[str], cwd: Path) -> None: + env = { + **os.environ, + "GIT_AUTHOR_NAME": "dispatchboard-dev", + "GIT_AUTHOR_EMAIL": "dev@dispatchboard.invalid", + "GIT_COMMITTER_NAME": "dispatchboard-dev", + "GIT_COMMITTER_EMAIL": "dev@dispatchboard.invalid", + "GIT_AUTHOR_DATE": "2026-08-02T09:00:00+08:00", + "GIT_COMMITTER_DATE": "2026-08-02T09:00:00+08:00", + } + subprocess.run(command, cwd=cwd, env=env, check=True, capture_output=True, text=True) + + +def build(out: Path) -> None: + if out.exists() and any(out.iterdir()): + raise SystemExit(f"拒绝构建：{out} 已存在且非空") + out.mkdir(parents=True, exist_ok=True) + for relative, content in FILES.items(): + path = out / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + epic = out / ".codestable/epics/sequence-rollout.md" + epic.parent.mkdir(parents=True, exist_ok=True) + epic.write_text(EPIC, encoding="utf-8") + approved = hashlib.sha256(epic.read_bytes()).hexdigest() + cursor = out / ".codestable/work/epic-sequence-rollout.md" + cursor.parent.mkdir(parents=True, exist_ok=True) + cursor.write_text(CURSOR.format(approved_revision=approved), encoding="utf-8") + _run(["git", "init", "-q", "-b", "main"], out) + _run(["git", "add", "-A"], out) + _run(["git", "commit", "-q", "-m", "seed dispatchboard learning scenarios"], out) + + +def verify() -> int: + with tempfile.TemporaryDirectory(prefix="dispatchboard-learning-") as tmp: + repo = Path(tmp) / "repo" + build(repo) + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider"], + cwd=repo, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, + check=False, + ) + epic = repo / ".codestable/epics/sequence-rollout.md" + cursor = repo / ".codestable/work/epic-sequence-rollout.md" + approved = hashlib.sha256(epic.read_bytes()).hexdigest() + return result.returncode if approved in cursor.read_text(encoding="utf-8") else 1 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--out") + parser.add_argument("--verify", action="store_true") + args = parser.parse_args() + if args.verify: + return verify() + if not args.out: + parser.error("--out 或 --verify 必选其一") + build(Path(args.out)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_cs_learning_transfer_oracle.py b/tests/test_cs_learning_transfer_oracle.py new file mode 100644 index 0000000..a2f4e2b --- /dev/null +++ b/tests/test_cs_learning_transfer_oracle.py @@ -0,0 +1,386 @@ +"""learning-transfer lesson oracle 的严格 schema 与迁移契约。""" + +from __future__ import annotations + +import json +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / ".claude/skills/eval-cs-skill/scripts" +EXPERIMENT = ROOT / "experiments/cs-learning-transfer-001" +SEED_BUILDER = ROOT / "experiments/seeds/dispatchboard-learning/build-seed.py" +sys.path.insert(0, str(SCRIPTS)) + +from scorers.learning_transfer import ( # noqa: E402 + validate_lesson_transition, + validate_observed_lesson, +) + + +VALID_LESSON = """--- +status: observed +scope: reference lookup +date: 2026-08-02 +--- +规则：修改 sibling lookup 前同时核对 storage 与 query normalization。 +适用 / 不适用：适用于引用查找；已有 canonical owner 时停止。 +证据：tests/test_references.py。 +候选归宿：project-doc +""" + + +def _write_lesson( + repo: Path, + text: str = VALID_LESSON, + relative: str = ".codestable/lessons/2026-08-02-reference-lookup.md", +) -> Path: + path = repo / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _validate(repo: Path, paths: list[str] | None = None) -> dict: + return validate_observed_lesson( + repo, + required_concepts=["storage", "query normalization"], + paths=paths, + ) + + +def test_observed_lesson_accepts_the_exact_schema(tmp_path: Path) -> None: + _write_lesson(tmp_path) + + result = _validate(tmp_path) + + assert result == { + "ok": True, + "path": ".codestable/lessons/2026-08-02-reference-lookup.md", + "errors": [], + } + + +@pytest.mark.parametrize( + "mutated", + [ + VALID_LESSON.replace("date: 2026-08-02", "date: 2026-08-02\nowner: docs"), + VALID_LESSON.replace("scope: reference lookup", "scope: first\nscope: second"), + VALID_LESSON.replace("status: observed", "status: [observed]"), + VALID_LESSON.replace("scope: reference lookup", "scope: 7"), + VALID_LESSON.replace("date: 2026-08-02", "date: [2026-08-02]"), + ], + ids=[ + "extra-frontmatter-key", + "duplicate-frontmatter-key", + "status-wrong-type", + "scope-wrong-type", + "date-wrong-type", + ], +) +def test_observed_lesson_rejects_non_exact_frontmatter( + tmp_path: Path, + mutated: str, +) -> None: + _write_lesson(tmp_path, mutated) + + assert _validate(tmp_path)["ok"] is False + + +@pytest.mark.parametrize( + "mutated", + [ + VALID_LESSON.replace( + "规则：修改 sibling lookup 前同时核对 storage 与 query normalization。\n", + "规则：修改 sibling lookup 前同时核对 storage 与 query normalization。\n" + "规则：不要改 display spelling。\n", + ), + VALID_LESSON.replace( + "候选归宿：project-doc\n", + "候选归宿：project-doc\n额外说明：不可进入 oracle。\n", + ), + VALID_LESSON.replace("适用 / 不适用：适用于引用查找；已有 canonical owner 时停止。", "适用 / 不适用："), + VALID_LESSON.replace("候选归宿：project-doc", "候选归宿：wiki"), + VALID_LESSON.replace( + "证据：tests/test_references.py。", + "证据：tests/a.py；tests/b.py；tests/c.py；tests/d.py。", + ), + ], + ids=[ + "duplicate-body-field", + "hidden-extra-body-line", + "empty-body-value", + "unknown-candidate-home", + "too-many-evidence-items", + ], +) +def test_observed_lesson_rejects_non_exact_body( + tmp_path: Path, + mutated: str, +) -> None: + _write_lesson(tmp_path, mutated) + + assert _validate(tmp_path)["ok"] is False + + +@pytest.mark.parametrize( + ("relative", "text"), + [ + ( + ".codestable/lessons/2026-08-03-reference-lookup.md", + VALID_LESSON, + ), + ( + ".codestable/lessons/2026-02-30-reference-lookup.md", + VALID_LESSON.replace("date: 2026-08-02", "date: '2026-02-30'"), + ), + ( + ".codestable/lessons/2026-08-02-Reference_lookup.md", + VALID_LESSON, + ), + ( + ".codestable/lessons/2026-08-02-abcdefghijklmnopqrstuvwxyz12345.md", + VALID_LESSON, + ), + ( + ".codestable/other/2026-08-02-reference-lookup.md", + VALID_LESSON, + ), + ], + ids=[ + "date-mismatch", + "invalid-calendar-date", + "invalid-slug-characters", + "slug-too-long", + "wrong-home", + ], +) +def test_observed_lesson_rejects_invalid_path_or_date( + tmp_path: Path, + relative: str, + text: str, +) -> None: + _write_lesson(tmp_path, text, relative) + + assert _validate(tmp_path, paths=[relative])["ok"] is False + + +def test_observed_lesson_rejects_escape_and_symlink_paths(tmp_path: Path) -> None: + outside = tmp_path.parent / "outside-lesson.md" + outside.write_text(VALID_LESSON, encoding="utf-8") + assert _validate(tmp_path, paths=["../outside-lesson.md"])["ok"] is False + + target = tmp_path / "target.md" + target.write_text(VALID_LESSON, encoding="utf-8") + link = tmp_path / ".codestable/lessons/2026-08-02-reference-lookup.md" + link.parent.mkdir(parents=True) + link.symlink_to(target) + assert _validate(tmp_path)["ok"] is False + + +def _transition_lessons() -> tuple[str, str]: + after = VALID_LESSON.replace("status: observed", "status: validated").replace( + "证据：tests/test_references.py。", + "证据：tests/test_references.py。；tests/test_channel_references.py。", + ) + return VALID_LESSON, after + + +def test_lesson_transition_accepts_exactly_one_appended_evidence_item() -> None: + before, after = _transition_lessons() + + assert validate_lesson_transition(before, after) == {"ok": True, "errors": []} + + +@pytest.mark.parametrize( + "mutation", + [ + lambda before, after: after.replace( + "tests/test_channel_references.py。", + "tests/test_channel_references.py。；tests/test_route_references.py。", + ), + lambda before, after: after.replace( + "tests/test_references.py。；tests/test_channel_references.py。", + "tests/test_channel_references.py。；tests/test_references.py。", + ), + lambda before, after: after.replace( + "tests/test_references.py。；", + "tests/test_references_v2.py。；", + ), + lambda before, after: after.replace( + "候选归宿：project-doc\n", + "候选归宿：project-doc\n隐藏正文：不允许。\n", + ), + lambda before, after: after.replace( + "tests/test_channel_references.py。", + "已再次验证。", + ), + lambda before, after: after.replace( + "tests/test_channel_references.py。", + "验证通过 1/1。", + ), + ], + ids=[ + "append-two", + "reorder-old-evidence", + "rewrite-old-evidence", + "hidden-body-change", + "new-evidence-without-pointer", + "numeric-fraction-is-not-a-pointer", + ], +) +def test_lesson_transition_rejects_non_incremental_changes(mutation) -> None: + before, after = _transition_lessons() + + result = validate_lesson_transition(before, mutation(before, after)) + + assert result["ok"] is False + + +def test_retirement_marker_must_be_in_the_new_evidence_item() -> None: + before = VALID_LESSON.replace("status: observed", "status: validated").replace( + "证据：tests/test_references.py。", + "证据：tests/test_references.py canonical owner 已确认。", + ) + after = before.replace("status: validated", "status: retired").replace( + "证据：tests/test_references.py canonical owner 已确认。", + "证据：tests/test_references.py canonical owner 已确认。；tests/test_policy.py。", + ) + + result = validate_lesson_transition(before, after) + + assert result["ok"] is False + assert "退役原因" in result["errors"] + + +def _build_seed(repo: Path) -> None: + subprocess.run( + [sys.executable, str(SEED_BUILDER), "--out", str(repo)], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def _apply_asset(script: str, repo: Path) -> None: + subprocess.run( + [sys.executable, str(EXPERIMENT / script), str(repo)], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def _references(repo: Path) -> dict: + return runpy.run_path(str(repo / "dispatchboard/references.py")) + + +def test_fixture_candidates_do_not_copy_canonical_owner_contracts(tmp_path: Path) -> None: + issue = json.loads( + (EXPERIMENT / "fixtures/positive/lt-issue-reference.json").read_text(encoding="utf-8") + )["scenario"] + epic = json.loads( + (EXPERIMENT / "fixtures/positive/lt-epic-sequence.json").read_text(encoding="utf-8") + )["scenario"] + stale = json.loads( + (EXPERIMENT / "fixtures/stale/lt-stale-reference-v2.json").read_text(encoding="utf-8") + )["scenario"] + + assert issue["candidate"]["required_concepts"] == [ + "storage", + "query", + "normalization", + "display spelling", + ] + assert "canonical policy" in issue["a"]["request"] + assert stale["candidate"]["required_concepts"] == issue["candidate"]["required_concepts"] + assert epic["candidate"]["required_concepts"] == [ + "persisted history sources", + "single snapshot", + "sibling allocator", + ] + + repo = tmp_path / "seed" + _build_seed(repo) + attention = (repo / ".codestable/attention.md").read_text(encoding="utf-8") + epic_body = (repo / ".codestable/epics/sequence-rollout.md").read_text( + encoding="utf-8" + ).split("# Sequence rollout", 1)[1] + assert "reference_policy" not in attention + assert "active" not in epic_body + assert "retired" not in epic_body + assert "max" not in epic_body + + +def test_reference_preflight_separates_naive_and_golden_siblings(tmp_path: Path) -> None: + naive = tmp_path / "naive" + golden = tmp_path / "golden" + _build_seed(naive) + _build_seed(golden) + _apply_asset("preflight/issue-reference-naive.py", naive) + _apply_asset("preflight/issue-reference-golden.py", golden) + + naive_refs = _references(naive) + golden_refs = _references(golden) + assert naive_refs["find_job"](" job_7 ")["id"] == "JOB-7" + assert golden_refs["find_job"](" job_7 ")["id"] == "JOB-7" + assert naive_refs["find_channel"](" channel_7 ") is None + assert golden_refs["find_channel"](" channel_7 ")["id"] == "Channel-7" + + +def test_stale_hook_migrates_a_side_before_leaving_the_sibling_stale(tmp_path: Path) -> None: + repo = tmp_path / "repo" + _build_seed(repo) + _apply_asset("preflight/issue-reference-golden.py", repo) + lesson = repo / ".codestable/lessons/2026-08-02-reference-lookup.md" + lesson.parent.mkdir(parents=True) + lesson.write_text(VALID_LESSON, encoding="utf-8") + lesson_before = lesson.read_bytes() + + _apply_asset("hooks/stale-reference-opaque-v2.py", repo) + + refs = _references(repo) + assert refs["REFERENCE_POLICY"] == "opaque-v2" + assert refs["find_job"]("JOB-7")["id"] == "JOB-7" + assert refs["find_job"](" job_7 ") is None + assert refs["find_job"]("job-7") is None + assert refs["find_job"]("JOB_7") is None + assert refs["find_route"](" route-7 ")["id"] == "Route_7" + assert "reference_policy: opaque-v2" in ( + repo / ".codestable/attention.md" + ).read_text(encoding="utf-8") + public_test = (repo / "tests/test_references.py").read_text(encoding="utf-8") + assert "test_job_reference_uses_opaque_v2_policy" in public_test + assert "test_route_reference_uses_v1_canonical_policy" not in public_test + assert lesson.read_bytes() == lesson_before + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider"], + cwd=repo, + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_stale_preflight_requires_all_opaque_variants_to_be_distinct(tmp_path: Path) -> None: + naive = tmp_path / "naive" + golden = tmp_path / "golden" + _build_seed(naive) + _build_seed(golden) + _apply_asset("preflight/stale-reference-v2-naive.py", naive) + _apply_asset("preflight/stale-reference-v2-golden.py", golden) + + naive_refs = _references(naive) + golden_refs = _references(golden) + variants = [" route_7 ", "route_7", "Route-7"] + assert all(naive_refs["find_route"](value) is not None for value in variants) + assert all(golden_refs["find_route"](value) is None for value in variants) + assert golden_refs["find_route"]("Route_7")["id"] == "Route_7" diff --git a/tests/test_cs_learning_transfer_sequence.py b/tests/test_cs_learning_transfer_sequence.py new file mode 100644 index 0000000..f1fa9df --- /dev/null +++ b/tests/test_cs_learning_transfer_sequence.py @@ -0,0 +1,2053 @@ +"""learning-transfer sequence 的恢复、隔离与清理契约。""" + +from __future__ import annotations + +import json +import hashlib +import subprocess +import sys +import time +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / ".claude/skills/eval-cs-skill/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import sequence # noqa: E402 +import e2e_env # noqa: E402 +import runner as runner_mod # noqa: E402 +from _model import ExecutionTarget, Fixture, HarnessResult # noqa: E402 +from config import ExperimentConfig # noqa: E402 + + +def _fixture() -> Fixture: + return Fixture.from_dict({ + "id": "lt-checkpoint", + "answerType": "learning-transfer", + "task": {"kind": "learning-transfer"}, + "scenario": { + "class": "positive", + "seed": "dispatchboard-learning", + "a": { + "skill": "cs-feat", + "request": "complete task A", + "candidate_source": "output", + }, + "candidate": { + "expected_home": "lesson", + "required_concepts": ["example"], + }, + "b": {"skill": "cs-feat", "request": "complete task B"}, + "preflight": {}, + "expect": {"lesson_transition": "observed->validated"}, + }, + }) + + +def _config(*, model: str = "mock-model") -> ExperimentConfig: + return ExperimentConfig( + name="learning-transfer-checkpoint", + skill_under_test="cs-feat", + execution_mode="learning-transfer", + model_targets=[{ + "id": "fake-target", + "family": "fake-family", + "harness": "fake-harness", + "model": model, + }], + ) + + +def _stub_sequence(monkeypatch: pytest.MonkeyPatch) -> None: + def build_seed(_seed: str, destination: Path, _root: Path) -> Path: + destination.mkdir(parents=True) + return destination + + def run_pair(**kwargs) -> dict: + kwargs["phase_callback"]({ + "phase": "a", + "status": "passed", + "metrics": {"cost_usd": {"value": 0.01, "tag": "measured"}}, + }) + return {"state": "completed", "phase_metrics": []} + + monkeypatch.setattr(sequence, "build_seed_repo", build_seed) + monkeypatch.setattr(sequence, "preflight_fixture", lambda *_args: {"ok": True}) + monkeypatch.setattr(sequence, "run_pair", run_pair) + + +def _structurally_valid_pair() -> dict: + return { + "state": "completed", + "phase_metrics": [], + "a_ok": True, + "a_mutation_ok": True, + "candidate_unique": True, + "lesson_schema_ok": True, + "lesson_only_mutation": True, + "prompt_equal": True, + "isolation_ok": True, + "lesson_transition_ok": True, + "lesson_expectation_ok": True, + "treatment_mutation_ok": True, + "control_mutation_ok": True, + "treatment_regression_ok": True, + "control_regression_ok": True, + } + + +@pytest.mark.parametrize( + "output,match", + [ + ("完成。\n晶化候选：example rule", "首个非空输出行"), + ("晶化候选：unrelated rule", "required concepts"), + ], +) +def test_output_candidate_requires_first_line_and_frozen_concepts( + tmp_path, output, match, +) -> None: + with pytest.raises(ValueError, match=match): + sequence.extract_candidate(_fixture(), output, tmp_path) + + +def test_epic_candidate_uses_only_the_new_evidence_delta(tmp_path) -> None: + fixture = _fixture() + fixture.raw["scenario"]["a"].update({ + "skill": "cs-epic", + "candidate_source": "epic-cursor", + }) + cursor = tmp_path / ".codestable/work/epic-example.md" + cursor.parent.mkdir(parents=True) + cursor.write_text( + "## 临时决策与证据\n\n- 晶化候选：old example rule\n", + encoding="utf-8", + ) + before = sequence.snapshot_candidates(fixture, tmp_path) + cursor.write_text( + cursor.read_text(encoding="utf-8") + + "- 晶化候选：new example rule\n", + encoding="utf-8", + ) + + assert sequence.extract_candidate( + fixture, + "Epic 子项不得展示候选", + tmp_path, + before=before, + ) == "new example rule" + + +def test_adapter_exception_emits_soft_invocation_cost(tmp_path) -> None: + class RaisingHarness: + name = "raising" + + def invoke(self, *_args, **_kwargs): + raise TimeoutError("provider detail must not enter the checkpoint") + + events: list[dict] = [] + with pytest.raises(sequence.RetryableSequenceError, match="TimeoutError"): + sequence._invoke_phase( + harness=RaisingHarness(), + prompt="paid prompt", + target=ExecutionTarget( + id="fake-target", + family="fake-family", + harness="raising", + model="mock-model", + ), + workdir=tmp_path, + phase="a", + emit=events.append, + ) + + assert [event["status"] for event in events] == [ + "invocation-started", + "retryable-error", + ] + assert events[0]["invocation_id"] == events[1]["invocation_id"] + assert events[1]["error_type"] == "TimeoutError" + assert events[1]["metrics"]["cost_usd"]["tag"] == "soft" + assert "provider detail" not in json.dumps(events) + + +def test_invocation_start_is_durable_before_the_provider_and_reuses_its_id(tmp_path) -> None: + events: list[dict] = [] + + class ObservingHarness: + name = "observing" + + def invoke(self, _prompt, model, _workdir, timeout_s): + assert timeout_s == 600 + assert events[0]["status"] == "invocation-started" + assert events[0]["metrics"]["cost_usd"]["tag"] == "soft" + return HarnessResult( + output="completed", + model=model, + harness=self.name, + wall_ms=1, + usage={"input_tokens": 10, "output_tokens": 5}, + ) + + sequence._invoke_phase( + harness=ObservingHarness(), + prompt="paid prompt", + target=ExecutionTarget( + id="fake-target", + family="fake-family", + harness="observing", + model="mock-model", + ), + workdir=tmp_path, + phase="a", + emit=events.append, + ) + + assert [event["status"] for event in events] == [ + "invocation-started", + "invocation-complete", + ] + assert events[0]["invocation_id"] == events[1]["invocation_id"] + + +@pytest.mark.parametrize( + "terminal,error_type", + [ + (None, "InterruptedInvocation"), + ("retryable-error", "TimeoutError"), + ], +) +def test_invocation_journal_recovers_operational_error_without_generic_error_event( + tmp_path, + terminal, + error_type, +) -> None: + checkpoint = tmp_path / "results.partial.jsonl" + base = { + "target_id": "fake-target", + "fixture_id": "lt-checkpoint", + "k_index": 0, + "phase": "a", + "invocation_id": "invocation-1", + "metrics": {"cost_usd": {"value": 0.25, "tag": "soft"}}, + } + sequence.append_checkpoint(checkpoint, {**base, "status": "invocation-started"}) + if terminal is not None: + sequence.append_checkpoint(checkpoint, { + **base, + "status": terminal, + "error_type": error_type, + }) + else: + with checkpoint.open("a", encoding="utf-8") as handle: + handle.write('{"phase":"a","status":"invocation-complete"') + + sequence._repair_checkpoint_tail(checkpoint) + operational = sequence.load_retryable_errors(checkpoint) + cost = sequence._checkpoint_actual_cost(checkpoint) + + assert len(operational) == 1 + assert operational[0]["error"] == error_type + assert cost == { + "invocation_count": 1, + "cost_usd": {"value": 0.25, "tag": "soft"}, + } + + +def test_checkpoint_tail_repair_preserves_original_if_atomic_replace_fails( + monkeypatch, + tmp_path, +) -> None: + checkpoint = tmp_path / "results.partial.jsonl" + original = b'{"phase":"header"}\n{"phase":"truncated"' + checkpoint.write_bytes(original) + + def fail_replace(_source, _destination): + raise OSError("replace interrupted") + + monkeypatch.setattr(sequence.os, "replace", fail_replace) + + with pytest.raises(OSError, match="replace interrupted"): + sequence._repair_checkpoint_tail(checkpoint) + + assert checkpoint.read_bytes() == original + assert not list(tmp_path.glob(f".{checkpoint.name}.*.tmp")) + + +def test_invocation_terminal_cannot_reuse_an_id_across_cells(tmp_path) -> None: + checkpoint = tmp_path / "results.partial.jsonl" + start = { + "target_id": "fake-target", + "fixture_id": "lt-checkpoint", + "k_index": 0, + "phase": "a", + "status": "invocation-started", + "invocation_id": "invocation-1", + "metrics": {"cost_usd": {"value": 0.25, "tag": "soft"}}, + } + sequence.append_checkpoint(checkpoint, start) + sequence.append_checkpoint(checkpoint, { + **start, + "fixture_id": "different-fixture", + "status": "invocation-complete", + }) + + with pytest.raises(ValueError, match="identity 不匹配"): + sequence._reduce_checkpoint(checkpoint) + + +def test_checkpoint_rejects_resume_when_execution_target_changes(monkeypatch, tmp_path) -> None: + _stub_sequence(monkeypatch) + checkpoint = tmp_path / "results.partial.jsonl" + kwargs = { + "fixtures": [_fixture()], + "k": 1, + "experiment_dir": tmp_path / "experiment", + "root": ROOT, + "run_root": tmp_path / "runs", + "checkpoint_path": checkpoint, + "harness_resolver": lambda _name: object(), + } + + sequence.run_sequence(config=_config(), **kwargs) + assert not (tmp_path / "runs/fake-target__lt-checkpoint__0").exists() + + with pytest.raises(ValueError, match="checkpoint.*不匹配"): + sequence.run_sequence(config=_config(model="different-model"), **kwargs) + + events = [json.loads(line) for line in checkpoint.read_text(encoding="utf-8").splitlines()] + assert events[0]["kind"] == "sequence-checkpoint-header" + assert set(events[0]["inputs"]) == { + "config", + "fixtures", + "skill_snapshots", + "pipeline", + "experiment_assets", + "seeds", + "targets", + "k", + "run_identity", + } + assert events[0]["inputs"]["pipeline"]["sequence.py"] == hashlib.sha256( + Path(sequence.__file__).read_bytes() + ).hexdigest() + assert { + "_model.py", + "buildprompt.py", + "config.py", + "e2e_env.py", + "fixtures.py", + "metrics.py", + "runner.py", + "scorers/__init__.py", + "scorers/base.py", + "scorers/learning_transfer.py", + "harness/adapter_claude.py", + "harness/adapter_codex.py", + } <= set(events[0]["inputs"]["pipeline"]) + a_event = next(event for event in events if event.get("phase") == "a") + assert a_event["phase_key"] == "fake-target|lt-checkpoint|0|a" + + +def test_freeze_scale_allows_only_proportional_k_calibration() -> None: + required = { + "model_families": 2, + "positive_fixtures": 4, + "guard_fixtures": 2, + "k": 5, + "pairs": 60, + "agent_invocations": 240, + "hook_runs": 20, + } + calibration = { + **required, + "k": 2, + "pairs": 24, + "agent_invocations": 96, + "hook_runs": 8, + } + + assert sequence._freeze_run_mode(required, required) == "final" + assert sequence._freeze_run_mode(required, calibration) == "calibration" + with pytest.raises(ValueError, match="完整 fixtures 与 model families"): + sequence._freeze_run_mode( + required, + {**calibration, "model_families": 1, "pairs": 12, "agent_invocations": 48}, + ) + + +@pytest.mark.parametrize( + "mutation,match", + [ + (lambda manifest: manifest.update({"schema_version": 2}), "schema_version"), + (lambda manifest: manifest.update({"hash_algorithm": "sha1"}), "hash_algorithm"), + ( + lambda manifest: manifest["primary_metric"].update({"aggregation": "per family"}), + "primary_metric", + ), + ( + lambda manifest: manifest["primary_metric"].update({"family_guard": "optional"}), + "primary_metric", + ), + ], +) +def test_freeze_metadata_must_match_the_preregistered_contract(mutation, match) -> None: + manifest = { + "schema_version": 1, + "hash_algorithm": "sha256", + "primary_metric": dict(sequence._PRIMARY_METRIC), + } + mutation(manifest) + + with pytest.raises(ValueError, match=match): + sequence._validate_freeze_metadata(manifest) + + +def test_actual_cost_includes_invocations_from_failed_half_pair(monkeypatch, tmp_path) -> None: + _stub_sequence(monkeypatch) + + def fail_after_a(**kwargs) -> dict: + kwargs["phase_callback"]({ + "phase": "a", + "status": "passed", + "metrics": {"cost_usd": {"value": 0.25, "tag": "measured"}}, + }) + raise RuntimeError("transient harness failure") + + monkeypatch.setattr(sequence, "run_pair", fail_after_a) + payload = sequence.run_sequence( + config=_config(), + fixtures=[_fixture()], + k=1, + experiment_dir=tmp_path / "experiment", + root=ROOT, + run_root=tmp_path / "runs", + checkpoint_path=tmp_path / "results.partial.jsonl", + harness_resolver=lambda _name: object(), + ) + + assert payload["aggregate"]["cost"] == { + "invocation_count": 1, + "cost_usd": {"value": 0.25, "tag": "measured"}, + } + + +def test_successful_invocation_is_checkpointed_before_deterministic_oracle_failure( + monkeypatch, + tmp_path, +) -> None: + _stub_sequence(monkeypatch) + + class PaidHarness: + name = "paid-harness" + + def invoke(self, _prompt, model, _workdir, timeout_s): + assert timeout_s == 600 + return HarnessResult( + output="completed", + model=model, + harness=self.name, + wall_ms=1, + usage={"input_tokens": 10, "output_tokens": 5}, + ) + + harness = PaidHarness() + + def fail_after_paid_invocation(**kwargs) -> dict: + sequence._invoke_phase( + harness=harness, + prompt="paid prompt", + target=ExecutionTarget( + id="fake-target", + family="fake-family", + harness=harness.name, + model="mock-model", + ), + workdir=tmp_path, + phase="a", + emit=kwargs["phase_callback"], + ) + raise ValueError("deterministic oracle failed") + + monkeypatch.setattr(sequence, "run_pair", fail_after_paid_invocation) + checkpoint = tmp_path / "results.partial.jsonl" + payload = sequence.run_sequence( + config=_config(), + fixtures=[_fixture()], + k=1, + experiment_dir=tmp_path / "experiment", + root=ROOT, + run_root=tmp_path / "runs", + checkpoint_path=checkpoint, + harness_resolver=lambda _name: harness, + ) + + assert payload["errors"][0]["state"] == "pipeline-error" + assert payload["aggregate"]["cost"]["invocation_count"] == 1 + events = [json.loads(line) for line in checkpoint.read_text(encoding="utf-8").splitlines()] + invocation = next( + event + for event in events + if event.get("phase") == "a" and event.get("status") == "invocation-complete" + ) + assert invocation["status"] == "invocation-complete" + + +def test_checkpoint_rejects_resume_when_experiment_asset_changes(monkeypatch, tmp_path) -> None: + _stub_sequence(monkeypatch) + experiment = tmp_path / "experiment" + check = experiment / "checks/a.py" + check.parent.mkdir(parents=True) + check.write_text("def test_a(): assert True\n", encoding="utf-8") + fixture = _fixture() + fixture.raw["scenario"]["a"]["checks"] = ["checks/a.py"] + checkpoint = tmp_path / "results.partial.jsonl" + kwargs = { + "config": _config(), + "fixtures": [fixture], + "k": 1, + "experiment_dir": experiment, + "root": ROOT, + "run_root": tmp_path / "runs", + "checkpoint_path": checkpoint, + "harness_resolver": lambda _name: object(), + } + + sequence.run_sequence(**kwargs) + check.write_text("def test_a(): assert False\n", encoding="utf-8") + + with pytest.raises(ValueError, match="checkpoint.*不匹配"): + sequence.run_sequence(**kwargs) + + +def test_curation_rejects_git_control_mutation(tmp_path) -> None: + fixture = _fixture() + fixture.raw["scenario"]["a"].update({ + "checks": [], + "allowed_paths": ["post-a.txt"], + }) + seed = tmp_path / "seed" + seed.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=seed, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=seed, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=seed, check=True) + (seed / "tracked.txt").write_text("base\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=seed, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=seed, check=True) + + class CurationGitMutatingHarness: + name = "curation-git-mutating" + + def __init__(self) -> None: + self.calls = 0 + + def invoke(self, prompt, model, workdir, timeout_s): + assert timeout_s == 600 + self.calls += 1 + if self.calls == 1: + (workdir / "post-a.txt").write_text("done\n", encoding="utf-8") + output = "晶化候选：example rule" + else: + lesson = workdir / ".codestable/lessons/2026-08-02-example.md" + lesson.parent.mkdir(parents=True) + lesson.write_text( + "---\nstatus: observed\nscope: example\ndate: 2026-08-02\n---\n" + "规则：example rule。\n" + "适用 / 不适用：适用于 example；已有 owner 时停止。\n" + "证据：post-a.txt。\n候选归宿：project-doc\n", + encoding="utf-8", + ) + subprocess.run( + ["git", "config", "user.name", "changed"], + cwd=workdir, + check=True, + ) + output = "recorded" + return HarnessResult( + output=output, + model=model, + harness=self.name, + wall_ms=1, + ) + + harness = CurationGitMutatingHarness() + pair = sequence.run_pair( + fixture=fixture, + target=ExecutionTarget( + id="fake-target", + family="fake-family", + harness=harness.name, + model="mock-model", + ), + k_index=0, + seed_repo=seed, + experiment_dir=tmp_path, + root=ROOT, + run_root=tmp_path / "run", + harness=harness, + ) + + assert harness.calls == 2 + assert pair["state"] == "pipeline-failed" + assert pair["curation_repo_integrity_ok"] is False + assert pair["lesson_only_mutation"] is False + + +def test_repo_manifest_only_ignores_root_git_control_directory(tmp_path) -> None: + root_git = tmp_path / ".git/config" + nested_git = tmp_path / "module/.git/payload.txt" + root_git.parent.mkdir(parents=True) + nested_git.parent.mkdir(parents=True) + root_git.write_text("ignored\n", encoding="utf-8") + nested_git.write_text("business data\n", encoding="utf-8") + + manifest = sequence.repo_manifest(tmp_path) + + assert ".git/config" not in manifest + assert "module/.git/payload.txt" in manifest + + +def test_repo_control_rejects_a_git_dir_outside_the_cell(tmp_path) -> None: + repo = tmp_path / "cell" + external_git = tmp_path / "external-git" + subprocess.run( + ["git", "init", "-q", "--separate-git-dir", str(external_git), str(repo)], + check=True, + ) + + snapshot = sequence.repo_control_snapshot(repo) + + assert snapshot["repository"] is True + assert snapshot["safe"] is False + assert sequence.repo_control_unchanged(snapshot, snapshot) is False + + +def test_pair_baseline_comparison_preserves_preexisting_lessons() -> None: + existing = ".codestable/lessons/2026-07-01-existing.md" + injected = ".codestable/lessons/2026-08-02-new.md" + control = {"app.py": "same", existing: "existing"} + treatment = {**control, injected: "new"} + + assert sequence._differs_only_by_path(treatment, control, injected) is True + + +def test_repo_control_snapshot_rejects_external_hooks_without_reading_them( + monkeypatch, + tmp_path, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=repo, check=True) + (repo / "tracked.txt").write_text("base\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=repo, check=True) + external = tmp_path / "external-hooks" + external.mkdir() + (external / "secret").write_text("must not be read\n", encoding="utf-8") + subprocess.run( + ["git", "config", "core.hooksPath", str(external)], + cwd=repo, + check=True, + ) + original_hash_tree = e2e_env._hash_tree + + def guarded_hash_tree(path: Path): + assert path.resolve() != external.resolve() + return original_hash_tree(path) + + monkeypatch.setattr(e2e_env, "_hash_tree", guarded_hash_tree) + + snapshot = sequence.repo_control_snapshot(repo) + + assert snapshot["safe"] is False + assert snapshot["hooks"] is None + + +def test_repo_control_snapshot_rejects_local_config_includes_before_running_git( + monkeypatch, + tmp_path, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=repo, check=True) + (repo / "tracked.txt").write_text("base\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=repo, check=True) + external = tmp_path / "external.gitconfig" + external.write_text("[alias]\n leaked = status\n", encoding="utf-8") + subprocess.run( + ["git", "config", "include.path", str(external)], + cwd=repo, + check=True, + ) + + def unexpected_git(*_args, **_kwargs): + raise AssertionError("local include 必须在任何 Git 子进程前被拒绝") + + monkeypatch.setattr(e2e_env, "_git_output", unexpected_git) + monkeypatch.setattr(e2e_env.subprocess, "run", unexpected_git) + + snapshot = sequence.repo_control_snapshot(repo) + + assert snapshot["safe"] is False + assert snapshot["local_config"] == hashlib.sha256( + (repo / ".git/config").read_bytes() + ).hexdigest() + + +def test_repo_control_rejects_bom_prefixed_include_before_running_git( + monkeypatch, + tmp_path, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=repo, check=True) + config = repo / ".git/config" + config.write_text( + "\ufeff[include]\n path = /tmp/outside.gitconfig\n" + config.read_text(encoding="utf-8"), + encoding="utf-8", + ) + + def unexpected_git(*_args, **_kwargs): + raise AssertionError("BOM include 必须在任何 Git 子进程前被拒绝") + + monkeypatch.setattr(e2e_env, "_git_output", unexpected_git) + monkeypatch.setattr(e2e_env.subprocess, "run", unexpected_git) + + snapshot = sequence.repo_control_snapshot(repo) + + assert snapshot["safe"] is False + assert snapshot["local_config"] == hashlib.sha256(config.read_bytes()).hexdigest() + + +def test_repo_control_git_commands_ignore_host_config_and_repo_injection( + monkeypatch, + tmp_path, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=repo, check=True) + (repo / "tracked.txt").write_text("base\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=repo, check=True) + external_hooks = tmp_path / "external-hooks" + external_hooks.mkdir() + (external_hooks / "host-only").write_text("do not read\n", encoding="utf-8") + global_config = tmp_path / "global.gitconfig" + global_config.write_text( + f"[core]\n hooksPath = {external_hooks}\n", + encoding="utf-8", + ) + external_git = tmp_path / "external-git" + external_git.mkdir() + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(global_config)) + monkeypatch.setenv("GIT_CONFIG_COUNT", "1") + monkeypatch.setenv("GIT_CONFIG_KEY_0", "core.hooksPath") + monkeypatch.setenv("GIT_CONFIG_VALUE_0", str(external_hooks)) + monkeypatch.setenv("GIT_DIR", str(external_git)) + + snapshot = sequence.repo_control_snapshot(repo) + + assert snapshot["safe"] is True + assert "host-only" not in snapshot["hooks"] + + +def test_repo_control_git_commands_receive_only_minimal_environment( + monkeypatch, + tmp_path, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=repo, check=True) + (repo / "tracked.txt").write_text("base\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=repo, check=True) + monkeypatch.setenv("OPENAI_API_KEY", "must-not-cross") + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-cross") + monkeypatch.setenv("PROJECT_SECRET", "must-not-cross") + monkeypatch.setenv("GIT_TRACE", "1") + monkeypatch.setenv("DYLD_INSERT_LIBRARIES", "/tmp/must-not-load.dylib") + observed: list[dict[str, str]] = [] + real_run = e2e_env.subprocess.run + + def capture_run(*args, **kwargs): + observed.append(dict(kwargs["env"])) + return real_run(*args, **kwargs) + + monkeypatch.setattr(e2e_env.subprocess, "run", capture_run) + + snapshot = sequence.repo_control_snapshot(repo) + + assert snapshot["safe"] is True + assert observed + for env in observed: + for forbidden in ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "PROJECT_SECRET", + "GIT_TRACE", + "DYLD_INSERT_LIBRARIES", + ): + assert forbidden not in env + assert env["GIT_CONFIG_COUNT"] == "0" + assert env["GIT_CONFIG_NOSYSTEM"] == "1" + assert "PYTHONPATH" not in env + assert Path(env["HOME"]) != Path.home() + + +def test_freeze_git_commands_receive_only_minimal_environment( + monkeypatch, +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "must-not-cross") + monkeypatch.setenv("PROJECT_SECRET", "must-not-cross") + monkeypatch.setenv("GIT_TRACE", "1") + observed: list[dict[str, str]] = [] + real_run = sequence.subprocess.run + + def capture_run(*args, **kwargs): + observed.append(dict(kwargs["env"])) + return real_run(*args, **kwargs) + + monkeypatch.setattr(sequence.subprocess, "run", capture_run) + + digest = sequence._git_blob_hash(ROOT, "HEAD", "AGENTS.md") + + assert digest is not None + assert observed + for env in observed: + assert "OPENAI_API_KEY" not in env + assert "PROJECT_SECRET" not in env + assert "GIT_TRACE" not in env + assert env["GIT_CONFIG_GLOBAL"] == "/dev/null" + assert "PYTHONPATH" not in env + assert Path(env["HOME"]) != Path.home() + + +def test_deterministic_hook_receives_no_host_credentials_or_python_plugins( + monkeypatch, + tmp_path, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + script = tmp_path / "hook.py" + (tmp_path / "hook_helper.py").write_text("VALUE = 'loaded'\n", encoding="utf-8") + script.write_text( + "import json, os, sys\n" + "from pathlib import Path\n" + "sys.path.insert(0, str(Path(__file__).parent))\n" + "from hook_helper import VALUE\n" + "assert VALUE == 'loaded'\n" + "Path(sys.argv[1], 'hook-env.json').write_text(json.dumps(dict(os.environ)))\n", + encoding="utf-8", + ) + for key in ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "PROJECT_SECRET"): + monkeypatch.setenv(key, "must-not-cross") + monkeypatch.setenv("PYTHONPATH", "/tmp/host-pythonpath") + monkeypatch.setenv("PYTEST_ADDOPTS", "--capture=no") + + result = sequence._run_repo_script(script, repo) + + assert result.returncode == 0, result.stderr + env = json.loads((repo / "hook-env.json").read_text(encoding="utf-8")) + for forbidden in ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "PROJECT_SECRET", + "PYTEST_ADDOPTS", + ): + assert forbidden not in env + assert env["PYTHONPATH"] == str(repo.resolve()) + assert env["PYTHONDONTWRITEBYTECODE"] == "1" + assert env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] == "1" + assert Path(env["HOME"]) != Path.home() + assert not (tmp_path / "__pycache__").exists() + + +def test_deterministic_hook_ignores_repo_sitecustomize(monkeypatch, tmp_path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + marker = repo / "sitecustomize-loaded" + (repo / "sitecustomize.py").write_text( + f"from pathlib import Path\nPath({str(marker)!r}).write_text('loaded')\n", + encoding="utf-8", + ) + script = tmp_path / "hook.py" + script.write_text("# deterministic no-op\n", encoding="utf-8") + monkeypatch.setenv("PYTHONPATH", str(repo)) + + result = sequence._run_repo_script(script, repo) + + assert result.returncode == 0 + assert not marker.exists() + + +def test_deterministic_pytest_receives_exact_repo_pythonpath_and_no_host_credentials( + monkeypatch, + tmp_path, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + check = tmp_path / "test_env.py" + check.write_text( + "import os\n" + f"EXPECTED = {str(repo.resolve())!r}\n" + "def test_environment_is_isolated():\n" + " assert os.environ['PYTHONPATH'] == EXPECTED\n" + " assert os.environ['PYTHONDONTWRITEBYTECODE'] == '1'\n" + " assert os.environ['PYTEST_DISABLE_PLUGIN_AUTOLOAD'] == '1'\n" + " assert 'OPENAI_API_KEY' not in os.environ\n" + " assert 'PROJECT_SECRET' not in os.environ\n", + encoding="utf-8", + ) + monkeypatch.setenv("OPENAI_API_KEY", "must-not-cross") + monkeypatch.setenv("PROJECT_SECRET", "must-not-cross") + monkeypatch.setenv("PYTHONPATH", "/tmp/host-pythonpath") + + result = sequence._run_check_files(repo, [check]) + + assert result["passed"] == 1 + assert result["total"] == 1 + assert set(result["evidence"][0]) == { + "check_sha256", + "passed", + "returncode", + "output_sha256", + } + assert not (tmp_path / "__pycache__").exists() + + +def test_seed_builder_receives_minimal_environment(monkeypatch, tmp_path) -> None: + root = tmp_path / "root" + builder = root / "experiments/seeds/example/build-seed.py" + builder.parent.mkdir(parents=True) + (builder.parent / "seed_helper.py").write_text("VALUE = 'loaded'\n", encoding="utf-8") + builder.write_text( + "import argparse, json, os, sys\n" + "from pathlib import Path\n" + "sys.path.insert(0, str(Path(__file__).parent))\n" + "from seed_helper import VALUE\n" + "assert VALUE == 'loaded'\n" + "parser = argparse.ArgumentParser()\n" + "parser.add_argument('--out', required=True)\n" + "out = Path(parser.parse_args().out)\n" + "out.mkdir(parents=True)\n" + "(out / 'seed-env.json').write_text(json.dumps(dict(os.environ)))\n", + encoding="utf-8", + ) + monkeypatch.setenv("OPENAI_API_KEY", "must-not-cross") + monkeypatch.setenv("PROJECT_SECRET", "must-not-cross") + destination = tmp_path / "cell/repo" + + e2e_env.build_seed_repo("example", destination, root) + + env = json.loads((destination / "seed-env.json").read_text(encoding="utf-8")) + assert "OPENAI_API_KEY" not in env + assert "PROJECT_SECRET" not in env + assert env["PYTHONPATH"] == str(root.resolve()) + assert env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] == "1" + assert not (builder.parent / "__pycache__").exists() + + +def test_bug_injector_receives_minimal_environment(monkeypatch, tmp_path) -> None: + experiment = tmp_path / "experiment" + injector = experiment / "bugs/example/inject.py" + injector.parent.mkdir(parents=True) + (injector.parent / "inject_helper.py").write_text("VALUE = 'loaded'\n", encoding="utf-8") + injector.write_text( + "import json, os, sys\n" + "from pathlib import Path\n" + "sys.path.insert(0, str(Path(__file__).parent))\n" + "from inject_helper import VALUE\n" + "assert VALUE == 'loaded'\n" + "Path(sys.argv[1], 'inject-env.json').write_text(json.dumps(dict(os.environ)))\n", + encoding="utf-8", + ) + + def fake_build_seed(_seed: str, destination: Path, _root: Path) -> Path: + destination.mkdir(parents=True) + return destination + + monkeypatch.setattr(e2e_env, "build_seed_repo", fake_build_seed) + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-cross") + monkeypatch.setenv("PROJECT_SECRET", "must-not-cross") + fixture = type("FixtureStub", (), { + "raw": {"scenario": {"seed": "example", "bug_id": "example"}}, + })() + + repo = e2e_env.prepare_e2e_workdir(fixture, str(tmp_path / "cell"), experiment) + + env = json.loads((repo / "inject-env.json").read_text(encoding="utf-8")) + assert "ANTHROPIC_API_KEY" not in env + assert "PROJECT_SECRET" not in env + assert env["PYTHONPATH"] == str(Path.cwd().resolve()) + assert not (injector.parent / "__pycache__").exists() + + +@pytest.mark.parametrize( + "relative,content", + [ + ("config.worktree", "[include]\n path = /tmp/outside.gitconfig\n"), + ("commondir", "../../outside-git\n"), + ("objects/info/alternates", "/tmp/outside-objects\n"), + ("objects/info/http-alternates", "https://example.invalid/objects\n"), + ], +) +def test_repo_control_rejects_extended_git_control_before_running_git( + monkeypatch, + tmp_path, + relative, + content, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=repo, check=True) + if relative == "config.worktree": + subprocess.run( + ["git", "config", "extensions.worktreeConfig", "true"], + cwd=repo, + check=True, + ) + control = repo / ".git" / relative + control.parent.mkdir(parents=True, exist_ok=True) + control.write_text(content, encoding="utf-8") + + def unexpected_git(*_args, **_kwargs): + raise AssertionError("扩展 Git 控制面必须在任何 Git 子进程前被拒绝") + + monkeypatch.setattr(e2e_env, "_git_output", unexpected_git) + monkeypatch.setattr(e2e_env.subprocess, "run", unexpected_git) + + snapshot = sequence.repo_control_snapshot(repo) + + assert snapshot["safe"] is False + assert relative in snapshot["unsafe_control_paths"] + + +def test_truncated_preflight_cache_is_rebuilt_atomically(monkeypatch, tmp_path) -> None: + _stub_sequence(monkeypatch) + run_root = tmp_path / "runs" + cache = run_root / "preflight/lt-checkpoint.json" + cache.parent.mkdir(parents=True) + cache.write_text('{"checkpoint_fingerprint":', encoding="utf-8") + + payload = sequence.run_sequence( + config=_config(), + fixtures=[_fixture()], + k=1, + experiment_dir=tmp_path / "experiment", + root=ROOT, + run_root=run_root, + checkpoint_path=tmp_path / "results.partial.jsonl", + harness_resolver=lambda _name: object(), + ) + + assert len(payload["pairs"]) == 1 + rebuilt = json.loads(cache.read_text(encoding="utf-8")) + assert rebuilt["result"] == {"ok": True} + assert not list(cache.parent.glob(f".{cache.name}.*.tmp")) + + +def test_fixture_invalid_preflight_is_persisted_in_the_checkpoint(monkeypatch, tmp_path) -> None: + _stub_sequence(monkeypatch) + monkeypatch.setattr( + sequence, + "preflight_fixture", + lambda *_args: {"ok": False, "state": "fixture-invalid", "reason": "golden failed"}, + ) + checkpoint = tmp_path / "results.partial.jsonl" + + payload = sequence.run_sequence( + config=_config(), + fixtures=[_fixture()], + k=1, + experiment_dir=tmp_path / "experiment", + root=ROOT, + run_root=tmp_path / "runs", + checkpoint_path=checkpoint, + harness_resolver=lambda _name: object(), + ) + + assert payload["invalid"][0]["state"] == "fixture-invalid" + events = [json.loads(line) for line in checkpoint.read_text(encoding="utf-8").splitlines()] + invalid = next(event for event in events if event.get("phase") == "fixture-invalid") + assert invalid["fixture_id"] == "lt-checkpoint" + assert invalid["preflight"]["reason"] == "golden failed" + + +def test_checkpoint_recovers_truncated_tail_and_preserves_retry_history( + monkeypatch, + tmp_path, +) -> None: + _stub_sequence(monkeypatch) + attempts = 0 + + def flaky_pair(**kwargs): + nonlocal attempts + attempts += 1 + if attempts == 1: + kwargs["phase_callback"]({ + "phase": "a", + "status": "retryable-error", + "metrics": {"cost_usd": {"value": 0.2, "tag": "soft"}}, + }) + raise sequence.RetryableSequenceError("temporary") + return _structurally_valid_pair() + + monkeypatch.setattr(sequence, "run_pair", flaky_pair) + checkpoint = tmp_path / "results.partial.jsonl" + kwargs = { + "config": _config(), + "fixtures": [_fixture()], + "k": 1, + "experiment_dir": tmp_path / "experiment", + "root": ROOT, + "run_root": tmp_path / "runs", + "checkpoint_path": checkpoint, + "harness_resolver": lambda _name: object(), + } + first = sequence.run_sequence(**kwargs) + assert len(first["errors"]) == 1 + with checkpoint.open("a", encoding="utf-8") as handle: + handle.write('{"phase":"truncated"') + + second = sequence.run_sequence(**kwargs) + + assert attempts == 2 + assert len(second["pairs"]) == 1 + assert second["errors"] == [] + assert second["aggregate"]["integrity"]["ok"] is True + assert second["aggregate"]["operational_errors"] == { + "attempts": 1, + "resolved": 1, + "unresolved": 0, + } + assert second["operational_errors"][0]["resolved"] is True + assert checkpoint.read_text(encoding="utf-8").endswith("\n") + + +def test_checkpoint_rejects_resume_when_custom_asset_or_seed_sibling_changes( + monkeypatch, + tmp_path, +) -> None: + _stub_sequence(monkeypatch) + root = tmp_path / "root" + for skill in ("cs-feat", "cs-keep"): + path = root / f"plugins/codestable/skills/{skill}/SKILL.md" + path.parent.mkdir(parents=True) + path.write_text(f"# {skill}\n", encoding="utf-8") + seed = root / "experiments/seeds/dispatchboard-learning" + seed.mkdir(parents=True) + (seed / "build-seed.py").write_text("# builder\n", encoding="utf-8") + sibling = seed / "template.txt" + sibling.write_text("v1\n", encoding="utf-8") + experiment = tmp_path / "experiment" + custom = experiment / "custom/oracle.py" + custom.parent.mkdir(parents=True) + custom.write_text("# v1\n", encoding="utf-8") + fixture = _fixture() + fixture.raw["scenario"]["a"]["checks"] = ["custom/oracle.py"] + checkpoint = tmp_path / "results.partial.jsonl" + kwargs = { + "config": _config(), + "fixtures": [fixture], + "k": 1, + "experiment_dir": experiment, + "root": root, + "run_root": tmp_path / "runs", + "checkpoint_path": checkpoint, + "harness_resolver": lambda _name: object(), + } + sequence.run_sequence(**kwargs) + + custom.write_text("# v2\n", encoding="utf-8") + with pytest.raises(ValueError, match="checkpoint.*不匹配"): + sequence.run_sequence(**kwargs) + + custom.write_text("# v1\n", encoding="utf-8") + sibling.write_text("v2\n", encoding="utf-8") + with pytest.raises(ValueError, match="checkpoint.*不匹配"): + sequence.run_sequence(**kwargs) + + +def test_real_campaign_rejects_prepared_freeze_before_reading_models(tmp_path) -> None: + experiment = tmp_path / "experiment" + experiment.mkdir() + (experiment / "freeze.json").write_text( + json.dumps({"state": "prepared-awaiting-commit", "source_commit": "pending"}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="state 必须为 frozen"): + sequence.validate_freeze_manifest( + experiment_dir=experiment, + root=tmp_path, + config=_config(), + fixtures=[_fixture()], + k=1, + targets=_config().model_targets, + ) + + +def _passed_probe_attestation(targets, source_commit: str) -> dict: + return { + "status": "passed", + "source_commit": source_commit, + "targets": [ + { + "id": target.id, + "family": target.family, + "harness": target.harness, + "model": target.model, + "cell_write": True, + "host_read_blocked": True, + "sibling_read_blocked": True, + "host_write_blocked": True, + "host_config_unchanged": True, + "runtime_removed": True, + } + for target in targets + ], + } + + +def test_probe_attestation_requires_exact_targets_source_and_all_oracles() -> None: + targets = _config().model_targets + source_commit = "a" * 40 + manifest = { + "model_target_probe": _passed_probe_attestation(targets, source_commit), + } + + sequence._validate_probe_attestation(manifest, targets, source_commit) + + wrong_source = json.loads(json.dumps(manifest)) + wrong_source["model_target_probe"]["source_commit"] = "b" * 40 + with pytest.raises(ValueError, match="source_commit"): + sequence._validate_probe_attestation(wrong_source, targets, source_commit) + + missing = json.loads(json.dumps(manifest)) + missing["model_target_probe"]["targets"] = [] + with pytest.raises(ValueError, match="targets 与 config"): + sequence._validate_probe_attestation(missing, targets, source_commit) + + failed = json.loads(json.dumps(manifest)) + failed["model_target_probe"]["targets"][0]["sibling_read_blocked"] = False + with pytest.raises(ValueError, match="oracle"): + sequence._validate_probe_attestation(failed, targets, source_commit) + + duplicate = json.loads(json.dumps(manifest)) + duplicate["model_target_probe"]["targets"].append( + duplicate["model_target_probe"]["targets"][0] + ) + with pytest.raises(ValueError, match="targets 与 config"): + sequence._validate_probe_attestation(duplicate, targets, source_commit) + + +@pytest.mark.parametrize( + ("scope", "field"), + [ + ("probe", "prompt"), + ("target", "output"), + ("target", "path"), + ("target", "secret"), + ("target", "session_id"), + ("target", "extra_oracle"), + ], +) +def test_probe_attestation_rejects_unregistered_fields(scope: str, field: str) -> None: + targets = _config().model_targets + source_commit = "a" * 40 + manifest = { + "model_target_probe": _passed_probe_attestation(targets, source_commit), + } + if scope == "probe": + manifest["model_target_probe"][field] = "must-not-persist" + else: + manifest["model_target_probe"]["targets"][0][field] = "must-not-persist" + + with pytest.raises(ValueError, match="字段"): + sequence._validate_probe_attestation(manifest, targets, source_commit) + + +def test_different_result_paths_have_independent_sequence_run_roots(tmp_path) -> None: + first = runner_mod._sequence_run_root(tmp_path / "calibration.json") + second = runner_mod._sequence_run_root(tmp_path / "final.json") + + assert first != second + assert first.parent == second.parent == tmp_path / "runs" + + +def test_same_result_path_uses_a_nonblocking_process_lock(tmp_path) -> None: + out_path = tmp_path / "calibration.json" + + with runner_mod._sequence_output_lock(out_path): + with pytest.raises(runner_mod.SequenceOutputBusyError, match="同一 --out"): + with runner_mod._sequence_output_lock(out_path): + pass + + with runner_mod._sequence_output_lock(out_path): + pass + + +def test_fresh_runner_clears_checkpoint_preflight_and_cell_state( + monkeypatch, + tmp_path, +) -> None: + experiment = tmp_path / "experiment" + experiment.mkdir() + out_path = tmp_path / "calibration.json" + checkpoint = out_path.parent / f"{out_path.name}.partial.jsonl" + run_root = runner_mod._sequence_run_root(out_path) + stale = run_root / "preflight/stale.json" + stale.parent.mkdir(parents=True) + stale.write_text("stale\n", encoding="utf-8") + checkpoint.write_text('{"phase":"header"}\n', encoding="utf-8") + config = _config() + config.variants = ["baseline"] + config.scorers = ["learning_transfer"] + targets = config.model_targets + + monkeypatch.setattr(runner_mod, "load_config", lambda _path: config) + monkeypatch.setattr(runner_mod, "load_fixtures", lambda *_args: [_fixture()]) + monkeypatch.setattr(runner_mod, "select_execution_targets", lambda *_args: targets) + monkeypatch.setattr(runner_mod, "repo_root", lambda: ROOT) + monkeypatch.setattr(sequence, "dry_run_sequence", lambda *_args: { + "est_total_usd": 0.1, + "budget_usd": 50.0, + "invocation_count": 4, + "hook_runs": 0, + }) + monkeypatch.setattr(sequence, "validate_freeze_manifest", lambda **_kwargs: {"ok": True}) + + def run_sequence(**kwargs): + assert kwargs["run_root"] == run_root + assert not stale.exists() + assert not checkpoint.exists() + return {"pairs": [], "errors": [], "invalid": []} + + monkeypatch.setattr(sequence, "run_sequence", run_sequence) + + exit_code = runner_mod.main([ + "--experiment", str(experiment), + "--fresh", + "--out", str(out_path), + ]) + + assert exit_code == 0 + assert not run_root.exists() + + +def test_fresh_runner_refuses_to_erase_paid_invocation_history( + monkeypatch, + tmp_path, +) -> None: + experiment = tmp_path / "experiment" + experiment.mkdir() + out_path = tmp_path / "calibration.json" + checkpoint = out_path.parent / f"{out_path.name}.partial.jsonl" + checkpoint.write_text( + json.dumps({ + "phase": "a", + "status": "retryable-error", + "metrics": {"cost_usd": {"value": 0.1, "tag": "soft"}}, + }) + '\n{"phase":"truncated"', + encoding="utf-8", + ) + config = _config() + config.variants = ["baseline"] + config.scorers = ["learning_transfer"] + monkeypatch.setattr(runner_mod, "load_config", lambda _path: config) + monkeypatch.setattr(runner_mod, "load_fixtures", lambda *_args: [_fixture()]) + monkeypatch.setattr( + runner_mod, + "select_execution_targets", + lambda *_args: config.model_targets, + ) + monkeypatch.setattr(runner_mod, "repo_root", lambda: ROOT) + monkeypatch.setattr(sequence, "dry_run_sequence", lambda *_args: { + "est_total_usd": 0.1, + "budget_usd": 50.0, + "invocation_count": 4, + "hook_runs": 0, + }) + monkeypatch.setattr(sequence, "validate_freeze_manifest", lambda **_kwargs: {"ok": True}) + monkeypatch.setattr( + sequence, + "run_sequence", + lambda **_kwargs: pytest.fail("paid history must block before sequence execution"), + ) + + exit_code = runner_mod.main([ + "--experiment", str(experiment), + "--fresh", + "--out", str(out_path), + ]) + + assert exit_code == 2 + assert checkpoint.exists() + + +@pytest.mark.parametrize("fresh", [False, True]) +def test_runner_refuses_to_overwrite_terminal_sequence_output( + monkeypatch, + tmp_path, + fresh, +) -> None: + experiment = tmp_path / "experiment" + experiment.mkdir() + out_path = tmp_path / "final.json" + original = '{"aggregate":{"cost":{"invocation_count":240}}}\n' + out_path.write_text(original, encoding="utf-8") + config = _config() + config.variants = ["baseline"] + config.scorers = ["learning_transfer"] + monkeypatch.setattr(runner_mod, "load_config", lambda _path: config) + monkeypatch.setattr(runner_mod, "load_fixtures", lambda *_args: [_fixture()]) + monkeypatch.setattr( + runner_mod, + "select_execution_targets", + lambda *_args: config.model_targets, + ) + monkeypatch.setattr(runner_mod, "repo_root", lambda: ROOT) + monkeypatch.setattr(sequence, "dry_run_sequence", lambda *_args: { + "est_total_usd": 0.1, + "budget_usd": 50.0, + "invocation_count": 4, + "hook_runs": 0, + }) + monkeypatch.setattr(sequence, "validate_freeze_manifest", lambda **_kwargs: {"ok": True}) + monkeypatch.setattr( + sequence, + "run_sequence", + lambda **_kwargs: pytest.fail("terminal output must block before sequence execution"), + ) + args = ["--experiment", str(experiment), "--out", str(out_path)] + if fresh: + args.append("--fresh") + + exit_code = runner_mod.main(args) + + assert exit_code == 2 + assert out_path.read_text(encoding="utf-8") == original + + +def test_learning_transfer_dry_run_refuses_to_overwrite_existing_output( + monkeypatch, + tmp_path, +) -> None: + experiment = tmp_path / "experiment" + experiment.mkdir() + out_path = tmp_path / "final.json" + original = '{"aggregate":{"cost":{"invocation_count":240}}}\n' + out_path.write_text(original, encoding="utf-8") + config = _config() + config.variants = ["baseline"] + config.scorers = ["learning_transfer"] + monkeypatch.setattr(runner_mod, "load_config", lambda _path: config) + monkeypatch.setattr(runner_mod, "load_fixtures", lambda *_args: [_fixture()]) + monkeypatch.setattr( + runner_mod, + "select_execution_targets", + lambda *_args: config.model_targets, + ) + monkeypatch.setattr(sequence, "dry_run_sequence", lambda *_args: { + "est_total_usd": 0.1, + "budget_usd": 50.0, + "invocation_count": 4, + "hook_runs": 0, + }) + + exit_code = runner_mod.main([ + "--experiment", str(experiment), + "--dry-run", + "--out", str(out_path), + ]) + + assert exit_code == 2 + assert out_path.read_text(encoding="utf-8") == original + + +@pytest.mark.parametrize( + "event", + [ + {"phase": "error", "state": "pipeline-error", "status": "pipeline-error"}, + {"phase": "score", "status": "pipeline-failed", "pair": {"state": "pipeline-failed"}}, + { + "phase": "fixture-invalid", + "status": "fixture-invalid", + "preflight": {"ok": False, "state": "fixture-invalid"}, + }, + ], +) +def test_fresh_runner_refuses_to_erase_irreversible_sequence_evidence( + monkeypatch, + tmp_path, + event, +) -> None: + experiment = tmp_path / "experiment" + experiment.mkdir() + out_path = tmp_path / "calibration.json" + checkpoint = runner_mod._sequence_checkpoint_path(out_path) + checkpoint.write_text(json.dumps(event) + "\n", encoding="utf-8") + config = _config() + config.variants = ["baseline"] + config.scorers = ["learning_transfer"] + monkeypatch.setattr(runner_mod, "load_config", lambda _path: config) + monkeypatch.setattr(runner_mod, "load_fixtures", lambda *_args: [_fixture()]) + monkeypatch.setattr( + runner_mod, + "select_execution_targets", + lambda *_args: config.model_targets, + ) + monkeypatch.setattr(runner_mod, "repo_root", lambda: ROOT) + monkeypatch.setattr(sequence, "dry_run_sequence", lambda *_args: { + "est_total_usd": 0.1, + "budget_usd": 50.0, + "invocation_count": 4, + "hook_runs": 0, + }) + monkeypatch.setattr(sequence, "validate_freeze_manifest", lambda **_kwargs: {"ok": True}) + monkeypatch.setattr( + sequence, + "run_sequence", + lambda **_kwargs: pytest.fail("irreversible evidence must block --fresh"), + ) + + exit_code = runner_mod.main([ + "--experiment", str(experiment), + "--fresh", + "--out", str(out_path), + ]) + + assert exit_code == 2 + assert checkpoint.exists() + + +def test_a_rejects_git_config_mutation_even_when_business_diff_is_allowed(tmp_path) -> None: + fixture = _fixture() + fixture.raw["scenario"]["a"]["allowed_paths"] = ["post-a.txt"] + seed = tmp_path / "seed" + seed.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=seed, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=seed, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=seed, check=True) + (seed / "tracked.txt").write_text("base\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=seed, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=seed, check=True) + + class GitMutatingHarness: + name = "git-mutating" + + def __init__(self) -> None: + self.calls = 0 + + def invoke(self, _prompt, model, workdir, timeout_s): + assert timeout_s == 600 + self.calls += 1 + (workdir / "post-a.txt").write_text("done\n", encoding="utf-8") + subprocess.run(["git", "config", "user.name", "changed"], cwd=workdir, check=True) + return HarnessResult( + output="晶化候选：example rule", + model=model, + harness=self.name, + wall_ms=1, + ) + + harness = GitMutatingHarness() + pair = sequence.run_pair( + fixture=fixture, + target=ExecutionTarget( + id="fake-target", + family="fake-family", + harness=harness.name, + model="mock-model", + ), + k_index=0, + seed_repo=seed, + experiment_dir=tmp_path, + root=ROOT, + run_root=tmp_path / "run", + harness=harness, + ) + + assert harness.calls == 1 + assert pair["state"] == "pipeline-failed" + assert pair["a_mutation_ok"] is False + assert pair["a_repo_integrity_ok"] is False + + +def test_epic_preflight_rejects_seed_with_mismatched_approved_revision(tmp_path) -> None: + fixture = _fixture() + scenario = fixture.raw["scenario"] + scenario["a"]["skill"] = "cs-epic" + scenario["b"]["skill"] = "cs-epic" + scenario["b"].update({ + "hidden_tests": ["hidden/result.py"], + "regression_tests": ["regression/base.py"], + }) + scenario["preflight"] = { + "naive_hook": "preflight/naive.py", + "golden_hook": "preflight/golden.py", + } + + experiment = tmp_path / "experiment" + for directory in ("hidden", "regression", "preflight"): + (experiment / directory).mkdir(parents=True) + (experiment / "preflight/naive.py").write_text("# no-op\n", encoding="utf-8") + (experiment / "preflight/golden.py").write_text( + "from pathlib import Path\nimport sys\n" + "(Path(sys.argv[1]) / 'result.txt').write_text('ok\\n', encoding='utf-8')\n", + encoding="utf-8", + ) + (experiment / "hidden/result.py").write_text( + "from pathlib import Path\ndef test_result(): assert Path('result.txt').exists()\n", + encoding="utf-8", + ) + (experiment / "regression/base.py").write_text( + "def test_base(): assert True\n", + encoding="utf-8", + ) + + seed = tmp_path / "seed" + epic = seed / ".codestable/epics/example.md" + cursor = seed / ".codestable/work/epic-example.md" + epic.parent.mkdir(parents=True) + cursor.parent.mkdir(parents=True) + epic.write_text( + "---\nstatus: active\nwork: ../work/epic-example.md\n---\n" + "# Example\n\n## 子项契约\n\n- `ITEM-A`：first\n- `ITEM-B`：second\n", + encoding="utf-8", + ) + cursor.write_text( + "---\nepic: ../epics/example.md\nphase: executing\n" + "approved_revision: deadbeef\ncurrent_item: ITEM-A\n" + "item_progression: continuous\nmilestone_commit: authorized\n" + "remote_publish: final\n---\n\n## 子项进度\n\n- [ ] ITEM-A\n- [ ] ITEM-B\n", + encoding="utf-8", + ) + + result = sequence.preflight_fixture( + fixture, + seed, + experiment, + tmp_path / "preflight-run", + ) + + assert result["ok"] is False + assert result["state"] == "fixture-invalid" + assert "approved_revision" in result["reason"] + + +def test_operational_exception_is_checkpointed_as_retryable_error(monkeypatch, tmp_path) -> None: + _stub_sequence(monkeypatch) + + def fail_operationally(**_kwargs) -> dict: + raise sequence.RetryableSequenceError("model transport timed out") + + monkeypatch.setattr(sequence, "run_pair", fail_operationally) + checkpoint = tmp_path / "results.partial.jsonl" + payload = sequence.run_sequence( + config=_config(), + fixtures=[_fixture()], + k=1, + experiment_dir=tmp_path / "experiment", + root=ROOT, + run_root=tmp_path / "runs", + checkpoint_path=checkpoint, + harness_resolver=lambda _name: object(), + ) + + assert payload["errors"][0]["state"] == "retryable-error" + assert payload["operational_errors"][0]["resolved"] is False + events = [json.loads(line) for line in checkpoint.read_text(encoding="utf-8").splitlines()] + error_event = next(event for event in events if event.get("phase") == "error") + assert error_event["status"] == "retryable-error" + assert error_event["phase_key"] == "fake-target|lt-checkpoint|0|error" + assert not (tmp_path / "runs/fake-target__lt-checkpoint__0").exists() + + +def test_non_retryable_pipeline_exception_is_checkpointed_and_not_retried( + monkeypatch, + tmp_path, +) -> None: + _stub_sequence(monkeypatch) + attempts = 0 + + def fail_deterministically(**_kwargs) -> dict: + nonlocal attempts + attempts += 1 + raise ValueError("pipeline contract broken") + + monkeypatch.setattr(sequence, "run_pair", fail_deterministically) + checkpoint = tmp_path / "results.partial.jsonl" + kwargs = { + "config": _config(), + "fixtures": [_fixture()], + "k": 1, + "experiment_dir": tmp_path / "experiment", + "root": ROOT, + "run_root": tmp_path / "runs", + "checkpoint_path": checkpoint, + "harness_resolver": lambda _name: object(), + } + + first = sequence.run_sequence(**kwargs) + second = sequence.run_sequence(**kwargs) + + assert attempts == 1 + assert first["errors"][0]["state"] == "pipeline-error" + assert second["errors"] == first["errors"] + assert second["operational_errors"] == [] + assert second["aggregate"]["integrity"]["ok"] is False + + +def test_retryable_error_followed_by_pipeline_error_remains_unresolved( + monkeypatch, + tmp_path, +) -> None: + _stub_sequence(monkeypatch) + attempts = 0 + + def fail_in_two_different_ways(**_kwargs) -> dict: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise sequence.RetryableSequenceError("transport timed out") + raise ValueError("deterministic pipeline failure") + + monkeypatch.setattr(sequence, "run_pair", fail_in_two_different_ways) + checkpoint = tmp_path / "results.partial.jsonl" + kwargs = { + "config": _config(), + "fixtures": [_fixture()], + "k": 1, + "experiment_dir": tmp_path / "experiment", + "root": ROOT, + "run_root": tmp_path / "runs", + "checkpoint_path": checkpoint, + "harness_resolver": lambda _name: object(), + } + + sequence.run_sequence(**kwargs) + second = sequence.run_sequence(**kwargs) + + assert attempts == 2 + assert second["operational_errors"][0]["resolved"] is False + assert second["aggregate"]["operational_errors"] == { + "attempts": 1, + "resolved": 0, + "unresolved": 1, + } + + +def test_preflight_hook_timeout_is_fast_fixture_invalid(monkeypatch, tmp_path) -> None: + fixture = _fixture() + scenario = fixture.raw["scenario"] + scenario["b"].update({ + "hidden_tests": ["hidden/result.py"], + "regression_tests": ["regression/base.py"], + }) + scenario["preflight"] = { + "naive_hook": "preflight/naive.py", + "golden_hook": "preflight/golden.py", + } + experiment = tmp_path / "experiment" + for directory in ("hidden", "regression", "preflight"): + (experiment / directory).mkdir(parents=True) + (experiment / "preflight/naive.py").write_text( + "import time\ntime.sleep(1)\n", + encoding="utf-8", + ) + (experiment / "preflight/golden.py").write_text( + "from pathlib import Path\nimport sys\n" + "(Path(sys.argv[1]) / 'result.txt').write_text('ok\\n', encoding='utf-8')\n", + encoding="utf-8", + ) + (experiment / "hidden/result.py").write_text( + "from pathlib import Path\ndef test_result(): assert Path('result.txt').exists()\n", + encoding="utf-8", + ) + (experiment / "regression/base.py").write_text( + "def test_base(): assert True\n", + encoding="utf-8", + ) + seed = tmp_path / "seed" + seed.mkdir() + monkeypatch.setattr(sequence, "_DETERMINISTIC_TIMEOUT_S", 0.05, raising=False) + + started = time.monotonic() + result = sequence.preflight_fixture( + fixture, + seed, + experiment, + tmp_path / "preflight-run", + ) + + assert time.monotonic() - started < 0.5 + assert result["ok"] is False + assert result["state"] == "fixture-invalid" + assert "timeout" in result["reason"] + + +def test_a_check_timeout_is_fast_pipeline_failure(monkeypatch, tmp_path) -> None: + fixture = _fixture() + fixture.raw["scenario"]["a"]["checks"] = ["checks/hang.py"] + experiment = tmp_path / "experiment" + (experiment / "checks").mkdir(parents=True) + (experiment / "checks/hang.py").write_text( + "import time\ndef test_hang(): time.sleep(1)\n", + encoding="utf-8", + ) + seed = tmp_path / "seed" + seed.mkdir() + + class CandidateHarness: + name = "candidate" + + def __init__(self) -> None: + self.calls = 0 + + def invoke(self, _prompt, model, _workdir, timeout_s): + assert timeout_s == 600 + self.calls += 1 + return HarnessResult( + output="晶化候选：example rule", + model=model, + harness=self.name, + wall_ms=1, + ) + + monkeypatch.setattr(sequence, "_DETERMINISTIC_TIMEOUT_S", 0.05) + harness = CandidateHarness() + started = time.monotonic() + pair = sequence.run_pair( + fixture=fixture, + target=ExecutionTarget( + id="fake-target", + family="fake-family", + harness=harness.name, + model="mock-model", + ), + k_index=0, + seed_repo=seed, + experiment_dir=experiment, + root=ROOT, + run_root=tmp_path / "run", + harness=harness, + ) + + assert time.monotonic() - started < 0.5 + assert harness.calls == 1 + assert pair["state"] == "pipeline-failed" + assert pair["failure_reason"] == "A oracle failed" + + +def test_between_tasks_hook_timeout_is_fast_and_symmetric(monkeypatch, tmp_path) -> None: + source = tmp_path / "source" + source.mkdir() + treatment = tmp_path / "treatment" + control = tmp_path / "control" + sequence.copy_repo(source, treatment) + sequence.copy_repo(source, control) + hook = tmp_path / "hang.py" + hook.write_text("import time\ntime.sleep(1)\n", encoding="utf-8") + monkeypatch.setattr(sequence, "_DETERMINISTIC_TIMEOUT_S", 0.05) + + started = time.monotonic() + result = sequence.apply_between_tasks_hook(hook, treatment, control, []) + + assert time.monotonic() - started < 0.5 + assert result["ok"] is False + assert result["runs"] == 2 + assert result["timed_out"] == 2 + + +def test_harness_result_error_is_retryable_not_pipeline_failure(monkeypatch, tmp_path) -> None: + def build_seed(_seed: str, destination: Path, _root: Path) -> Path: + destination.mkdir(parents=True) + return destination + + class ErrorHarness: + name = "error-harness" + + def invoke(self, _prompt, model, _workdir, timeout_s): + assert timeout_s == 600 + return HarnessResult( + output="", + model=model, + harness=self.name, + wall_ms=1, + error="transport unavailable", + ) + + monkeypatch.setattr(sequence, "build_seed_repo", build_seed) + monkeypatch.setattr(sequence, "preflight_fixture", lambda *_args: {"ok": True}) + payload = sequence.run_sequence( + config=_config(), + fixtures=[_fixture()], + k=1, + experiment_dir=tmp_path / "experiment", + root=ROOT, + run_root=tmp_path / "runs", + checkpoint_path=tmp_path / "results.partial.jsonl", + harness_resolver=lambda _name: ErrorHarness(), + ) + + assert payload["pairs"] == [] + assert payload["errors"][0]["state"] == "retryable-error" + assert payload["errors"][0]["error"] == "RetryableSequenceError" + + +def test_b_rejects_index_mutation_while_control_business_diff_stays_valid(tmp_path) -> None: + fixture = _fixture() + scenario = fixture.raw["scenario"] + scenario["a"]["allowed_paths"] = ["post-a.txt"] + scenario["b"]["allowed_paths"] = ["result.txt"] + scenario["expect"]["lesson_transition"] = "unchanged-observed" + seed = tmp_path / "seed" + seed.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=seed, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=seed, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=seed, check=True) + (seed / "tracked.txt").write_text("base\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=seed, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=seed, check=True) + + class IndexMutatingHarness: + name = "index-mutating" + + def invoke(self, prompt, model, workdir, timeout_s): + assert timeout_s == 600 + if "complete task A" in prompt: + (workdir / "post-a.txt").write_text("done\n", encoding="utf-8") + output = "晶化候选：example rule" + elif "请记录为 observed lesson" in prompt: + lesson = workdir / ".codestable/lessons/2026-08-02-example.md" + lesson.parent.mkdir(parents=True) + lesson.write_text( + "---\nstatus: observed\nscope: example\ndate: 2026-08-02\n---\n" + "规则：example rule。\n适用 / 不适用：example scope。\n" + "证据：post-a.txt。\n候选归宿：project-doc\n", + encoding="utf-8", + ) + output = "recorded" + else: + (workdir / "result.txt").write_text("done\n", encoding="utf-8") + if (workdir / ".codestable/lessons/2026-08-02-example.md").exists(): + subprocess.run(["git", "add", "result.txt"], cwd=workdir, check=True) + output = "done" + return HarnessResult( + output=output, + model=model, + harness=self.name, + wall_ms=1, + ) + + pair = sequence.run_pair( + fixture=fixture, + target=ExecutionTarget( + id="fake-target", + family="fake-family", + harness=IndexMutatingHarness.name, + model="mock-model", + ), + k_index=0, + seed_repo=seed, + experiment_dir=tmp_path, + root=ROOT, + run_root=tmp_path / "run", + harness=IndexMutatingHarness(), + ) + + assert pair["treatment_repo_integrity_ok"] is False + assert pair["treatment_mutation_ok"] is False + assert pair["control_repo_integrity_ok"] is True + assert pair["control_mutation_ok"] is True + + +@pytest.mark.parametrize("mutation", ["head", "index", "config", "hooks"]) +def test_repo_control_snapshot_detects_every_protected_git_surface(tmp_path, mutation) -> None: + repo = tmp_path / mutation + repo.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=repo, check=True) + tracked = repo / "tracked.txt" + tracked.write_text("base\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=repo, check=True) + before = sequence.repo_control_snapshot(repo) + + if mutation == "head": + (repo / "second.txt").write_text("second\n", encoding="utf-8") + subprocess.run(["git", "add", "second.txt"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-q", "-m", "second"], cwd=repo, check=True) + elif mutation == "index": + tracked.write_text("staged\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=repo, check=True) + elif mutation == "config": + subprocess.run(["git", "config", "user.name", "changed"], cwd=repo, check=True) + else: + hooks = subprocess.run( + ["git", "rev-parse", "--git-path", "hooks"], + cwd=repo, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + hook_dir = Path(hooks) if Path(hooks).is_absolute() else repo / hooks + (hook_dir / "pre-commit").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + + assert sequence.repo_control_snapshot(repo) != before + + +@pytest.mark.parametrize("changed_input", ["skill", "seed"]) +def test_checkpoint_rejects_resume_when_frozen_file_changes( + monkeypatch, + tmp_path, + changed_input, +) -> None: + _stub_sequence(monkeypatch) + root = tmp_path / "root" + for skill in ("cs-feat", "cs-keep"): + skill_path = root / f"plugins/codestable/skills/{skill}/SKILL.md" + skill_path.parent.mkdir(parents=True) + skill_path.write_text(f"# {skill}\n", encoding="utf-8") + seed_builder = root / "experiments/seeds/dispatchboard-learning/build-seed.py" + seed_builder.parent.mkdir(parents=True) + seed_builder.write_text("# frozen seed\n", encoding="utf-8") + checkpoint = tmp_path / "results.partial.jsonl" + kwargs = { + "config": _config(), + "fixtures": [_fixture()], + "k": 1, + "experiment_dir": tmp_path / "experiment", + "root": root, + "run_root": tmp_path / "runs", + "checkpoint_path": checkpoint, + "harness_resolver": lambda _name: object(), + } + sequence.run_sequence(**kwargs) + + changed = ( + root / "plugins/codestable/skills/cs-feat/SKILL.md" + if changed_input == "skill" + else seed_builder + ) + changed.write_text(changed.read_text(encoding="utf-8") + "# changed\n", encoding="utf-8") + + with pytest.raises(ValueError, match="checkpoint.*不匹配"): + sequence.run_sequence(**kwargs) diff --git a/tests/test_cs_skill_eval.py b/tests/test_cs_skill_eval.py index 243d7f5..8d7e015 100644 --- a/tests/test_cs_skill_eval.py +++ b/tests/test_cs_skill_eval.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import sys +from argparse import Namespace from pathlib import Path import pytest @@ -563,3 +564,1580 @@ def test_e2e_scorer_applies_only_e2e(): assert scorers_pkg.applies("e2e_outcome", "e2e-outcome") assert not scorers_pkg.applies("e2e_outcome", "findings-recall") assert not scorers_pkg.applies("e2e_outcome", "routing-decision") + + +# ---- learning-transfer sequence ---- + +def test_learning_targets_bind_each_model_to_one_harness() -> None: + from config import ExperimentConfig + + config = ExperimentConfig( + name="learning-transfer", + skill_under_test="cs-feat", + execution_mode="learning-transfer", + model_targets=[ + { + "id": "claude-haiku", + "family": "claude", + "harness": "claude-headless", + "model": "claude-haiku-4-5", + }, + { + "id": "codex-terra", + "family": "codex", + "harness": "codex-cli", + "model": "gpt-5.6-terra", + }, + ], + ) + + cells = runner_mod.build_matrix( + config, + Namespace(variant=None, harness=None, model=None), + ) + + assert cells == [ + ("baseline", "claude-headless", "claude-haiku-4-5"), + ("baseline", "codex-cli", "gpt-5.6-terra"), + ] + + +@pytest.mark.parametrize( + "overrides,cli", + [ + ({"variants": ["candidate"]}, []), + ({"scorers": ["planted_defect"]}, []), + ({}, ["--scorer", "planted_defect"]), + ], +) +def test_learning_transfer_runner_fixes_baseline_and_scorer( + monkeypatch, tmp_path, overrides, cli, +) -> None: + from config import ExperimentConfig + + values = { + "name": "learning-contract", + "skill_under_test": "cs-feat", + "execution_mode": "learning-transfer", + "variants": ["baseline"], + "scorers": ["learning_transfer"], + "fixture_classes": ["positive"], + "model_targets": [{ + "id": "mock-target", + "family": "mock-family", + "harness": "mock", + "model": "mock-model", + }], + } + values.update(overrides) + monkeypatch.setattr(runner_mod, "load_config", lambda _path: ExperimentConfig(**values)) + monkeypatch.setattr(runner_mod, "load_fixtures", lambda *_args: []) + + assert runner_mod.main([ + "--experiment", str(tmp_path), "--dry-run", *cli, + ]) == 2 + + +def _learning_fixture_dict() -> dict: + return { + "id": "lt-feat-01", + "answerType": "learning-transfer", + "task": {"kind": "learning-transfer"}, + "scenario": { + "class": "positive", + "seed": "learning-lab", + "a": { + "skill": "cs-feat", + "request": "实现第一条选择路径", + "candidate_source": "output", + "checks": ["checks/a.py"], + "allowed_paths": ["post-a.txt", ".codestable/work/**"], + }, + "candidate": { + "expected_home": "lesson", + "required_concepts": ["empty collection", "sibling behavior"], + }, + "b": { + "skill": "cs-feat", + "request": "实现第二条选择路径", + "hidden_tests": ["hidden/feat.py"], + "regression_tests": ["regression/feat.py"], + "allowed_paths": ["learning_lab/notes.py", ".codestable/work/**"], + }, + "preflight": { + "naive_hook": "preflight/feat-naive.py", + "golden_hook": "preflight/feat-golden.py", + }, + "expect": { + "lesson_transition": "observed->validated", + }, + }, + } + + +def test_learning_transfer_fixture_validation_accepts_complete_positive_scenario() -> None: + fixture = _learning_fixture_dict() + + assert fx_mod.validate_fixture_dict(fixture) == [] + + +@pytest.mark.parametrize( + "mutation,expected", + [ + ( + lambda fixture: fixture["scenario"]["a"].update( + {"allowed_paths": [".codestable/**"]} + ), + "a.allowed_paths 不得允许 lesson mutation", + ), + ( + lambda fixture: fixture["scenario"]["b"].update( + {"skill": "cs-refactor"} + ), + "A/B 必须由同一个 owning skill", + ), + ( + lambda fixture: fixture["scenario"]["candidate"].update( + {"required_concepts": []} + ), + "required_concepts 必须是非空 list", + ), + ], +) +def test_learning_transfer_fixture_rejects_confounding_contracts( + mutation, expected, +) -> None: + fixture = _learning_fixture_dict() + mutation(fixture) + + assert any(expected in problem for problem in fx_mod.validate_fixture_dict(fixture)) + + +def test_learning_transfer_fixture_rejects_paths_outside_the_experiment_or_repo() -> None: + fixture = _learning_fixture_dict() + fixture["scenario"]["b"]["allowed_paths"] = ["../outside.py"] + + problems = fx_mod.validate_fixture_dict(fixture) + + assert any("相对路径" in problem for problem in problems) + + +def test_learning_transfer_fixture_rejects_unsafe_ids_and_seed_paths() -> None: + fixture = _learning_fixture_dict() + fixture["id"] = "../../escape" + fixture["scenario"]["seed"] = "../outside" + + problems = fx_mod.validate_fixture_dict(fixture) + + assert any("id" in problem and "slug" in problem for problem in problems) + assert any("seed" in problem and "slug" in problem for problem in problems) + + +def test_learning_transfer_fixture_requires_regression_oracles() -> None: + fixture = _learning_fixture_dict() + del fixture["scenario"]["b"]["regression_tests"] + + problems = fx_mod.validate_fixture_dict(fixture) + + assert any("regression_tests" in problem for problem in problems) + + +def test_learning_transfer_fixture_requires_a_mutation_allowlist() -> None: + fixture = _learning_fixture_dict() + del fixture["scenario"]["a"]["allowed_paths"] + + problems = fx_mod.validate_fixture_dict(fixture) + + assert any("a.allowed_paths" in problem for problem in problems) + + +def test_learning_transfer_fixture_requires_explicit_lesson_transition() -> None: + fixture = _learning_fixture_dict() + del fixture["scenario"]["expect"] + + problems = fx_mod.validate_fixture_dict(fixture) + + assert any("lesson_transition" in problem for problem in problems) + + +def test_learning_transfer_stale_fixture_requires_hook_allowlist() -> None: + fixture = _learning_fixture_dict() + fixture["scenario"]["class"] = "stale" + fixture["scenario"]["expect"]["lesson_transition"] = "observed->retired" + fixture["scenario"]["between_tasks"] = {"hook": "hooks/stale.py"} + + problems = fx_mod.validate_fixture_dict(fixture) + + assert any("between_tasks.allowed_paths" in problem for problem in problems) + + +def test_claude_harness_disables_session_persistence(monkeypatch, tmp_path) -> None: + import subprocess + import harness.adapter_claude as adapter_claude + + observed: dict[str, list[str]] = {} + + def fake_run(command, **kwargs): + observed["command"] = command + observed["env"] = kwargs["env"] + return subprocess.CompletedProcess( + command, + 0, + stdout=json.dumps({"result": "done", "num_turns": 1}), + stderr="", + ) + + monkeypatch.setattr( + adapter_claude.shutil, + "which", + lambda name: f"/usr/bin/{name}" if name in {"claude", "sandbox-exec"} else None, + ) + monkeypatch.setattr(adapter_claude.subprocess, "run", fake_run) + monkeypatch.setenv("OPENAI_API_KEY", "must-not-cross-provider") + + adapter_claude.ClaudeHarness().invoke("task", "claude-haiku-4-5", tmp_path, 30) + + assert observed["command"][0] == "/usr/bin/sandbox-exec" + profile_index = observed["command"].index("-p") + profile = observed["command"][profile_index + 1] + assert "(allow default)" in profile + assert "(deny file-read*" in profile + assert "(deny file-write*" in profile + assert str(tmp_path.resolve()) in profile + assert "--no-session-persistence" in observed["command"] + assert "--safe-mode" in observed["command"] + permission_index = observed["command"].index("--permission-mode") + assert observed["command"][permission_index + 1] == "bypassPermissions" + assert "OPENAI_API_KEY" not in observed["env"] + + +def test_codex_harness_runs_ephemerally(monkeypatch, tmp_path) -> None: + import subprocess + import harness.adapter_codex as adapter_codex + + observed: dict[str, list[str]] = {} + + def fake_run(command, **kwargs): + observed["command"] = command + observed["env"] = kwargs["env"] + return subprocess.CompletedProcess(command, 0, stdout="done", stderr="") + + monkeypatch.setattr( + adapter_codex.shutil, + "which", + lambda name: f"/usr/bin/{name}" if name in {"codex", "sandbox-exec"} else None, + ) + monkeypatch.setattr(adapter_codex.subprocess, "run", fake_run) + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-cross-provider") + + adapter_codex.CodexHarness().invoke("task", "gpt-5.6-terra", tmp_path, 30) + + assert observed["command"][0] == "/usr/bin/sandbox-exec" + profile = observed["command"][observed["command"].index("-p") + 1] + assert "(deny file-read*" in profile + assert "(deny file-write*" in profile + assert str(tmp_path.resolve()) in profile + assert "--ephemeral" in observed["command"] + assert "--json" in observed["command"] + assert "--ignore-user-config" in observed["command"] + assert "--ignore-rules" in observed["command"] + sandbox_index = observed["command"].index("--sandbox") + assert observed["command"][sandbox_index + 1] == "workspace-write" + assert "ANTHROPIC_API_KEY" not in observed["env"] + + +def test_codex_harness_parses_jsonl_usage_as_measured_tokens() -> None: + import harness.adapter_codex as adapter_codex + + output, usage = adapter_codex._parse("\n".join([ + json.dumps({"type": "thread.started", "thread_id": "session-secret"}), + json.dumps({ + "type": "item.completed", + "item": {"type": "agent_message", "text": "done"}, + }), + json.dumps({ + "type": "turn.completed", + "usage": {"input_tokens": 120, "cached_input_tokens": 20, "output_tokens": 30}, + }), + ])) + + assert output == "done" + assert usage == { + "input_tokens": 120, + "output_tokens": 30, + "source": "codex-json", + } + captured = metrics_mod.capture(HarnessResult( + output=output, + model="gpt-5.6-terra", + harness="codex-cli", + wall_ms=1, + usage=usage, + ), prompt="task") + assert captured["input_tokens"]["tag"] == MEASURED + assert captured["output_tokens"]["tag"] == MEASURED + assert captured["cost_usd"]["tag"] == SOFT + + +def test_learning_transfer_b_prompt_does_not_leak_the_candidate() -> None: + from buildprompt import build_sequence_task_prompt + + fixture = Fixture.from_dict(_learning_fixture_dict()) + + prompt = build_sequence_task_prompt(fixture, "B SKILL SNAPSHOT", phase="b") + + assert "B SKILL SNAPSHOT" in prompt + assert "实现第二条选择路径" in prompt + assert "empty collection" not in prompt + assert "sibling behavior" not in prompt + assert "晶化候选" not in prompt + assert "treatment" not in prompt + assert "control" not in prompt + + +def test_learning_transfer_extracts_one_candidate_from_ordinary_task_output(tmp_path) -> None: + from sequence import extract_candidate + + fixture = Fixture.from_dict(_learning_fixture_dict()) + output = ( + "晶化候选：修改选择接口前先核对同域 empty collection 与 sibling behavior。\n" + "完成实现和验证。\n证据：tests。" + ) + + candidate = extract_candidate(fixture, output, tmp_path) + + assert candidate == "修改选择接口前先核对同域 empty collection 与 sibling behavior。" + + +def test_learning_transfer_extracts_epic_candidate_from_existing_cursor(tmp_path) -> None: + from sequence import extract_candidate + + fixture_dict = _learning_fixture_dict() + fixture_dict["scenario"]["a"]["skill"] = "cs-epic" + fixture_dict["scenario"]["a"]["candidate_source"] = "epic-cursor" + fixture_dict["scenario"]["candidate"]["required_concepts"] = ["rollback", "fixture"] + cursor = tmp_path / ".codestable/work/epic-learning.md" + cursor.parent.mkdir(parents=True) + cursor.write_text( + "## 临时决策与证据\n- 晶化候选：迁移子项开工前先核对 rollback fixture。\n", + encoding="utf-8", + ) + + candidate = extract_candidate(Fixture.from_dict(fixture_dict), "不得展示候选", tmp_path) + + assert candidate == "迁移子项开工前先核对 rollback fixture。" + + +def test_learning_transfer_curation_prompt_carries_exact_authorized_candidate() -> None: + from buildprompt import build_curation_prompt + + fixture = Fixture.from_dict(_learning_fixture_dict()) + + prompt = build_curation_prompt( + fixture, + "CS-KEEP SNAPSHOT", + "修改选择接口前先核对同域空集合语义。", + "A checks passed: 2/2", + ) + + assert "CS-KEEP SNAPSHOT" in prompt + assert "修改选择接口前先核对同域空集合语义。" in prompt + assert "A checks passed: 2/2" in prompt + assert "显式授权" in prompt + assert "记录" in prompt + assert "lesson 类归宿" in prompt + assert "不得修改业务代码" in prompt + + +def test_learning_transfer_repo_copy_has_identical_content_manifest(tmp_path) -> None: + from e2e_env import copy_repo, repo_manifest + + source = tmp_path / "source" + (source / "learning_lab").mkdir(parents=True) + (source / "learning_lab/module.py").write_text("VALUE = 1\n", encoding="utf-8") + (source / ".git").mkdir() + (source / ".git/index").write_text("volatile", encoding="utf-8") + + treatment = copy_repo(source, tmp_path / "treatment") + control = copy_repo(source, tmp_path / "control") + + assert repo_manifest(treatment) == repo_manifest(control) + assert repo_manifest(treatment) == { + "learning_lab/module.py": repo_manifest(source)["learning_lab/module.py"] + } + + +def test_learning_transfer_curation_allows_only_lesson_mutation(tmp_path) -> None: + from e2e_env import changed_paths, paths_match_allowlist, repo_manifest + + repo = tmp_path / "repo" + (repo / "learning_lab").mkdir(parents=True) + (repo / "learning_lab/module.py").write_text("VALUE = 1\n", encoding="utf-8") + before = repo_manifest(repo) + lesson = repo / ".codestable/lessons/2026-08-02-selection.md" + lesson.parent.mkdir(parents=True) + lesson.write_text("---\nstatus: observed\n---\n规则：x\n", encoding="utf-8") + + lesson_changes = changed_paths(before, repo_manifest(repo)) + + assert paths_match_allowlist(lesson_changes, [".codestable/lessons/**"]) + + (repo / "learning_lab/module.py").write_text("VALUE = 2\n", encoding="utf-8") + code_changes = changed_paths(before, repo_manifest(repo)) + + assert not paths_match_allowlist(code_changes, [".codestable/lessons/**"]) + + +def test_learning_transfer_lesson_oracle_requires_observed_schema(tmp_path) -> None: + from scorers.learning_transfer import validate_observed_lesson + + lesson = tmp_path / ".codestable/lessons/2026-08-02-selection.md" + lesson.parent.mkdir(parents=True) + lesson.write_text( + """--- +status: observed +scope: selection interfaces +date: 2026-08-02 +--- +规则：修改选择接口前先核对同域 empty collection 语义。 +适用 / 不适用：适用于 sibling behavior；有项目文档 owner 时停止。 +证据：tests/test_selection.py。 +候选归宿：project-doc +""", + encoding="utf-8", + ) + + result = validate_observed_lesson( + tmp_path, + required_concepts=["empty collection", "sibling behavior"], + ) + + assert result["ok"] is True + assert result["path"] == ".codestable/lessons/2026-08-02-selection.md" + + +def test_learning_transfer_hidden_checks_are_measured_per_file(tmp_path) -> None: + from sequence import _run_check_files + + repo = tmp_path / "repo" + repo.mkdir() + hidden = tmp_path / "hidden" + hidden.mkdir() + (hidden / "test_green.py").write_text("def test_green(): assert True\n", encoding="utf-8") + (hidden / "test_red.py").write_text("def test_red(): assert False\n", encoding="utf-8") + + result = _run_check_files(repo, [hidden / "test_green.py", hidden / "test_red.py"]) + + assert result["passed"] == 1 + assert result["total"] == 2 + assert result["rate"] == 0.5 + + +def test_learning_transfer_checkpoint_key_includes_phase() -> None: + from sequence import phase_key + + a_key = phase_key("claude-haiku", "lt-feat-01", 2, "a") + b_key = phase_key("claude-haiku", "lt-feat-01", 2, "b-treatment") + + assert a_key == "claude-haiku|lt-feat-01|2|a" + assert b_key == "claude-haiku|lt-feat-01|2|b-treatment" + assert a_key != b_key + + +def test_learning_transfer_checkpoint_excludes_half_pairs(tmp_path) -> None: + from sequence import append_checkpoint, load_completed_pairs + + checkpoint = tmp_path / "sequence.partial.jsonl" + base = {"target_id": "claude-haiku", "fixture_id": "lt-feat-01", "k_index": 0} + for phase in ("a", "curation", "b-treatment", "b-control"): + append_checkpoint(checkpoint, {**base, "phase": phase, "status": "passed"}) + + assert load_completed_pairs(checkpoint) == [] + + pair = {**base, "phase": "score", "status": "passed", "pair": {"treatment_ok": 1, "control_ok": 0}} + append_checkpoint(checkpoint, pair) + + assert load_completed_pairs(checkpoint) == [pair["pair"]] + + +def test_learning_transfer_alternates_paired_branch_order() -> None: + from sequence import branch_order + + assert branch_order(0) == ("treatment", "control") + assert branch_order(1) == ("control", "treatment") + assert branch_order(2) == ("treatment", "control") + + +def test_learning_transfer_dry_run_counts_all_pair_invocations() -> None: + from config import ExperimentConfig + from sequence import dry_run_sequence + + fixture_dict = _learning_fixture_dict() + fixture_dict["scenario"]["between_tasks"] = { + "hook": "hooks/stale.py", + "allowed_paths": ["facts/version.txt"], + } + config = ExperimentConfig( + name="learning-transfer", + skill_under_test="cs-feat", + execution_mode="learning-transfer", + model_targets=[{ + "id": "claude-haiku", + "family": "claude", + "harness": "claude-headless", + "model": "claude-haiku-4-5", + }], + ) + + estimate = dry_run_sequence( + config, + [Fixture.from_dict(fixture_dict)], + k=3, + root=ROOT, + ) + + assert estimate["invocation_count"] == 12 + assert estimate["phase_invocations"] == { + "a": 3, + "curation": 3, + "b-treatment": 3, + "b-control": 3, + } + assert estimate["hook_runs"] == 6 + assert estimate["est_total_usd"] > 0 + + +def test_learning_transfer_run_pair_uses_only_lesson_as_treatment(tmp_path) -> None: + from _model import ExecutionTarget + from sequence import run_pair + + fixture_dict = _learning_fixture_dict() + fixture_dict["scenario"]["a"]["checks"] = ["checks/a.py"] + fixture_dict["scenario"]["b"]["hidden_tests"] = ["hidden/b.py"] + fixture_dict["scenario"]["b"]["allowed_paths"] = ["learning_lab/result.txt"] + fixture_dict["scenario"]["between_tasks"] = { + "hook": "hooks/between.py", + "allowed_paths": ["facts/version.txt"], + } + fixture = Fixture.from_dict(fixture_dict) + experiment = tmp_path / "experiment" + (experiment / "checks").mkdir(parents=True) + (experiment / "hidden").mkdir() + (experiment / "regression").mkdir() + (experiment / "hooks").mkdir() + (experiment / "checks/a.py").write_text( + "from pathlib import Path\n\ndef test_a(): assert Path('post-a.txt').exists()\n", + encoding="utf-8", + ) + (experiment / "hidden/b.py").write_text( + "from pathlib import Path\n\ndef test_b():\n" + " assert Path('learning_lab/result.txt').read_text() == 'correct\\n'\n" + " assert Path('facts/version.txt').read_text() == 'v2\\n'\n", + encoding="utf-8", + ) + (experiment / "hooks/between.py").write_text( + "from pathlib import Path\nimport sys\n" + "repo = Path(sys.argv[1])\n(repo / 'facts').mkdir(exist_ok=True)\n" + "(repo / 'facts/version.txt').write_text('v2\\n', encoding='utf-8')\n", + encoding="utf-8", + ) + (experiment / "regression/feat.py").write_text( + "from pathlib import Path\n\ndef test_existing(): assert Path('post-a.txt').exists()\n", + encoding="utf-8", + ) + seed = tmp_path / "seed" + (seed / "learning_lab").mkdir(parents=True) + (seed / "learning_lab/__init__.py").write_text("", encoding="utf-8") + + class FakeLearningHarness: + name = "fake-learning" + + def __init__(self) -> None: + self.calls: list[tuple[str, Path]] = [] + + def invoke(self, prompt, model, workdir, timeout_s): + self.calls.append((prompt, workdir)) + if "实现第一条选择路径" in prompt: + (workdir / "post-a.txt").write_text("passed\n", encoding="utf-8") + output = "晶化候选：修改选择接口前先核对同域 empty collection 与 sibling behavior。" + elif "请记录为 observed lesson" in prompt: + lesson = workdir / ".codestable/lessons/2026-08-02-selection.md" + lesson.parent.mkdir(parents=True) + lesson.write_text( + "---\nstatus: observed\nscope: selection\ndate: 2026-08-02\n---\n" + "规则：核对 empty collection。\n" + "适用 / 不适用：适用于 sibling behavior。\n" + "证据：post-a.txt。\n候选归宿：project-doc\n", + encoding="utf-8", + ) + output = "recorded" + else: + lesson = workdir / ".codestable/lessons/2026-08-02-selection.md" + learned = lesson.exists() + if learned: + text = lesson.read_text(encoding="utf-8") + lesson.write_text( + text.replace("status: observed", "status: validated").replace( + "证据：post-a.txt。", + "证据：post-a.txt；learning_lab/result.txt。", + ), + encoding="utf-8", + ) + (workdir / "learning_lab/result.txt").write_text( + "correct\n" if learned else "wrong\n", + encoding="utf-8", + ) + output = "done" + return HarnessResult( + output=output, + model=model, + harness=self.name, + wall_ms=1, + ) + + harness = FakeLearningHarness() + phase_events: list[dict] = [] + pair = run_pair( + fixture=fixture, + target=ExecutionTarget( + id="fake", + family="fake", + harness=harness.name, + model="mock-model", + ), + k_index=0, + seed_repo=seed, + experiment_dir=experiment, + root=ROOT, + run_root=tmp_path / "run", + harness=harness, + phase_callback=phase_events.append, + ) + + assert len(harness.calls) == 4 + assert pair["a_ok"] is True + assert pair["a_mutation_ok"] is True + assert pair["candidate_unique"] is True + assert pair["lesson_schema_ok"] is True + assert pair["lesson_only_mutation"] is True + assert pair["prompt_equal"] is True + assert pair["hook_ok"] is True + assert pair["lesson_transition_ok"] is True + assert pair["lesson_expectation_ok"] is True + assert pair["lesson_status"] == "validated" + assert pair["stale_retired"] is None + assert pair["treatment_regression_ok"] is True + assert pair["control_regression_ok"] is True + assert pair["treatment_ok"] is True + assert pair["control_ok"] is False + assert harness.calls[2][0] == harness.calls[3][0] + assert [item["phase"] for item in pair["phase_metrics"]] == [ + "a", + "curation", + "b-treatment", + "b-control", + ] + assert all("wall_ms" in item["metrics"] for item in pair["phase_metrics"]) + assert [ + event["phase"] + for event in phase_events + if event.get("status") != "invocation-started" + ] == [ + "a", + "curation", + "hook", + "b-treatment", + "b-control", + ] + for phase in ("a", "curation", "b-treatment", "b-control"): + invocation_events = [event for event in phase_events if event["phase"] == phase] + assert [event["status"] for event in invocation_events] == [ + "invocation-started", + "invocation-complete", + ] + assert invocation_events[0]["invocation_id"] == invocation_events[1]["invocation_id"] + assert all("output" not in event and "candidate" not in event for event in phase_events) + + +def test_learning_transfer_a_without_unique_candidate_is_terminal_pipeline_failure(tmp_path) -> None: + from _model import ExecutionTarget + from sequence import run_pair + + fixture_dict = _learning_fixture_dict() + fixture_dict["scenario"]["a"]["checks"] = [] + fixture = Fixture.from_dict(fixture_dict) + seed = tmp_path / "seed" + seed.mkdir() + + class MissingCandidateHarness: + name = "missing-candidate" + + def __init__(self) -> None: + self.calls = 0 + + def invoke(self, prompt, model, workdir, timeout_s): + self.calls += 1 + return HarnessResult( + output="任务完成，但没有形成候选。", + model=model, + harness=self.name, + wall_ms=1, + ) + + harness = MissingCandidateHarness() + pair = run_pair( + fixture=fixture, + target=ExecutionTarget( + id="fake", family="fake", harness=harness.name, model="mock-model", + ), + k_index=0, + seed_repo=seed, + experiment_dir=tmp_path, + root=ROOT, + run_root=tmp_path / "run", + harness=harness, + ) + + assert harness.calls == 1 + assert pair["state"] == "pipeline-failed" + assert pair["a_ok"] is True + assert pair["candidate_unique"] is False + + +def test_learning_transfer_invalid_curation_stops_before_b(tmp_path) -> None: + from _model import ExecutionTarget + from sequence import run_pair + + fixture_dict = _learning_fixture_dict() + fixture_dict["scenario"]["a"]["checks"] = [] + fixture = Fixture.from_dict(fixture_dict) + seed = tmp_path / "seed" + seed.mkdir() + + class InvalidCurationHarness: + name = "invalid-curation" + + def __init__(self) -> None: + self.calls = 0 + + def invoke(self, prompt, model, workdir, timeout_s): + self.calls += 1 + if self.calls == 1: + output = "晶化候选：核对 empty collection 与 sibling behavior。" + else: + lesson = workdir / ".codestable/lessons/bad.md" + lesson.parent.mkdir(parents=True) + lesson.write_text("无 schema\n", encoding="utf-8") + output = "recorded" + return HarnessResult( + output=output, + model=model, + harness=self.name, + wall_ms=1, + ) + + harness = InvalidCurationHarness() + pair = run_pair( + fixture=fixture, + target=ExecutionTarget( + id="fake", family="fake", harness=harness.name, model="mock-model", + ), + k_index=0, + seed_repo=seed, + experiment_dir=tmp_path, + root=ROOT, + run_root=tmp_path / "run", + harness=harness, + ) + + assert harness.calls == 2 + assert pair["state"] == "pipeline-failed" + assert pair["candidate_unique"] is True + assert pair["lesson_schema_ok"] is False + + +def test_learning_transfer_lesson_transition_allows_status_and_evidence_only() -> None: + from scorers.learning_transfer import validate_lesson_transition + + before = """--- +status: observed +scope: selection +date: 2026-08-02 +--- +规则：先核对空集合语义。 +适用 / 不适用：选择接口；已有文档 owner 时停止。 +证据：tests/test_first.py。 +候选归宿：project-doc +""" + validated = before.replace("status: observed", "status: validated").replace( + "证据：tests/test_first.py。", + "证据：tests/test_first.py；tests/test_second.py。", + ) + rewritten = validated.replace("规则：先核对空集合语义。", "规则：所有接口都返回空集合。") + + assert validate_lesson_transition(before, validated)["ok"] is True + result = validate_lesson_transition(before, rewritten) + assert result["ok"] is False + assert "规则" in result["errors"] + + +def test_learning_transfer_lesson_transition_allows_narrow_retirement() -> None: + from scorers.learning_transfer import validate_lesson_transition + + before = """--- +status: validated +scope: selection +date: 2026-08-02 +--- +规则：先核对空集合语义。 +适用 / 不适用：选择接口；已有文档 owner 时停止。 +证据：tests/test_first.py。 +候选归宿：project-doc +""" + retired = before.replace("status: validated", "status: retired").replace( + "证据：tests/test_first.py。", + "证据：tests/test_first.py；hooks/stale.py 反证，替代为 docs/selection.md。", + ) + + result = validate_lesson_transition(before, retired) + + assert result["ok"] is True + + +def test_learning_transfer_between_hook_keeps_both_branches_in_sync(tmp_path) -> None: + from e2e_env import copy_repo + from sequence import apply_between_tasks_hook + + source = tmp_path / "source" + (source / "facts").mkdir(parents=True) + (source / "facts/version.txt").write_text("v1\n", encoding="utf-8") + treatment = copy_repo(source, tmp_path / "treatment") + control = copy_repo(source, tmp_path / "control") + lesson = treatment / ".codestable/lessons/2026-08-02-selection.md" + lesson.parent.mkdir(parents=True) + lesson.write_text("---\nstatus: observed\n---\n规则：v1。\n", encoding="utf-8") + hook = tmp_path / "stale.py" + hook.write_text( + "from pathlib import Path\nimport sys\n" + "repo = Path(sys.argv[1])\n" + "(repo / 'facts/version.txt').write_text('v2\\n', encoding='utf-8')\n", + encoding="utf-8", + ) + + result = apply_between_tasks_hook(hook, treatment, control, ["facts/version.txt"]) + + assert result["ok"] is True + assert result["runs"] == 2 + assert lesson.read_text(encoding="utf-8") == "---\nstatus: observed\n---\n规则：v1。\n" + assert (treatment / "facts/version.txt").read_text(encoding="utf-8") == "v2\n" + assert (control / "facts/version.txt").read_text(encoding="utf-8") == "v2\n" + + +def test_learning_transfer_scores_a_positive_pair_with_measured_oracles() -> None: + from scorers.learning_transfer import score_pair + + result = score_pair({ + "fixture_class": "positive", + "a_ok": True, + "a_mutation_ok": True, + "candidate_unique": True, + "lesson_schema_ok": True, + "lesson_only_mutation": True, + "prompt_equal": True, + "isolation_ok": True, + "lesson_transition_ok": True, + "lesson_expectation_ok": True, + "treatment_mutation_ok": True, + "control_mutation_ok": True, + "treatment_regression_ok": True, + "control_regression_ok": True, + "treatment_hidden": 1.0, + "control_hidden": 0.0, + }) + + assert result["status"] == "passed" + assert result["scores"]["paired_delta"] == {"value": 1.0, "tag": MEASURED} + assert result["scores"]["paired_win"]["value"] == 1 + assert result["scores"]["paired_loss"]["value"] == 0 + assert result["scores"]["paired_tie"]["value"] == 0 + assert all(score["tag"] == MEASURED for score in result["scores"].values()) + + +def test_learning_transfer_aggregate_sums_phase_cost_instead_of_averaging() -> None: + from scorers.learning_transfer import aggregate_pairs + + records = [] + for k_index in range(2): + pair = _completed_learning_pair( + family="claude", + fixture_id="feat", + owning_skill="cs-feat", + k_index=k_index, + ) + pair["phase_metrics"] = [ + {"phase": phase, "metrics": {"cost_usd": {"value": 0.125, "tag": MEASURED}}} + for phase in ("a", "curation", "b-treatment", "b-control") + ] + records.append(pair) + + aggregate = aggregate_pairs(records) + + assert aggregate["cost"] == { + "invocation_count": 8, + "cost_usd": {"value": 1.0, "tag": MEASURED}, + } + + +def _completed_learning_pair( + *, + family: str, + fixture_id: str, + owning_skill: str, + k_index: int, + fixture_class: str = "positive", + treatment: float = 1.0, + control: float = 0.0, +) -> dict: + pair = { + "state": "completed", + "family": family, + "fixture_id": fixture_id, + "owning_skill": owning_skill, + "k_index": k_index, + "fixture_class": fixture_class, + "a_ok": True, + "a_mutation_ok": True, + "candidate_unique": True, + "lesson_schema_ok": True, + "lesson_only_mutation": True, + "prompt_equal": True, + "isolation_ok": True, + "lesson_transition_ok": True, + "lesson_expectation_ok": True, + "treatment_mutation_ok": True, + "control_mutation_ok": True, + "treatment_regression_ok": True, + "control_regression_ok": True, + "treatment_hidden": treatment, + "control_hidden": control, + } + if fixture_class == "stale": + pair["stale_retired"] = True + return pair + + +def test_learning_transfer_aggregate_keeps_family_effects_and_guards_separate() -> None: + from scorers.learning_transfer import aggregate_pairs + + records = [ + _completed_learning_pair( + family="claude", fixture_id="feat", owning_skill="cs-feat", k_index=0, + ), + _completed_learning_pair( + family="codex", fixture_id="feat", owning_skill="cs-feat", k_index=0, + treatment=0.5, + ), + _completed_learning_pair( + family="claude", fixture_id="unrelated", owning_skill="cs-feat", k_index=0, + fixture_class="unrelated", treatment=1.0, control=1.0, + ), + ] + + aggregate = aggregate_pairs(records) + + assert aggregate["overall"]["paired_delta"] == 0.75 + assert aggregate["families"]["claude"]["paired_delta"] == 1.0 + assert aggregate["families"]["codex"]["paired_delta"] == 0.5 + assert aggregate["overall"]["wins"] == 2 + assert aggregate["guards"]["unrelated"]["pairs"] == 1 + assert aggregate["power"]["ok"] is False + + +def _powered_learning_records(treatment: float = 1.0, control: float = 0.0) -> list[dict]: + records = [] + skills = ("cs-feat", "cs-issue", "cs-refactor", "cs-epic") + for family in ("claude", "codex"): + for skill in skills: + for k_index in range(5): + records.append(_completed_learning_pair( + family=family, + fixture_id=skill, + owning_skill=skill, + k_index=k_index, + treatment=treatment, + control=control, + )) + for guard_class in ("unrelated", "stale"): + for k_index in range(5): + records.append(_completed_learning_pair( + family=family, + fixture_id=guard_class, + owning_skill="cs-feat", + k_index=k_index, + fixture_class=guard_class, + treatment=1.0, + control=1.0, + )) + return records + + +def test_learning_transfer_power_requires_two_families_four_skills_and_k5() -> None: + from scorers.learning_transfer import aggregate_pairs + + records = _powered_learning_records() + + assert aggregate_pairs(records)["power"]["ok"] is True + + missing_repeat = [ + record for record in records + if not ( + record["family"] == "codex" + and record["owning_skill"] == "cs-epic" + and record["fixture_class"] == "positive" + and record["k_index"] == 4 + ) + ] + assert aggregate_pairs(missing_repeat)["power"]["ok"] is False + + +def test_learning_transfer_verdict_confirms_at_the_25pp_boundary() -> None: + from scorers.learning_transfer import aggregate_pairs, transfer_verdict + + aggregate = aggregate_pairs(_powered_learning_records(treatment=0.75, control=0.5)) + + result = transfer_verdict(aggregate) + + assert result["accepted"] is True + assert result["verdict"] == { + "direction": "CONFIRMED", + "observed": 0.25, + "threshold": 0.25, + "confidence": "high", + } + assert result["reasons"] == [] + + +def test_resolved_operational_error_is_reported_without_poisoning_acceptance() -> None: + from scorers.learning_transfer import aggregate_pairs, transfer_verdict + + aggregate = aggregate_pairs( + _powered_learning_records(treatment=0.75, control=0.5), + operational_errors=[{ + "target_id": "claude", + "fixture_id": "cs-feat", + "k_index": 0, + "state": "retryable-error", + "error": "RetryableSequenceError", + "resolved": True, + }], + ) + + assert aggregate["integrity"]["ok"] is True + assert aggregate["operational_errors"] == { + "attempts": 1, + "resolved": 1, + "unresolved": 0, + } + assert transfer_verdict(aggregate)["accepted"] is True + + +def test_unresolved_operational_error_is_underpowered_not_structural_corruption() -> None: + from scorers.learning_transfer import aggregate_pairs, transfer_verdict + + records = _powered_learning_records(treatment=0.75, control=0.5) + missing = records.pop() + operational = { + "target_id": missing["family"], + "fixture_id": missing["fixture_id"], + "k_index": missing["k_index"], + "state": "retryable-error", + "error": "RetryableSequenceError", + "resolved": False, + } + aggregate = aggregate_pairs( + [*records, {key: value for key, value in operational.items() if key != "resolved"}], + operational_errors=[operational], + ) + result = transfer_verdict(aggregate) + + assert aggregate["integrity"]["ok"] is True + assert aggregate["operational_errors"]["unresolved"] == 1 + assert result["accepted"] is False + assert result["verdict"]["confidence"] == "underpowered" + assert "存在未解决 operational error" in result["reasons"] + + +@pytest.mark.parametrize("state", ["pipeline-failed", "pipeline-error", "fixture-invalid"]) +def test_deterministic_failure_states_block_structural_integrity(state: str) -> None: + from scorers.learning_transfer import aggregate_pairs + + aggregate = aggregate_pairs([ + *_powered_learning_records(), + {"state": state}, + ]) + + assert aggregate["integrity"]["ok"] is False + assert aggregate["integrity"]["blockers"] == 1 + + +def test_missing_structural_oracle_fails_closed() -> None: + from scorers.learning_transfer import aggregate_pairs + + records = _powered_learning_records() + records[0].pop("prompt_equal") + + assert aggregate_pairs(records)["integrity"]["ok"] is False + + +def test_learning_transfer_verdict_rejects_underpowered_family_and_guard_failures() -> None: + from scorers.learning_transfer import aggregate_pairs, transfer_verdict + + powered = _powered_learning_records(treatment=0.75, control=0.5) + underpowered = powered[:-1] + underpowered_result = transfer_verdict(aggregate_pairs(underpowered)) + assert underpowered_result["accepted"] is False + assert underpowered_result["verdict"]["confidence"] == "underpowered" + assert "统计功效不足" in underpowered_result["reasons"] + + flat_family = [dict(record) for record in powered] + for record in flat_family: + if record["family"] == "codex" and record["fixture_class"] == "positive": + record["treatment_hidden"] = record["control_hidden"] + family_result = transfer_verdict(aggregate_pairs(flat_family)) + assert family_result["accepted"] is False + assert "至少一个 model family 未呈正向迁移" in family_result["reasons"] + + guard_regression = [dict(record) for record in powered] + guard = next(record for record in guard_regression if record["fixture_class"] == "unrelated") + guard["treatment_hidden"] = 0.0 + guard["control_hidden"] = 1.0 + guard_result = transfer_verdict(aggregate_pairs(guard_regression)) + assert guard_result["accepted"] is False + assert "unrelated guard 回退或缺失" in guard_result["reasons"] + + +def test_learning_transfer_scorer_is_registered_only_for_sequence_fixtures() -> None: + assert "learning_transfer" in scorers_pkg.available() + assert scorers_pkg.applies("learning_transfer", "learning-transfer") + assert not scorers_pkg.applies("learning_transfer", "e2e-outcome") + + +def test_learning_transfer_runner_delegates_dry_run_to_sequence(tmp_path) -> None: + experiment = tmp_path / "learning-transfer" + fixtures_dir = experiment / "fixtures/learning-transfer" + fixtures_dir.mkdir(parents=True) + (experiment / "config.json").write_text(json.dumps({ + "name": "learning-transfer", + "skill_under_test": "cs-feat", + "execution_mode": "learning-transfer", + "model_targets": [{ + "id": "claude-haiku", + "family": "claude", + "harness": "claude-headless", + "model": "claude-haiku-4-5", + }], + "fixture_classes": ["learning-transfer"], + "scorers": ["learning_transfer"], + "k": 1, + "budget_usd": 10, + }), encoding="utf-8") + (fixtures_dir / "feat.json").write_text( + json.dumps(_learning_fixture_dict(), ensure_ascii=False), + encoding="utf-8", + ) + out = tmp_path / "estimate.json" + + rc = runner_mod.main([ + "--experiment", str(experiment), + "--dry-run", + "--out", str(out), + ]) + + assert rc == 0 + estimate = json.loads(out.read_text(encoding="utf-8")) + assert estimate["invocation_count"] == 4 + assert estimate["phase_invocations"] == { + "a": 1, + "curation": 1, + "b-treatment": 1, + "b-control": 1, + } + + +def test_learning_transfer_runner_filters_explicit_execution_targets(tmp_path) -> None: + experiment = tmp_path / "learning-transfer" + fixtures_dir = experiment / "fixtures/learning-transfer" + fixtures_dir.mkdir(parents=True) + (experiment / "config.json").write_text(json.dumps({ + "name": "learning-transfer", + "skill_under_test": "cs-feat", + "execution_mode": "learning-transfer", + "model_targets": [ + { + "id": "claude-haiku", + "family": "claude", + "harness": "claude-headless", + "model": "claude-haiku-4-5", + }, + { + "id": "codex-terra", + "family": "codex", + "harness": "codex-cli", + "model": "gpt-5.6-terra", + }, + ], + "fixture_classes": ["learning-transfer"], + "scorers": ["learning_transfer"], + "k": 1, + "budget_usd": 10, + }), encoding="utf-8") + (fixtures_dir / "feat.json").write_text( + json.dumps(_learning_fixture_dict(), ensure_ascii=False), + encoding="utf-8", + ) + out = tmp_path / "estimate.json" + + rc = runner_mod.main([ + "--experiment", str(experiment), + "--harness", "codex-cli", + "--model", "gpt-5.6-terra", + "--dry-run", + "--out", str(out), + ]) + + assert rc == 0 + estimate = json.loads(out.read_text(encoding="utf-8")) + assert estimate["targets"] == 1 + assert estimate["per_target"] == [ + {"target_id": "codex-terra", "est_usd": estimate["per_target"][0]["est_usd"]}, + ] + assert estimate["invocation_count"] == 4 + + +def test_learning_transfer_runner_rejects_empty_execution_target_filter(tmp_path, capsys) -> None: + experiment = tmp_path / "learning-transfer" + fixtures_dir = experiment / "fixtures/learning-transfer" + fixtures_dir.mkdir(parents=True) + (experiment / "config.json").write_text(json.dumps({ + "name": "learning-transfer", + "skill_under_test": "cs-feat", + "execution_mode": "learning-transfer", + "model_targets": [{ + "id": "claude-haiku", + "family": "claude", + "harness": "claude-headless", + "model": "claude-haiku-4-5", + }], + "fixture_classes": ["learning-transfer"], + "scorers": ["learning_transfer"], + }), encoding="utf-8") + (fixtures_dir / "feat.json").write_text( + json.dumps(_learning_fixture_dict(), ensure_ascii=False), + encoding="utf-8", + ) + + rc = runner_mod.main([ + "--experiment", str(experiment), + "--harness", "codex-cli", + "--dry-run", + ]) + + assert rc == 2 + assert "没有匹配的 model target" in capsys.readouterr().err + + +def test_learning_transfer_builds_each_seed_repo_from_its_builder(tmp_path) -> None: + from e2e_env import build_seed_repo + + root = tmp_path / "project" + seed_dir = root / "experiments/seeds/learning-lab" + seed_dir.mkdir(parents=True) + (seed_dir / "build-seed.py").write_text( + "from pathlib import Path\nimport argparse\n" + "p=argparse.ArgumentParser(); p.add_argument('--out', required=True); a=p.parse_args()\n" + "out=Path(a.out); out.mkdir(parents=True); (out/'seed.txt').write_text('fresh\\n')\n", + encoding="utf-8", + ) + + repo = build_seed_repo("learning-lab", tmp_path / "cell/repo", root) + + assert repo == tmp_path / "cell/repo" + assert (repo / "seed.txt").read_text(encoding="utf-8") == "fresh\n" + + +def test_learning_transfer_repo_manifest_rejects_symlinks(tmp_path) -> None: + from e2e_env import repo_manifest + + outside = tmp_path / "outside.txt" + outside.write_text("secret\n", encoding="utf-8") + repo = tmp_path / "repo" + repo.mkdir() + (repo / "escape").symlink_to(outside) + + with pytest.raises(ValueError, match="symlink"): + repo_manifest(repo) + + +def test_learning_transfer_rebuilds_treatment_from_post_a_git_state(tmp_path) -> None: + import subprocess + from e2e_env import copy_repo, repo_manifest + from sequence import materialize_paired_repos + + post_a = tmp_path / "post-a" + post_a.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=post_a, check=True) + subprocess.run(["git", "config", "user.name", "eval"], cwd=post_a, check=True) + subprocess.run(["git", "config", "user.email", "eval@example.invalid"], cwd=post_a, check=True) + (post_a / "code.py").write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "code.py"], cwd=post_a, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=post_a, check=True) + + curation = copy_repo(post_a, tmp_path / "curation") + lesson = curation / ".codestable/lessons/2026-08-02-example.md" + lesson.parent.mkdir(parents=True) + lesson.write_text("lesson\n", encoding="utf-8") + subprocess.run(["git", "config", "user.name", "changed-by-curation"], cwd=curation, check=True) + subprocess.run(["git", "add", lesson.relative_to(curation).as_posix()], cwd=curation, check=True) + subprocess.run(["git", "commit", "-q", "-m", "must-not-propagate"], cwd=curation, check=True) + + treatment, control = materialize_paired_repos( + post_a, + curation, + lesson.relative_to(curation).as_posix(), + tmp_path / "treatment", + tmp_path / "control", + ) + + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=post_a, check=True, capture_output=True, text=True, + ).stdout.strip() + for repo in (treatment, control): + assert subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=repo, check=True, capture_output=True, text=True, + ).stdout.strip() == head + assert subprocess.run( + ["git", "config", "user.name"], cwd=repo, check=True, capture_output=True, text=True, + ).stdout.strip() == "eval" + assert (treatment / lesson.relative_to(curation)).read_text(encoding="utf-8") == "lesson\n" + assert not (control / lesson.relative_to(curation)).exists() + treatment_without_lesson = repo_manifest(treatment) + treatment_without_lesson.pop(lesson.relative_to(curation).as_posix()) + assert treatment_without_lesson == repo_manifest(control) + + +def test_learning_transfer_sequence_resumes_only_complete_pairs(monkeypatch, tmp_path) -> None: + import sequence + from config import ExperimentConfig + + config = ExperimentConfig( + name="learning-transfer", + skill_under_test="cs-feat", + execution_mode="learning-transfer", + model_targets=[{ + "id": "fake", + "family": "fake-family", + "harness": "fake-harness", + "model": "mock-model", + }], + ) + fixture = Fixture.from_dict(_learning_fixture_dict()) + calls: list[int] = [] + + def fake_build_seed(_seed, destination, _root): + destination.mkdir(parents=True) + return destination + + def fake_run_pair(**kwargs): + calls.append(kwargs["k_index"]) + kwargs["phase_callback"]({"phase": "a", "status": "passed"}) + pair = _completed_learning_pair( + family="fake-family", + fixture_id=fixture.id, + owning_skill="cs-feat", + k_index=kwargs["k_index"], + ) + if kwargs["k_index"] == 1: + pair["state"] = "pipeline-failed" + return pair + + monkeypatch.setattr(sequence, "build_seed_repo", fake_build_seed) + monkeypatch.setattr(sequence, "run_pair", fake_run_pair) + monkeypatch.setattr(sequence, "preflight_fixture", lambda *_args: {"ok": True}) + checkpoint = tmp_path / "results.partial.jsonl" + run_root = tmp_path / "runs" + + first = sequence.run_sequence( + config=config, + fixtures=[fixture], + k=2, + experiment_dir=tmp_path, + root=ROOT, + run_root=run_root, + checkpoint_path=checkpoint, + harness_resolver=lambda _name: object(), + ) + second = sequence.run_sequence( + config=config, + fixtures=[fixture], + k=2, + experiment_dir=tmp_path, + root=ROOT, + run_root=run_root, + checkpoint_path=checkpoint, + harness_resolver=lambda _name: object(), + ) + + assert calls == [0, 1] + assert len(first["pairs"]) == 2 + assert len(second["pairs"]) == 2 + assert [pair["state"] for pair in first["pairs"]] == ["completed", "pipeline-failed"] + assert first["preflight"][fixture.id]["ok"] is True + assert len(sequence.load_completed_pairs(checkpoint)) == 2 + events = [json.loads(line) for line in checkpoint.read_text(encoding="utf-8").splitlines()] + assert [event["phase"] for event in events].count("a") == 2 + assert [event["phase"] for event in events].count("score") == 2 + + +def test_learning_transfer_runner_executes_sequence_and_writes_results(monkeypatch, tmp_path) -> None: + import sequence + + experiment = tmp_path / "learning-transfer" + fixtures_dir = experiment / "fixtures/learning-transfer" + fixtures_dir.mkdir(parents=True) + (experiment / "config.json").write_text(json.dumps({ + "name": "learning-transfer", + "skill_under_test": "cs-feat", + "execution_mode": "learning-transfer", + "model_targets": [{ + "id": "fake", + "family": "fake-family", + "harness": "mock", + "model": "mock-model", + }], + "fixture_classes": ["learning-transfer"], + "scorers": ["learning_transfer"], + "k": 1, + "budget_usd": 10, + }), encoding="utf-8") + (fixtures_dir / "feat.json").write_text( + json.dumps(_learning_fixture_dict(), ensure_ascii=False), + encoding="utf-8", + ) + + def fake_build_seed(_seed, destination, _root): + destination.mkdir(parents=True) + return destination + + def fake_run_pair(**kwargs): + return _completed_learning_pair( + family="fake-family", + fixture_id="lt-feat-01", + owning_skill="cs-feat", + k_index=kwargs["k_index"], + ) + + monkeypatch.setattr(sequence, "build_seed_repo", fake_build_seed) + monkeypatch.setattr(sequence, "run_pair", fake_run_pair) + monkeypatch.setattr(sequence, "preflight_fixture", lambda *_args: {"ok": True}) + out = tmp_path / "results.json" + + rc = runner_mod.main([ + "--experiment", str(experiment), + "--out", str(out), + ]) + + assert rc == 0 + payload = json.loads(out.read_text(encoding="utf-8")) + assert payload["execution_mode"] == "learning-transfer" + assert len(payload["pairs"]) == 1 + assert not Path(str(out) + ".partial.jsonl").exists() + + +def test_learning_transfer_preflight_requires_golden_green_and_naive_target_red(tmp_path) -> None: + from sequence import preflight_fixture + + fixture_dict = _learning_fixture_dict() + fixture_dict["scenario"]["b"]["hidden_tests"] = ["hidden/feat.py"] + fixture_dict["scenario"]["b"]["regression_tests"] = ["regression/feat.py"] + fixture = Fixture.from_dict(fixture_dict) + experiment = tmp_path / "experiment" + for dirname in ("hidden", "regression", "preflight"): + (experiment / dirname).mkdir(parents=True) + (experiment / "hidden/feat.py").write_text( + "from pathlib import Path\n\ndef test_target(): assert Path('behavior.txt').read_text() == 'golden\\n'\n", + encoding="utf-8", + ) + (experiment / "regression/feat.py").write_text( + "from pathlib import Path\n\ndef test_existing(): assert Path('existing.txt').read_text() == 'stable\\n'\n", + encoding="utf-8", + ) + for name, value in (("feat-naive.py", "naive"), ("feat-golden.py", "golden")): + (experiment / "preflight" / name).write_text( + "from pathlib import Path\nimport sys\nrepo=Path(sys.argv[1])\n" + f"(repo/'behavior.txt').write_text('{value}\\n', encoding='utf-8')\n", + encoding="utf-8", + ) + seed = tmp_path / "seed" + seed.mkdir() + (seed / "existing.txt").write_text("stable\n", encoding="utf-8") + + result = preflight_fixture( + fixture, + seed, + experiment, + tmp_path / "preflight-runs", + ) + + assert result["ok"] is True + assert result["golden_hidden"] == 1.0 + assert result["naive_hidden"] == 0.0 + assert result["golden_regression"] == 1.0 + assert result["naive_regression"] == 1.0 + + +def test_learning_transfer_invalid_fixture_skips_models_and_blocks_verdict(monkeypatch, tmp_path) -> None: + import sequence + from config import ExperimentConfig + + config = ExperimentConfig( + name="learning-transfer", + skill_under_test="cs-feat", + execution_mode="learning-transfer", + model_targets=[{ + "id": "fake", + "family": "fake-family", + "harness": "fake-harness", + "model": "mock-model", + }], + ) + fixture = Fixture.from_dict(_learning_fixture_dict()) + + def fake_build_seed(_seed, destination, _root): + destination.mkdir(parents=True) + return destination + + monkeypatch.setattr(sequence, "build_seed_repo", fake_build_seed) + monkeypatch.setattr(sequence, "preflight_fixture", lambda *_args: { + "ok": False, + "golden_hidden": 0.0, + }) + monkeypatch.setattr( + sequence, + "run_pair", + lambda **_kwargs: pytest.fail("invalid fixture 不得调用模型"), + ) + + payload = sequence.run_sequence( + config=config, + fixtures=[fixture], + k=1, + experiment_dir=tmp_path, + root=ROOT, + run_root=tmp_path / "runs", + checkpoint_path=tmp_path / "partial.jsonl", + harness_resolver=lambda _name: object(), + ) + + assert payload["pairs"] == [] + assert payload["invalid"][0]["state"] == "fixture-invalid" + assert payload["verdict"]["accepted"] is False diff --git a/tests/test_cs_skill_eval_harness.py b/tests/test_cs_skill_eval_harness.py new file mode 100644 index 0000000..67f32aa --- /dev/null +++ b/tests/test_cs_skill_eval_harness.py @@ -0,0 +1,588 @@ +"""eval-cs-skill harness 的宿主隔离契约。""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / ".claude/skills/eval-cs-skill/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def test_target_probe_requires_explicit_blocked_markers( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import probe_targets + from _model import ExecutionTarget, HarnessResult + + host_home = tmp_path / "host-home" + host_home.mkdir() + + class SkippingHarness: + name = "skipping" + + def invoke(self, prompt, model, workdir, timeout_s): + assert timeout_s == 180 + token = re.search(r"exact token ([0-9a-f]+)", prompt).group(1) + (workdir / "inside.txt").write_text(token, encoding="utf-8") + return HarnessResult( + output="done", + model=model, + harness=self.name, + wall_ms=1, + ) + + monkeypatch.setattr(probe_targets, "physical_home", lambda: host_home) + monkeypatch.setattr(probe_targets, "get_harness", lambda _name: SkippingHarness()) + target = ExecutionTarget( + id="fake-target", + family="fake-family", + harness="skipping", + model="mock-model", + ) + + with pytest.raises(RuntimeError, match="host_read_blocked,sibling_read_blocked,host_write_blocked"): + probe_targets._probe_target(target) + + assert not list(host_home.glob(".cs-eval-probe-*")) + + +def test_target_probe_removes_host_sentinel_when_initial_snapshot_fails( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import probe_targets + from _model import ExecutionTarget + + host_home = tmp_path / "host-home" + host_home.mkdir() + monkeypatch.setattr(probe_targets, "physical_home", lambda: host_home) + monkeypatch.setattr(probe_targets, "get_harness", lambda _name: object()) + monkeypatch.setattr( + probe_targets, + "_config_snapshot", + lambda: (_ for _ in ()).throw(OSError("snapshot failed")), + ) + target = ExecutionTarget( + id="fake-target", + family="fake-family", + harness="failing-snapshot", + model="mock-model", + ) + + with pytest.raises(OSError, match="snapshot failed"): + probe_targets._probe_target(target) + + assert not list(host_home.glob(".cs-eval-probe-*")) + + +def test_target_probe_snapshots_the_host_claude_root_config( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import probe_targets + + host_home = tmp_path / "host-home" + host_home.mkdir() + root_config = host_home / ".claude.json" + root_config.write_text('{"version":1}\n', encoding="utf-8") + monkeypatch.setattr(probe_targets, "physical_home", lambda: host_home) + + before = probe_targets._config_snapshot() + root_config.write_text('{"version":2}\n', encoding="utf-8") + + assert probe_targets._config_snapshot() != before + + +@pytest.mark.parametrize( + "relative", + [ + Path(".codex/state_5.sqlite"), + Path(".codex/sessions/2026/run.jsonl"), + Path(".codex/rollout/run.jsonl"), + Path(".claude/projects/repo/session.jsonl"), + ], +) +def test_target_probe_snapshots_known_host_session_state( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + relative: Path, +) -> None: + import probe_targets + + host_home = tmp_path / "host-home" + state_file = host_home / relative + state_file.parent.mkdir(parents=True) + state_file.write_text("before\n", encoding="utf-8") + monkeypatch.setattr(probe_targets, "physical_home", lambda: host_home) + monkeypatch.delenv("CODEX_HOME", raising=False) + + before = probe_targets._config_snapshot() + state_file.write_text("after-state-change\n", encoding="utf-8") + + assert probe_targets._config_snapshot() != before + + +def test_sandbox_profile_rejects_control_characters_and_protects_physical_home( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from harness import base + + fake_home = tmp_path / "fake-home" + fake_home.mkdir() + monkeypatch.setenv("HOME", str(fake_home)) + workdir = tmp_path / "pair/cell" + runtime = tmp_path / "pair/runtime" + binary = tmp_path / "binary" + for path in (workdir, runtime): + path.mkdir(parents=True) + binary.touch() + + profile = base.macos_sandbox_profile(workdir, runtime, binary) + + assert str(base.physical_home()) in profile + assert str(fake_home.resolve()) in profile + with pytest.raises(ValueError, match="NUL 或换行"): + base.macos_sandbox_profile(tmp_path / "bad\npath", runtime, binary) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS Seatbelt 专用") +def test_claude_seatbelt_runs_cli_and_confines_file_access(tmp_path: Path) -> None: + import harness.adapter_claude as adapter_claude + + sandbox = shutil.which("sandbox-exec") + binary = shutil.which("claude") + if not sandbox or not binary: + pytest.skip("需要本机 sandbox-exec 与 claude CLI") + + workdir = tmp_path / "cell" + runtime = tmp_path / "runtime" + home = runtime / "home" + runtime_tmp = runtime / "tmp" + for path in (workdir, runtime, home, runtime_tmp): + path.mkdir() + secret = ROOT / "AGENTS.md" + secret_marker = "# Agent Rules" + + resolved_binary = Path(binary).resolve() + profile = adapter_claude._sandbox_profile(workdir, runtime, resolved_binary) + sandbox_env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(home), + "TMPDIR": str(runtime_tmp), + } + + version = subprocess.run( + [sandbox, "-p", profile, str(resolved_binary), "--version"], + cwd=workdir, + env=sandbox_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=30, + check=False, + ) + assert version.returncode == 0, version.stderr + + inside = subprocess.run( + [ + sandbox, + "-p", + profile, + "/bin/sh", + "-c", + 'printf cell > "$1" && test "$(cat "$1")" = cell', + "sh", + str(workdir / "inside.txt"), + ], + cwd=workdir, + env=sandbox_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert inside.returncode == 0, inside.stderr + + secret_link = workdir / "host-secret-link" + secret_link.symlink_to(secret) + outside_read = subprocess.run( + [sandbox, "-p", profile, "/bin/sh", "-c", 'cat "$1"', "sh", str(secret_link)], + cwd=workdir, + env=sandbox_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert outside_read.returncode != 0 + assert secret_marker not in outside_read.stdout + + sibling_secret = tmp_path / "sibling-secret.txt" + sibling_secret.write_text("sibling-must-not-cross\n", encoding="utf-8") + sibling_read = subprocess.run( + [sandbox, "-p", profile, "/bin/sh", "-c", 'cat "$1"', "sh", str(sibling_secret)], + cwd=workdir, + env=sandbox_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert sibling_read.returncode != 0 + assert "sibling-must-not-cross" not in sibling_read.stdout + + forbidden = ROOT / f".cs-eval-forbidden-{os.getpid()}-{tmp_path.name}" + assert not forbidden.exists() + forbidden_link = workdir / "host-write-link" + forbidden_link.symlink_to(forbidden) + try: + outside_write = subprocess.run( + [sandbox, "-p", profile, "/bin/sh", "-c", ': > "$1"', "sh", str(forbidden_link)], + cwd=workdir, + env=sandbox_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert outside_write.returncode != 0 + assert not forbidden.exists() + finally: + forbidden.unlink(missing_ok=True) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS Seatbelt 专用") +def test_codex_seatbelt_runs_cli_and_confines_file_access(tmp_path: Path) -> None: + import harness.adapter_codex as adapter_codex + + sandbox = shutil.which("sandbox-exec") + binary = shutil.which("codex") + if not sandbox or not binary: + pytest.skip("需要本机 sandbox-exec 与 codex CLI") + + workdir = tmp_path / "cell" + runtime = tmp_path / "runtime" + home = runtime / "home" + runtime_tmp = runtime / "tmp" + codex_home = runtime / "codex-home" + for path in (workdir, runtime, home, runtime_tmp, codex_home): + path.mkdir() + secret = ROOT / "AGENTS.md" + secret_marker = "# Agent Rules" + + resolved_binary, binary_read_roots = adapter_codex._native_codex_runtime(binary) + profile = adapter_codex._sandbox_profile( + workdir, + runtime, + resolved_binary, + binary_read_roots, + ) + sandbox_env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(home), + "CODEX_HOME": str(codex_home), + "TMPDIR": str(runtime_tmp), + } + + version = subprocess.run( + [sandbox, "-p", profile, str(resolved_binary), "--version"], + cwd=workdir, + env=sandbox_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=30, + check=False, + ) + assert version.returncode == 0, version.stderr + + inside = subprocess.run( + [ + sandbox, + "-p", + profile, + "/bin/sh", + "-c", + 'printf cell > "$1" && test "$(cat "$1")" = cell', + "sh", + str(workdir / "inside.txt"), + ], + cwd=workdir, + env=sandbox_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert inside.returncode == 0, inside.stderr + + secret_link = workdir / "host-secret-link" + secret_link.symlink_to(secret) + outside_read = subprocess.run( + [sandbox, "-p", profile, "/bin/sh", "-c", 'cat "$1"', "sh", str(secret_link)], + cwd=workdir, + env=sandbox_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert outside_read.returncode != 0 + assert secret_marker not in outside_read.stdout + + sibling_secret = tmp_path / "sibling-secret.txt" + sibling_secret.write_text("sibling-must-not-cross\n", encoding="utf-8") + sibling_read = subprocess.run( + [sandbox, "-p", profile, "/bin/sh", "-c", 'cat "$1"', "sh", str(sibling_secret)], + cwd=workdir, + env=sandbox_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert sibling_read.returncode != 0 + assert "sibling-must-not-cross" not in sibling_read.stdout + + forbidden = ROOT / f".cs-eval-codex-forbidden-{os.getpid()}-{tmp_path.name}" + assert not forbidden.exists() + forbidden_link = workdir / "host-write-link" + forbidden_link.symlink_to(forbidden) + try: + outside_write = subprocess.run( + [sandbox, "-p", profile, "/bin/sh", "-c", ': > "$1"', "sh", str(forbidden_link)], + cwd=workdir, + env=sandbox_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert outside_write.returncode != 0 + assert not forbidden.exists() + finally: + forbidden.unlink(missing_ok=True) + + +def test_claude_harness_uses_an_ephemeral_provider_scoped_environment( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import harness.adapter_claude as adapter_claude + + binary_target = tmp_path / "claude-native" + binary_target.touch() + binary_link = tmp_path / "claude" + binary_link.symlink_to(binary_target) + workdir = tmp_path / "repo" + observed: dict[str, object] = {} + + def fake_which(name: str) -> str | None: + if name == "claude": + return str(binary_link) + if name == "sandbox-exec": + return "/usr/bin/sandbox-exec" + return None + + def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + observed["command"] = command + observed["env"] = dict(kwargs["env"]) + return subprocess.CompletedProcess( + command, + 0, + stdout=json.dumps({"result": "done", "num_turns": 1}), + stderr="", + ) + + monkeypatch.setattr(adapter_claude.shutil, "which", fake_which) + monkeypatch.setattr(adapter_claude.subprocess, "run", fake_run) + monkeypatch.setenv("ANTHROPIC_API_KEY", "anthropic-test-token") + monkeypatch.setenv("OPENAI_API_KEY", "openai-must-not-cross") + monkeypatch.setenv("CODEX_API_KEY", "codex-must-not-cross") + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "host-codex")) + monkeypatch.setenv("PWD", str(ROOT)) + + result = adapter_claude.ClaudeHarness().invoke( + "task", + "claude-haiku-4-5", + workdir, + 30, + ) + + assert result.output == "done" + command = observed["command"] + assert isinstance(command, list) + assert command[3] == str(binary_target.resolve()) + assert "--no-session-persistence" in command + assert "--safe-mode" in command + assert command[command.index("--permission-mode") + 1] == "bypassPermissions" + + env = observed["env"] + assert isinstance(env, dict) + assert env["ANTHROPIC_API_KEY"] == "anthropic-test-token" + for forbidden_key in ("OPENAI_API_KEY", "CODEX_API_KEY", "CODEX_HOME", "PWD"): + assert forbidden_key not in env + for key in ("HOME", "TMPDIR"): + runtime_path = Path(env[key]) + assert runtime_path.parent.parent == tmp_path + assert not runtime_path.exists() + + +def test_claude_harness_imports_only_provider_env_from_host_settings( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import harness.adapter_claude as adapter_claude + + host_home = tmp_path / "host-home" + settings = host_home / ".claude/settings.json" + settings.parent.mkdir(parents=True) + settings.write_text( + json.dumps({ + "env": { + "ANTHROPIC_AUTH_TOKEN": "claude-settings-token", + "ANTHROPIC_BASE_URL": "https://api.example.invalid", + "OPENAI_API_KEY": "must-not-cross", + "PROJECT_SECRET": "must-not-cross", + } + }), + encoding="utf-8", + ) + observed: dict[str, object] = {} + + def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + observed["env"] = dict(kwargs["env"]) + return subprocess.CompletedProcess( + command, + 0, + stdout=json.dumps({"result": "done"}), + stderr="", + ) + + monkeypatch.setattr(adapter_claude.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr(adapter_claude.subprocess, "run", fake_run) + monkeypatch.setenv("HOME", str(host_home)) + for key in adapter_claude._SETTINGS_ENV_KEYS: + monkeypatch.delenv(key, raising=False) + + adapter_claude.ClaudeHarness().invoke( + "task", + "claude-haiku-4-5", + tmp_path / "repo", + 30, + ) + + env = observed["env"] + assert isinstance(env, dict) + assert env["ANTHROPIC_AUTH_TOKEN"] == "claude-settings-token" + assert env["ANTHROPIC_BASE_URL"] == "https://api.example.invalid" + assert "OPENAI_API_KEY" not in env + assert "PROJECT_SECRET" not in env + assert Path(env["HOME"]) != host_home + + +def test_codex_harness_uses_ephemeral_config_and_provider_scoped_environment( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import harness.adapter_codex as adapter_codex + + workdir = tmp_path / "repo" + observed: dict[str, object] = {} + + def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + observed["command"] = command + observed["env"] = dict(kwargs["env"]) + return subprocess.CompletedProcess(command, 0, stdout="done", stderr="") + + monkeypatch.setattr( + adapter_codex.shutil, + "which", + lambda name: f"/usr/bin/{name}" if name in {"codex", "sandbox-exec"} else None, + ) + monkeypatch.setattr(adapter_codex.subprocess, "run", fake_run) + monkeypatch.setenv("OPENAI_API_KEY", "openai-test-token") + monkeypatch.setenv("ANTHROPIC_API_KEY", "anthropic-must-not-cross") + monkeypatch.setenv("ANTHROPIC_AUTH_TOKEN", "anthropic-must-not-cross") + monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN", "claude-must-not-cross") + monkeypatch.setenv("HOME", str(tmp_path / "host-home")) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "host-codex")) + monkeypatch.setenv("PWD", str(ROOT)) + + result = adapter_codex.CodexHarness().invoke("task", "gpt-5.6-terra", workdir, 30) + + assert result.output == "done" + command = observed["command"] + assert isinstance(command, list) + assert command[0] == "/usr/bin/sandbox-exec" + profile = command[command.index("-p") + 1] + assert "(deny file-read*" in profile + assert "(deny file-write*" in profile + assert str(workdir.resolve()) in profile + for flag in ("--ephemeral", "--json", "--ignore-user-config", "--ignore-rules"): + assert flag in command + assert command[command.index("--sandbox") + 1] == "workspace-write" + + env = observed["env"] + assert isinstance(env, dict) + assert env["OPENAI_API_KEY"] == "openai-test-token" + for forbidden_key in ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "CLAUDE_CODE_OAUTH_TOKEN", + "PWD", + ): + assert forbidden_key not in env + for key in ("HOME", "CODEX_HOME", "TMPDIR"): + runtime_path = Path(env[key]) + assert str(runtime_path).startswith(str(tmp_path)) + assert not runtime_path.exists() + + +def test_codex_harness_copies_auth_into_the_ephemeral_codex_home( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import harness.adapter_codex as adapter_codex + + source_home = tmp_path / "source-codex" + source_home.mkdir() + source_auth = source_home / "auth.json" + source_auth.write_text('{"auth_mode":"test"}', encoding="utf-8") + observed: dict[str, object] = {} + + def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + env = dict(kwargs["env"]) + isolated_home = Path(env["CODEX_HOME"]) + observed["env"] = env + observed["copied_auth"] = (isolated_home / "auth.json").read_text(encoding="utf-8") + return subprocess.CompletedProcess(command, 0, stdout="done", stderr="") + + monkeypatch.setattr( + adapter_codex.shutil, + "which", + lambda name: f"/usr/bin/{name}" if name in {"codex", "sandbox-exec"} else None, + ) + monkeypatch.setattr(adapter_codex.subprocess, "run", fake_run) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setenv("CODEX_HOME", str(source_home)) + + adapter_codex.CodexHarness().invoke("task", "gpt-5.6-terra", tmp_path / "repo", 30) + + assert observed["copied_auth"] == source_auth.read_text(encoding="utf-8") + env = observed["env"] + assert isinstance(env, dict) + assert Path(env["CODEX_HOME"]) != source_home + assert not Path(env["CODEX_HOME"]).exists()