From 1d60f0da147a47da331ad4b60d8dbb5d2de9be8a Mon Sep 17 00:00:00 2001 From: calesthio Date: Thu, 2 Jul 2026 08:22:03 -0700 Subject: [PATCH] backlot: fix all five dogfood findings (F-01..F-05) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - F-01: cost bar crit (red) state past 90% of budget - F-02: normalize() hardens fetched board state against sparse payloads - F-03: /thumb 404s for videos with no extractable poster frame instead of serving raw video bytes - F-04: checkpoint artifact path refs only resolve inside the project dir - F-05 (board half): stall detection — in_progress stage with no disk activity >10min renders red 'stalled?' + header badge flips to STALLED?; verified against the real wedged why-cities-glow project - eval harness from dogfood session committed (visual regression + interaction smoke, capture watcher, server/gate test suites) + regression tests for each finding; 46 backlot tests green, visual eval green (restage-before-capture note logged) --- backlot/server.py | 6 +- backlot/state.py | 26 ++- backlot/ui/board.css | 8 + backlot/ui/board.js | 36 ++++- scripts/backlot_visual_eval.py | 234 +++++++++++++++++++++++++++ scripts/backlot_watch_captures.py | 195 ++++++++++++++++++++++ tests/backlot/test_gate_scenarios.py | 99 ++++++++++++ tests/backlot/test_server.py | 205 +++++++++++++++++++++++ tests/backlot/test_state.py | 56 +++++++ tests/backlot/test_visual_eval.py | 43 +++++ tests/backlot/test_watch_captures.py | 47 ++++++ 11 files changed, 950 insertions(+), 5 deletions(-) create mode 100644 scripts/backlot_visual_eval.py create mode 100644 scripts/backlot_watch_captures.py create mode 100644 tests/backlot/test_gate_scenarios.py create mode 100644 tests/backlot/test_server.py create mode 100644 tests/backlot/test_visual_eval.py create mode 100644 tests/backlot/test_watch_captures.py diff --git a/backlot/server.py b/backlot/server.py index b957c2ac..854cc938 100644 --- a/backlot/server.py +++ b/backlot/server.py @@ -239,7 +239,11 @@ def create_app() -> FastAPI: width = min(THUMB_WIDTHS, key=lambda x: abs(x - w)) cached = await asyncio.to_thread(_thumbnail_for, target, width) if cached is None: - return FileResponse(target) # not an image we can thumb — serve as-is + # Never fall back to raw video bytes for an consumer (F-03); + # non-thumbable images are safe to serve as-is. + if target.suffix.lower() in {".mp4", ".webm", ".mov"}: + raise HTTPException(status_code=404, detail="no poster frame available") + return FileResponse(target) return FileResponse(cached, media_type="image/jpeg") # ---- Media (range requests handled by FileResponse) --------------- diff --git a/backlot/state.py b/backlot/state.py index 94c9caf8..ee891878 100644 --- a/backlot/state.py +++ b/backlot/state.py @@ -31,6 +31,11 @@ FALLBACK_STAGES = [ # How long (seconds) without filesystem activity before a board reads "idle". LIVE_WINDOW_SECONDS = 5 * 60 +# An in_progress stage with no filesystem activity for this long is flagged +# as possibly stalled (F-05: a wedged agent must be visible, not silent — +# heartbeat checkpoints and tool events both reset the clock). +STALL_WINDOW_SECONDS = 10 * 60 + def _read_json(path: Path) -> Optional[dict]: """Read a JSON file, returning None on any failure.""" @@ -84,13 +89,22 @@ def _load_pipeline_meta(pipeline_type: Optional[str]) -> dict[str, Any]: def _resolve_artifact(project_dir: Path, value: Any) -> Optional[dict]: - """Checkpoint artifacts may be inline dicts or path strings — resolve both.""" + """Checkpoint artifacts may be inline dicts or path strings — resolve both. + + Path references are only followed INSIDE the project directory: a + checkpoint must not be able to pull arbitrary JSON from elsewhere on + disk onto the board (F-04). + """ if isinstance(value, dict): return value if isinstance(value, str) and value: p = Path(value) if not p.is_absolute(): p = project_dir / value + try: + p.resolve().relative_to(Path(project_dir).resolve()) + except (ValueError, OSError): + return None return _read_json(p) return None @@ -553,6 +567,16 @@ def load_board_state(project_dir: Path) -> dict[str, Any]: last_activity = _last_activity(project_dir) now = time.time() + # Stall detection: an in_progress stage that stopped writing anything. + for stage_entry in stages: + if ( + stage_entry["status"] == "in_progress" + and last_activity + and (now - last_activity) > STALL_WINDOW_SECONDS + ): + stage_entry["stalled"] = True + stage_entry["stalled_minutes"] = int((now - last_activity) / 60) + state: dict[str, Any] = { "project_id": project_id, "title": marker.get("title") or meta_json.get("name") or project_id.replace("-", " ").title(), diff --git a/backlot/ui/board.css b/backlot/ui/board.css index df7de240..9ae3c2f8 100644 --- a/backlot/ui/board.css +++ b/backlot/ui/board.css @@ -524,3 +524,11 @@ body:not(.first) .lib-card, body:not(.first) .drawer { animation: none; } /* stages that ran but aren't declared by the pipeline manifest */ .stage.undeclared .node { border-style: dashed; opacity: .85; } .stage.undeclared .name { font-style: italic; } + +/* spend past 90% of budget */ +.cost .bar i.crit { background: var(--red); } + +/* in_progress stage with no filesystem activity for a while (F-05) */ +.stage.stalled .node { border-color: var(--red); color: var(--red); background: var(--red-dim); animation: none; } +.stage.stalled .name { color: var(--red); } +.stage.stalled .sub { color: var(--red); } diff --git a/backlot/ui/board.js b/backlot/ui/board.js index c6878c40..1f2bcc30 100644 --- a/backlot/ui/board.js +++ b/backlot/ui/board.js @@ -32,9 +32,13 @@ function renderSlate(s) { const awaiting = s.stages.find((x) => x.status === "awaiting_human"); const inProgress = s.stages.find((x) => x.status === "in_progress"); + const stalled = s.stages.find((x) => x.stalled); let liveEl; if (awaiting) { liveEl = el("span", { class: "live" }, el("span", { class: "dot" }), "◈ AWAITING YOU"); + } else if (stalled) { + liveEl = el("span", { class: "live", style: "color:var(--red)" }, + el("span", { class: "dot", style: "background:var(--red);animation:none" }), "⚠ STALLED?"); } else if (s.live || inProgress) { liveEl = el("span", { class: "live" }, el("span", { class: "dot" }), "LIVE"); } else { @@ -52,7 +56,7 @@ function renderSlate(s) { hasBudget ? el("span", {}, ` / ${fmtMoney(budget)}`) : "")); if (hasBudget) { cost.append(el("div", { class: "bar" }, el("i", { - class: pct > 75 ? "warn" : "", style: `width:${pct}%`, + class: pct > 90 ? "crit" : pct > 75 ? "warn" : "", style: `width:${pct}%`, }))); } cost.append(el("div", { class: "label" }, "generation spend")); @@ -77,6 +81,9 @@ function renderSlate(s) { function stageSub(st) { if (st.status === "awaiting_human") return "awaiting your approval\nreply in chat to continue"; + if (st.status === "in_progress" && st.stalled) { + return `stalled? no activity for ${st.stalled_minutes}m\nask the agent for status`; + } if (st.status === "in_progress" && st.partial_progress) { const done = st.partial_progress.completed_scene_ids; if (Array.isArray(done)) return `${done.length} scene${done.length === 1 ? "" : "s"} done`; @@ -96,7 +103,7 @@ function renderRail(s) { let pendingIndex = 1; for (const st of s.stages) { const cls = st.status === "completed" ? "done" - : st.status === "in_progress" ? "active" + : st.status === "in_progress" ? (st.stalled ? "active stalled" : "active") : st.status === "awaiting_human" ? "await" : st.status === "failed" ? "failed" : ""; const icon = STAGE_ICONS[st.status] || String(pendingIndex); @@ -717,8 +724,31 @@ function render() { if (renders) app.append(renders); } +// Defensive normalization (F-02): the server contract guarantees these +// fields, but a sparse/legacy payload must degrade, never crash the board. +function normalize(s) { + s.pipeline = s.pipeline || { pipeline_type: "unknown", stages: [], known: false }; + s.stages = Array.isArray(s.stages) ? s.stages : []; + s.artifacts = s.artifacts || {}; + s.media = s.media || {}; + s.media.renders = Array.isArray(s.media.renders) ? s.media.renders : []; + s.media.snapshots = Array.isArray(s.media.snapshots) ? s.media.snapshots : []; + s.media.music = Array.isArray(s.media.music) ? s.media.music : []; + s.events = Array.isArray(s.events) ? s.events : []; + if (s.storyboard && Array.isArray(s.storyboard.scenes)) { + for (const c of s.storyboard.scenes) { + c.takes = Array.isArray(c.takes) ? c.takes : []; + c.audio = Array.isArray(c.audio) ? c.audio : []; + c.required_assets = Array.isArray(c.required_assets) ? c.required_assets : []; + } + } else { + s.storyboard = null; + } + return s; +} + async function refresh() { - state = await getJSON(`/api/project/${encodeURIComponent(projectId)}/state`); + state = normalize(await getJSON(`/api/project/${encodeURIComponent(projectId)}/state`)); render(); } diff --git a/scripts/backlot_visual_eval.py b/scripts/backlot_visual_eval.py new file mode 100644 index 00000000..34bd0438 --- /dev/null +++ b/scripts/backlot_visual_eval.py @@ -0,0 +1,234 @@ +"""Deterministic visual eval for Backlot. + +Stages the fictional Backlot projects, captures canonical browser screenshots, +optionally compares them to goldens, and can run a small Playwright interaction +smoke against the staged board. + +Examples: + python scripts/backlot_visual_eval.py + python scripts/backlot_visual_eval.py --bless + python scripts/backlot_visual_eval.py --interactions +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import time +import urllib.request +from datetime import datetime +from pathlib import Path +from typing import Any + +from PIL import Image, ImageChops + +REPO_ROOT = Path(__file__).resolve().parent.parent +STAGE_DIR = REPO_ROOT / ".backlot" / "screenshot-stage" +GOLDENS_DIR = REPO_ROOT / "internal" / "evals" / "goldens" +CAPTURE_ROOT = REPO_ROOT / "internal" / "evals" / "captures" +PORT = 4791 + +SHOTS = [ + ("library", "/?static=1", 1560, 500, 4200, [ + (1370, 20, 1510, 62), # live/idle badge + (90, 106, 422, 380), # card border/status animation variance + (440, 106, 772, 380), + (790, 106, 1122, 380), + (1140, 106, 1472, 380), + ]), + ("board-live", "/p/signal-in-the-static?static=1", 1560, 1150, 4200, []), + ("script-gate", "/p/the-slow-orchard?static=1", 1560, 760, 3200, []), + ("storyboard", "/p/the-last-lighthouse?static=1", 1560, 1500, 4200, []), +] + + +def compare_images( + expected_path: Path, + actual_path: Path, + diff_path: Path, + *, + threshold: float = 0.015, + masks: list[tuple[int, int, int, int]] | None = None, +) -> dict[str, Any]: + """Compare screenshots by changed-pixel ratio and write a red diff image.""" + expected = Image.open(expected_path).convert("RGB") + actual = Image.open(actual_path).convert("RGB") + if expected.size != actual.size: + diff_path.parent.mkdir(parents=True, exist_ok=True) + actual.save(diff_path) + return {"passed": False, "changed_ratio": 1.0, "reason": f"size {expected.size} != {actual.size}"} + + masks = masks or [] + for box in masks: + patch = expected.crop(box) + actual.paste(patch, box) + + delta = ImageChops.difference(expected, actual) + changed = 0 + pixels = delta.load() + width, height = delta.size + diff = Image.new("RGB", delta.size, (0, 0, 0)) + diff_px = diff.load() + for y in range(height): + for x in range(width): + if max(pixels[x, y]) > 8: + changed += 1 + diff_px[x, y] = (255, 40, 40) + else: + diff_px[x, y] = actual.getpixel((x, y)) + ratio = changed / float(width * height) + diff_path.parent.mkdir(parents=True, exist_ok=True) + diff.save(diff_path) + return {"passed": ratio <= threshold, "changed_ratio": round(ratio, 6), "threshold": threshold} + + +def run_stage() -> None: + subprocess.run( + [sys.executable, "scripts/backlot_screenshot_stage.py", "--stage-only"], + cwd=REPO_ROOT, + check=True, + timeout=180, + ) + + +def start_server() -> subprocess.Popen: + env = dict(os.environ) + env["OPENMONTAGE_PROJECTS_DIR"] = str(STAGE_DIR) + server = subprocess.Popen( + [sys.executable, "-m", "backlot", "serve", "--port", str(PORT)], + cwd=REPO_ROOT, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + deadline = time.time() + 20 + while time.time() < deadline: + try: + with urllib.request.urlopen(f"http://127.0.0.1:{PORT}/api/health", timeout=1): + return server + except Exception: + time.sleep(0.3) + server.terminate() + raise RuntimeError("Backlot server did not become healthy") + + +def capture_screenshot(url: str, output: Path, width: int, height: int, wait_ms: int) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + "npx", + "playwright", + "screenshot", + "--viewport-size", + f"{width},{height}", + "--wait-for-timeout", + str(wait_ms), + url, + str(output), + ], + cwd=REPO_ROOT, + check=True, + timeout=120, + shell=(os.name == "nt"), + ) + + +def capture_shots(capture_dir: Path) -> list[dict[str, Any]]: + results = [] + for name, path, width, height, wait_ms, _masks in SHOTS: + out = capture_dir / f"{name}.png" + capture_screenshot(f"http://127.0.0.1:{PORT}{path}", out, width, height, wait_ms) + results.append({"name": name, "path": out}) + return results + + +def compare_or_bless(capture_dir: Path, *, bless: bool, threshold: float) -> list[dict[str, Any]]: + GOLDENS_DIR.mkdir(parents=True, exist_ok=True) + report = [] + for name, _path, _width, _height, _wait_ms, masks in SHOTS: + actual = capture_dir / f"{name}.png" + golden = GOLDENS_DIR / f"{name}.png" + if bless or not golden.exists(): + shutil.copyfile(actual, golden) + report.append({"name": name, "status": "blessed", "golden": str(golden)}) + continue + diff = capture_dir / "diffs" / f"{name}.png" + result = compare_images(golden, actual, diff, threshold=threshold, masks=masks) + result.update({"name": name, "diff": str(diff)}) + report.append(result) + return report + + +def run_interactions(capture_dir: Path) -> dict[str, Any]: + """Run browser interaction smoke through Python Playwright.""" + from playwright.sync_api import sync_playwright + + screenshot = capture_dir / "interaction-smoke.png" + with sync_playwright() as pw: + browser = pw.chromium.launch(headless=True) + page = browser.new_page(viewport={"width": 1560, "height": 1000}) + page.goto(f"http://127.0.0.1:{PORT}/p/the-last-lighthouse?static=1") + page.wait_for_selector(".stage") + page.locator(".stage").first.click() + page.wait_for_selector(".drawer") + drawer_text = page.locator(".drawer").inner_text() + if "research" not in drawer_text: + raise RuntimeError("stage drawer did not open") + page.locator(".script-card").first.click() + page.wait_for_selector(".modal-bg.open") + page.keyboard.press("Escape") + page.wait_for_function("() => !document.querySelector('.modal-bg')?.classList.contains('open')") + if page.locator(".takes").count() < 1: + raise RuntimeError("takes drawer not present on staged takes scene") + replay_button = page.locator(".rp-btn", has_text="REPLAY RUN") + if replay_button.count(): + replay_button.first.click() + page.wait_for_selector('input[type="range"]') + page.locator('input[type="range"]').fill("500") + page.screenshot(path=str(screenshot), full_page=True) + browser.close() + return {"status": "passed", "screenshot": str(capture_dir / "interaction-smoke.png")} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bless", action="store_true", help="Write current captures as goldens") + parser.add_argument("--no-stage", action="store_true", help="Reuse existing .backlot/screenshot-stage") + parser.add_argument("--interactions", action="store_true", help="Run Playwright interaction smoke") + parser.add_argument("--threshold", type=float, default=0.015) + parser.add_argument("--out-dir", type=Path, default=None) + args = parser.parse_args(argv) + + if not args.no_stage: + run_stage() + + stamp = datetime.now().strftime("visual-%Y%m%d-%H%M%S") + capture_dir = args.out_dir or (CAPTURE_ROOT / stamp) + capture_dir.mkdir(parents=True, exist_ok=True) + + server = start_server() + try: + capture_shots(capture_dir) + report = compare_or_bless(capture_dir, bless=args.bless, threshold=args.threshold) + interaction_report = run_interactions(capture_dir) if args.interactions else None + finally: + server.terminate() + try: + server.wait(timeout=5) + except subprocess.TimeoutExpired: + server.kill() + + passed = all(item.get("passed", item.get("status") == "blessed") for item in report) + payload = {"capture_dir": str(capture_dir), "shots": report, "interactions": interaction_report} + report_path = capture_dir / "report.json" + report_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + print(json.dumps(payload, indent=2)) + return 0 if passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/backlot_watch_captures.py b/scripts/backlot_watch_captures.py new file mode 100644 index 00000000..0885276b --- /dev/null +++ b/scripts/backlot_watch_captures.py @@ -0,0 +1,195 @@ +"""Capture Backlot board screenshots whenever watched project state changes. + +This is the Half-B dogfood watcher from internal/evals/BACKLOT_EVAL_PLAN.md. +It polls the Backlot API, fingerprints board-relevant state, and captures the +library plus the changed project board through Playwright. + +Example: + python scripts/backlot_watch_captures.py --projects why-cities-glow rain-on-glass +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_BASE_URL = "http://127.0.0.1:4750" +DEFAULT_CAPTURE_ROOT = REPO_ROOT / "internal" / "evals" / "captures" + + +def capture_slug(project_id: str, stage: str | None, status: str | None) -> str: + """Stable, filesystem-safe screenshot name stem.""" + raw = "-".join(part for part in (project_id, stage or "unknown", status or "unknown") if part) + raw = raw.replace("\\", "-").replace("/", "-").replace("..", "") + slug = re.sub(r"[^A-Za-z0-9._-]+", "-", raw).strip(".-") + slug = re.sub(r"-{2,}", "-", slug) + return slug or "capture" + + +def state_fingerprint(state: dict[str, Any]) -> str: + """Hashable representation of board-visible state. + + Intentionally ignores mtime-ish noise such as last_activity while keeping + the pieces that should trigger a capture: stage transitions, generating + flags, scene visual changes, costs, renders, and event count/tail. + """ + scenes = [] + storyboard = state.get("storyboard") or {} + for card in storyboard.get("scenes") or []: + visual = card.get("visual") or {} + scenes.append({ + "id": card.get("id"), + "generating": bool(card.get("generating")), + "generating_tool": card.get("generating_tool"), + "visual": { + "path": visual.get("path"), + "exists": visual.get("exists"), + "type": visual.get("type"), + }, + "takes": [take.get("path") for take in (card.get("takes") or [])], + "audio": [asset.get("path") for asset in (card.get("audio") or [])], + }) + + media = state.get("media") or {} + events = state.get("events") or [] + visible = { + "stages": [ + { + "name": stage.get("name"), + "status": stage.get("status"), + "gate_skipped": stage.get("gate_skipped"), + "versions": stage.get("versions"), + "partial_progress": stage.get("partial_progress"), + } + for stage in state.get("stages") or [] + ], + "scenes": scenes, + "cost": state.get("cost"), + "renders": [r.get("path") for r in media.get("renders") or []], + "snapshots": [s.get("path") for s in media.get("snapshots") or []], + "event_count": len(events), + "event_tail": events[-3:], + } + return json.dumps(visible, sort_keys=True, default=str, separators=(",", ":")) + + +def active_stage(state: dict[str, Any]) -> tuple[str | None, str | None]: + for stage in state.get("stages") or []: + if stage.get("status") in {"in_progress", "awaiting_human", "failed", "blocked"}: + return stage.get("name"), stage.get("status") + for stage in reversed(state.get("stages") or []): + if stage.get("status") == "completed": + return stage.get("name"), stage.get("status") + return None, None + + +def fetch_json(base_url: str, path: str) -> dict[str, Any] | list[Any]: + with urllib.request.urlopen(f"{base_url.rstrip('/')}{path}", timeout=10) as response: + return json.loads(response.read().decode("utf-8")) + + +def capture_url(url: str, output: Path, *, width: int = 1560, height: int = 1150, wait_ms: int = 1200) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + "npx", + "playwright", + "screenshot", + "--viewport-size", + f"{width},{height}", + "--wait-for-timeout", + str(wait_ms), + url, + str(output), + ], + cwd=REPO_ROOT, + check=True, + timeout=120, + shell=(os.name == "nt"), + ) + + +def capture_project(base_url: str, capture_dir: Path, project_id: str, seq: int, state: dict[str, Any]) -> None: + stage, status = active_stage(state) + stem = f"{seq:03d}-{capture_slug(project_id, stage, status)}" + capture_url(f"{base_url.rstrip('/')}/?static=1", capture_dir / "library" / f"{stem}.png", height=620) + capture_url( + f"{base_url.rstrip('/')}/p/{project_id}?static=1", + capture_dir / project_id / f"{stem}.png", + ) + + +def watch( + projects: list[str], + *, + base_url: str, + capture_dir: Path, + interval_s: float, + once: bool = False, + no_screenshots: bool = False, +) -> int: + fingerprints: dict[str, str] = {} + seq = 0 + capture_dir.mkdir(parents=True, exist_ok=True) + print(f"[watch] base={base_url} captures={capture_dir}") + while True: + changed = False + for project_id in projects: + try: + state = fetch_json(base_url, f"/api/project/{project_id}/state") + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: + print(f"[watch] {project_id}: state fetch failed: {exc}", file=sys.stderr) + continue + fp = state_fingerprint(state) + if fingerprints.get(project_id) == fp: + continue + fingerprints[project_id] = fp + changed = True + seq += 1 + stage, status = active_stage(state) + print(f"[watch] change {project_id}: {stage or 'unknown'} -> {status or 'unknown'}") + if not no_screenshots: + capture_project(base_url, capture_dir, project_id, seq, state) + if once: + return 0 + if not changed: + time.sleep(interval_s) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--projects", nargs="+", required=True, help="Project ids to watch") + parser.add_argument("--base-url", default=DEFAULT_BASE_URL) + parser.add_argument("--interval", type=float, default=20.0, help="Polling interval in seconds") + parser.add_argument("--out-dir", type=Path, default=None) + parser.add_argument("--once", action="store_true", help="Poll once and exit") + parser.add_argument("--no-screenshots", action="store_true", help="Exercise polling without Playwright") + args = parser.parse_args(argv) + + out_dir = args.out_dir + if out_dir is None: + stamp = datetime.now().strftime("dogfood-%Y%m%d-%H%M%S") + out_dir = DEFAULT_CAPTURE_ROOT / stamp + return watch( + args.projects, + base_url=args.base_url, + capture_dir=out_dir, + interval_s=args.interval, + once=args.once, + no_screenshots=args.no_screenshots, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/backlot/test_gate_scenarios.py b/tests/backlot/test_gate_scenarios.py new file mode 100644 index 00000000..d00920c3 --- /dev/null +++ b/tests/backlot/test_gate_scenarios.py @@ -0,0 +1,99 @@ +"""Gate-integrity scenarios for Backlot and checkpoint hardening.""" + +import json +from pathlib import Path + +import pytest + +from backlot import state as state_mod +from backlot.state import load_board_state +from lib.checkpoint import CheckpointValidationError, write_checkpoint + + +def _script_artifact() -> dict: + return { + "version": "1.0", + "title": "Gate Test", + "total_duration_seconds": 5, + "sections": [{"id": "s1", "text": "Hello.", "start_seconds": 0, "end_seconds": 5}], + } + + +def _manifest_artifact() -> dict: + return {"version": "1.0", "assets": [], "total_cost_usd": 0.0} + + +def _write(path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data), encoding="utf-8") + + +def test_completed_gated_stage_without_approval_is_rejected(tmp_path): + with pytest.raises(CheckpointValidationError, match="GATE VIOLATION"): + write_checkpoint( + tmp_path, + "film", + "script", + "completed", + {"script": _script_artifact()}, + pipeline_type="cinematic", + ) + + +def test_typo_pipeline_type_fails_closed(tmp_path): + with pytest.raises(CheckpointValidationError, match="Unknown pipeline_type"): + write_checkpoint( + tmp_path, + "film", + "script", + "completed", + {"script": _script_artifact()}, + pipeline_type="cinemtaic", + human_approved=True, + ) + + +def test_handwritten_completed_checkpoint_surfaces_gate_skip(tmp_path, monkeypatch): + monkeypatch.setattr(state_mod, "PROJECTS_DIR", tmp_path) + project = tmp_path / "film" + _write(project / "checkpoint_script.json", { + "version": "1.0", + "project_id": "film", + "pipeline_type": "cinematic", + "stage": "script", + "status": "completed", + "timestamp": "2026-07-02T00:00:00Z", + "artifacts": {"script": _script_artifact()}, + }) + + state = load_board_state(project) + + script = next(stage for stage in state["stages"] if stage["name"] == "script") + assert script["gate_skipped"] is True + + +def test_awaiting_then_approved_archives_history_without_gate_skip(tmp_path): + write_checkpoint( + tmp_path, + "film", + "assets", + "awaiting_human", + {"asset_manifest": _manifest_artifact()}, + pipeline_type="cinematic", + ) + write_checkpoint( + tmp_path, + "film", + "assets", + "completed", + {"asset_manifest": _manifest_artifact()}, + pipeline_type="cinematic", + human_approved=True, + ) + + state = load_board_state(tmp_path / "film") + + assets = next(stage for stage in state["stages"] if stage["name"] == "assets") + assert assets.get("gate_skipped") in (None, False) + assert assets["versions"] == 2 + assert assets["history_entries"][0]["status"] == "awaiting_human" diff --git a/tests/backlot/test_server.py b/tests/backlot/test_server.py new file mode 100644 index 00000000..7dfed8ed --- /dev/null +++ b/tests/backlot/test_server.py @@ -0,0 +1,205 @@ +"""Server/API tests for Backlot. + +These cover the deterministic eval surface in internal/evals/BACKLOT_EVAL_PLAN.md: +API shape, path safety, media/thumb serving, range requests, and loose +performance budgets. +""" + +from __future__ import annotations + +import io +import json +import time +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from PIL import Image + +from backlot import server as server_mod +from backlot import state as state_mod + + +@pytest.fixture +def projects_root(tmp_path, monkeypatch): + root = tmp_path / "projects" + root.mkdir() + monkeypatch.setattr(state_mod, "PROJECTS_DIR", root) + monkeypatch.setattr(server_mod, "PROJECTS_DIR", root) + monkeypatch.setattr(server_mod, "_summary_cache", {}) + monkeypatch.setattr(server_mod, "_PROJECTS_ROOT_STR", __import__("os").path.normcase(str(root.resolve()))) + monkeypatch.setattr(server_mod, "THUMB_CACHE_DIR", tmp_path / "thumbs") + return root + + +@pytest.fixture +def client(projects_root, monkeypatch): + async def no_watch(): + return None + + monkeypatch.setattr(server_mod, "_watch_projects", no_watch) + with TestClient(server_mod.create_app()) as c: + yield c + + +def _write_json(path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data), encoding="utf-8") + + +def _make_project(root: Path, project_id: str = "film") -> Path: + project = root / project_id + (project / "artifacts").mkdir(parents=True) + (project / "assets" / "images").mkdir(parents=True) + (project / "assets" / "video").mkdir(parents=True) + (project / "renders").mkdir(parents=True) + _write_json( + project / "project.json", + { + "project_id": project_id, + "title": "Film", + "pipeline_type": "cinematic", + "created_at": "2026-07-02T00:00:00Z", + }, + ) + _write_json( + project / "checkpoint_script.json", + { + "version": "1.0", + "project_id": project_id, + "pipeline_type": "cinematic", + "stage": "script", + "status": "awaiting_human", + "timestamp": "2026-07-02T00:01:00Z", + "artifacts": {}, + }, + ) + return project + + +def _write_png(path: Path, color: tuple[int, int, int] = (200, 40, 80)) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + img = Image.new("RGB", (24, 16), color) + buf = io.BytesIO() + img.save(buf, format="PNG") + path.write_bytes(buf.getvalue()) + + +class TestBacklotServerApi: + def test_health(self, client): + response = client.get("/api/health") + assert response.status_code == 200 + assert response.json() == {"ok": True, "app": "backlot"} + + def test_projects_shape_and_state(self, client, projects_root): + _make_project(projects_root, "film") + + projects = client.get("/api/projects") + assert projects.status_code == 200 + body = projects.json() + assert len(body) == 1 + assert body[0]["project_id"] == "film" + assert body[0]["awaiting_human"] is True + assert "stage_states" in body[0] + + state = client.get("/api/project/film/state") + assert state.status_code == 200 + state_body = state.json() + assert state_body["project_id"] == "film" + assert state_body["title"] == "Film" + assert state_body["stages"] + + @pytest.mark.parametrize( + ("url", "status"), + [ + ("/api/project/../state", 404), + ("/api/project/C:/state", 400), + ("/api/project/nope/state", 404), + ], + ) + def test_project_id_rejects_bad_or_unknown_ids(self, client, url, status): + response = client.get(url) + assert response.status_code == status + + def test_media_rejects_path_traversal(self, client, projects_root): + _make_project(projects_root, "film") + response = client.get("/media/film/%2E%2E/project.json") + assert response.status_code == 403 + + def test_media_serves_range_requests(self, client, projects_root): + project = _make_project(projects_root, "film") + media = project / "renders" / "final.mp4" + media.write_bytes(b"0123456789") + + response = client.get("/media/film/renders/final.mp4", headers={"Range": "bytes=2-5"}) + + assert response.status_code == 206 + assert response.content == b"2345" + assert response.headers["content-range"].startswith("bytes 2-5/10") + + def test_thumb_downscales_image_and_passes_through_non_media(self, client, projects_root): + project = _make_project(projects_root, "film") + _write_png(project / "assets" / "images" / "sc1.png") + text = project / "artifacts" / "note.txt" + text.write_text("hello", encoding="utf-8") + + image = client.get("/thumb/film/assets/images/sc1.png?w=320") + assert image.status_code == 200 + assert image.headers["content-type"] == "image/jpeg" + assert image.content.startswith(b"\xff\xd8") + + passthrough = client.get("/thumb/film/artifacts/note.txt") + assert passthrough.status_code == 200 + assert passthrough.content == b"hello" + + +class TestBacklotPerformanceBudgets: + def test_projects_and_state_stay_within_loose_budgets(self, client, projects_root): + for i in range(25): + project = _make_project(projects_root, f"film-{i:02d}") + _write_json( + project / "artifacts" / "scene_plan.json", + {"version": "1.0", "scenes": [{"id": "sc1", "start_seconds": 0, "end_seconds": 1}]}, + ) + + t0 = time.perf_counter() + cold = client.get("/api/projects") + cold_s = time.perf_counter() - t0 + assert cold.status_code == 200 + assert cold_s < 2.0 + + t1 = time.perf_counter() + warm = client.get("/api/projects") + warm_s = time.perf_counter() - t1 + assert warm.status_code == 200 + assert warm_s < 0.150 + + t2 = time.perf_counter() + state = client.get("/api/project/film-00/state") + state_s = time.perf_counter() - t2 + assert state.status_code == 200 + assert state_s < 0.400 + + def test_image_thumb_generation_stays_within_budget(self, client, projects_root): + project = _make_project(projects_root, "film") + _write_png(project / "assets" / "images" / "sc1.png") + + t0 = time.perf_counter() + response = client.get("/thumb/film/assets/images/sc1.png?w=640") + elapsed = time.perf_counter() - t0 + + assert response.status_code == 200 + assert elapsed < 1.5 + + +class TestFindingsFixes: + """Regression tests for dogfood findings F-03 (thumb video fallback).""" + + def test_thumb_never_serves_raw_video_bytes(self, client, projects_root): + p = _make_project(projects_root, "vid") + fake_video = p / "renders" / "final.mp4" + fake_video.parent.mkdir(parents=True, exist_ok=True) + # Not a real video: ffmpeg poster extraction will fail. + fake_video.write_bytes(b"\x00" * 4096) + res = client.get("/thumb/vid/renders/final.mp4") + assert res.status_code == 404 # never the raw video bytes (F-03) diff --git a/tests/backlot/test_state.py b/tests/backlot/test_state.py index a5382cf9..35827b35 100644 --- a/tests/backlot/test_state.py +++ b/tests/backlot/test_state.py @@ -188,3 +188,59 @@ class TestLibrary: summary = summarize_project(p) assert summary["awaiting_human"] is True assert summary["active_stage"] == "script" + + +class TestFindingsFixes: + """Regression tests for dogfood findings F-04/F-05.""" + + def test_artifact_refs_outside_project_are_not_followed(self, projects_root, tmp_path): + # F-04: a checkpoint pointing at JSON outside the project tree + # must not surface that file on the board. + secret = tmp_path / "secret.json" + secret.write_text(json.dumps({"version": "1.0", "leaked": True}), encoding="utf-8") + p = _make_project(projects_root, "sneaky-ref") + _write(p / "checkpoint_script.json", { + "stage": "script", "status": "completed", + "timestamp": "2026-01-01T01:00:00Z", + "artifacts": {"script": str(secret)}, + }) + s = load_board_state(p) + assert "script" not in s["artifacts"] + + def test_inside_project_absolute_refs_still_resolve(self, projects_root): + p = _make_project(projects_root, "abs-ref") + _write(p / "artifacts" / "inline_script.json", SCRIPT) + _write(p / "checkpoint_script.json", { + "stage": "script", "status": "completed", + "timestamp": "2026-01-01T01:00:00Z", + "artifacts": {"script": str((p / "artifacts" / "inline_script.json").resolve())}, + }) + s = load_board_state(p) + assert s["artifacts"]["script"]["title"] == "Test Film" + + def test_stalled_in_progress_stage_flagged(self, projects_root): + # F-05: an in_progress stage with no recent activity reads stalled. + import os + p = _make_project(projects_root, "wedged") + _write(p / "checkpoint_research.json", { + "stage": "research", "status": "in_progress", + "timestamp": "2026-01-01T01:00:00Z", "artifacts": {}, + }) + past = time.time() - 30 * 60 + for f in p.rglob("*"): + if f.is_file(): + os.utime(f, (past, past)) + s = load_board_state(p) + research = next(x for x in s["stages"] if x["name"] == "research") + assert research["stalled"] is True + assert research["stalled_minutes"] >= 29 + + def test_fresh_in_progress_not_stalled(self, projects_root): + p = _make_project(projects_root, "busy") + _write(p / "checkpoint_research.json", { + "stage": "research", "status": "in_progress", + "timestamp": "2026-01-01T01:00:00Z", "artifacts": {}, + }) + s = load_board_state(p) + research = next(x for x in s["stages"] if x["name"] == "research") + assert "stalled" not in research diff --git a/tests/backlot/test_visual_eval.py b/tests/backlot/test_visual_eval.py new file mode 100644 index 00000000..70d376ac --- /dev/null +++ b/tests/backlot/test_visual_eval.py @@ -0,0 +1,43 @@ +"""Tests for Backlot visual eval image comparison helpers.""" + +from pathlib import Path + +from PIL import Image + +from scripts.backlot_visual_eval import compare_images + + +def _img(path: Path, color: tuple[int, int, int]) -> None: + Image.new("RGB", (10, 10), color).save(path) + + +def test_compare_images_detects_large_drift(tmp_path): + expected = tmp_path / "expected.png" + actual = tmp_path / "actual.png" + diff = tmp_path / "diff.png" + _img(expected, (0, 0, 0)) + _img(actual, (255, 255, 255)) + + result = compare_images(expected, actual, diff, threshold=0.015) + + assert result["passed"] is False + assert result["changed_ratio"] == 1.0 + assert diff.exists() + + +def test_compare_images_can_mask_regions(tmp_path): + expected = tmp_path / "expected.png" + actual = tmp_path / "actual.png" + diff = tmp_path / "diff.png" + _img(expected, (0, 0, 0)) + _img(actual, (0, 0, 0)) + img = Image.open(actual) + for x in range(5): + for y in range(5): + img.putpixel((x, y), (255, 255, 255)) + img.save(actual) + + result = compare_images(expected, actual, diff, threshold=0.015, masks=[(0, 0, 5, 5)]) + + assert result["passed"] is True + assert result["changed_ratio"] == 0.0 diff --git a/tests/backlot/test_watch_captures.py b/tests/backlot/test_watch_captures.py new file mode 100644 index 00000000..bcc2487a --- /dev/null +++ b/tests/backlot/test_watch_captures.py @@ -0,0 +1,47 @@ +"""Tests for the Backlot dogfood screenshot watcher helpers.""" + +from scripts.backlot_watch_captures import capture_slug, state_fingerprint + + +def test_capture_slug_keeps_names_filesystem_safe(): + assert capture_slug("why-cities-glow", "scene_plan", "awaiting_human") == ( + "why-cities-glow-scene_plan-awaiting_human" + ) + assert capture_slug("../bad id", "C:\\stage", "in progress!") == "bad-id-C-stage-in-progress" + + +def test_state_fingerprint_changes_on_board_relevant_state_only(): + state = { + "stages": [ + {"name": "script", "status": "completed", "partial_progress": None}, + {"name": "assets", "status": "in_progress", "partial_progress": {"done": ["sc1"]}}, + ], + "storyboard": { + "scenes": [ + { + "id": "sc1", + "generating": False, + "visual": {"path": "assets/images/sc1.png", "exists": True}, + "takes": [{"path": "assets/images/sc1.png"}], + }, + {"id": "sc2", "generating": True, "generating_tool": "flux_image", "visual": None}, + ] + }, + "cost": {"total_spent_usd": 0.1}, + "media": {"renders": []}, + "events": [{"event": "start", "tool": "flux_image"}], + "last_activity": 123, + } + same = dict(state) + same["last_activity"] = 999 + + changed = dict(state) + changed["storyboard"] = { + "scenes": [ + state["storyboard"]["scenes"][0], + {"id": "sc2", "generating": False, "visual": {"path": "assets/images/sc2.png", "exists": True}}, + ] + } + + assert state_fingerprint(state) == state_fingerprint(same) + assert state_fingerprint(state) != state_fingerprint(changed)