mirror of
https://github.com/calesthio/OpenMontage.git
synced 2026-08-05 15:20:40 +08:00
backlot: fix all five dogfood findings (F-01..F-05)
- F-01: cost bar crit (red) state past 90% of budget - F-02: normalize() hardens fetched board state against sparse payloads - F-03: /thumb 404s for videos with no extractable poster frame instead of serving raw video bytes - F-04: checkpoint artifact path refs only resolve inside the project dir - F-05 (board half): stall detection — in_progress stage with no disk activity >10min renders red 'stalled?' + header badge flips to STALLED?; verified against the real wedged why-cities-glow project - eval harness from dogfood session committed (visual regression + interaction smoke, capture watcher, server/gate test suites) + regression tests for each finding; 46 backlot tests green, visual eval green (restage-before-capture note logged)
This commit is contained in:
calesthio
234
scripts/backlot_visual_eval.py
Normal file
234
scripts/backlot_visual_eval.py
Normal file
@@ -0,0 +1,234 @@
|
||||
"""Deterministic visual eval for Backlot.
|
||||
|
||||
Stages the fictional Backlot projects, captures canonical browser screenshots,
|
||||
optionally compares them to goldens, and can run a small Playwright interaction
|
||||
smoke against the staged board.
|
||||
|
||||
Examples:
|
||||
python scripts/backlot_visual_eval.py
|
||||
python scripts/backlot_visual_eval.py --bless
|
||||
python scripts/backlot_visual_eval.py --interactions
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image, ImageChops
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
STAGE_DIR = REPO_ROOT / ".backlot" / "screenshot-stage"
|
||||
GOLDENS_DIR = REPO_ROOT / "internal" / "evals" / "goldens"
|
||||
CAPTURE_ROOT = REPO_ROOT / "internal" / "evals" / "captures"
|
||||
PORT = 4791
|
||||
|
||||
SHOTS = [
|
||||
("library", "/?static=1", 1560, 500, 4200, [
|
||||
(1370, 20, 1510, 62), # live/idle badge
|
||||
(90, 106, 422, 380), # card border/status animation variance
|
||||
(440, 106, 772, 380),
|
||||
(790, 106, 1122, 380),
|
||||
(1140, 106, 1472, 380),
|
||||
]),
|
||||
("board-live", "/p/signal-in-the-static?static=1", 1560, 1150, 4200, []),
|
||||
("script-gate", "/p/the-slow-orchard?static=1", 1560, 760, 3200, []),
|
||||
("storyboard", "/p/the-last-lighthouse?static=1", 1560, 1500, 4200, []),
|
||||
]
|
||||
|
||||
|
||||
def compare_images(
|
||||
expected_path: Path,
|
||||
actual_path: Path,
|
||||
diff_path: Path,
|
||||
*,
|
||||
threshold: float = 0.015,
|
||||
masks: list[tuple[int, int, int, int]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Compare screenshots by changed-pixel ratio and write a red diff image."""
|
||||
expected = Image.open(expected_path).convert("RGB")
|
||||
actual = Image.open(actual_path).convert("RGB")
|
||||
if expected.size != actual.size:
|
||||
diff_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
actual.save(diff_path)
|
||||
return {"passed": False, "changed_ratio": 1.0, "reason": f"size {expected.size} != {actual.size}"}
|
||||
|
||||
masks = masks or []
|
||||
for box in masks:
|
||||
patch = expected.crop(box)
|
||||
actual.paste(patch, box)
|
||||
|
||||
delta = ImageChops.difference(expected, actual)
|
||||
changed = 0
|
||||
pixels = delta.load()
|
||||
width, height = delta.size
|
||||
diff = Image.new("RGB", delta.size, (0, 0, 0))
|
||||
diff_px = diff.load()
|
||||
for y in range(height):
|
||||
for x in range(width):
|
||||
if max(pixels[x, y]) > 8:
|
||||
changed += 1
|
||||
diff_px[x, y] = (255, 40, 40)
|
||||
else:
|
||||
diff_px[x, y] = actual.getpixel((x, y))
|
||||
ratio = changed / float(width * height)
|
||||
diff_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
diff.save(diff_path)
|
||||
return {"passed": ratio <= threshold, "changed_ratio": round(ratio, 6), "threshold": threshold}
|
||||
|
||||
|
||||
def run_stage() -> None:
|
||||
subprocess.run(
|
||||
[sys.executable, "scripts/backlot_screenshot_stage.py", "--stage-only"],
|
||||
cwd=REPO_ROOT,
|
||||
check=True,
|
||||
timeout=180,
|
||||
)
|
||||
|
||||
|
||||
def start_server() -> subprocess.Popen:
|
||||
env = dict(os.environ)
|
||||
env["OPENMONTAGE_PROJECTS_DIR"] = str(STAGE_DIR)
|
||||
server = subprocess.Popen(
|
||||
[sys.executable, "-m", "backlot", "serve", "--port", str(PORT)],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
deadline = time.time() + 20
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{PORT}/api/health", timeout=1):
|
||||
return server
|
||||
except Exception:
|
||||
time.sleep(0.3)
|
||||
server.terminate()
|
||||
raise RuntimeError("Backlot server did not become healthy")
|
||||
|
||||
|
||||
def capture_screenshot(url: str, output: Path, width: int, height: int, wait_ms: int) -> None:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run(
|
||||
[
|
||||
"npx",
|
||||
"playwright",
|
||||
"screenshot",
|
||||
"--viewport-size",
|
||||
f"{width},{height}",
|
||||
"--wait-for-timeout",
|
||||
str(wait_ms),
|
||||
url,
|
||||
str(output),
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
check=True,
|
||||
timeout=120,
|
||||
shell=(os.name == "nt"),
|
||||
)
|
||||
|
||||
|
||||
def capture_shots(capture_dir: Path) -> list[dict[str, Any]]:
|
||||
results = []
|
||||
for name, path, width, height, wait_ms, _masks in SHOTS:
|
||||
out = capture_dir / f"{name}.png"
|
||||
capture_screenshot(f"http://127.0.0.1:{PORT}{path}", out, width, height, wait_ms)
|
||||
results.append({"name": name, "path": out})
|
||||
return results
|
||||
|
||||
|
||||
def compare_or_bless(capture_dir: Path, *, bless: bool, threshold: float) -> list[dict[str, Any]]:
|
||||
GOLDENS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
report = []
|
||||
for name, _path, _width, _height, _wait_ms, masks in SHOTS:
|
||||
actual = capture_dir / f"{name}.png"
|
||||
golden = GOLDENS_DIR / f"{name}.png"
|
||||
if bless or not golden.exists():
|
||||
shutil.copyfile(actual, golden)
|
||||
report.append({"name": name, "status": "blessed", "golden": str(golden)})
|
||||
continue
|
||||
diff = capture_dir / "diffs" / f"{name}.png"
|
||||
result = compare_images(golden, actual, diff, threshold=threshold, masks=masks)
|
||||
result.update({"name": name, "diff": str(diff)})
|
||||
report.append(result)
|
||||
return report
|
||||
|
||||
|
||||
def run_interactions(capture_dir: Path) -> dict[str, Any]:
|
||||
"""Run browser interaction smoke through Python Playwright."""
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
screenshot = capture_dir / "interaction-smoke.png"
|
||||
with sync_playwright() as pw:
|
||||
browser = pw.chromium.launch(headless=True)
|
||||
page = browser.new_page(viewport={"width": 1560, "height": 1000})
|
||||
page.goto(f"http://127.0.0.1:{PORT}/p/the-last-lighthouse?static=1")
|
||||
page.wait_for_selector(".stage")
|
||||
page.locator(".stage").first.click()
|
||||
page.wait_for_selector(".drawer")
|
||||
drawer_text = page.locator(".drawer").inner_text()
|
||||
if "research" not in drawer_text:
|
||||
raise RuntimeError("stage drawer did not open")
|
||||
page.locator(".script-card").first.click()
|
||||
page.wait_for_selector(".modal-bg.open")
|
||||
page.keyboard.press("Escape")
|
||||
page.wait_for_function("() => !document.querySelector('.modal-bg')?.classList.contains('open')")
|
||||
if page.locator(".takes").count() < 1:
|
||||
raise RuntimeError("takes drawer not present on staged takes scene")
|
||||
replay_button = page.locator(".rp-btn", has_text="REPLAY RUN")
|
||||
if replay_button.count():
|
||||
replay_button.first.click()
|
||||
page.wait_for_selector('input[type="range"]')
|
||||
page.locator('input[type="range"]').fill("500")
|
||||
page.screenshot(path=str(screenshot), full_page=True)
|
||||
browser.close()
|
||||
return {"status": "passed", "screenshot": str(capture_dir / "interaction-smoke.png")}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--bless", action="store_true", help="Write current captures as goldens")
|
||||
parser.add_argument("--no-stage", action="store_true", help="Reuse existing .backlot/screenshot-stage")
|
||||
parser.add_argument("--interactions", action="store_true", help="Run Playwright interaction smoke")
|
||||
parser.add_argument("--threshold", type=float, default=0.015)
|
||||
parser.add_argument("--out-dir", type=Path, default=None)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not args.no_stage:
|
||||
run_stage()
|
||||
|
||||
stamp = datetime.now().strftime("visual-%Y%m%d-%H%M%S")
|
||||
capture_dir = args.out_dir or (CAPTURE_ROOT / stamp)
|
||||
capture_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
server = start_server()
|
||||
try:
|
||||
capture_shots(capture_dir)
|
||||
report = compare_or_bless(capture_dir, bless=args.bless, threshold=args.threshold)
|
||||
interaction_report = run_interactions(capture_dir) if args.interactions else None
|
||||
finally:
|
||||
server.terminate()
|
||||
try:
|
||||
server.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
server.kill()
|
||||
|
||||
passed = all(item.get("passed", item.get("status") == "blessed") for item in report)
|
||||
payload = {"capture_dir": str(capture_dir), "shots": report, "interactions": interaction_report}
|
||||
report_path = capture_dir / "report.json"
|
||||
report_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
print(json.dumps(payload, indent=2))
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
195
scripts/backlot_watch_captures.py
Normal file
195
scripts/backlot_watch_captures.py
Normal file
@@ -0,0 +1,195 @@
|
||||
"""Capture Backlot board screenshots whenever watched project state changes.
|
||||
|
||||
This is the Half-B dogfood watcher from internal/evals/BACKLOT_EVAL_PLAN.md.
|
||||
It polls the Backlot API, fingerprints board-relevant state, and captures the
|
||||
library plus the changed project board through Playwright.
|
||||
|
||||
Example:
|
||||
python scripts/backlot_watch_captures.py --projects why-cities-glow rain-on-glass
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_BASE_URL = "http://127.0.0.1:4750"
|
||||
DEFAULT_CAPTURE_ROOT = REPO_ROOT / "internal" / "evals" / "captures"
|
||||
|
||||
|
||||
def capture_slug(project_id: str, stage: str | None, status: str | None) -> str:
|
||||
"""Stable, filesystem-safe screenshot name stem."""
|
||||
raw = "-".join(part for part in (project_id, stage or "unknown", status or "unknown") if part)
|
||||
raw = raw.replace("\\", "-").replace("/", "-").replace("..", "")
|
||||
slug = re.sub(r"[^A-Za-z0-9._-]+", "-", raw).strip(".-")
|
||||
slug = re.sub(r"-{2,}", "-", slug)
|
||||
return slug or "capture"
|
||||
|
||||
|
||||
def state_fingerprint(state: dict[str, Any]) -> str:
|
||||
"""Hashable representation of board-visible state.
|
||||
|
||||
Intentionally ignores mtime-ish noise such as last_activity while keeping
|
||||
the pieces that should trigger a capture: stage transitions, generating
|
||||
flags, scene visual changes, costs, renders, and event count/tail.
|
||||
"""
|
||||
scenes = []
|
||||
storyboard = state.get("storyboard") or {}
|
||||
for card in storyboard.get("scenes") or []:
|
||||
visual = card.get("visual") or {}
|
||||
scenes.append({
|
||||
"id": card.get("id"),
|
||||
"generating": bool(card.get("generating")),
|
||||
"generating_tool": card.get("generating_tool"),
|
||||
"visual": {
|
||||
"path": visual.get("path"),
|
||||
"exists": visual.get("exists"),
|
||||
"type": visual.get("type"),
|
||||
},
|
||||
"takes": [take.get("path") for take in (card.get("takes") or [])],
|
||||
"audio": [asset.get("path") for asset in (card.get("audio") or [])],
|
||||
})
|
||||
|
||||
media = state.get("media") or {}
|
||||
events = state.get("events") or []
|
||||
visible = {
|
||||
"stages": [
|
||||
{
|
||||
"name": stage.get("name"),
|
||||
"status": stage.get("status"),
|
||||
"gate_skipped": stage.get("gate_skipped"),
|
||||
"versions": stage.get("versions"),
|
||||
"partial_progress": stage.get("partial_progress"),
|
||||
}
|
||||
for stage in state.get("stages") or []
|
||||
],
|
||||
"scenes": scenes,
|
||||
"cost": state.get("cost"),
|
||||
"renders": [r.get("path") for r in media.get("renders") or []],
|
||||
"snapshots": [s.get("path") for s in media.get("snapshots") or []],
|
||||
"event_count": len(events),
|
||||
"event_tail": events[-3:],
|
||||
}
|
||||
return json.dumps(visible, sort_keys=True, default=str, separators=(",", ":"))
|
||||
|
||||
|
||||
def active_stage(state: dict[str, Any]) -> tuple[str | None, str | None]:
|
||||
for stage in state.get("stages") or []:
|
||||
if stage.get("status") in {"in_progress", "awaiting_human", "failed", "blocked"}:
|
||||
return stage.get("name"), stage.get("status")
|
||||
for stage in reversed(state.get("stages") or []):
|
||||
if stage.get("status") == "completed":
|
||||
return stage.get("name"), stage.get("status")
|
||||
return None, None
|
||||
|
||||
|
||||
def fetch_json(base_url: str, path: str) -> dict[str, Any] | list[Any]:
|
||||
with urllib.request.urlopen(f"{base_url.rstrip('/')}{path}", timeout=10) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
|
||||
|
||||
def capture_url(url: str, output: Path, *, width: int = 1560, height: int = 1150, wait_ms: int = 1200) -> None:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run(
|
||||
[
|
||||
"npx",
|
||||
"playwright",
|
||||
"screenshot",
|
||||
"--viewport-size",
|
||||
f"{width},{height}",
|
||||
"--wait-for-timeout",
|
||||
str(wait_ms),
|
||||
url,
|
||||
str(output),
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
check=True,
|
||||
timeout=120,
|
||||
shell=(os.name == "nt"),
|
||||
)
|
||||
|
||||
|
||||
def capture_project(base_url: str, capture_dir: Path, project_id: str, seq: int, state: dict[str, Any]) -> None:
|
||||
stage, status = active_stage(state)
|
||||
stem = f"{seq:03d}-{capture_slug(project_id, stage, status)}"
|
||||
capture_url(f"{base_url.rstrip('/')}/?static=1", capture_dir / "library" / f"{stem}.png", height=620)
|
||||
capture_url(
|
||||
f"{base_url.rstrip('/')}/p/{project_id}?static=1",
|
||||
capture_dir / project_id / f"{stem}.png",
|
||||
)
|
||||
|
||||
|
||||
def watch(
|
||||
projects: list[str],
|
||||
*,
|
||||
base_url: str,
|
||||
capture_dir: Path,
|
||||
interval_s: float,
|
||||
once: bool = False,
|
||||
no_screenshots: bool = False,
|
||||
) -> int:
|
||||
fingerprints: dict[str, str] = {}
|
||||
seq = 0
|
||||
capture_dir.mkdir(parents=True, exist_ok=True)
|
||||
print(f"[watch] base={base_url} captures={capture_dir}")
|
||||
while True:
|
||||
changed = False
|
||||
for project_id in projects:
|
||||
try:
|
||||
state = fetch_json(base_url, f"/api/project/{project_id}/state")
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
print(f"[watch] {project_id}: state fetch failed: {exc}", file=sys.stderr)
|
||||
continue
|
||||
fp = state_fingerprint(state)
|
||||
if fingerprints.get(project_id) == fp:
|
||||
continue
|
||||
fingerprints[project_id] = fp
|
||||
changed = True
|
||||
seq += 1
|
||||
stage, status = active_stage(state)
|
||||
print(f"[watch] change {project_id}: {stage or 'unknown'} -> {status or 'unknown'}")
|
||||
if not no_screenshots:
|
||||
capture_project(base_url, capture_dir, project_id, seq, state)
|
||||
if once:
|
||||
return 0
|
||||
if not changed:
|
||||
time.sleep(interval_s)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--projects", nargs="+", required=True, help="Project ids to watch")
|
||||
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
|
||||
parser.add_argument("--interval", type=float, default=20.0, help="Polling interval in seconds")
|
||||
parser.add_argument("--out-dir", type=Path, default=None)
|
||||
parser.add_argument("--once", action="store_true", help="Poll once and exit")
|
||||
parser.add_argument("--no-screenshots", action="store_true", help="Exercise polling without Playwright")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
out_dir = args.out_dir
|
||||
if out_dir is None:
|
||||
stamp = datetime.now().strftime("dogfood-%Y%m%d-%H%M%S")
|
||||
out_dir = DEFAULT_CAPTURE_ROOT / stamp
|
||||
return watch(
|
||||
args.projects,
|
||||
base_url=args.base_url,
|
||||
capture_dir=out_dir,
|
||||
interval_s=args.interval,
|
||||
once=args.once,
|
||||
no_screenshots=args.no_screenshots,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user