Add run cost & time reporting to the test scorecard.

SCORING.md

@@ -334,6 +334,9 @@ scripts/scoring/run-eval-matrix.sh --models sonnet,haiku --reps 2
 scripts/scoring/extract-run-signals.sh --out-dir evals/weekly/<date>
 
 # 3. Judge: grade every valid run's final answer against its rubric (Opus, blind).
+#    Resumable: re-running skips runs already in scores.csv, so an interrupted
+#    judge pass continues without re-grading (or re-paying for) completed runs.
+#    Pass --fresh to discard scores.csv and grade every run from scratch.
 scripts/scoring/judge-runs.sh --out-dir evals/weekly/<date>
 
 # 4. Summarize: aggregate to the per-model lift + efficiency scorecard.
@@ -374,9 +377,9 @@ Under `evals/weekly/<date>/`:
 - **`manifest.csv`** — one row per run: `prompt, skill_family, skill_leaf, model,
   condition, rep, run_id, exit_code, skills_sha, timestamp, skill_available,
   skill_activation, reached_leaf, fetched_site, fetched_count, model_snapshot,
-  cli_version, total_cost_usd, num_turns, result_subtype, budget_hit, signals_ok`.
-  The runner writes the first ten (base) columns; `extract-run-signals.sh` derives
-  the rest from each transcript. `budget_hit` is `1` when the run hit the per-run
+  cli_version, total_cost_usd, num_turns, result_subtype, budget_hit, signals_ok,
+  duration_ms`. The runner writes the first ten (base) columns;
+  `extract-run-signals.sh` derives the rest from each transcript. `budget_hit` is `1` when the run hit the per-run
   spend cap (`result_subtype = error_max_budget_usd`): such a run is truncated, so
   it is excluded from scoring and the cost mean and reported as budget-capped.
   `signals_ok` is `0` when the transcript did not parse into the expected shape (a
@@ -387,7 +390,9 @@ Under `evals/weekly/<date>/`:
   target SKILL.md (so `reached_leaf` records whether progressive disclosure
   actually fired); `fetched_site`/`fetched_count` replace a raw URL list and count
   reaches to `skills.qdrant.tech`, net of denied attempts; `total_cost_usd` and
-  `num_turns` are the per-run efficiency signals, straight from the result event.
+  `num_turns` are the per-run efficiency signals, straight from the result event;
+  `duration_ms` is the run's wall-clock duration (also from the result event),
+  aggregated into the scorecard's generation-time stats.
 - **`scores.csv`** — one row per graded rubric item: `prompt, skill, model,
   condition, rep, item_type, item_text, verdict, credit, contribution,
   contested, evidence_quote`. This is the raw grade ledger; everything else is
@@ -434,6 +439,11 @@ Under `evals/weekly/<date>/`:
   - **coverage** — `graded X of Y` runs, and every dropped run listed with its
     reason (invalid install / errored / no gradeable answer). A partial run must
     not read as a clean one (refer to the No silent caps guardrail).
+  - **cost & time** — actual spend (not the cost *mean*): generation `$` (per
+    model) + **Opus judge `$`** (summed from each run's `judge_cost.txt`) + grand
+    total; and generation timing — compute-time (Σ per-run `duration_ms`, with
+    mean/median/max) and wall-clock, whose ratio is the **parallel speedup** from
+    `--jobs`. Answers "what did this run cost and how long did it take".
   - contested-item and harness-failure counts.
   - **provenance** — exact resolved model snapshot string per model label, CLI
     version(s), skills commit(s), and the UTC run window (the only correlate for a

interpret-stats.md

@@ -78,6 +78,14 @@ Traces a regression to a specific skill.
 - `runs graded`: how many of the attempted runs were actually scored (`X of Y`).
 - `dropped`: runs excluded, each with a reason (`budget-capped (truncated)` / `invalid: skill unavailable` / `errored` / `no gradeable answer`). Stops a partial week from reading as a clean one.
 
+### Cost & time
+
+- `generation`: actual dollars spent on the generation runs (all of them, incl. truncated ones that still cost), broken down per model — not the cost *mean*.
+- `judge (Opus)`: dollars spent grading, summed from each run's `judge_cost.txt`.
+- `total`: generation + judge — what this run cost end to end.
+- `compute-time (Σ per-run)`: sum of per-run wall durations, with mean/median/max.
+- `wall-clock`: elapsed time of the generation phase; `compute-time ÷ wall-clock` is the **parallel speedup** delivered by `--jobs`.
+
 ### Run health
 
 - `contested items`: items where two judge samples disagreed; the rubric-ambiguity backlog.

scripts/scoring/extract-run-signals.sh

@@ -57,7 +57,7 @@ done
 [[ -z "$MANIFEST" ]] && MANIFEST="$OUT_DIR/manifest.csv"
 [[ -f "$MANIFEST" ]] || { echo "Manifest not found: $MANIFEST" >&2; exit 66; }
 
-OUT_HEADER="prompt,skill_family,skill_leaf,model,condition,rep,run_id,exit_code,skills_sha,timestamp,skill_available,skill_activation,reached_leaf,fetched_site,fetched_count,model_snapshot,cli_version,total_cost_usd,num_turns,result_subtype,budget_hit,signals_ok"
+OUT_HEADER="prompt,skill_family,skill_leaf,model,condition,rep,run_id,exit_code,skills_sha,timestamp,skill_available,skill_activation,reached_leaf,fetched_site,fetched_count,model_snapshot,cli_version,total_cost_usd,num_turns,result_subtype,budget_hit,signals_ok,duration_ms"
 
 tmp="$(mktemp)"
 echo "$OUT_HEADER" > "$tmp"
@@ -81,6 +81,7 @@ tail -n +2 "$MANIFEST" | while IFS= read -r line; do
   result_subtype=""
   budget_hit=0
   signals_ok=1
+  duration_ms=""
 
   if [[ -f "$stdout" ]]; then
     init="$(grep -m1 '"subtype":"init"' "$stdout" || true)"
@@ -98,6 +99,7 @@ tail -n +2 "$MANIFEST" | while IFS= read -r line; do
     result_ev="$(grep '"type":"result"' "$stdout" 2>/dev/null | tail -1)"
     total_cost_usd="$(printf '%s' "$result_ev" | jq -r '.total_cost_usd // ""' 2>/dev/null || echo "")"
     num_turns="$(printf '%s' "$result_ev" | jq -r '.num_turns // ""' 2>/dev/null || echo "")"
+    duration_ms="$(printf '%s' "$result_ev" | jq -r '.duration_ms // ""' 2>/dev/null || echo "")"
     result_subtype="$(printf '%s' "$result_ev" | jq -r '.subtype // ""' 2>/dev/null || echo "")"
     # A run that hit the per-run spend cap: definitive markers from the result event.
     terminal_reason="$(printf '%s' "$result_ev" | jq -r '.terminal_reason // ""' 2>/dev/null || echo "")"
@@ -154,10 +156,10 @@ tail -n +2 "$MANIFEST" | while IFS= read -r line; do
     signals_ok=0
   fi
 
-  printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \
+  printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \
     "$base" "$skill_available" "$activation" "$reached_leaf" \
     "$fetched_site" "$fetched_count" "$model_snapshot" "$cli_version" \
-    "$total_cost_usd" "$num_turns" "$result_subtype" "$budget_hit" "$signals_ok" >> "$tmp"
+    "$total_cost_usd" "$num_turns" "$result_subtype" "$budget_hit" "$signals_ok" "$duration_ms" >> "$tmp"
 done
 
 mv "$tmp" "$MANIFEST"

scripts/scoring/judge-runs.sh

@@ -14,17 +14,23 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 OUT_DIR=""
 JUDGE_MODEL="opus"
 SCORES=""
+FRESH="0"
 
 usage() {
   cat <<'USAGE'
-Usage: scripts/scoring/judge-runs.sh --out-dir DIR [--judge-model M] [--scores FILE]
+Usage: scripts/scoring/judge-runs.sh --out-dir DIR [--judge-model M] [--scores FILE] [--fresh]
 
 Grades every valid run under DIR into DIR/scores.csv (blind Opus judge).
 
+By default this RESUMES: an existing scores.csv is kept and any run already graded
+in it is skipped, so an interrupted judging pass can be re-run without re-grading
+(and re-paying for) completed runs. Use --fresh to grade from scratch.
+
 Options:
   --out-dir DIR       Weekly dir with manifest.csv and run subdirs.
   --judge-model M     Grader model. Default: opus
   --scores FILE       Output ledger. Default: <out-dir>/scores.csv
+  --fresh             Delete any existing scores.csv and grade every run anew.
   -h, --help          Show this help.
 USAGE
 }
@@ -34,6 +40,7 @@ while [[ $# -gt 0 ]]; do
     --out-dir) OUT_DIR="${2:?}"; shift 2 ;;
     --judge-model) JUDGE_MODEL="${2:?}"; shift 2 ;;
     --scores) SCORES="${2:?}"; shift 2 ;;
+    --fresh) FRESH="1"; shift ;;
     -h|--help) usage; exit 0 ;;
     *) echo "Unknown option: $1" >&2; usage >&2; exit 64 ;;
   esac
@@ -56,7 +63,17 @@ C_PROMPT=$(col prompt); C_FAMILY=$(col skill_family); C_MODEL=$(col model)
 C_COND=$(col condition); C_REP=$(col rep); C_RUNID=$(col run_id)
 C_EXIT=$(col exit_code); C_AVAIL=$(col skill_available); C_BUDGET=$(col budget_hit)
 
-rm -f "$SCORES"
+# Fresh start deletes the ledger; otherwise resume (keep it, skip graded runs).
+[[ "$FRESH" == "1" ]] && rm -f "$SCORES"
+
+# A run counts as already graded if a scores.csv row matches its
+# (prompt, model, condition, rep) exactly. Exact field compare (quoting-safe).
+already_graded() {
+  [[ -f "$SCORES" ]] || return 1
+  awk -F, -v p="$1" -v m="$2" -v c="$3" -v r="$4" \
+    'NR>1 && $1==p && $3==m && $4==c && $5==r {found=1; exit} END{exit !found}' "$SCORES"
+}
+
 graded=0; skip_invalid=0; skip_error=0; skip_noanswer=0
 
 tail -n +2 "$MANIFEST" | while IFS= read -r line; do
@@ -67,6 +84,10 @@ tail -n +2 "$MANIFEST" | while IFS= read -r line; do
   exit_code=$(get "$C_EXIT"); avail=$(get "$C_AVAIL")
   run_dir="$OUT_DIR/$run_id"
 
+  if already_graded "$prompt" "$model" "$cond" "$rep"; then
+    echo "  skip (already graded) $model/$cond/rep$rep $prompt" >&2; continue
+  fi
+
   budget=""; [[ -n "${C_BUDGET:-}" ]] && budget=$(get "$C_BUDGET")
   if [[ "$budget" == "1" ]]; then
     echo "  skip (budget-capped: truncated) $run_id" >&2; skip_error=$((skip_error+1)); continue

scripts/scoring/judge.py

@@ -235,24 +235,22 @@ JudgeFn = Callable[[str], str]
 
 
 def load_env_key(repo_root: Path) -> None:
-    """Populate ANTHROPIC_API_KEY from a .env if not already in the environment.
-    Checks the repo root and the embedded harness's .env (skill-test/.env). CI
-    passes the key as an env var, so this is only a local-run convenience."""
+    """Populate ANTHROPIC_API_KEY from .env if not already in the environment."""
     if os.environ.get("ANTHROPIC_API_KEY"):
         return
-    for env in (repo_root / ".env", repo_root / "skill-test" / ".env"):
-        if not env.exists():
-            continue
-        for line in env.read_text().splitlines():
-            line = line.strip()
-            if line.startswith("ANTHROPIC_API_KEY="):
-                val = line.split("=", 1)[1].strip().strip("'\"")
-                if val:
-                    os.environ["ANTHROPIC_API_KEY"] = val
-                return
+    env = repo_root / ".env"
+    if not env.exists():
+        return
+    for line in env.read_text().splitlines():
+        line = line.strip()
+        if line.startswith("ANTHROPIC_API_KEY="):
+            val = line.split("=", 1)[1].strip().strip("'\"")
+            if val:
+                os.environ["ANTHROPIC_API_KEY"] = val
+            return
 
 
-def claude_cli_backend(model: str, repo_root: Path) -> JudgeFn:
+def claude_cli_backend(model: str, repo_root: Path, cost_sink: list | None = None) -> JudgeFn:
     load_env_key(repo_root)
 
     def run(prompt: str) -> str:
@@ -271,12 +269,21 @@ def claude_cli_backend(model: str, repo_root: Path) -> JudgeFn:
         )
         if proc.returncode != 0:
             raise RuntimeError(f"claude judge failed: {proc.stderr[:300]}")
-        # The CLI's json envelope carries the assistant text in `.result`.
+        # The CLI's json envelope carries the assistant text in `.result` and the
+        # grading call's own dollar cost in `.total_cost_usd` — record the latter
+        # so the scorecard can report judge spend (otherwise silently discarded).
         try:
             env = json.loads(proc.stdout)
-            return env.get("result", "") if isinstance(env, dict) else proc.stdout
         except json.JSONDecodeError:
             return proc.stdout
+        if not isinstance(env, dict):
+            return proc.stdout
+        if cost_sink is not None:
+            try:
+                cost_sink.append(float(env.get("total_cost_usd") or 0.0))
+            except (TypeError, ValueError):
+                pass
+        return env.get("result", "")
 
     return run
 
@@ -429,10 +436,11 @@ def main() -> int:
         return 2
 
     repo_root = Path(__file__).resolve().parents[2]
+    cost_sink: list = []
     if args.canned:
         backend = canned_backend(args.canned)
     else:
-        backend = claude_cli_backend(args.judge_model, repo_root)
+        backend = claude_cli_backend(args.judge_model, repo_root, cost_sink)
 
     meta = infer_meta(run_dir, args)
 
@@ -442,6 +450,11 @@ def main() -> int:
 
     rows = grade_run(run_dir, backend, meta)
 
+    # Record this run's judge (Opus) spend next to its transcript so the
+    # summarizer can total judge cost across the week.
+    if cost_sink:
+        (run_dir / "judge_cost.txt").write_text(f"{sum(cost_sink):.6f}\n")
+
     invalid = [r for r in rows if not r["valid"]]
     if invalid:
         print(f"warning: {len(invalid)} item(s) failed to parse a valid verdict", file=sys.stderr)

scripts/scoring/summarize-eval.py

@@ -25,8 +25,9 @@ import argparse
 import csv
 import math
 from collections import Counter, defaultdict
+from datetime import datetime, timezone
 from pathlib import Path
-from statistics import mean, pstdev, stdev
+from statistics import mean, median, pstdev, stdev
 
 MODELS_ORDER = ["sonnet", "haiku"]
 CONDITIONS = ["no-skill", "with-skill"]
@@ -251,6 +252,81 @@ def provenance(manifest):
 # --- new sections ----------------------------------------------------------
 
 
+def _stamp_epoch(ts):
+    """Parse a run's YYYYmmddTHHMMSSZ start timestamp to epoch seconds."""
+    try:
+        return datetime.strptime(ts, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc).timestamp()
+    except (ValueError, TypeError):
+        return None
+
+
+def cost_time_section(weekdir, manifest):
+    """Total spend (generation + Opus judge) and generation timing — the run's
+    money/time summary. Spend is actual dollars spent (all runs, incl. truncated
+    ones that still cost), not the cost *mean*. Judge spend is read from each
+    run's judge_cost.txt; generation timing from manifest duration_ms + start."""
+    L = ["## Cost & time\n"]
+
+    gen_by_model = defaultdict(float)
+    gen_total = 0.0
+    for r in manifest:
+        c = r.get("total_cost_usd", "")
+        if c in ("", None):
+            continue
+        try:
+            v = float(c)
+        except ValueError:
+            continue
+        gen_total += v
+        gen_by_model[r.get("model", "?")] += v
+
+    judge_total = 0.0
+    for r in manifest:
+        f = weekdir / (r.get("run_id") or "") / "judge_cost.txt"
+        if f.exists():
+            try:
+                judge_total += float(f.read_text().strip())
+            except ValueError:
+                pass
+
+    grand = gen_total + judge_total
+    by_model_str = ", ".join(f"{m} ${gen_by_model[m]:.2f}" for m in sorted(gen_by_model)) or "n/a"
+    L.append("**Spend (actual $ spent, all runs):**")
+    L.append(f"- generation: ${gen_total:.2f}  ({by_model_str})")
+    L.append(f"- judge (Opus): ${judge_total:.2f}")
+    L.append(f"- **total: ${grand:.2f}**")
+    L.append("")
+
+    durs, starts, finishes = [], [], []
+    for r in manifest:
+        d = r.get("duration_ms", "")
+        if d in ("", None):
+            continue
+        try:
+            ds = float(d) / 1000.0
+        except ValueError:
+            continue
+        durs.append(ds)
+        e = _stamp_epoch(r.get("timestamp", ""))
+        if e is not None:
+            starts.append(e)
+            finishes.append(e + ds)
+    L.append("**Time (generation phase):**")
+    if durs:
+        compute = sum(durs)
+        L.append(f"- runs timed: {len(durs)}")
+        L.append(f"- compute-time (Σ per-run): {compute/60:.1f} min  "
+                 f"(mean {mean(durs):.0f}s, median {median(durs):.0f}s, max {max(durs):.0f}s)")
+        if starts and finishes:
+            wall = max(finishes) - min(starts)
+            speed = f"  (parallel speedup ~{compute/wall:.1f}×)" if wall > 0 else ""
+            L.append(f"- wall-clock: {wall/60:.1f} min{speed}")
+    else:
+        L.append("- (no per-run durations recorded)")
+    L.append("")
+    return L
+
+
 def models_present(cell_q, cell_cost):
     seen = {m for (m, _c) in cell_q} | {m for (m, _c) in cell_cost}
     return [m for m in MODELS_ORDER if m in seen] + sorted(m for m in seen if m not in MODELS_ORDER)
@@ -518,6 +594,9 @@ def build_scorecard(weekdir, prompt_q, cell_q, cell_cost, contested, ungraded,
     L += avoid_section(scores)
     L += coverage_section(manifest, scores)
 
+    # cost & time summary
+    L += cost_time_section(weekdir, manifest)
+
     # run health
     budget_capped = sum(1 for r in manifest if r.get("budget_hit") == "1")
     signals_bad = sum(1 for r in manifest if r.get("signals_ok") == "0")