From 39740d66616b9e1af0522196dbdc56db1c7a8eb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Mon, 10 Aug 2026 07:51:07 +0200 Subject: [PATCH] Add run cost & time reporting to the test scorecard. --- SCORING.md | 18 ++++-- interpret-stats.md | 8 +++ scripts/scoring/extract-run-signals.sh | 8 ++- scripts/scoring/judge-runs.sh | 25 +++++++- scripts/scoring/judge.py | 47 +++++++++------ scripts/scoring/summarize-eval.py | 81 +++++++++++++++++++++++++- 6 files changed, 160 insertions(+), 27 deletions(-) diff --git a/SCORING.md b/SCORING.md index c09a198..1f1514d 100644 --- a/SCORING.md +++ b/SCORING.md @@ -334,6 +334,9 @@ scripts/scoring/run-eval-matrix.sh --models sonnet,haiku --reps 2 scripts/scoring/extract-run-signals.sh --out-dir evals/weekly/ # 3. Judge: grade every valid run's final answer against its rubric (Opus, blind). +# Resumable: re-running skips runs already in scores.csv, so an interrupted +# judge pass continues without re-grading (or re-paying for) completed runs. +# Pass --fresh to discard scores.csv and grade every run from scratch. scripts/scoring/judge-runs.sh --out-dir evals/weekly/ # 4. Summarize: aggregate to the per-model lift + efficiency scorecard. @@ -374,9 +377,9 @@ Under `evals/weekly//`: - **`manifest.csv`** — one row per run: `prompt, skill_family, skill_leaf, model, condition, rep, run_id, exit_code, skills_sha, timestamp, skill_available, skill_activation, reached_leaf, fetched_site, fetched_count, model_snapshot, - cli_version, total_cost_usd, num_turns, result_subtype, budget_hit, signals_ok`. - The runner writes the first ten (base) columns; `extract-run-signals.sh` derives - the rest from each transcript. `budget_hit` is `1` when the run hit the per-run + cli_version, total_cost_usd, num_turns, result_subtype, budget_hit, signals_ok, + duration_ms`. The runner writes the first ten (base) columns; + `extract-run-signals.sh` derives the rest from each transcript. `budget_hit` is `1` when the run hit the per-run spend cap (`result_subtype = error_max_budget_usd`): such a run is truncated, so it is excluded from scoring and the cost mean and reported as budget-capped. `signals_ok` is `0` when the transcript did not parse into the expected shape (a @@ -387,7 +390,9 @@ Under `evals/weekly//`: target SKILL.md (so `reached_leaf` records whether progressive disclosure actually fired); `fetched_site`/`fetched_count` replace a raw URL list and count reaches to `skills.qdrant.tech`, net of denied attempts; `total_cost_usd` and - `num_turns` are the per-run efficiency signals, straight from the result event. + `num_turns` are the per-run efficiency signals, straight from the result event; + `duration_ms` is the run's wall-clock duration (also from the result event), + aggregated into the scorecard's generation-time stats. - **`scores.csv`** — one row per graded rubric item: `prompt, skill, model, condition, rep, item_type, item_text, verdict, credit, contribution, contested, evidence_quote`. This is the raw grade ledger; everything else is @@ -434,6 +439,11 @@ Under `evals/weekly//`: - **coverage** — `graded X of Y` runs, and every dropped run listed with its reason (invalid install / errored / no gradeable answer). A partial run must not read as a clean one (refer to the No silent caps guardrail). + - **cost & time** — actual spend (not the cost *mean*): generation `$` (per + model) + **Opus judge `$`** (summed from each run's `judge_cost.txt`) + grand + total; and generation timing — compute-time (Σ per-run `duration_ms`, with + mean/median/max) and wall-clock, whose ratio is the **parallel speedup** from + `--jobs`. Answers "what did this run cost and how long did it take". - contested-item and harness-failure counts. - **provenance** — exact resolved model snapshot string per model label, CLI version(s), skills commit(s), and the UTC run window (the only correlate for a diff --git a/interpret-stats.md b/interpret-stats.md index a8d9a09..c4eae9b 100644 --- a/interpret-stats.md +++ b/interpret-stats.md @@ -78,6 +78,14 @@ Traces a regression to a specific skill. - `runs graded`: how many of the attempted runs were actually scored (`X of Y`). - `dropped`: runs excluded, each with a reason (`budget-capped (truncated)` / `invalid: skill unavailable` / `errored` / `no gradeable answer`). Stops a partial week from reading as a clean one. +### Cost & time + +- `generation`: actual dollars spent on the generation runs (all of them, incl. truncated ones that still cost), broken down per model — not the cost *mean*. +- `judge (Opus)`: dollars spent grading, summed from each run's `judge_cost.txt`. +- `total`: generation + judge — what this run cost end to end. +- `compute-time (Σ per-run)`: sum of per-run wall durations, with mean/median/max. +- `wall-clock`: elapsed time of the generation phase; `compute-time ÷ wall-clock` is the **parallel speedup** delivered by `--jobs`. + ### Run health - `contested items`: items where two judge samples disagreed; the rubric-ambiguity backlog. diff --git a/scripts/scoring/extract-run-signals.sh b/scripts/scoring/extract-run-signals.sh index 6937ef2..d879d6b 100755 --- a/scripts/scoring/extract-run-signals.sh +++ b/scripts/scoring/extract-run-signals.sh @@ -57,7 +57,7 @@ done [[ -z "$MANIFEST" ]] && MANIFEST="$OUT_DIR/manifest.csv" [[ -f "$MANIFEST" ]] || { echo "Manifest not found: $MANIFEST" >&2; exit 66; } -OUT_HEADER="prompt,skill_family,skill_leaf,model,condition,rep,run_id,exit_code,skills_sha,timestamp,skill_available,skill_activation,reached_leaf,fetched_site,fetched_count,model_snapshot,cli_version,total_cost_usd,num_turns,result_subtype,budget_hit,signals_ok" +OUT_HEADER="prompt,skill_family,skill_leaf,model,condition,rep,run_id,exit_code,skills_sha,timestamp,skill_available,skill_activation,reached_leaf,fetched_site,fetched_count,model_snapshot,cli_version,total_cost_usd,num_turns,result_subtype,budget_hit,signals_ok,duration_ms" tmp="$(mktemp)" echo "$OUT_HEADER" > "$tmp" @@ -81,6 +81,7 @@ tail -n +2 "$MANIFEST" | while IFS= read -r line; do result_subtype="" budget_hit=0 signals_ok=1 + duration_ms="" if [[ -f "$stdout" ]]; then init="$(grep -m1 '"subtype":"init"' "$stdout" || true)" @@ -98,6 +99,7 @@ tail -n +2 "$MANIFEST" | while IFS= read -r line; do result_ev="$(grep '"type":"result"' "$stdout" 2>/dev/null | tail -1)" total_cost_usd="$(printf '%s' "$result_ev" | jq -r '.total_cost_usd // ""' 2>/dev/null || echo "")" num_turns="$(printf '%s' "$result_ev" | jq -r '.num_turns // ""' 2>/dev/null || echo "")" + duration_ms="$(printf '%s' "$result_ev" | jq -r '.duration_ms // ""' 2>/dev/null || echo "")" result_subtype="$(printf '%s' "$result_ev" | jq -r '.subtype // ""' 2>/dev/null || echo "")" # A run that hit the per-run spend cap: definitive markers from the result event. terminal_reason="$(printf '%s' "$result_ev" | jq -r '.terminal_reason // ""' 2>/dev/null || echo "")" @@ -154,10 +156,10 @@ tail -n +2 "$MANIFEST" | while IFS= read -r line; do signals_ok=0 fi - printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \ + printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \ "$base" "$skill_available" "$activation" "$reached_leaf" \ "$fetched_site" "$fetched_count" "$model_snapshot" "$cli_version" \ - "$total_cost_usd" "$num_turns" "$result_subtype" "$budget_hit" "$signals_ok" >> "$tmp" + "$total_cost_usd" "$num_turns" "$result_subtype" "$budget_hit" "$signals_ok" "$duration_ms" >> "$tmp" done mv "$tmp" "$MANIFEST" diff --git a/scripts/scoring/judge-runs.sh b/scripts/scoring/judge-runs.sh index a73a738..7b5267f 100755 --- a/scripts/scoring/judge-runs.sh +++ b/scripts/scoring/judge-runs.sh @@ -14,17 +14,23 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" OUT_DIR="" JUDGE_MODEL="opus" SCORES="" +FRESH="0" usage() { cat <<'USAGE' -Usage: scripts/scoring/judge-runs.sh --out-dir DIR [--judge-model M] [--scores FILE] +Usage: scripts/scoring/judge-runs.sh --out-dir DIR [--judge-model M] [--scores FILE] [--fresh] Grades every valid run under DIR into DIR/scores.csv (blind Opus judge). +By default this RESUMES: an existing scores.csv is kept and any run already graded +in it is skipped, so an interrupted judging pass can be re-run without re-grading +(and re-paying for) completed runs. Use --fresh to grade from scratch. + Options: --out-dir DIR Weekly dir with manifest.csv and run subdirs. --judge-model M Grader model. Default: opus --scores FILE Output ledger. Default: /scores.csv + --fresh Delete any existing scores.csv and grade every run anew. -h, --help Show this help. USAGE } @@ -34,6 +40,7 @@ while [[ $# -gt 0 ]]; do --out-dir) OUT_DIR="${2:?}"; shift 2 ;; --judge-model) JUDGE_MODEL="${2:?}"; shift 2 ;; --scores) SCORES="${2:?}"; shift 2 ;; + --fresh) FRESH="1"; shift ;; -h|--help) usage; exit 0 ;; *) echo "Unknown option: $1" >&2; usage >&2; exit 64 ;; esac @@ -56,7 +63,17 @@ C_PROMPT=$(col prompt); C_FAMILY=$(col skill_family); C_MODEL=$(col model) C_COND=$(col condition); C_REP=$(col rep); C_RUNID=$(col run_id) C_EXIT=$(col exit_code); C_AVAIL=$(col skill_available); C_BUDGET=$(col budget_hit) -rm -f "$SCORES" +# Fresh start deletes the ledger; otherwise resume (keep it, skip graded runs). +[[ "$FRESH" == "1" ]] && rm -f "$SCORES" + +# A run counts as already graded if a scores.csv row matches its +# (prompt, model, condition, rep) exactly. Exact field compare (quoting-safe). +already_graded() { + [[ -f "$SCORES" ]] || return 1 + awk -F, -v p="$1" -v m="$2" -v c="$3" -v r="$4" \ + 'NR>1 && $1==p && $3==m && $4==c && $5==r {found=1; exit} END{exit !found}' "$SCORES" +} + graded=0; skip_invalid=0; skip_error=0; skip_noanswer=0 tail -n +2 "$MANIFEST" | while IFS= read -r line; do @@ -67,6 +84,10 @@ tail -n +2 "$MANIFEST" | while IFS= read -r line; do exit_code=$(get "$C_EXIT"); avail=$(get "$C_AVAIL") run_dir="$OUT_DIR/$run_id" + if already_graded "$prompt" "$model" "$cond" "$rep"; then + echo " skip (already graded) $model/$cond/rep$rep $prompt" >&2; continue + fi + budget=""; [[ -n "${C_BUDGET:-}" ]] && budget=$(get "$C_BUDGET") if [[ "$budget" == "1" ]]; then echo " skip (budget-capped: truncated) $run_id" >&2; skip_error=$((skip_error+1)); continue diff --git a/scripts/scoring/judge.py b/scripts/scoring/judge.py index 3eeadbf..cb5be69 100755 --- a/scripts/scoring/judge.py +++ b/scripts/scoring/judge.py @@ -235,24 +235,22 @@ JudgeFn = Callable[[str], str] def load_env_key(repo_root: Path) -> None: - """Populate ANTHROPIC_API_KEY from a .env if not already in the environment. - Checks the repo root and the embedded harness's .env (skill-test/.env). CI - passes the key as an env var, so this is only a local-run convenience.""" + """Populate ANTHROPIC_API_KEY from .env if not already in the environment.""" if os.environ.get("ANTHROPIC_API_KEY"): return - for env in (repo_root / ".env", repo_root / "skill-test" / ".env"): - if not env.exists(): - continue - for line in env.read_text().splitlines(): - line = line.strip() - if line.startswith("ANTHROPIC_API_KEY="): - val = line.split("=", 1)[1].strip().strip("'\"") - if val: - os.environ["ANTHROPIC_API_KEY"] = val - return + env = repo_root / ".env" + if not env.exists(): + return + for line in env.read_text().splitlines(): + line = line.strip() + if line.startswith("ANTHROPIC_API_KEY="): + val = line.split("=", 1)[1].strip().strip("'\"") + if val: + os.environ["ANTHROPIC_API_KEY"] = val + return -def claude_cli_backend(model: str, repo_root: Path) -> JudgeFn: +def claude_cli_backend(model: str, repo_root: Path, cost_sink: list | None = None) -> JudgeFn: load_env_key(repo_root) def run(prompt: str) -> str: @@ -271,12 +269,21 @@ def claude_cli_backend(model: str, repo_root: Path) -> JudgeFn: ) if proc.returncode != 0: raise RuntimeError(f"claude judge failed: {proc.stderr[:300]}") - # The CLI's json envelope carries the assistant text in `.result`. + # The CLI's json envelope carries the assistant text in `.result` and the + # grading call's own dollar cost in `.total_cost_usd` — record the latter + # so the scorecard can report judge spend (otherwise silently discarded). try: env = json.loads(proc.stdout) - return env.get("result", "") if isinstance(env, dict) else proc.stdout except json.JSONDecodeError: return proc.stdout + if not isinstance(env, dict): + return proc.stdout + if cost_sink is not None: + try: + cost_sink.append(float(env.get("total_cost_usd") or 0.0)) + except (TypeError, ValueError): + pass + return env.get("result", "") return run @@ -429,10 +436,11 @@ def main() -> int: return 2 repo_root = Path(__file__).resolve().parents[2] + cost_sink: list = [] if args.canned: backend = canned_backend(args.canned) else: - backend = claude_cli_backend(args.judge_model, repo_root) + backend = claude_cli_backend(args.judge_model, repo_root, cost_sink) meta = infer_meta(run_dir, args) @@ -442,6 +450,11 @@ def main() -> int: rows = grade_run(run_dir, backend, meta) + # Record this run's judge (Opus) spend next to its transcript so the + # summarizer can total judge cost across the week. + if cost_sink: + (run_dir / "judge_cost.txt").write_text(f"{sum(cost_sink):.6f}\n") + invalid = [r for r in rows if not r["valid"]] if invalid: print(f"warning: {len(invalid)} item(s) failed to parse a valid verdict", file=sys.stderr) diff --git a/scripts/scoring/summarize-eval.py b/scripts/scoring/summarize-eval.py index 417c372..a1d445c 100755 --- a/scripts/scoring/summarize-eval.py +++ b/scripts/scoring/summarize-eval.py @@ -25,8 +25,9 @@ import argparse import csv import math from collections import Counter, defaultdict +from datetime import datetime, timezone from pathlib import Path -from statistics import mean, pstdev, stdev +from statistics import mean, median, pstdev, stdev MODELS_ORDER = ["sonnet", "haiku"] CONDITIONS = ["no-skill", "with-skill"] @@ -251,6 +252,81 @@ def provenance(manifest): # --- new sections ---------------------------------------------------------- +def _stamp_epoch(ts): + """Parse a run's YYYYmmddTHHMMSSZ start timestamp to epoch seconds.""" + try: + return datetime.strptime(ts, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc).timestamp() + except (ValueError, TypeError): + return None + + +def cost_time_section(weekdir, manifest): + """Total spend (generation + Opus judge) and generation timing — the run's + money/time summary. Spend is actual dollars spent (all runs, incl. truncated + ones that still cost), not the cost *mean*. Judge spend is read from each + run's judge_cost.txt; generation timing from manifest duration_ms + start.""" + L = ["## Cost & time\n"] + + gen_by_model = defaultdict(float) + gen_total = 0.0 + for r in manifest: + c = r.get("total_cost_usd", "") + if c in ("", None): + continue + try: + v = float(c) + except ValueError: + continue + gen_total += v + gen_by_model[r.get("model", "?")] += v + + judge_total = 0.0 + for r in manifest: + f = weekdir / (r.get("run_id") or "") / "judge_cost.txt" + if f.exists(): + try: + judge_total += float(f.read_text().strip()) + except ValueError: + pass + + grand = gen_total + judge_total + by_model_str = ", ".join(f"{m} ${gen_by_model[m]:.2f}" for m in sorted(gen_by_model)) or "n/a" + L.append("**Spend (actual $ spent, all runs):**") + L.append(f"- generation: ${gen_total:.2f} ({by_model_str})") + L.append(f"- judge (Opus): ${judge_total:.2f}") + L.append(f"- **total: ${grand:.2f}**") + L.append("") + + durs, starts, finishes = [], [], [] + for r in manifest: + d = r.get("duration_ms", "") + if d in ("", None): + continue + try: + ds = float(d) / 1000.0 + except ValueError: + continue + durs.append(ds) + e = _stamp_epoch(r.get("timestamp", "")) + if e is not None: + starts.append(e) + finishes.append(e + ds) + L.append("**Time (generation phase):**") + if durs: + compute = sum(durs) + L.append(f"- runs timed: {len(durs)}") + L.append(f"- compute-time (Σ per-run): {compute/60:.1f} min " + f"(mean {mean(durs):.0f}s, median {median(durs):.0f}s, max {max(durs):.0f}s)") + if starts and finishes: + wall = max(finishes) - min(starts) + speed = f" (parallel speedup ~{compute/wall:.1f}×)" if wall > 0 else "" + L.append(f"- wall-clock: {wall/60:.1f} min{speed}") + else: + L.append("- (no per-run durations recorded)") + L.append("") + return L + + def models_present(cell_q, cell_cost): seen = {m for (m, _c) in cell_q} | {m for (m, _c) in cell_cost} return [m for m in MODELS_ORDER if m in seen] + sorted(m for m in seen if m not in MODELS_ORDER) @@ -518,6 +594,9 @@ def build_scorecard(weekdir, prompt_q, cell_q, cell_cost, contested, ungraded, L += avoid_section(scores) L += coverage_section(manifest, scores) + # cost & time summary + L += cost_time_section(weekdir, manifest) + # run health budget_capped = sum(1 for r in manifest if r.get("budget_hit") == "1") signals_bad = sum(1 for r in manifest if r.get("signals_ok") == "0")