From e6a58d5fc3f446c0e2189172b34e84c41f8310dd Mon Sep 17 00:00:00 2001
From: "Raghu :)" <218665445+RogueTex@users.noreply.github.com>
Date: Sun, 6 Sep 2026 23:20:26 -0700
Subject: [PATCH] fix(evals): reject null grader expectations without crashing

---
 scripts/run-evals-test.js | 19 +++++++++++++++++++
 scripts/run-evals.js      |  3 ++-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/scripts/run-evals-test.js b/scripts/run-evals-test.js
index 381a8cf..75fa70f 100644
--- a/scripts/run-evals-test.js
+++ b/scripts/run-evals-test.js
@@ -90,6 +90,25 @@ test('rejects grader results that omit expectations', () => {
   assert.equal(parseGrading(raw, 2), null);
 });
 
+test('rejects null expectation entries without throwing', () => {
+  for (const expectations of [
+    [null],
+    [{ text: 'first expectation', passed: true, evidence: 'observed' }, null],
+  ]) {
+    const raw = JSON.stringify({
+      expectations,
+      summary: {
+        passed: expectations.length - 1,
+        failed: 1,
+        total: expectations.length,
+        pass_rate: (expectations.length - 1) / expectations.length,
+      },
+    });
+
+    assert.equal(parseGrading(raw, expectations.length), null);
+  }
+});
+
 test('rejects incomplete or inconsistent grader summaries', () => {
   const expectation = { text: 'expected behavior', passed: false, evidence: 'not observed' };
   const cases = [
diff --git a/scripts/run-evals.js b/scripts/run-evals.js
index 8013887..0d543b3 100644
--- a/scripts/run-evals.js
+++ b/scripts/run-evals.js
@@ -439,12 +439,13 @@ function parseGrading(raw, expectedCount) {
   const expectations = g.expectations;
   const summary = g.summary;
   const passed = Array.isArray(expectations)
-    ? expectations.filter((expectation) => expectation.passed === true).length
+    ? expectations.filter((expectation) => expectation?.passed === true).length
     : 0;
   const ok =
     Number.isInteger(expectedCount) && expectedCount > 0 &&
     Array.isArray(expectations) && expectations.length === expectedCount &&
     expectations.every((expectation) =>
+      expectation !== null &&
       typeof expectation.text === 'string' &&
       typeof expectation.passed === 'boolean' &&
       typeof expectation.evidence === 'string') &&