fix(qe-court): ship executable referee assets

This commit is contained in:
Dragan Spiridonov
2026-08-30 13:45:16 +00:00
parent fb687f2bac
commit cef23d4d31
11 changed files with 219 additions and 25 deletions
+4 -4
View File
@@ -161,7 +161,7 @@ is Phase 2). To run a court now:
the panel before seating it** — this step is not optional:
```ts
import { validateCourtConfig } from 'src/skills/qe-court/referee';
import { validateCourtConfig } from 'agentic-qe/skills/qe-court/referee';
const violations = validateCourtConfig(config); // [] == valid
if (violations.length) throw new Error(`Cannot convene: ${violations.join(', ')}`);
```
@@ -188,9 +188,9 @@ Durable, attestable evidence — the "jury waiting for everything you ship."
## Trust tier
**Tier 3 (verified).** The court's falsifiable invariants are enforced in code
(`src/skills/qe-court/referee.ts`) and covered by an oracle suite
(`tests/unit/skills/qe-court/referee.test.ts`) that the acceptance eval
**Tier 3 (verified).** The court's falsifiable invariants are enforced by the
published `agentic-qe/skills/qe-court/referee` entrypoint and covered by a
consumer-runnable `aqe-court-referee self-test <oracle>` suite that the acceptance eval
(`evals/qe-court.yaml`, command-eval mode) runs through the `aqe eval` CLI —
15/15 green as of 2026-07-29. That suite now validates the **shipped `config.json`
itself**, so a routing edit that seats a colluding panel fails in CI rather than in
@@ -0,0 +1,38 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://agentic-qe.dev/schemas/qe-court-config.json",
"title": "QE-Court configuration",
"type": "object",
"required": ["prosecutors", "overturnDepth", "routing", "options"],
"properties": {
"prosecutors": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
"overturnDepth": { "type": "integer", "minimum": 0 },
"emitScore": { "type": "boolean" },
"routing": {
"type": "object",
"required": ["jury"],
"additionalProperties": {
"anyOf": [
{ "type": "string" },
{
"type": "object",
"properties": { "provider": { "type": "string", "minLength": 1 }, "model": { "type": "string" } },
"additionalProperties": false
}
]
}
},
"options": {
"type": "object",
"required": ["writerIsNeverJuror", "minDistinctVendors"],
"properties": {
"blindFiling": { "type": "boolean" },
"writerIsNeverJuror": { "const": true },
"minDistinctVendors": { "type": "integer", "minimum": 2 },
"requireDoeGatedScore": { "type": "boolean" },
"signCourtRecord": { "type": "boolean" }
},
"additionalProperties": false
}
}
}
+8 -8
View File
@@ -3,8 +3,8 @@ version: 0.3.0
status: active
description: >
Acceptance eval for the QE-Court adversarial review protocol (ADR-124).
Command-eval suite: each case runs a real oracle from the referee suite
(tests/unit/skills/qe-court/referee.test.ts) through the `aqe eval` CLI and
Command-eval suite: each case runs a real oracle through the published
`aqe-court-referee` executable and
asserts it passes. Each oracle passes on correct referee logic and FAILS on a
seeded regression, so a green run is genuine tier-3 evidence — not a self-report.
The seeded-mutant is mirrored in fixtures/seeded-mutant-delivery/ (canSpend
@@ -16,7 +16,7 @@ test_cases:
category: overturn
priority: critical
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "overturn_catches_mutant"
command: aqe-court-referee self-test overturn-catches-mutant
expected:
exit_code: 0
@@ -25,7 +25,7 @@ test_cases:
category: overturn
priority: critical
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "overturn_disabled"
command: aqe-court-referee self-test overturn-disabled
expected:
exit_code: 0
@@ -34,7 +34,7 @@ test_cases:
category: anti-collusion
priority: critical
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "writer_not_juror_enforced"
command: aqe-court-referee self-test writer-not-juror
expected:
exit_code: 0
@@ -43,7 +43,7 @@ test_cases:
category: anti-collusion
priority: high
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "vendor_diversity_enforced"
command: aqe-court-referee self-test vendor-diversity
expected:
exit_code: 0
@@ -52,7 +52,7 @@ test_cases:
category: doe-gate
priority: high
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "no_score_without_doe_gate"
command: aqe-court-referee self-test doe-score-gate
expected:
exit_code: 0
@@ -61,7 +61,7 @@ test_cases:
category: verdict
priority: high
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "verdict_classes"
command: aqe-court-referee self-test verdict-classes
expected:
exit_code: 0
+4 -4
View File
@@ -161,7 +161,7 @@ is Phase 2). To run a court now:
the panel before seating it** — this step is not optional:
```ts
import { validateCourtConfig } from 'src/skills/qe-court/referee';
import { validateCourtConfig } from 'agentic-qe/skills/qe-court/referee';
const violations = validateCourtConfig(config); // [] == valid
if (violations.length) throw new Error(`Cannot convene: ${violations.join(', ')}`);
```
@@ -188,9 +188,9 @@ Durable, attestable evidence — the "jury waiting for everything you ship."
## Trust tier
**Tier 3 (verified).** The court's falsifiable invariants are enforced in code
(`src/skills/qe-court/referee.ts`) and covered by an oracle suite
(`tests/unit/skills/qe-court/referee.test.ts`) that the acceptance eval
**Tier 3 (verified).** The court's falsifiable invariants are enforced by the
published `agentic-qe/skills/qe-court/referee` entrypoint and covered by a
consumer-runnable `aqe-court-referee self-test <oracle>` suite that the acceptance eval
(`evals/qe-court.yaml`, command-eval mode) runs through the `aqe eval` CLI —
15/15 green as of 2026-07-29. That suite now validates the **shipped `config.json`
itself**, so a routing edit that seats a colluding panel fails in CI rather than in
+38
View File
@@ -0,0 +1,38 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://agentic-qe.dev/schemas/qe-court-config.json",
"title": "QE-Court configuration",
"type": "object",
"required": ["prosecutors", "overturnDepth", "routing", "options"],
"properties": {
"prosecutors": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
"overturnDepth": { "type": "integer", "minimum": 0 },
"emitScore": { "type": "boolean" },
"routing": {
"type": "object",
"required": ["jury"],
"additionalProperties": {
"anyOf": [
{ "type": "string" },
{
"type": "object",
"properties": { "provider": { "type": "string", "minLength": 1 }, "model": { "type": "string" } },
"additionalProperties": false
}
]
}
},
"options": {
"type": "object",
"required": ["writerIsNeverJuror", "minDistinctVendors"],
"properties": {
"blindFiling": { "type": "boolean" },
"writerIsNeverJuror": { "const": true },
"minDistinctVendors": { "type": "integer", "minimum": 2 },
"requireDoeGatedScore": { "type": "boolean" },
"signCourtRecord": { "type": "boolean" }
},
"additionalProperties": false
}
}
}
+8 -8
View File
@@ -3,8 +3,8 @@ version: 0.3.0
status: active
description: >
Acceptance eval for the QE-Court adversarial review protocol (ADR-124).
Command-eval suite: each case runs a real oracle from the referee suite
(tests/unit/skills/qe-court/referee.test.ts) through the `aqe eval` CLI and
Command-eval suite: each case runs a real oracle through the published
`aqe-court-referee` executable and
asserts it passes. Each oracle passes on correct referee logic and FAILS on a
seeded regression, so a green run is genuine tier-3 evidence — not a self-report.
The seeded-mutant is mirrored in fixtures/seeded-mutant-delivery/ (canSpend
@@ -16,7 +16,7 @@ test_cases:
category: overturn
priority: critical
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "overturn_catches_mutant"
command: aqe-court-referee self-test overturn-catches-mutant
expected:
exit_code: 0
@@ -25,7 +25,7 @@ test_cases:
category: overturn
priority: critical
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "overturn_disabled"
command: aqe-court-referee self-test overturn-disabled
expected:
exit_code: 0
@@ -34,7 +34,7 @@ test_cases:
category: anti-collusion
priority: critical
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "writer_not_juror_enforced"
command: aqe-court-referee self-test writer-not-juror
expected:
exit_code: 0
@@ -43,7 +43,7 @@ test_cases:
category: anti-collusion
priority: high
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "vendor_diversity_enforced"
command: aqe-court-referee self-test vendor-diversity
expected:
exit_code: 0
@@ -52,7 +52,7 @@ test_cases:
category: doe-gate
priority: high
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "no_score_without_doe_gate"
command: aqe-court-referee self-test doe-score-gate
expected:
exit_code: 0
@@ -61,7 +61,7 @@ test_cases:
category: verdict
priority: high
input:
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "verdict_classes"
command: aqe-court-referee self-test verdict-classes
expected:
exit_code: 0
+1
View File
@@ -37,6 +37,7 @@
"bin": {
"agentic-qe": "dist/cli/bundle.js",
"aqe": "dist/cli/bundle.js",
"aqe-court-referee": "dist/skills/qe-court/cli.js",
"aqe-mcp": "dist/mcp/bundle.js",
"aqe-v3": "dist/cli/bundle.js"
},
+6 -1
View File
@@ -52,13 +52,18 @@
"./verification/adversarial-verify": {
"import": "./dist/verification/adversarial-verify/index.js",
"types": "./dist/verification/adversarial-verify/index.d.ts"
},
"./skills/qe-court/referee": {
"import": "./dist/skills/qe-court/referee.js",
"types": "./dist/skills/qe-court/referee.d.ts"
}
},
"bin": {
"agentic-qe": "./dist/cli/bundle.js",
"aqe": "./dist/cli/bundle.js",
"aqe-v3": "./dist/cli/bundle.js",
"aqe-mcp": "./dist/mcp/bundle.js"
"aqe-mcp": "./dist/mcp/bundle.js",
"aqe-court-referee": "./dist/skills/qe-court/cli.js"
},
"scripts": {
"preinstall": "node scripts/preinstall.cjs || true",
+2
View File
@@ -89,6 +89,8 @@ const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : '';
if (invokedPath === fileURLToPath(import.meta.url)) {
const distDir = path.resolve(process.argv[2] ?? 'dist');
const result = normalizeEmittedESM(distDir);
const courtRefereeCli = path.join(distDir, 'skills', 'qe-court', 'cli.js');
if (fs.existsSync(courtRefereeCli)) fs.chmodSync(courtRefereeCli, 0o755);
console.log(
`Normalized ESM specifiers: ${result.rewrittenSpecifiers} in `
+ `${result.rewrittenFiles}/${result.scannedFiles} emitted files`,
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env node
import { readFileSync, realpathSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import {
resolveVerdict,
shouldEmitScore,
validateCourtConfig,
validatePanel,
type Charge,
type CourtConfig,
} from './referee.js';
const MUTANT: Charge = {
id: 'boundary-off-by-one', fatal: true, reproduced: true, depthFound: 1,
};
export function runRefereeCli(argv: string[]): number {
const [command, value] = argv;
if (command === 'validate-config' && value) {
const config = JSON.parse(readFileSync(value, 'utf8')) as CourtConfig;
const violations = validateCourtConfig(config);
process.stdout.write(JSON.stringify({ valid: violations.length === 0, violations }) + '\n');
return violations.length === 0 ? 0 : 2;
}
if (command === 'self-test' && value) {
const passed = runOracle(value);
process.stdout.write(JSON.stringify({ oracle: value, passed }) + '\n');
return passed ? 0 : 3;
}
process.stderr.write(
'Usage: aqe-court-referee validate-config <config.json> | self-test <oracle>\n',
);
return 64;
}
function runOracle(name: string): boolean {
switch (name) {
case 'overturn-catches-mutant':
return resolveVerdict([MUTANT], 2) === 'BLOCK';
case 'overturn-disabled':
return resolveVerdict([MUTANT], 0) === 'SHIP';
case 'writer-not-juror':
return validatePanel([
{ role: 'writer', provider: 'codex' },
{ role: 'jury', provider: 'codex' },
{ role: 'prosecutor', provider: 'claude-code' },
]).includes('writerIsNeverJuror');
case 'vendor-diversity':
return validatePanel([
{ role: 'writer', provider: 'cognitum-low' },
{ role: 'jury', provider: 'cognitum-high' },
]).includes('vendor-diversity');
case 'doe-score-gate':
return !shouldEmitScore(false, true) && shouldEmitScore(true, true);
case 'verdict-classes':
return resolveVerdict([{ ...MUTANT, depthFound: 0 }], 2) === 'BLOCK'
&& resolveVerdict([{ ...MUTANT, fatal: false, depthFound: 0 }], 2) === 'REMAND'
&& resolveVerdict([], 2) === 'SHIP';
default:
return false;
}
}
if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {
process.exitCode = runRefereeCli(process.argv.slice(2));
}
+43
View File
@@ -0,0 +1,43 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { runRefereeCli } from '../../../../src/skills/qe-court/cli';
const dirs: string[] = [];
afterEach(() => {
vi.restoreAllMocks();
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
describe('aqe-court-referee consumer entrypoint (#632)', () => {
it('validates a host-neutral court config', () => {
const dir = mkdtempSync(join(tmpdir(), 'aqe-court-'));
dirs.push(dir);
const file = join(dir, 'config.json');
writeFileSync(file, JSON.stringify({
routing: {
writer: { provider: 'codex' },
jury: { provider: 'claude-code' },
},
options: { writerIsNeverJuror: true, minDistinctVendors: 2 },
}));
const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
expect(runRefereeCli(['validate-config', file])).toBe(0);
expect(stdout).toHaveBeenCalledWith(expect.stringContaining('"valid":true'));
});
it.each([
'overturn-catches-mutant',
'overturn-disabled',
'writer-not-juror',
'vendor-diversity',
'doe-score-gate',
'verdict-classes',
])('runs shipped oracle %s without repository tests', (oracle) => {
vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
expect(runRefereeCli(['self-test', oracle])).toBe(0);
});
});