mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
fix(qe-court): ship executable referee assets
This commit is contained in:
@@ -161,7 +161,7 @@ is Phase 2). To run a court now:
|
||||
the panel before seating it** — this step is not optional:
|
||||
|
||||
```ts
|
||||
import { validateCourtConfig } from 'src/skills/qe-court/referee';
|
||||
import { validateCourtConfig } from 'agentic-qe/skills/qe-court/referee';
|
||||
const violations = validateCourtConfig(config); // [] == valid
|
||||
if (violations.length) throw new Error(`Cannot convene: ${violations.join(', ')}`);
|
||||
```
|
||||
@@ -188,9 +188,9 @@ Durable, attestable evidence — the "jury waiting for everything you ship."
|
||||
|
||||
## Trust tier
|
||||
|
||||
**Tier 3 (verified).** The court's falsifiable invariants are enforced in code
|
||||
(`src/skills/qe-court/referee.ts`) and covered by an oracle suite
|
||||
(`tests/unit/skills/qe-court/referee.test.ts`) that the acceptance eval
|
||||
**Tier 3 (verified).** The court's falsifiable invariants are enforced by the
|
||||
published `agentic-qe/skills/qe-court/referee` entrypoint and covered by a
|
||||
consumer-runnable `aqe-court-referee self-test <oracle>` suite that the acceptance eval
|
||||
(`evals/qe-court.yaml`, command-eval mode) runs through the `aqe eval` CLI —
|
||||
15/15 green as of 2026-07-29. That suite now validates the **shipped `config.json`
|
||||
itself**, so a routing edit that seats a colluding panel fails in CI rather than in
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://agentic-qe.dev/schemas/qe-court-config.json",
|
||||
"title": "QE-Court configuration",
|
||||
"type": "object",
|
||||
"required": ["prosecutors", "overturnDepth", "routing", "options"],
|
||||
"properties": {
|
||||
"prosecutors": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
|
||||
"overturnDepth": { "type": "integer", "minimum": 0 },
|
||||
"emitScore": { "type": "boolean" },
|
||||
"routing": {
|
||||
"type": "object",
|
||||
"required": ["jury"],
|
||||
"additionalProperties": {
|
||||
"anyOf": [
|
||||
{ "type": "string" },
|
||||
{
|
||||
"type": "object",
|
||||
"properties": { "provider": { "type": "string", "minLength": 1 }, "model": { "type": "string" } },
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"type": "object",
|
||||
"required": ["writerIsNeverJuror", "minDistinctVendors"],
|
||||
"properties": {
|
||||
"blindFiling": { "type": "boolean" },
|
||||
"writerIsNeverJuror": { "const": true },
|
||||
"minDistinctVendors": { "type": "integer", "minimum": 2 },
|
||||
"requireDoeGatedScore": { "type": "boolean" },
|
||||
"signCourtRecord": { "type": "boolean" }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,8 @@ version: 0.3.0
|
||||
status: active
|
||||
description: >
|
||||
Acceptance eval for the QE-Court adversarial review protocol (ADR-124).
|
||||
Command-eval suite: each case runs a real oracle from the referee suite
|
||||
(tests/unit/skills/qe-court/referee.test.ts) through the `aqe eval` CLI and
|
||||
Command-eval suite: each case runs a real oracle through the published
|
||||
`aqe-court-referee` executable and
|
||||
asserts it passes. Each oracle passes on correct referee logic and FAILS on a
|
||||
seeded regression, so a green run is genuine tier-3 evidence — not a self-report.
|
||||
The seeded-mutant is mirrored in fixtures/seeded-mutant-delivery/ (canSpend
|
||||
@@ -16,7 +16,7 @@ test_cases:
|
||||
category: overturn
|
||||
priority: critical
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "overturn_catches_mutant"
|
||||
command: aqe-court-referee self-test overturn-catches-mutant
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
@@ -25,7 +25,7 @@ test_cases:
|
||||
category: overturn
|
||||
priority: critical
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "overturn_disabled"
|
||||
command: aqe-court-referee self-test overturn-disabled
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
@@ -34,7 +34,7 @@ test_cases:
|
||||
category: anti-collusion
|
||||
priority: critical
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "writer_not_juror_enforced"
|
||||
command: aqe-court-referee self-test writer-not-juror
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
@@ -43,7 +43,7 @@ test_cases:
|
||||
category: anti-collusion
|
||||
priority: high
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "vendor_diversity_enforced"
|
||||
command: aqe-court-referee self-test vendor-diversity
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
@@ -52,7 +52,7 @@ test_cases:
|
||||
category: doe-gate
|
||||
priority: high
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "no_score_without_doe_gate"
|
||||
command: aqe-court-referee self-test doe-score-gate
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
@@ -61,7 +61,7 @@ test_cases:
|
||||
category: verdict
|
||||
priority: high
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "verdict_classes"
|
||||
command: aqe-court-referee self-test verdict-classes
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
|
||||
@@ -161,7 +161,7 @@ is Phase 2). To run a court now:
|
||||
the panel before seating it** — this step is not optional:
|
||||
|
||||
```ts
|
||||
import { validateCourtConfig } from 'src/skills/qe-court/referee';
|
||||
import { validateCourtConfig } from 'agentic-qe/skills/qe-court/referee';
|
||||
const violations = validateCourtConfig(config); // [] == valid
|
||||
if (violations.length) throw new Error(`Cannot convene: ${violations.join(', ')}`);
|
||||
```
|
||||
@@ -188,9 +188,9 @@ Durable, attestable evidence — the "jury waiting for everything you ship."
|
||||
|
||||
## Trust tier
|
||||
|
||||
**Tier 3 (verified).** The court's falsifiable invariants are enforced in code
|
||||
(`src/skills/qe-court/referee.ts`) and covered by an oracle suite
|
||||
(`tests/unit/skills/qe-court/referee.test.ts`) that the acceptance eval
|
||||
**Tier 3 (verified).** The court's falsifiable invariants are enforced by the
|
||||
published `agentic-qe/skills/qe-court/referee` entrypoint and covered by a
|
||||
consumer-runnable `aqe-court-referee self-test <oracle>` suite that the acceptance eval
|
||||
(`evals/qe-court.yaml`, command-eval mode) runs through the `aqe eval` CLI —
|
||||
15/15 green as of 2026-07-29. That suite now validates the **shipped `config.json`
|
||||
itself**, so a routing edit that seats a colluding panel fails in CI rather than in
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://agentic-qe.dev/schemas/qe-court-config.json",
|
||||
"title": "QE-Court configuration",
|
||||
"type": "object",
|
||||
"required": ["prosecutors", "overturnDepth", "routing", "options"],
|
||||
"properties": {
|
||||
"prosecutors": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
|
||||
"overturnDepth": { "type": "integer", "minimum": 0 },
|
||||
"emitScore": { "type": "boolean" },
|
||||
"routing": {
|
||||
"type": "object",
|
||||
"required": ["jury"],
|
||||
"additionalProperties": {
|
||||
"anyOf": [
|
||||
{ "type": "string" },
|
||||
{
|
||||
"type": "object",
|
||||
"properties": { "provider": { "type": "string", "minLength": 1 }, "model": { "type": "string" } },
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"type": "object",
|
||||
"required": ["writerIsNeverJuror", "minDistinctVendors"],
|
||||
"properties": {
|
||||
"blindFiling": { "type": "boolean" },
|
||||
"writerIsNeverJuror": { "const": true },
|
||||
"minDistinctVendors": { "type": "integer", "minimum": 2 },
|
||||
"requireDoeGatedScore": { "type": "boolean" },
|
||||
"signCourtRecord": { "type": "boolean" }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,8 @@ version: 0.3.0
|
||||
status: active
|
||||
description: >
|
||||
Acceptance eval for the QE-Court adversarial review protocol (ADR-124).
|
||||
Command-eval suite: each case runs a real oracle from the referee suite
|
||||
(tests/unit/skills/qe-court/referee.test.ts) through the `aqe eval` CLI and
|
||||
Command-eval suite: each case runs a real oracle through the published
|
||||
`aqe-court-referee` executable and
|
||||
asserts it passes. Each oracle passes on correct referee logic and FAILS on a
|
||||
seeded regression, so a green run is genuine tier-3 evidence — not a self-report.
|
||||
The seeded-mutant is mirrored in fixtures/seeded-mutant-delivery/ (canSpend
|
||||
@@ -16,7 +16,7 @@ test_cases:
|
||||
category: overturn
|
||||
priority: critical
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "overturn_catches_mutant"
|
||||
command: aqe-court-referee self-test overturn-catches-mutant
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
@@ -25,7 +25,7 @@ test_cases:
|
||||
category: overturn
|
||||
priority: critical
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "overturn_disabled"
|
||||
command: aqe-court-referee self-test overturn-disabled
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
@@ -34,7 +34,7 @@ test_cases:
|
||||
category: anti-collusion
|
||||
priority: critical
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "writer_not_juror_enforced"
|
||||
command: aqe-court-referee self-test writer-not-juror
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
@@ -43,7 +43,7 @@ test_cases:
|
||||
category: anti-collusion
|
||||
priority: high
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "vendor_diversity_enforced"
|
||||
command: aqe-court-referee self-test vendor-diversity
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
@@ -52,7 +52,7 @@ test_cases:
|
||||
category: doe-gate
|
||||
priority: high
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "no_score_without_doe_gate"
|
||||
command: aqe-court-referee self-test doe-score-gate
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
@@ -61,7 +61,7 @@ test_cases:
|
||||
category: verdict
|
||||
priority: high
|
||||
input:
|
||||
command: npx vitest run tests/unit/skills/qe-court/referee.test.ts -t "verdict_classes"
|
||||
command: aqe-court-referee self-test verdict-classes
|
||||
expected:
|
||||
exit_code: 0
|
||||
|
||||
|
||||
Generated
+1
@@ -37,6 +37,7 @@
|
||||
"bin": {
|
||||
"agentic-qe": "dist/cli/bundle.js",
|
||||
"aqe": "dist/cli/bundle.js",
|
||||
"aqe-court-referee": "dist/skills/qe-court/cli.js",
|
||||
"aqe-mcp": "dist/mcp/bundle.js",
|
||||
"aqe-v3": "dist/cli/bundle.js"
|
||||
},
|
||||
|
||||
+6
-1
@@ -52,13 +52,18 @@
|
||||
"./verification/adversarial-verify": {
|
||||
"import": "./dist/verification/adversarial-verify/index.js",
|
||||
"types": "./dist/verification/adversarial-verify/index.d.ts"
|
||||
},
|
||||
"./skills/qe-court/referee": {
|
||||
"import": "./dist/skills/qe-court/referee.js",
|
||||
"types": "./dist/skills/qe-court/referee.d.ts"
|
||||
}
|
||||
},
|
||||
"bin": {
|
||||
"agentic-qe": "./dist/cli/bundle.js",
|
||||
"aqe": "./dist/cli/bundle.js",
|
||||
"aqe-v3": "./dist/cli/bundle.js",
|
||||
"aqe-mcp": "./dist/mcp/bundle.js"
|
||||
"aqe-mcp": "./dist/mcp/bundle.js",
|
||||
"aqe-court-referee": "./dist/skills/qe-court/cli.js"
|
||||
},
|
||||
"scripts": {
|
||||
"preinstall": "node scripts/preinstall.cjs || true",
|
||||
|
||||
@@ -89,6 +89,8 @@ const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : '';
|
||||
if (invokedPath === fileURLToPath(import.meta.url)) {
|
||||
const distDir = path.resolve(process.argv[2] ?? 'dist');
|
||||
const result = normalizeEmittedESM(distDir);
|
||||
const courtRefereeCli = path.join(distDir, 'skills', 'qe-court', 'cli.js');
|
||||
if (fs.existsSync(courtRefereeCli)) fs.chmodSync(courtRefereeCli, 0o755);
|
||||
console.log(
|
||||
`Normalized ESM specifiers: ${result.rewrittenSpecifiers} in `
|
||||
+ `${result.rewrittenFiles}/${result.scannedFiles} emitted files`,
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFileSync, realpathSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
resolveVerdict,
|
||||
shouldEmitScore,
|
||||
validateCourtConfig,
|
||||
validatePanel,
|
||||
type Charge,
|
||||
type CourtConfig,
|
||||
} from './referee.js';
|
||||
|
||||
const MUTANT: Charge = {
|
||||
id: 'boundary-off-by-one', fatal: true, reproduced: true, depthFound: 1,
|
||||
};
|
||||
|
||||
export function runRefereeCli(argv: string[]): number {
|
||||
const [command, value] = argv;
|
||||
if (command === 'validate-config' && value) {
|
||||
const config = JSON.parse(readFileSync(value, 'utf8')) as CourtConfig;
|
||||
const violations = validateCourtConfig(config);
|
||||
process.stdout.write(JSON.stringify({ valid: violations.length === 0, violations }) + '\n');
|
||||
return violations.length === 0 ? 0 : 2;
|
||||
}
|
||||
if (command === 'self-test' && value) {
|
||||
const passed = runOracle(value);
|
||||
process.stdout.write(JSON.stringify({ oracle: value, passed }) + '\n');
|
||||
return passed ? 0 : 3;
|
||||
}
|
||||
process.stderr.write(
|
||||
'Usage: aqe-court-referee validate-config <config.json> | self-test <oracle>\n',
|
||||
);
|
||||
return 64;
|
||||
}
|
||||
|
||||
function runOracle(name: string): boolean {
|
||||
switch (name) {
|
||||
case 'overturn-catches-mutant':
|
||||
return resolveVerdict([MUTANT], 2) === 'BLOCK';
|
||||
case 'overturn-disabled':
|
||||
return resolveVerdict([MUTANT], 0) === 'SHIP';
|
||||
case 'writer-not-juror':
|
||||
return validatePanel([
|
||||
{ role: 'writer', provider: 'codex' },
|
||||
{ role: 'jury', provider: 'codex' },
|
||||
{ role: 'prosecutor', provider: 'claude-code' },
|
||||
]).includes('writerIsNeverJuror');
|
||||
case 'vendor-diversity':
|
||||
return validatePanel([
|
||||
{ role: 'writer', provider: 'cognitum-low' },
|
||||
{ role: 'jury', provider: 'cognitum-high' },
|
||||
]).includes('vendor-diversity');
|
||||
case 'doe-score-gate':
|
||||
return !shouldEmitScore(false, true) && shouldEmitScore(true, true);
|
||||
case 'verdict-classes':
|
||||
return resolveVerdict([{ ...MUTANT, depthFound: 0 }], 2) === 'BLOCK'
|
||||
&& resolveVerdict([{ ...MUTANT, fatal: false, depthFound: 0 }], 2) === 'REMAND'
|
||||
&& resolveVerdict([], 2) === 'SHIP';
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
process.exitCode = runRefereeCli(process.argv.slice(2));
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { runRefereeCli } from '../../../../src/skills/qe-court/cli';
|
||||
|
||||
const dirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('aqe-court-referee consumer entrypoint (#632)', () => {
|
||||
it('validates a host-neutral court config', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'aqe-court-'));
|
||||
dirs.push(dir);
|
||||
const file = join(dir, 'config.json');
|
||||
writeFileSync(file, JSON.stringify({
|
||||
routing: {
|
||||
writer: { provider: 'codex' },
|
||||
jury: { provider: 'claude-code' },
|
||||
},
|
||||
options: { writerIsNeverJuror: true, minDistinctVendors: 2 },
|
||||
}));
|
||||
const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
|
||||
|
||||
expect(runRefereeCli(['validate-config', file])).toBe(0);
|
||||
expect(stdout).toHaveBeenCalledWith(expect.stringContaining('"valid":true'));
|
||||
});
|
||||
|
||||
it.each([
|
||||
'overturn-catches-mutant',
|
||||
'overturn-disabled',
|
||||
'writer-not-juror',
|
||||
'vendor-diversity',
|
||||
'doe-score-gate',
|
||||
'verdict-classes',
|
||||
])('runs shipped oracle %s without repository tests', (oracle) => {
|
||||
vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
|
||||
expect(runRefereeCli(['self-test', oracle])).toBe(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user