mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
feat: implement Six Hats Tier 3 Phase 1 — YAML pipelines, heartbeat CLI, economic routing (#334)
Three strategic features from Issue #334 with full CLI-MCP parity: Imp-9: YAML Deterministic QE Pipelines - Approval gate step type with auto-approve timeout and cancel cleanup - 4 deterministic actions (quality-gate, coverage, pattern-health, routing) with real SQL queries and graceful DB degradation - 7 CLI commands (aqe pipeline load/validate/run/list/status/approve/reject) - YAML loader validation for approval fields Imp-10: Token-Free Heartbeat CLI & MCP - 6 CLI commands (aqe heartbeat status/run-now/history/log/pause/resume) - 3 MCP tools (heartbeat_status/trigger/log) - History persistence, date validation, timeout support Imp-18: Economic Routing Model - EconomicRoutingModel with quality-per-dollar scoring and EMA estimates - Budget-aware tier selection with fallback strategies - Cost-adjusted rewards for neural router (opt-in) - CLI commands (aqe routing economics/accuracy/metrics) - routing_economics MCP tool with kv_store persistence Security: path traversal fix on CLI log reader, config weight validation 71 new tests across 6 test files, all passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dragan Spiridonov
+47
-44
@@ -1,55 +1,58 @@
|
||||
# Agentic QE v3 - Daily QE Pipeline Example
|
||||
# Per ADR-041: V3 QE CLI Enhancement
|
||||
# Deterministic pipeline: zero LLM tokens (Imp-9)
|
||||
|
||||
name: daily-qe-pipeline
|
||||
description: Comprehensive daily quality engineering pipeline
|
||||
id: daily-qe-pipeline
|
||||
name: Daily QE Pipeline
|
||||
description: Deterministic daily quality engineering pipeline (zero LLM tokens)
|
||||
version: "1.0.0"
|
||||
schedule: 0 2 * * *
|
||||
defaultMode: sequential
|
||||
tags:
|
||||
- testing
|
||||
- quality
|
||||
- daily
|
||||
- token-free
|
||||
|
||||
stages:
|
||||
- name: test-generation
|
||||
command: aqe test generate
|
||||
params:
|
||||
target: src/
|
||||
coverage-goal: 90
|
||||
ai-enhanced: true
|
||||
timeout: 120
|
||||
steps:
|
||||
- id: coverage-check
|
||||
name: Coverage Threshold Check
|
||||
domain: coverage-analysis
|
||||
action: threshold-check
|
||||
inputMapping:
|
||||
minCoverage: "input.coverageMin"
|
||||
timeout: 30000
|
||||
|
||||
- name: parallel-execution
|
||||
command: aqe test execute
|
||||
depends_on:
|
||||
- test-generation
|
||||
params:
|
||||
parallel: 8
|
||||
retry: 3
|
||||
timeout: 300
|
||||
- id: pattern-health
|
||||
name: Pattern Health Check
|
||||
domain: learning-optimization
|
||||
action: health-check
|
||||
timeout: 30000
|
||||
|
||||
- name: coverage-analysis
|
||||
command: aqe coverage analyze
|
||||
depends_on:
|
||||
- parallel-execution
|
||||
params:
|
||||
gap-detection: true
|
||||
report-format: html
|
||||
- id: quality-gate
|
||||
name: Quality Gate
|
||||
domain: quality-assessment
|
||||
action: gate-check
|
||||
dependsOn: [coverage-check]
|
||||
inputMapping:
|
||||
coverageMin: "input.coverageMin"
|
||||
currentCoverage: "results.coverage-check.currentCoverage"
|
||||
continueOnFailure: false
|
||||
timeout: 30000
|
||||
|
||||
- name: quality-gate
|
||||
command: aqe quality gate
|
||||
depends_on:
|
||||
- coverage-analysis
|
||||
params:
|
||||
fail-threshold: 80
|
||||
coverage-min: 85
|
||||
- id: routing-check
|
||||
name: Routing Accuracy Check
|
||||
domain: learning-optimization
|
||||
action: routing-check
|
||||
timeout: 30000
|
||||
|
||||
triggers:
|
||||
- event: push
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- event: pull_request
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- id: deploy-approval
|
||||
name: Deploy Approval
|
||||
domain: quality-assessment
|
||||
action: gate-check
|
||||
dependsOn: [quality-gate]
|
||||
approval:
|
||||
autoApproveAfter: 300000
|
||||
message: "All quality checks passed. Approve deployment?"
|
||||
condition:
|
||||
path: "results.quality-gate.passed"
|
||||
operator: eq
|
||||
value: true
|
||||
timeout: 600000
|
||||
|
||||
@@ -18,6 +18,8 @@ import {
|
||||
createProtocolHandler,
|
||||
createBrainHandler,
|
||||
createHypergraphHandler,
|
||||
createHeartbeatHandler,
|
||||
createRoutingHandler,
|
||||
} from './handlers/index.js';
|
||||
|
||||
// ============================================================================
|
||||
@@ -117,6 +119,12 @@ export class CommandRegistry {
|
||||
|
||||
// Hypergraph queries
|
||||
this.register(createHypergraphHandler(this.cleanupAndExit, this.ensureInitialized));
|
||||
|
||||
// Heartbeat scheduler (Imp-10)
|
||||
this.register(createHeartbeatHandler(this.cleanupAndExit));
|
||||
|
||||
// Routing economics & accuracy (Imp-18)
|
||||
this.register(createRoutingHandler(this.cleanupAndExit));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
/**
|
||||
* Agentic QE v3 - Pipeline Command (Imp-9)
|
||||
*
|
||||
* CLI commands for YAML deterministic pipeline management:
|
||||
* aqe pipeline load <file.yaml> [--vars key=value]
|
||||
* aqe pipeline validate <file.yaml>
|
||||
* aqe pipeline run <pipeline-id> [--input key=value] [--wait]
|
||||
* aqe pipeline list
|
||||
* aqe pipeline status <execution-id>
|
||||
* aqe pipeline approve <execution-id> <step-id>
|
||||
* aqe pipeline reject <execution-id> <step-id> [--reason "..."]
|
||||
*/
|
||||
|
||||
import { Command } from 'commander';
|
||||
import { resolve } from 'path';
|
||||
import chalk from 'chalk';
|
||||
import type { CLIContext } from '../handlers/interfaces.js';
|
||||
import { YamlPipelineLoader } from '../../coordination/yaml-pipeline-loader.js';
|
||||
import type { WorkflowOrchestrator } from '../../coordination/workflow-orchestrator.js';
|
||||
|
||||
// ============================================================================
|
||||
// Helpers
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Parse `--vars key1=value1 key2=value2` into a Record.
|
||||
*/
|
||||
function parseVars(rawVars: string[]): Record<string, unknown> {
|
||||
const vars: Record<string, unknown> = {};
|
||||
for (const entry of rawVars) {
|
||||
const eqIndex = entry.indexOf('=');
|
||||
if (eqIndex === -1) {
|
||||
vars[entry] = true;
|
||||
} else {
|
||||
const key = entry.slice(0, eqIndex);
|
||||
const value = entry.slice(eqIndex + 1);
|
||||
// Try to parse as number or boolean
|
||||
if (value === 'true') vars[key] = true;
|
||||
else if (value === 'false') vars[key] = false;
|
||||
else if (!isNaN(Number(value)) && value !== '') vars[key] = Number(value);
|
||||
else vars[key] = value;
|
||||
}
|
||||
}
|
||||
return vars;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `--input key1=value1 key2=value2` into a Record.
|
||||
*/
|
||||
function parseInput(rawInput: string[]): Record<string, unknown> {
|
||||
return parseVars(rawInput);
|
||||
}
|
||||
|
||||
function getOrchestrator(context: CLIContext): WorkflowOrchestrator | null {
|
||||
return context.workflowOrchestrator;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Command Factory
|
||||
// ============================================================================
|
||||
|
||||
export function createPipelineCommand(
|
||||
context: CLIContext,
|
||||
cleanupAndExit: (code: number) => Promise<never>,
|
||||
ensureInitialized: () => Promise<boolean>,
|
||||
): Command {
|
||||
const loader = new YamlPipelineLoader();
|
||||
|
||||
const pipelineCmd = new Command('pipeline')
|
||||
.description('Manage YAML deterministic pipelines (Imp-9)');
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// pipeline load <file.yaml>
|
||||
// ---------------------------------------------------------------
|
||||
pipelineCmd
|
||||
.command('load <file>')
|
||||
.description('Load and register a pipeline from a YAML file')
|
||||
.option('--vars <entries...>', 'Variable substitutions (key=value)')
|
||||
.action(async (file: string, options: { vars?: string[] }) => {
|
||||
if (!await ensureInitialized()) return;
|
||||
const orchestrator = getOrchestrator(context);
|
||||
if (!orchestrator) {
|
||||
console.error(chalk.red(' Workflow orchestrator not available. Run "aqe fleet init" first.'));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const filePath = resolve(file);
|
||||
const vars = options.vars ? parseVars(options.vars) : undefined;
|
||||
|
||||
console.log(chalk.blue(`\n Loading pipeline from: ${file}\n`));
|
||||
|
||||
const result = await loader.loadFromFile(filePath, vars);
|
||||
if (!result.success) {
|
||||
console.error(chalk.red(` Parse error: ${result.error.message}`));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const definition = result.value;
|
||||
const registerResult = orchestrator.registerWorkflow(definition);
|
||||
if (!registerResult.success) {
|
||||
console.error(chalk.red(` Registration error: ${registerResult.error.message}`));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(chalk.green(' Pipeline loaded successfully.'));
|
||||
console.log(` ID: ${chalk.cyan(definition.id)}`);
|
||||
console.log(` Name: ${chalk.cyan(definition.name)}`);
|
||||
console.log(` Steps: ${chalk.cyan(definition.steps.length)}`);
|
||||
console.log(` Version: ${chalk.cyan(definition.version)}`);
|
||||
if (definition.tags?.length) {
|
||||
console.log(` Tags: ${chalk.cyan(definition.tags.join(', '))}`);
|
||||
}
|
||||
console.log('');
|
||||
await cleanupAndExit(0);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// pipeline validate <file.yaml>
|
||||
// ---------------------------------------------------------------
|
||||
pipelineCmd
|
||||
.command('validate <file>')
|
||||
.description('Validate a YAML pipeline without registering it')
|
||||
.option('--vars <entries...>', 'Variable substitutions (key=value)')
|
||||
.action(async (file: string, options: { vars?: string[] }) => {
|
||||
const filePath = resolve(file);
|
||||
const vars = options.vars ? parseVars(options.vars) : undefined;
|
||||
|
||||
console.log(chalk.blue(`\n Validating pipeline: ${file}\n`));
|
||||
|
||||
const result = await loader.loadFromFile(filePath, vars);
|
||||
if (!result.success) {
|
||||
console.log(chalk.red(` Invalid: ${result.error.message}`));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const def = result.value;
|
||||
console.log(chalk.green(' Valid pipeline.'));
|
||||
console.log(` ID: ${chalk.cyan(def.id)}`);
|
||||
console.log(` Name: ${chalk.cyan(def.name)}`);
|
||||
console.log(` Steps: ${chalk.cyan(def.steps.length)}`);
|
||||
console.log('');
|
||||
await cleanupAndExit(0);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// pipeline run <pipeline-id>
|
||||
// ---------------------------------------------------------------
|
||||
pipelineCmd
|
||||
.command('run <pipelineId>')
|
||||
.description('Execute a registered pipeline')
|
||||
.option('--input <entries...>', 'Input parameters (key=value)')
|
||||
.option('--wait', 'Wait for execution to complete')
|
||||
.action(async (pipelineId: string, options: { input?: string[]; wait?: boolean }) => {
|
||||
if (!await ensureInitialized()) return;
|
||||
const orchestrator = getOrchestrator(context);
|
||||
if (!orchestrator) {
|
||||
console.error(chalk.red(' Workflow orchestrator not available. Run "aqe fleet init" first.'));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const input = options.input ? parseInput(options.input) : {};
|
||||
|
||||
console.log(chalk.blue(`\n Running pipeline: ${pipelineId}\n`));
|
||||
|
||||
const result = await orchestrator.executeWorkflow(pipelineId, input);
|
||||
if (!result.success) {
|
||||
console.error(chalk.red(` Failed: ${result.error.message}`));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const executionId = result.value;
|
||||
console.log(chalk.green(' Pipeline started.'));
|
||||
console.log(` Execution ID: ${chalk.cyan(executionId)}`);
|
||||
|
||||
if (options.wait) {
|
||||
console.log(chalk.gray(' Waiting for completion...'));
|
||||
let status = orchestrator.getWorkflowStatus(executionId);
|
||||
while (status && (status.status === 'running' || status.status === 'paused')) {
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
status = orchestrator.getWorkflowStatus(executionId);
|
||||
}
|
||||
if (status) {
|
||||
const statusColor = status.status === 'completed' ? chalk.green : chalk.red;
|
||||
console.log(` Status: ${statusColor(status.status)}`);
|
||||
if (status.duration) console.log(` Duration: ${chalk.cyan(`${status.duration}ms`)}`);
|
||||
if (status.error) console.log(` Error: ${chalk.red(status.error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('');
|
||||
await cleanupAndExit(0);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// pipeline list
|
||||
// ---------------------------------------------------------------
|
||||
pipelineCmd
|
||||
.command('list')
|
||||
.description('List all registered pipelines')
|
||||
.action(async () => {
|
||||
if (!await ensureInitialized()) return;
|
||||
const orchestrator = getOrchestrator(context);
|
||||
if (!orchestrator) {
|
||||
console.error(chalk.red(' Workflow orchestrator not available. Run "aqe fleet init" first.'));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const workflows = orchestrator.listWorkflows();
|
||||
console.log(chalk.blue(`\n Registered Pipelines (${workflows.length})\n`));
|
||||
|
||||
if (workflows.length === 0) {
|
||||
console.log(chalk.gray(' No pipelines registered.'));
|
||||
} else {
|
||||
for (const wf of workflows) {
|
||||
console.log(` ${chalk.cyan(wf.id)} — ${wf.name} (${wf.stepCount} steps, v${wf.version})`);
|
||||
if (wf.tags?.length) {
|
||||
console.log(` Tags: ${chalk.gray(wf.tags.join(', '))}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('');
|
||||
await cleanupAndExit(0);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// pipeline status <execution-id>
|
||||
// ---------------------------------------------------------------
|
||||
pipelineCmd
|
||||
.command('status <executionId>')
|
||||
.description('Show the status of a pipeline execution')
|
||||
.action(async (executionId: string) => {
|
||||
if (!await ensureInitialized()) return;
|
||||
const orchestrator = getOrchestrator(context);
|
||||
if (!orchestrator) {
|
||||
console.error(chalk.red(' Workflow orchestrator not available. Run "aqe fleet init" first.'));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const status = orchestrator.getWorkflowStatus(executionId);
|
||||
if (!status) {
|
||||
console.error(chalk.red(` Execution not found: ${executionId}`));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const statusColor =
|
||||
status.status === 'completed' ? chalk.green :
|
||||
status.status === 'failed' ? chalk.red :
|
||||
status.status === 'running' ? chalk.yellow : chalk.gray;
|
||||
|
||||
console.log(chalk.blue(`\n Pipeline Execution Status\n`));
|
||||
console.log(` Execution: ${chalk.cyan(executionId)}`);
|
||||
console.log(` Pipeline: ${chalk.cyan(status.workflowName)} (${status.workflowId})`);
|
||||
console.log(` Status: ${statusColor(status.status)}`);
|
||||
console.log(` Progress: ${chalk.cyan(`${status.progress}%`)}`);
|
||||
console.log(` Completed: ${chalk.cyan(status.completedSteps.join(', ') || 'none')}`);
|
||||
if (status.failedSteps.length > 0) {
|
||||
console.log(` Failed: ${chalk.red(status.failedSteps.join(', '))}`);
|
||||
}
|
||||
if (status.skippedSteps.length > 0) {
|
||||
console.log(` Skipped: ${chalk.gray(status.skippedSteps.join(', '))}`);
|
||||
}
|
||||
if (status.currentSteps.length > 0) {
|
||||
console.log(` Running: ${chalk.yellow(status.currentSteps.join(', '))}`);
|
||||
}
|
||||
if (status.duration) {
|
||||
console.log(` Duration: ${chalk.cyan(`${status.duration}ms`)}`);
|
||||
}
|
||||
if (status.error) {
|
||||
console.log(` Error: ${chalk.red(status.error)}`);
|
||||
}
|
||||
|
||||
console.log('');
|
||||
await cleanupAndExit(0);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// pipeline approve <execution-id> <step-id>
|
||||
// ---------------------------------------------------------------
|
||||
pipelineCmd
|
||||
.command('approve <executionId> <stepId>')
|
||||
.description('Approve a step that is awaiting approval')
|
||||
.action(async (executionId: string, stepId: string) => {
|
||||
if (!await ensureInitialized()) return;
|
||||
const orchestrator = getOrchestrator(context);
|
||||
if (!orchestrator) {
|
||||
console.error(chalk.red(' Workflow orchestrator not available.'));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const success = orchestrator.approveStep(executionId, stepId);
|
||||
if (success) {
|
||||
console.log(chalk.green(`\n Step '${stepId}' approved.\n`));
|
||||
} else {
|
||||
console.error(chalk.red(`\n No pending approval found for step '${stepId}' in execution '${executionId}'.\n`));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
await cleanupAndExit(0);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// pipeline reject <execution-id> <step-id>
|
||||
// ---------------------------------------------------------------
|
||||
pipelineCmd
|
||||
.command('reject <executionId> <stepId>')
|
||||
.description('Reject a step that is awaiting approval')
|
||||
.option('--reason <reason>', 'Rejection reason')
|
||||
.action(async (executionId: string, stepId: string, options: { reason?: string }) => {
|
||||
if (!await ensureInitialized()) return;
|
||||
const orchestrator = getOrchestrator(context);
|
||||
if (!orchestrator) {
|
||||
console.error(chalk.red(' Workflow orchestrator not available.'));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const success = orchestrator.rejectStep(executionId, stepId, options.reason);
|
||||
if (success) {
|
||||
console.log(chalk.green(`\n Step '${stepId}' rejected.`));
|
||||
if (options.reason) console.log(` Reason: ${chalk.gray(options.reason)}`);
|
||||
console.log('');
|
||||
} else {
|
||||
console.error(chalk.red(`\n No pending approval found for step '${stepId}' in execution '${executionId}'.\n`));
|
||||
await cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
await cleanupAndExit(0);
|
||||
});
|
||||
|
||||
return pipelineCmd;
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
/**
|
||||
* Agentic QE v3 - Heartbeat Command Handler
|
||||
* Imp-10: Token-Free Heartbeat Scheduler CLI Integration
|
||||
*
|
||||
* Handles the 'aqe heartbeat' command with subcommands:
|
||||
* status, run-now, history, log, pause, resume
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { Command } from 'commander';
|
||||
import chalk from 'chalk';
|
||||
import { ICommandHandler, CLIContext, formatDuration } from './interfaces.js';
|
||||
import { HeartbeatSchedulerWorker } from '../../workers/workers/heartbeat-scheduler.js';
|
||||
import type { WorkerResult } from '../../workers/interfaces.js';
|
||||
import { toErrorMessage } from '../../shared/error-utils.js';
|
||||
import { findProjectRoot } from '../../kernel/unified-memory.js';
|
||||
|
||||
// ============================================================================
|
||||
// Heartbeat Handler
|
||||
// ============================================================================
|
||||
|
||||
export class HeartbeatHandler implements ICommandHandler {
|
||||
readonly name = 'heartbeat';
|
||||
readonly description = 'Manage the token-free heartbeat scheduler';
|
||||
|
||||
private cleanupAndExit: (code: number) => Promise<never>;
|
||||
private worker: HeartbeatSchedulerWorker;
|
||||
|
||||
constructor(cleanupAndExit: (code: number) => Promise<never>) {
|
||||
this.cleanupAndExit = cleanupAndExit;
|
||||
this.worker = new HeartbeatSchedulerWorker();
|
||||
}
|
||||
|
||||
register(program: Command, _context: CLIContext): void {
|
||||
const heartbeat = program
|
||||
.command('heartbeat')
|
||||
.description(this.description);
|
||||
|
||||
heartbeat
|
||||
.command('status')
|
||||
.description('Show heartbeat worker status, health, and schedule')
|
||||
.action(async () => {
|
||||
await this.executeStatus();
|
||||
});
|
||||
|
||||
heartbeat
|
||||
.command('run-now')
|
||||
.description('Trigger an immediate heartbeat cycle')
|
||||
.option('-t, --timeout <ms>', 'Timeout in milliseconds (default: worker built-in 60s)')
|
||||
.action(async (options: { timeout?: string }) => {
|
||||
const timeout = options.timeout ? parseInt(options.timeout, 10) : undefined;
|
||||
await this.executeRunNow(timeout);
|
||||
});
|
||||
|
||||
heartbeat
|
||||
.command('history')
|
||||
.description('Show recent heartbeat results')
|
||||
.option('-n, --count <count>', 'Number of entries to show', '10')
|
||||
.action(async (options: { count: string }) => {
|
||||
await this.executeHistory(parseInt(options.count, 10) || 10);
|
||||
});
|
||||
|
||||
heartbeat
|
||||
.command('log')
|
||||
.description("Show today's daily log entries")
|
||||
.option('-d, --date <date>', 'Show log for specific date (YYYY-MM-DD)')
|
||||
.action(async (options: { date?: string }) => {
|
||||
await this.executeLog(options.date);
|
||||
});
|
||||
|
||||
heartbeat
|
||||
.command('pause')
|
||||
.description('Pause the heartbeat worker')
|
||||
.action(async () => {
|
||||
await this.executePause();
|
||||
});
|
||||
|
||||
heartbeat
|
||||
.command('resume')
|
||||
.description('Resume the heartbeat worker')
|
||||
.action(async () => {
|
||||
await this.executeResume();
|
||||
});
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Subcommand Implementations
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
private async executeStatus(): Promise<void> {
|
||||
try {
|
||||
await this.worker.initialize();
|
||||
const health = this.worker.getHealth();
|
||||
const lastResult = this.worker.lastResult;
|
||||
|
||||
console.log(chalk.blue('\n Heartbeat Scheduler Status'));
|
||||
console.log(chalk.gray(' ' + '\u2500'.repeat(35)));
|
||||
|
||||
console.log(` Status: ${statusColor(health.status)}`);
|
||||
console.log(` Health Score: ${scoreColor(health.healthScore)}${chalk.gray('/100')}`);
|
||||
|
||||
if (this.worker.lastRunAt) {
|
||||
const ago = formatRelativeTime(this.worker.lastRunAt);
|
||||
console.log(` Last Run: ${chalk.cyan(this.worker.lastRunAt.toISOString().replace('T', ' ').slice(0, 19))} ${chalk.gray(`(${ago})`)}`);
|
||||
} else {
|
||||
console.log(` Last Run: ${chalk.gray('never')}`);
|
||||
}
|
||||
|
||||
if (this.worker.nextRunAt) {
|
||||
const until = formatRelativeTime(this.worker.nextRunAt, true);
|
||||
console.log(` Next Run: ${chalk.cyan(this.worker.nextRunAt.toISOString().replace('T', ' ').slice(0, 19))} ${chalk.gray(`(${until})`)}`);
|
||||
}
|
||||
|
||||
console.log(` Total Runs: ${chalk.cyan(String(health.totalExecutions))}`);
|
||||
const successRate = health.totalExecutions > 0
|
||||
? ((health.successfulExecutions / health.totalExecutions) * 100).toFixed(1)
|
||||
: '100.0';
|
||||
console.log(` Success Rate: ${chalk.cyan(successRate + '%')}`);
|
||||
|
||||
if (lastResult?.metrics?.domainMetrics) {
|
||||
const dm = lastResult.metrics.domainMetrics;
|
||||
console.log('');
|
||||
console.log(chalk.blue(' Last Result:'));
|
||||
console.log(` Promoted: ${chalk.cyan(String(dm.promoted ?? 0))} patterns`);
|
||||
console.log(` Deprecated: ${chalk.cyan(String(dm.deprecated ?? 0))} patterns`);
|
||||
console.log(` Decayed: ${chalk.cyan(String(dm.decayed ?? 0))} patterns`);
|
||||
console.log(` Pending Exp: ${chalk.cyan(String(dm.pendingExperiences ?? 0))}`);
|
||||
console.log(` Avg Conf: ${chalk.cyan(String(dm.avgConfidence ?? 0))}`);
|
||||
}
|
||||
|
||||
console.log('');
|
||||
await this.cleanupAndExit(0);
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n Failed to get heartbeat status:'), toErrorMessage(error));
|
||||
await this.cleanupAndExit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private async executeRunNow(timeoutMs?: number): Promise<void> {
|
||||
try {
|
||||
console.log(chalk.blue('\n Triggering heartbeat cycle...\n'));
|
||||
|
||||
await this.worker.initialize();
|
||||
|
||||
const abortController = new AbortController();
|
||||
|
||||
// Only add external timeout if the user explicitly requests one shorter
|
||||
// than the worker's built-in 60s timeout.
|
||||
const timeoutHandle = timeoutMs && timeoutMs > 0
|
||||
? setTimeout(() => abortController.abort(), timeoutMs)
|
||||
: null;
|
||||
|
||||
// Use a real logger that outputs to the console, and a lightweight
|
||||
// event bus / memory — the heartbeat worker only needs DB access
|
||||
// (which it gets via getUnifiedMemory() internally).
|
||||
const result: WorkerResult = await this.worker.execute({
|
||||
eventBus: { publish: async () => {} },
|
||||
memory: {
|
||||
get: async () => undefined,
|
||||
set: async () => {},
|
||||
search: async () => [],
|
||||
},
|
||||
logger: {
|
||||
debug: () => {},
|
||||
info: (...args: unknown[]) => console.log(chalk.gray(' [heartbeat]'), ...args),
|
||||
warn: (...args: unknown[]) => console.warn(chalk.yellow(' [heartbeat]'), ...args),
|
||||
error: (...args: unknown[]) => console.error(chalk.red(' [heartbeat]'), ...args),
|
||||
},
|
||||
domains: {
|
||||
getDomainAPI: () => undefined,
|
||||
getDomainHealth: () => ({ status: 'healthy', errors: [] }),
|
||||
},
|
||||
signal: abortController.signal,
|
||||
});
|
||||
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle);
|
||||
|
||||
if (result.success) {
|
||||
const dm = result.metrics.domainMetrics;
|
||||
console.log(chalk.green(' Heartbeat cycle complete.'));
|
||||
console.log(` Duration: ${chalk.cyan(formatDuration(result.durationMs))}`);
|
||||
console.log(` Health Score: ${scoreColor(result.metrics.healthScore)}${chalk.gray('/100')}`);
|
||||
console.log(` Trend: ${trendColor(result.metrics.trend)}`);
|
||||
console.log(` Promoted: ${chalk.cyan(String(dm.promoted ?? 0))}`);
|
||||
console.log(` Deprecated: ${chalk.cyan(String(dm.deprecated ?? 0))}`);
|
||||
console.log(` Decayed: ${chalk.cyan(String(dm.decayed ?? 0))}`);
|
||||
console.log(` Findings: ${chalk.cyan(String(result.findings.length))}`);
|
||||
} else {
|
||||
console.error(chalk.red(' Heartbeat cycle failed:'), result.error);
|
||||
}
|
||||
|
||||
// Store history entry
|
||||
storeHistoryEntry(result);
|
||||
|
||||
console.log('');
|
||||
await this.cleanupAndExit(result.success ? 0 : 1);
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n Failed to run heartbeat:'), toErrorMessage(error));
|
||||
await this.cleanupAndExit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private async executeHistory(count: number): Promise<void> {
|
||||
try {
|
||||
const entries = loadHistoryEntries(count);
|
||||
|
||||
if (entries.length === 0) {
|
||||
console.log(chalk.yellow('\n No heartbeat history found. Run `aqe heartbeat run-now` first.\n'));
|
||||
await this.cleanupAndExit(0);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(chalk.blue(`\n Heartbeat History (last ${entries.length})`));
|
||||
console.log(chalk.gray(' ' + '\u2500'.repeat(60)));
|
||||
|
||||
for (const entry of entries) {
|
||||
const status = entry.success ? chalk.green('OK') : chalk.red('FAIL');
|
||||
const ts = entry.timestamp.slice(0, 19).replace('T', ' ');
|
||||
const dm = entry.domainMetrics || {};
|
||||
console.log(
|
||||
` ${chalk.gray(ts)} ${status} ` +
|
||||
`score:${chalk.cyan(String(entry.healthScore))} ` +
|
||||
`+${dm.promoted ?? 0}/-${dm.deprecated ?? 0} ` +
|
||||
`${chalk.gray(formatDuration(entry.durationMs))}`
|
||||
);
|
||||
}
|
||||
|
||||
console.log('');
|
||||
await this.cleanupAndExit(0);
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n Failed to load history:'), toErrorMessage(error));
|
||||
await this.cleanupAndExit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private async executeLog(date?: string): Promise<void> {
|
||||
try {
|
||||
const targetDate = date || new Date().toISOString().split('T')[0];
|
||||
|
||||
// Validate date format to prevent path traversal (CLI-MCP parity with heartbeat-handlers.ts)
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(targetDate)) {
|
||||
console.error(chalk.red(`\n Invalid date format: "${targetDate}". Use YYYY-MM-DD.\n`));
|
||||
await this.cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
const logDir = path.join(findProjectRoot(), '.agentic-qe', 'logs');
|
||||
const logPath = path.join(logDir, `${targetDate}.md`);
|
||||
|
||||
if (!fs.existsSync(logPath)) {
|
||||
console.log(chalk.yellow(`\n No daily log found for ${targetDate}.\n`));
|
||||
await this.cleanupAndExit(0);
|
||||
return;
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(logPath, 'utf-8');
|
||||
console.log(chalk.blue(`\n Daily Log \u2014 ${targetDate}`));
|
||||
console.log(chalk.gray(' ' + '\u2500'.repeat(40)));
|
||||
// Indent and display each line
|
||||
for (const line of content.split('\n')) {
|
||||
if (line.trim()) {
|
||||
console.log(` ${line}`);
|
||||
}
|
||||
}
|
||||
console.log('');
|
||||
await this.cleanupAndExit(0);
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n Failed to read daily log:'), toErrorMessage(error));
|
||||
await this.cleanupAndExit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private async executePause(): Promise<void> {
|
||||
try {
|
||||
this.worker.pause();
|
||||
console.log(chalk.yellow('\n Heartbeat worker paused.\n'));
|
||||
await this.cleanupAndExit(0);
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n Failed to pause heartbeat:'), toErrorMessage(error));
|
||||
await this.cleanupAndExit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private async executeResume(): Promise<void> {
|
||||
try {
|
||||
this.worker.resume();
|
||||
console.log(chalk.green('\n Heartbeat worker resumed.\n'));
|
||||
await this.cleanupAndExit(0);
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n Failed to resume heartbeat:'), toErrorMessage(error));
|
||||
await this.cleanupAndExit(1);
|
||||
}
|
||||
}
|
||||
|
||||
getHelp(): string {
|
||||
return `
|
||||
Manage the token-free heartbeat scheduler (Imp-10).
|
||||
|
||||
The heartbeat runs every 30 minutes performing SQL-only maintenance:
|
||||
- Pattern promotion checks
|
||||
- Stale pattern deprecation
|
||||
- Confidence decay application
|
||||
- Experience buffer monitoring
|
||||
- Daily Markdown log entries
|
||||
|
||||
Subcommands:
|
||||
status Show heartbeat worker status, health, and schedule
|
||||
run-now Trigger an immediate heartbeat cycle
|
||||
history Show recent heartbeat results (last 10)
|
||||
log Show today's daily log entries
|
||||
pause Pause the heartbeat worker
|
||||
resume Resume the heartbeat worker
|
||||
|
||||
Examples:
|
||||
aqe heartbeat status
|
||||
aqe heartbeat run-now
|
||||
aqe heartbeat history -n 5
|
||||
aqe heartbeat log
|
||||
aqe heartbeat log --date 2026-03-25
|
||||
aqe heartbeat pause
|
||||
aqe heartbeat resume
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// History Persistence
|
||||
// ============================================================================
|
||||
|
||||
interface HeartbeatHistoryEntry {
|
||||
timestamp: string;
|
||||
success: boolean;
|
||||
durationMs: number;
|
||||
healthScore: number;
|
||||
domainMetrics: Record<string, number | string>;
|
||||
}
|
||||
|
||||
const MAX_HISTORY_ENTRIES = 100;
|
||||
|
||||
function getHistoryPath(): string {
|
||||
return path.join(findProjectRoot(), '.agentic-qe', 'heartbeat-history.json');
|
||||
}
|
||||
|
||||
function storeHistoryEntry(result: WorkerResult): void {
|
||||
try {
|
||||
const historyPath = getHistoryPath();
|
||||
const dir = path.dirname(historyPath);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
let entries: HeartbeatHistoryEntry[] = [];
|
||||
if (fs.existsSync(historyPath)) {
|
||||
try {
|
||||
entries = JSON.parse(fs.readFileSync(historyPath, 'utf-8'));
|
||||
} catch {
|
||||
entries = [];
|
||||
}
|
||||
}
|
||||
|
||||
entries.unshift({
|
||||
timestamp: result.timestamp.toISOString(),
|
||||
success: result.success,
|
||||
durationMs: result.durationMs,
|
||||
healthScore: result.metrics.healthScore,
|
||||
domainMetrics: result.metrics.domainMetrics,
|
||||
});
|
||||
|
||||
// Prune to max entries
|
||||
if (entries.length > MAX_HISTORY_ENTRIES) {
|
||||
entries = entries.slice(0, MAX_HISTORY_ENTRIES);
|
||||
}
|
||||
|
||||
fs.writeFileSync(historyPath, JSON.stringify(entries, null, 2));
|
||||
} catch {
|
||||
// Non-critical: don't fail the command if history persistence fails
|
||||
}
|
||||
}
|
||||
|
||||
function loadHistoryEntries(count: number): HeartbeatHistoryEntry[] {
|
||||
try {
|
||||
const historyPath = getHistoryPath();
|
||||
if (!fs.existsSync(historyPath)) {
|
||||
return [];
|
||||
}
|
||||
const entries: HeartbeatHistoryEntry[] = JSON.parse(fs.readFileSync(historyPath, 'utf-8'));
|
||||
return entries.slice(0, count);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Display Helpers
|
||||
// ============================================================================
|
||||
|
||||
function statusColor(status: string): string {
|
||||
switch (status) {
|
||||
case 'idle': return chalk.cyan(status);
|
||||
case 'running': return chalk.yellow(status);
|
||||
case 'paused': return chalk.yellow(status);
|
||||
case 'stopped': return chalk.gray(status);
|
||||
case 'error': return chalk.red(status);
|
||||
default: return chalk.white(status);
|
||||
}
|
||||
}
|
||||
|
||||
function scoreColor(score: number): string {
|
||||
if (score >= 80) return chalk.green(String(score));
|
||||
if (score >= 50) return chalk.yellow(String(score));
|
||||
return chalk.red(String(score));
|
||||
}
|
||||
|
||||
function trendColor(trend: string): string {
|
||||
switch (trend) {
|
||||
case 'improving': return chalk.green(trend);
|
||||
case 'stable': return chalk.cyan(trend);
|
||||
case 'degrading': return chalk.red(trend);
|
||||
default: return chalk.gray(trend);
|
||||
}
|
||||
}
|
||||
|
||||
function formatRelativeTime(date: Date, future = false): string {
|
||||
const diffMs = future ? date.getTime() - Date.now() : Date.now() - date.getTime();
|
||||
if (diffMs < 0) return future ? 'now' : 'just now';
|
||||
if (diffMs < 60_000) return `${Math.floor(diffMs / 1000)}s ${future ? 'from now' : 'ago'}`;
|
||||
if (diffMs < 3_600_000) return `${Math.floor(diffMs / 60_000)}m ${future ? 'from now' : 'ago'}`;
|
||||
return `${Math.floor(diffMs / 3_600_000)}h ${future ? 'from now' : 'ago'}`;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Factory
|
||||
// ============================================================================
|
||||
|
||||
export function createHeartbeatHandler(
|
||||
cleanupAndExit: (code: number) => Promise<never>
|
||||
): HeartbeatHandler {
|
||||
return new HeartbeatHandler(cleanupAndExit);
|
||||
}
|
||||
@@ -16,3 +16,5 @@ export { DomainHandler, createDomainHandler } from './domain-handler.js';
|
||||
export { ProtocolHandler, createProtocolHandler } from './protocol-handler.js';
|
||||
export { BrainHandler, createBrainHandler } from './brain-handler.js';
|
||||
export { HypergraphHandler, createHypergraphHandler } from './hypergraph-handler.js';
|
||||
export { HeartbeatHandler, createHeartbeatHandler } from './heartbeat-handler.js';
|
||||
export { RoutingHandler, createRoutingHandler } from './routing-handler.js';
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
/**
|
||||
* Agentic QE v3 - Routing Command Handler
|
||||
* Imp-18: Economic Routing Model CLI Integration
|
||||
*
|
||||
* Handles the 'aqe routing' command with subcommands:
|
||||
* economics, accuracy, metrics
|
||||
*/
|
||||
|
||||
import { Command } from 'commander';
|
||||
import chalk from 'chalk';
|
||||
import { ICommandHandler, CLIContext } from './interfaces.js';
|
||||
import { toErrorMessage } from '../../shared/error-utils.js';
|
||||
import { createRoutingFeedbackCollector } from '../../routing/routing-feedback.js';
|
||||
import { getGlobalCostTracker } from '../../shared/llm/cost-tracker.js';
|
||||
import type { EconomicReport, EconomicScore } from '../../routing/economic-routing.js';
|
||||
|
||||
// ============================================================================
|
||||
// Routing Handler
|
||||
// ============================================================================
|
||||
|
||||
export class RoutingHandler implements ICommandHandler {
|
||||
readonly name = 'routing';
|
||||
readonly description = 'View routing performance, economics, and accuracy';
|
||||
|
||||
private cleanupAndExit: (code: number) => Promise<never>;
|
||||
|
||||
constructor(cleanupAndExit: (code: number) => Promise<never>) {
|
||||
this.cleanupAndExit = cleanupAndExit;
|
||||
}
|
||||
|
||||
getHelp(): string {
|
||||
return [
|
||||
'aqe routing economics [--complexity <0-1>] [--json] Show tier efficiency & budget',
|
||||
'aqe routing accuracy [--json] Show routing accuracy analysis',
|
||||
'aqe routing metrics [--json] Show per-agent performance',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
register(program: Command, _context: CLIContext): void {
|
||||
const routing = program
|
||||
.command('routing')
|
||||
.description(this.description);
|
||||
|
||||
routing
|
||||
.command('economics')
|
||||
.description('Show economic routing report: tier efficiency, budget, savings')
|
||||
.option('-c, --complexity <value>', 'Task complexity for scoring (0-1)', '0.5')
|
||||
.option('--json', 'Output as JSON')
|
||||
.action(async (options: { complexity: string; json?: boolean }) => {
|
||||
await this.executeEconomics(parseFloat(options.complexity) || 0.5, !!options.json);
|
||||
});
|
||||
|
||||
routing
|
||||
.command('accuracy')
|
||||
.description('Show routing accuracy analysis')
|
||||
.option('--json', 'Output as JSON')
|
||||
.action(async (options: { json?: boolean }) => {
|
||||
await this.executeAccuracy(!!options.json);
|
||||
});
|
||||
|
||||
routing
|
||||
.command('metrics')
|
||||
.description('Show per-agent performance metrics')
|
||||
.option('--json', 'Output as JSON')
|
||||
.action(async (options: { json?: boolean }) => {
|
||||
await this.executeMetrics(!!options.json);
|
||||
});
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Economics
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
private async executeEconomics(complexity: number, json: boolean): Promise<void> {
|
||||
try {
|
||||
const collector = createRoutingFeedbackCollector(100);
|
||||
await collector.initialize();
|
||||
collector.enableEconomicRouting({}, getGlobalCostTracker());
|
||||
|
||||
const report = collector.getEconomicReport();
|
||||
if (!report) {
|
||||
console.error(chalk.red('\n Economic routing is not available.\n'));
|
||||
await this.cleanupAndExit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify(report, (_k, v) => (v === Infinity ? 'Infinity' : v), 2));
|
||||
await this.cleanupAndExit(0);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(chalk.blue('\n Economic Routing Report'));
|
||||
console.log(chalk.gray(' ' + '\u2500'.repeat(50)));
|
||||
|
||||
// Tier efficiency table
|
||||
console.log(chalk.white('\n Tier Efficiency (complexity=' + complexity.toFixed(1) + '):\n'));
|
||||
console.log(chalk.gray(' Tier Quality Cost/Task Q/$ Score'));
|
||||
console.log(chalk.gray(' ' + '\u2500'.repeat(50)));
|
||||
|
||||
const scores = collector.getEconomicScore(complexity) ?? report.tierEfficiency;
|
||||
for (const s of scores) {
|
||||
const qpd = isFinite(s.qualityPerDollar) ? s.qualityPerDollar.toFixed(1) : '\u221E';
|
||||
console.log(
|
||||
` ${padRight(s.tier, 10)}` +
|
||||
`${chalk.cyan(s.qualityScore.toFixed(2))} ` +
|
||||
`$${s.estimatedCostUsd.toFixed(4)} ` +
|
||||
`${chalk.yellow(padLeft(qpd, 8))} ` +
|
||||
`${scoreColor(s.economicScore)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Budget
|
||||
console.log(chalk.white('\n Budget:'));
|
||||
console.log(` Hourly cost: $${report.currentHourlyCostUsd.toFixed(4)}`);
|
||||
console.log(` Daily cost: $${report.currentDailyCostUsd.toFixed(4)}`);
|
||||
if (report.budgetRemaining.hourly !== null) {
|
||||
console.log(` Hourly left: $${report.budgetRemaining.hourly.toFixed(4)}`);
|
||||
}
|
||||
if (report.budgetRemaining.daily !== null) {
|
||||
console.log(` Daily left: $${report.budgetRemaining.daily.toFixed(4)}`);
|
||||
}
|
||||
|
||||
// Recommendation
|
||||
console.log(chalk.white('\n Recommendation:'));
|
||||
console.log(` ${chalk.green(report.recommendation)}`);
|
||||
|
||||
if (report.savingsOpportunity) {
|
||||
console.log(chalk.white('\n Savings Opportunity:'));
|
||||
console.log(` ${chalk.yellow(report.savingsOpportunity.description)}`);
|
||||
}
|
||||
|
||||
console.log('');
|
||||
await this.cleanupAndExit(0);
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n Failed to get economic report:'), toErrorMessage(error));
|
||||
await this.cleanupAndExit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Accuracy
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
private async executeAccuracy(json: boolean): Promise<void> {
|
||||
try {
|
||||
const collector = createRoutingFeedbackCollector(10000);
|
||||
await collector.initialize();
|
||||
|
||||
const accuracy = collector.analyzeRoutingAccuracy();
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify(accuracy, null, 2));
|
||||
await this.cleanupAndExit(0);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(chalk.blue('\n Routing Accuracy Analysis'));
|
||||
console.log(chalk.gray(' ' + '\u2500'.repeat(40)));
|
||||
console.log(` Total outcomes: ${chalk.cyan(String(accuracy.totalOutcomes))}`);
|
||||
console.log(` Followed recs: ${chalk.cyan(String(accuracy.followedRecommendations))}`);
|
||||
console.log(` Override rate: ${chalk.yellow((accuracy.overrideRate * 100).toFixed(1) + '%')}`);
|
||||
console.log(` Rec success rate: ${scoreColor100(accuracy.recommendationSuccessRate * 100)}`);
|
||||
console.log(` Override success rate: ${scoreColor100(accuracy.overrideSuccessRate * 100)}`);
|
||||
console.log(` Confidence correlation: ${chalk.cyan(accuracy.confidenceCorrelation.toFixed(3))}`);
|
||||
|
||||
// Recommendations
|
||||
const recs = collector.getImprovementRecommendations();
|
||||
if (recs.length > 0) {
|
||||
console.log(chalk.white('\n Recommendations:'));
|
||||
for (const rec of recs) {
|
||||
console.log(` ${chalk.gray('\u2022')} ${rec}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('');
|
||||
await this.cleanupAndExit(0);
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n Failed to analyze routing accuracy:'), toErrorMessage(error));
|
||||
await this.cleanupAndExit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Metrics
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
private async executeMetrics(json: boolean): Promise<void> {
|
||||
try {
|
||||
const collector = createRoutingFeedbackCollector(10000);
|
||||
await collector.initialize();
|
||||
|
||||
const metrics = collector.getAllAgentMetrics();
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify(metrics, null, 2));
|
||||
await this.cleanupAndExit(0);
|
||||
return;
|
||||
}
|
||||
|
||||
if (metrics.length === 0) {
|
||||
console.log(chalk.yellow('\n No routing metrics available yet. Run some QE tasks first.\n'));
|
||||
await this.cleanupAndExit(0);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(chalk.blue('\n Agent Routing Metrics'));
|
||||
console.log(chalk.gray(' ' + '\u2500'.repeat(60)));
|
||||
console.log(chalk.gray(' Agent Tasks Success Quality Trend'));
|
||||
console.log(chalk.gray(' ' + '\u2500'.repeat(60)));
|
||||
|
||||
for (const m of metrics.slice(0, 20)) {
|
||||
const trend = m.trend === 'improving' ? chalk.green('\u2191')
|
||||
: m.trend === 'declining' ? chalk.red('\u2193')
|
||||
: chalk.gray('\u2192');
|
||||
console.log(
|
||||
` ${padRight(m.agentId, 24)}` +
|
||||
`${padLeft(String(m.totalTasks), 5)} ` +
|
||||
`${scoreColor100(m.successRate * 100)} ` +
|
||||
`${chalk.cyan(m.avgQualityScore.toFixed(2))} ` +
|
||||
`${trend} ${m.trend}`,
|
||||
);
|
||||
}
|
||||
|
||||
const stats = collector.getStats();
|
||||
console.log(chalk.gray(`\n ${stats.totalOutcomes} total outcomes, ${stats.uniqueAgentsUsed} agents`));
|
||||
console.log('');
|
||||
await this.cleanupAndExit(0);
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n Failed to get agent metrics:'), toErrorMessage(error));
|
||||
await this.cleanupAndExit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helpers
|
||||
// ============================================================================
|
||||
|
||||
function padRight(str: string, len: number): string {
|
||||
return str.length >= len ? str : str + ' '.repeat(len - str.length);
|
||||
}
|
||||
|
||||
function padLeft(str: string, len: number): string {
|
||||
return str.length >= len ? str : ' '.repeat(len - str.length) + str;
|
||||
}
|
||||
|
||||
function scoreColor(score: number): string {
|
||||
const pct = score * 100;
|
||||
const str = score.toFixed(3);
|
||||
if (pct >= 70) return chalk.green(str);
|
||||
if (pct >= 40) return chalk.yellow(str);
|
||||
return chalk.red(str);
|
||||
}
|
||||
|
||||
function scoreColor100(pct: number): string {
|
||||
const str = pct.toFixed(1) + '%';
|
||||
if (pct >= 70) return chalk.green(str);
|
||||
if (pct >= 40) return chalk.yellow(str);
|
||||
return chalk.red(str);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Factory
|
||||
// ============================================================================
|
||||
|
||||
export function createRoutingHandler(
|
||||
cleanupAndExit: (code: number) => Promise<never>,
|
||||
): RoutingHandler {
|
||||
return new RoutingHandler(cleanupAndExit);
|
||||
}
|
||||
@@ -991,6 +991,7 @@ import { createPlatformCommand } from './commands/platform.js';
|
||||
import { createProveCommand } from './commands/prove.js';
|
||||
import { createRuVectorCommand } from './commands/ruvector-commands.js';
|
||||
import { createAuditCommand } from './commands/audit.js';
|
||||
import { createPipelineCommand } from './commands/pipeline.js';
|
||||
|
||||
program.addCommand(createTokenUsageCommand());
|
||||
program.addCommand(createLLMRouterCommand());
|
||||
@@ -1003,6 +1004,7 @@ program.addCommand(createPlatformCommand());
|
||||
program.addCommand(createProveCommand(context, cleanupAndExit, ensureInitialized));
|
||||
program.addCommand(createRuVectorCommand());
|
||||
program.addCommand(createAuditCommand(context, cleanupAndExit, ensureInitialized));
|
||||
program.addCommand(createPipelineCommand(context, cleanupAndExit, ensureInitialized));
|
||||
|
||||
// ============================================================================
|
||||
// Shutdown Handlers
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
/**
|
||||
* Deterministic Step Actions (Imp-9)
|
||||
*
|
||||
* Built-in actions that execute WITHOUT LLM tokens. Each action maps to a
|
||||
* domain + action pair and is automatically wired into the WorkflowOrchestrator
|
||||
* as a fallback before domain service delegation.
|
||||
*
|
||||
* When the caller supplies explicit input values they are used directly.
|
||||
* When inputs are omitted, the actions query the unified SQLite database
|
||||
* for live metrics — keeping execution fully deterministic (SQL-only).
|
||||
*/
|
||||
|
||||
import { Result, ok, err, DomainName } from '../shared/types/index.js';
|
||||
import type { WorkflowContext } from './workflow-types.js';
|
||||
import { toErrorMessage } from '../shared/error-utils.js';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* A deterministic action that runs without any LLM tokens.
|
||||
*/
|
||||
export interface DeterministicAction {
|
||||
/** Unique action identifier */
|
||||
id: string;
|
||||
/** Target domain */
|
||||
domain: DomainName;
|
||||
/** Action name within the domain */
|
||||
action: string;
|
||||
/** Execute the action with the given input */
|
||||
execute(
|
||||
input: Record<string, unknown>,
|
||||
context: WorkflowContext,
|
||||
): Promise<Result<Record<string, unknown>, Error>>;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// DB helpers (fail-safe: return null when DB unavailable)
|
||||
// ============================================================================
|
||||
|
||||
interface DbAccessor {
|
||||
prepare(sql: string): { get(...params: unknown[]): unknown; all(...params: unknown[]): unknown[] };
|
||||
}
|
||||
|
||||
function tryGetDb(): DbAccessor | null {
|
||||
try {
|
||||
// Dynamic import to avoid hard dependency — the module may not be initialised
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const { getUnifiedMemory } = require('../kernel/unified-memory.js');
|
||||
const um = getUnifiedMemory();
|
||||
if (!um.isInitialized()) return null;
|
||||
return um.getDatabase() as DbAccessor;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Quality Gate Check
|
||||
// ============================================================================
|
||||
|
||||
const qualityGateCheck: DeterministicAction = {
|
||||
id: 'quality-gate-check',
|
||||
domain: 'quality-assessment',
|
||||
action: 'gate-check',
|
||||
async execute(input) {
|
||||
const coverageMin = typeof input.coverageMin === 'number' ? input.coverageMin : 80;
|
||||
const testsPassingMin = typeof input.testsPassingMin === 'number' ? input.testsPassingMin : 90;
|
||||
const maxBugs = typeof input.maxBugs === 'number' ? input.maxBugs : 5;
|
||||
|
||||
// ----- Fetch live data from DB when not supplied by caller -----
|
||||
let currentCoverage = typeof input.currentCoverage === 'number' ? input.currentCoverage : null;
|
||||
let currentTestsPassingRate = typeof input.currentTestsPassingRate === 'number' ? input.currentTestsPassingRate : null;
|
||||
let currentBugs = typeof input.currentBugs === 'number' ? input.currentBugs : null;
|
||||
|
||||
if (currentCoverage === null || currentTestsPassingRate === null || currentBugs === null) {
|
||||
const db = tryGetDb();
|
||||
if (db) {
|
||||
try {
|
||||
// Latest coverage from coverage_sessions
|
||||
if (currentCoverage === null) {
|
||||
const row = db.prepare(
|
||||
`SELECT after_lines FROM coverage_sessions ORDER BY created_at DESC LIMIT 1`,
|
||||
).get() as { after_lines: number } | undefined;
|
||||
currentCoverage = row?.after_lines ?? 0;
|
||||
}
|
||||
|
||||
// Test pass rate from recent test_outcomes
|
||||
if (currentTestsPassingRate === null) {
|
||||
const row = db.prepare(
|
||||
`SELECT COUNT(*) as total,
|
||||
SUM(CASE WHEN passed = 1 THEN 1 ELSE 0 END) as passed
|
||||
FROM test_outcomes
|
||||
WHERE created_at > datetime('now', '-7 days')`,
|
||||
).get() as { total: number; passed: number } | undefined;
|
||||
currentTestsPassingRate =
|
||||
row && row.total > 0 ? (row.passed / row.total) * 100 : 0;
|
||||
}
|
||||
|
||||
// Bug count: failed non-flaky tests in the last 7 days
|
||||
if (currentBugs === null) {
|
||||
const row = db.prepare(
|
||||
`SELECT COUNT(*) as bugs FROM test_outcomes
|
||||
WHERE passed = 0 AND flaky = 0
|
||||
AND created_at > datetime('now', '-7 days')`,
|
||||
).get() as { bugs: number } | undefined;
|
||||
currentBugs = row?.bugs ?? 0;
|
||||
}
|
||||
} catch {
|
||||
// Graceful degradation — fall through to defaults
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply defaults for anything still null
|
||||
currentCoverage = currentCoverage ?? 0;
|
||||
currentTestsPassingRate = currentTestsPassingRate ?? 0;
|
||||
currentBugs = currentBugs ?? 0;
|
||||
|
||||
const coveragePassed = currentCoverage >= coverageMin;
|
||||
const testsPassed = currentTestsPassingRate >= testsPassingMin;
|
||||
const bugsPassed = currentBugs <= maxBugs;
|
||||
const passed = coveragePassed && testsPassed && bugsPassed;
|
||||
|
||||
// Score is a normalized 0-1 value
|
||||
const coverageScore = Math.min(currentCoverage / coverageMin, 1);
|
||||
const testsScore = Math.min(currentTestsPassingRate / testsPassingMin, 1);
|
||||
const bugsScore = maxBugs > 0 ? Math.max(0, 1 - currentBugs / maxBugs) : (currentBugs === 0 ? 1 : 0);
|
||||
const score = (coverageScore + testsScore + bugsScore) / 3;
|
||||
|
||||
return ok({
|
||||
passed,
|
||||
score: Math.round(score * 100) / 100,
|
||||
source: typeof input.currentCoverage === 'number' ? 'input' : 'database',
|
||||
details: {
|
||||
coverage: { current: currentCoverage, threshold: coverageMin, passed: coveragePassed },
|
||||
testsPassing: { current: currentTestsPassingRate, threshold: testsPassingMin, passed: testsPassed },
|
||||
bugs: { current: currentBugs, threshold: maxBugs, passed: bugsPassed },
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Coverage Threshold Check
|
||||
// ============================================================================
|
||||
|
||||
const coverageThresholdCheck: DeterministicAction = {
|
||||
id: 'coverage-threshold',
|
||||
domain: 'coverage-analysis',
|
||||
action: 'threshold-check',
|
||||
async execute(input) {
|
||||
const minCoverage = typeof input.minCoverage === 'number' ? input.minCoverage : 80;
|
||||
|
||||
// ----- Fetch from DB when not supplied -----
|
||||
let currentCoverage = typeof input.currentCoverage === 'number' ? input.currentCoverage : null;
|
||||
let source: 'input' | 'database' = 'input';
|
||||
|
||||
if (currentCoverage === null) {
|
||||
source = 'database';
|
||||
const db = tryGetDb();
|
||||
if (db) {
|
||||
try {
|
||||
const row = db.prepare(
|
||||
`SELECT after_lines FROM coverage_sessions ORDER BY created_at DESC LIMIT 1`,
|
||||
).get() as { after_lines: number } | undefined;
|
||||
currentCoverage = row?.after_lines ?? 0;
|
||||
} catch {
|
||||
currentCoverage = 0;
|
||||
}
|
||||
} else {
|
||||
currentCoverage = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const passed = currentCoverage >= minCoverage;
|
||||
const gap = passed ? 0 : Math.round((minCoverage - currentCoverage) * 100) / 100;
|
||||
|
||||
return ok({
|
||||
currentCoverage,
|
||||
passed,
|
||||
gap,
|
||||
minCoverage,
|
||||
source,
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Pattern Health Check
|
||||
// ============================================================================
|
||||
|
||||
const patternHealthCheck: DeterministicAction = {
|
||||
id: 'pattern-health',
|
||||
domain: 'learning-optimization',
|
||||
action: 'health-check',
|
||||
async execute(input) {
|
||||
// ----- Fetch from DB when not supplied -----
|
||||
let totalPatterns = typeof input.totalPatterns === 'number' ? input.totalPatterns : null;
|
||||
let activePatterns = typeof input.activePatterns === 'number' ? input.activePatterns : null;
|
||||
let avgConfidence = typeof input.avgConfidence === 'number' ? input.avgConfidence : null;
|
||||
let source: 'input' | 'database' = 'input';
|
||||
|
||||
if (totalPatterns === null || activePatterns === null || avgConfidence === null) {
|
||||
source = 'database';
|
||||
const db = tryGetDb();
|
||||
if (db) {
|
||||
try {
|
||||
const row = db.prepare(`
|
||||
SELECT COUNT(*) as total,
|
||||
SUM(CASE WHEN deprecated_at IS NULL AND confidence >= 0.3 THEN 1 ELSE 0 END) as active,
|
||||
AVG(confidence) as avg_conf
|
||||
FROM qe_patterns
|
||||
`).get() as { total: number; active: number; avg_conf: number | null } | undefined;
|
||||
|
||||
totalPatterns = totalPatterns ?? (row?.total ?? 0);
|
||||
activePatterns = activePatterns ?? (row?.active ?? 0);
|
||||
avgConfidence = avgConfidence ?? (row?.avg_conf ?? 0);
|
||||
} catch {
|
||||
// Graceful degradation
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply defaults for anything still null
|
||||
totalPatterns = totalPatterns ?? 0;
|
||||
activePatterns = activePatterns ?? 0;
|
||||
avgConfidence = avgConfidence ?? 0;
|
||||
|
||||
// Health score: weighted combination of volume, activity ratio, and confidence
|
||||
const volumeScore = Math.min(totalPatterns / 100, 1); // 100 patterns = max volume
|
||||
const activityRatio = totalPatterns > 0 ? activePatterns / totalPatterns : 0;
|
||||
const healthScore = Math.round(
|
||||
(volumeScore * 0.3 + activityRatio * 0.3 + avgConfidence * 0.4) * 100,
|
||||
) / 100;
|
||||
|
||||
return ok({
|
||||
totalPatterns,
|
||||
activePatterns,
|
||||
avgConfidence,
|
||||
healthScore,
|
||||
source,
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Routing Accuracy Check
|
||||
// ============================================================================
|
||||
|
||||
const routingAccuracyCheck: DeterministicAction = {
|
||||
id: 'routing-accuracy',
|
||||
domain: 'learning-optimization',
|
||||
action: 'routing-check',
|
||||
async execute(input) {
|
||||
// ----- Fetch from DB when not supplied -----
|
||||
let totalOutcomes = typeof input.totalOutcomes === 'number' ? input.totalOutcomes : null;
|
||||
let successfulOutcomes = typeof input.successfulOutcomes === 'number' ? input.successfulOutcomes : null;
|
||||
let confidenceCorrelation = typeof input.confidenceCorrelation === 'number' ? input.confidenceCorrelation : null;
|
||||
let source: 'input' | 'database' = 'input';
|
||||
|
||||
if (totalOutcomes === null || successfulOutcomes === null) {
|
||||
source = 'database';
|
||||
const db = tryGetDb();
|
||||
if (db) {
|
||||
try {
|
||||
const row = db.prepare(`
|
||||
SELECT COUNT(*) as total,
|
||||
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful
|
||||
FROM routing_outcomes
|
||||
`).get() as { total: number; successful: number } | undefined;
|
||||
|
||||
totalOutcomes = totalOutcomes ?? (row?.total ?? 0);
|
||||
successfulOutcomes = successfulOutcomes ?? (row?.successful ?? 0);
|
||||
} catch {
|
||||
// Graceful degradation
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply defaults
|
||||
totalOutcomes = totalOutcomes ?? 0;
|
||||
successfulOutcomes = successfulOutcomes ?? 0;
|
||||
confidenceCorrelation = confidenceCorrelation ?? 0;
|
||||
|
||||
const successRate = totalOutcomes > 0
|
||||
? Math.round((successfulOutcomes / totalOutcomes) * 10000) / 100
|
||||
: 0;
|
||||
|
||||
return ok({
|
||||
successRate,
|
||||
totalOutcomes,
|
||||
successfulOutcomes,
|
||||
confidenceCorrelation,
|
||||
source,
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Registry
|
||||
// ============================================================================
|
||||
|
||||
/** All built-in deterministic actions */
|
||||
const DETERMINISTIC_ACTIONS: DeterministicAction[] = [
|
||||
qualityGateCheck,
|
||||
coverageThresholdCheck,
|
||||
patternHealthCheck,
|
||||
routingAccuracyCheck,
|
||||
];
|
||||
|
||||
/**
|
||||
* Look up a deterministic action by domain + action.
|
||||
* Returns undefined if no built-in action matches.
|
||||
*/
|
||||
export function findDeterministicAction(
|
||||
domain: DomainName,
|
||||
action: string,
|
||||
): DeterministicAction | undefined {
|
||||
return DETERMINISTIC_ACTIONS.find(
|
||||
(a) => a.domain === domain && a.action === action,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registered deterministic actions.
|
||||
*/
|
||||
export function getAllDeterministicActions(): readonly DeterministicAction[] {
|
||||
return DETERMINISTIC_ACTIONS;
|
||||
}
|
||||
@@ -45,6 +45,7 @@ export type {
|
||||
WorkflowCompletedPayload,
|
||||
WorkflowFailedPayload,
|
||||
StepEventPayload,
|
||||
StepAwaitingApprovalPayload,
|
||||
IWorkflowOrchestrator,
|
||||
DomainAction,
|
||||
DomainActionRegistry,
|
||||
@@ -73,10 +74,14 @@ import type {
|
||||
WorkflowCompletedPayload,
|
||||
WorkflowFailedPayload,
|
||||
StepEventPayload,
|
||||
StepAwaitingApprovalPayload,
|
||||
} from './workflow-types.js';
|
||||
|
||||
import { WorkflowEvents, DEFAULT_WORKFLOW_CONFIG } from './workflow-types.js';
|
||||
|
||||
// Import deterministic actions
|
||||
import { findDeterministicAction } from './deterministic-actions.js';
|
||||
|
||||
// Import built-in workflows
|
||||
import { getBuiltInWorkflows, BUILTIN_WORKFLOW_IDS } from './workflow-builtin.js';
|
||||
|
||||
@@ -90,6 +95,11 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator {
|
||||
private readonly executions: Map<string, WorkflowExecutionStatus> = new Map();
|
||||
private readonly actionRegistry: DomainActionRegistry = {};
|
||||
private readonly eventSubscriptions: Subscription[] = [];
|
||||
/** Pending approval gates: Map<`${executionId}:${stepId}`, gate> */
|
||||
private readonly approvalGates: Map<string, {
|
||||
resolve: (result: { approved: boolean; reason?: string }) => void;
|
||||
}> = new Map();
|
||||
|
||||
private initialized = false;
|
||||
|
||||
constructor(
|
||||
@@ -225,6 +235,14 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator {
|
||||
execution.completedAt = new Date();
|
||||
execution.duration = execution.completedAt.getTime() - execution.startedAt.getTime();
|
||||
|
||||
// Clean up any pending approval gates for this execution to prevent timer leaks
|
||||
for (const [key, gate] of this.approvalGates.entries()) {
|
||||
if (key.startsWith(`${executionId}:`)) {
|
||||
gate.resolve({ approved: false, reason: 'Workflow cancelled' });
|
||||
this.approvalGates.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
await this.publishEvent(WorkflowEvents.WorkflowCancelled, {
|
||||
executionId, workflowId: execution.workflowId, workflowName: execution.workflowName,
|
||||
}, execution.context.metadata.correlationId);
|
||||
@@ -277,6 +295,38 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator {
|
||||
return this.workflows.get(workflowId);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Approval Gate Methods
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Approve a step that is awaiting approval. Returns true if the step
|
||||
* was found and approved, false otherwise.
|
||||
*/
|
||||
approveStep(executionId: string, stepId: string): boolean {
|
||||
const key = `${executionId}:${stepId}`;
|
||||
const gate = this.approvalGates.get(key);
|
||||
if (!gate) return false;
|
||||
|
||||
gate.resolve({ approved: true });
|
||||
this.approvalGates.delete(key);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject a step that is awaiting approval. Returns true if the step
|
||||
* was found and rejected, false otherwise.
|
||||
*/
|
||||
rejectStep(executionId: string, stepId: string, reason?: string): boolean {
|
||||
const key = `${executionId}:${stepId}`;
|
||||
const gate = this.approvalGates.get(key);
|
||||
if (!gate) return false;
|
||||
|
||||
gate.resolve({ approved: false, reason });
|
||||
this.approvalGates.delete(key);
|
||||
return true;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Public Utility Methods
|
||||
// ============================================================================
|
||||
@@ -521,6 +571,21 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator {
|
||||
|
||||
this.mapStepOutput(step, output, execution.context);
|
||||
|
||||
// Handle approval gate if configured
|
||||
if (step.approval) {
|
||||
const approvalResult = await this.waitForApproval(step, execution);
|
||||
if (!approvalResult.approved) {
|
||||
result.status = 'failed';
|
||||
result.error = approvalResult.reason || 'Step rejected at approval gate';
|
||||
result.output = output;
|
||||
result.completedAt = new Date();
|
||||
result.duration = result.completedAt.getTime() - startedAt.getTime();
|
||||
execution.stepResults.set(step.id, result);
|
||||
await this.publishStepFailed(execution, step, result.error);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
result.status = 'completed'; result.output = output; result.completedAt = new Date();
|
||||
result.duration = result.completedAt.getTime() - startedAt.getTime();
|
||||
execution.stepResults.set(step.id, result);
|
||||
@@ -554,12 +619,21 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator {
|
||||
private async executeStepAction(
|
||||
step: WorkflowStepDefinition, input: Record<string, unknown>, context: WorkflowContext, timeout: number
|
||||
): Promise<unknown> {
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
setTimeout(() => reject(new Error(`Step timeout after ${timeout}ms`)), timeout);
|
||||
});
|
||||
|
||||
// Check for a deterministic action first (zero LLM tokens)
|
||||
const deterministicAction = findDeterministicAction(step.domain, step.action);
|
||||
if (deterministicAction) {
|
||||
const actionResult = await Promise.race([deterministicAction.execute(input, context), timeoutPromise]);
|
||||
if (!actionResult.success) throw actionResult.error;
|
||||
return actionResult.value;
|
||||
}
|
||||
|
||||
// Fall back to registered domain actions
|
||||
const domainActions = this.actionRegistry[step.domain];
|
||||
if (domainActions?.[step.action]) {
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
setTimeout(() => reject(new Error(`Step timeout after ${timeout}ms`)), timeout);
|
||||
});
|
||||
|
||||
const actionResult = await Promise.race([domainActions[step.action](input, context), timeoutPromise]);
|
||||
if (!actionResult.success) throw actionResult.error;
|
||||
return actionResult.value;
|
||||
@@ -887,6 +961,67 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Private Methods - Approval Gates
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Wait for external approval (or auto-approve after timeout).
|
||||
* Returns an object with approved status and optional rejection reason.
|
||||
*/
|
||||
private async waitForApproval(
|
||||
step: WorkflowStepDefinition,
|
||||
execution: WorkflowExecutionStatus,
|
||||
): Promise<{ approved: boolean; reason?: string }> {
|
||||
const approvalConfig = typeof step.approval === 'object' ? step.approval : {};
|
||||
const autoApproveAfter = approvalConfig.autoApproveAfter ?? 300000; // 5 min default
|
||||
const message = approvalConfig.message ?? `Awaiting approval for step: ${step.name}`;
|
||||
|
||||
// Update step result status
|
||||
const stepResult = execution.stepResults.get(step.id);
|
||||
if (stepResult) stepResult.status = 'awaiting_approval';
|
||||
|
||||
// Emit StepAwaitingApproval event
|
||||
await this.publishEvent<StepAwaitingApprovalPayload>(
|
||||
WorkflowEvents.StepAwaitingApproval,
|
||||
{
|
||||
executionId: execution.executionId,
|
||||
workflowId: execution.workflowId,
|
||||
stepId: step.id,
|
||||
stepName: step.name,
|
||||
domain: step.domain,
|
||||
message,
|
||||
autoApproveAfter: autoApproveAfter > 0 ? autoApproveAfter : undefined,
|
||||
},
|
||||
execution.context.metadata.correlationId,
|
||||
);
|
||||
|
||||
const key = `${execution.executionId}:${step.id}`;
|
||||
|
||||
return new Promise<{ approved: boolean; reason?: string }>((resolve) => {
|
||||
// Store the gate so approveStep/rejectStep can resolve it
|
||||
const gate: { resolve: (result: { approved: boolean; reason?: string }) => void } = { resolve };
|
||||
this.approvalGates.set(key, gate);
|
||||
|
||||
// Auto-approve timer (0 = never auto-approve)
|
||||
if (autoApproveAfter > 0) {
|
||||
const timer = setTimeout(() => {
|
||||
if (this.approvalGates.has(key)) {
|
||||
this.approvalGates.delete(key);
|
||||
resolve({ approved: true }); // auto-approve on timeout
|
||||
}
|
||||
}, autoApproveAfter);
|
||||
|
||||
// Wrap resolve to also clear the timer
|
||||
const originalResolve = gate.resolve;
|
||||
gate.resolve = (result) => {
|
||||
clearTimeout(timer);
|
||||
originalResolve(result);
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Private Methods - Utilities
|
||||
// ============================================================================
|
||||
|
||||
@@ -22,7 +22,7 @@ export type StepExecutionMode = 'sequential' | 'parallel';
|
||||
/**
|
||||
* Step status
|
||||
*/
|
||||
export type StepStatus = 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
|
||||
export type StepStatus = 'pending' | 'running' | 'completed' | 'failed' | 'skipped' | 'awaiting_approval';
|
||||
|
||||
/**
|
||||
* Workflow status
|
||||
@@ -84,6 +84,13 @@ export interface WorkflowStepDefinition {
|
||||
};
|
||||
/** Continue workflow on failure */
|
||||
continueOnFailure?: boolean;
|
||||
/** Approval gate configuration */
|
||||
approval?: boolean | {
|
||||
/** Auto-approve after this many ms (0 = never auto-approve) */
|
||||
autoApproveAfter?: number;
|
||||
/** Approval prompt message */
|
||||
message?: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -203,6 +210,9 @@ export const WorkflowEvents = {
|
||||
StepCompleted: 'workflow.StepCompleted',
|
||||
StepFailed: 'workflow.StepFailed',
|
||||
StepSkipped: 'workflow.StepSkipped',
|
||||
StepAwaitingApproval: 'workflow.StepAwaitingApproval',
|
||||
StepApproved: 'workflow.StepApproved',
|
||||
StepRejected: 'workflow.StepRejected',
|
||||
} as const;
|
||||
|
||||
export interface WorkflowStartedPayload {
|
||||
@@ -237,6 +247,11 @@ export interface StepEventPayload {
|
||||
domain: DomainName;
|
||||
}
|
||||
|
||||
export interface StepAwaitingApprovalPayload extends StepEventPayload {
|
||||
message?: string;
|
||||
autoApproveAfter?: number;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Workflow Orchestrator Interface
|
||||
// ============================================================================
|
||||
@@ -270,6 +285,10 @@ export interface IWorkflowOrchestrator {
|
||||
getActiveExecutions(): WorkflowExecutionStatus[];
|
||||
/** Get workflow definition */
|
||||
getWorkflow(workflowId: string): WorkflowDefinition | undefined;
|
||||
/** Approve a step that is awaiting approval */
|
||||
approveStep(executionId: string, stepId: string): boolean;
|
||||
/** Reject a step that is awaiting approval */
|
||||
rejectStep(executionId: string, stepId: string, reason?: string): boolean;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
|
||||
@@ -257,6 +257,14 @@ export class YamlPipelineLoader {
|
||||
? s.continueOnFailure
|
||||
: undefined;
|
||||
|
||||
// Validate approval gate configuration
|
||||
let approval: WorkflowStepDefinition['approval'];
|
||||
if (s.approval !== undefined) {
|
||||
const approvalResult = this.validateApproval(s.approval, s.id as string);
|
||||
if (!approvalResult.success) return approvalResult;
|
||||
approval = approvalResult.value;
|
||||
}
|
||||
|
||||
const step: WorkflowStepDefinition = {
|
||||
id: s.id,
|
||||
name: s.name,
|
||||
@@ -271,6 +279,7 @@ export class YamlPipelineLoader {
|
||||
...(retry !== undefined && { retry }),
|
||||
...(rollback !== undefined && { rollback }),
|
||||
...(continueOnFailure !== undefined && { continueOnFailure }),
|
||||
...(approval !== undefined && { approval }),
|
||||
};
|
||||
|
||||
steps.push(step);
|
||||
@@ -404,6 +413,40 @@ export class YamlPipelineLoader {
|
||||
});
|
||||
}
|
||||
|
||||
private validateApproval(
|
||||
raw: unknown,
|
||||
stepId: string,
|
||||
): Result<NonNullable<WorkflowStepDefinition['approval']>, Error> {
|
||||
// Simple boolean form: approval: true
|
||||
if (typeof raw === 'boolean') {
|
||||
return ok(raw);
|
||||
}
|
||||
|
||||
// Object form: approval: { autoApproveAfter, message }
|
||||
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
||||
return err(new Error(`Step '${stepId}': 'approval' must be a boolean or object`));
|
||||
}
|
||||
|
||||
const a = raw as Record<string, unknown>;
|
||||
const result: { autoApproveAfter?: number; message?: string } = {};
|
||||
|
||||
if (a.autoApproveAfter !== undefined) {
|
||||
if (typeof a.autoApproveAfter !== 'number' || a.autoApproveAfter < 0) {
|
||||
return err(new Error(`Step '${stepId}': approval.autoApproveAfter must be a non-negative number`));
|
||||
}
|
||||
result.autoApproveAfter = a.autoApproveAfter;
|
||||
}
|
||||
|
||||
if (a.message !== undefined) {
|
||||
if (typeof a.message !== 'string') {
|
||||
return err(new Error(`Step '${stepId}': approval.message must be a string`));
|
||||
}
|
||||
result.message = a.message;
|
||||
}
|
||||
|
||||
return ok(result);
|
||||
}
|
||||
|
||||
/** Validate that a value is Record<string, string>, undefined, or return an Error. */
|
||||
private validateStringRecord(
|
||||
value: unknown,
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
/**
|
||||
* Heartbeat MCP Handlers (Imp-10)
|
||||
*
|
||||
* Exposes the token-free heartbeat scheduler through MCP tools:
|
||||
* - heartbeat_status: Returns current heartbeat health and metrics
|
||||
* - heartbeat_trigger: Triggers an immediate heartbeat cycle
|
||||
* - heartbeat_log: Returns daily log entries for a given date
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import type { ToolResult } from '../types.js';
|
||||
import { HeartbeatSchedulerWorker } from '../../workers/workers/heartbeat-scheduler.js';
|
||||
import type { WorkerResult } from '../../workers/interfaces.js';
|
||||
import { toErrorMessage } from '../../shared/error-utils.js';
|
||||
import { findProjectRoot } from '../../kernel/unified-memory.js';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
export type HeartbeatStatusParams = Record<string, never>;
|
||||
|
||||
export interface HeartbeatStatusResult {
|
||||
status: string;
|
||||
healthScore: number;
|
||||
totalExecutions: number;
|
||||
successfulExecutions: number;
|
||||
failedExecutions: number;
|
||||
avgDurationMs: number;
|
||||
lastRunAt: string | null;
|
||||
nextRunAt: string | null;
|
||||
lastResult: {
|
||||
success: boolean;
|
||||
durationMs: number;
|
||||
healthScore: number;
|
||||
trend: string;
|
||||
domainMetrics: Record<string, number | string>;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface HeartbeatTriggerParams {
|
||||
/** Optional: timeout in milliseconds (default: 60000) */
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
export interface HeartbeatTriggerResult {
|
||||
success: boolean;
|
||||
durationMs: number;
|
||||
healthScore: number;
|
||||
trend: string;
|
||||
promoted: number;
|
||||
deprecated: number;
|
||||
decayed: number;
|
||||
pendingExperiences: number;
|
||||
avgConfidence: number;
|
||||
findingsCount: number;
|
||||
recommendationsCount: number;
|
||||
}
|
||||
|
||||
export interface HeartbeatLogParams {
|
||||
/** Date in YYYY-MM-DD format (defaults to today) */
|
||||
date?: string;
|
||||
}
|
||||
|
||||
export interface HeartbeatLogResult {
|
||||
date: string;
|
||||
exists: boolean;
|
||||
content: string;
|
||||
lineCount: number;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Shared State
|
||||
// ============================================================================
|
||||
|
||||
let sharedWorker: HeartbeatSchedulerWorker | null = null;
|
||||
|
||||
function getWorker(): HeartbeatSchedulerWorker {
|
||||
if (!sharedWorker) {
|
||||
sharedWorker = new HeartbeatSchedulerWorker();
|
||||
}
|
||||
return sharedWorker;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Handlers
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Get the current heartbeat scheduler status and health metrics.
|
||||
*/
|
||||
export async function handleHeartbeatStatus(
|
||||
_params: HeartbeatStatusParams,
|
||||
): Promise<ToolResult<HeartbeatStatusResult>> {
|
||||
try {
|
||||
const worker = getWorker();
|
||||
await worker.initialize();
|
||||
const health = worker.getHealth();
|
||||
const lastResult = worker.lastResult;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
status: health.status,
|
||||
healthScore: health.healthScore,
|
||||
totalExecutions: health.totalExecutions,
|
||||
successfulExecutions: health.successfulExecutions,
|
||||
failedExecutions: health.failedExecutions,
|
||||
avgDurationMs: health.avgDurationMs,
|
||||
lastRunAt: worker.lastRunAt?.toISOString() ?? null,
|
||||
nextRunAt: worker.nextRunAt?.toISOString() ?? null,
|
||||
lastResult: lastResult
|
||||
? {
|
||||
success: lastResult.success,
|
||||
durationMs: lastResult.durationMs,
|
||||
healthScore: lastResult.metrics.healthScore,
|
||||
trend: lastResult.metrics.trend,
|
||||
domainMetrics: lastResult.metrics.domainMetrics,
|
||||
}
|
||||
: null,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to get heartbeat status: ${toErrorMessage(error)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger an immediate heartbeat cycle and return the result.
|
||||
*/
|
||||
export async function handleHeartbeatTrigger(
|
||||
params: HeartbeatTriggerParams,
|
||||
): Promise<ToolResult<HeartbeatTriggerResult>> {
|
||||
try {
|
||||
const worker = getWorker();
|
||||
await worker.initialize();
|
||||
|
||||
const abortController = new AbortController();
|
||||
|
||||
// The worker already enforces its own 60s timeout via BaseWorker.executeWithTimeout().
|
||||
// Only add an external abort if the caller explicitly requests a *shorter* timeout.
|
||||
const userTimeout = params.timeout && params.timeout > 0 ? params.timeout : null;
|
||||
const timeoutHandle = userTimeout
|
||||
? setTimeout(() => abortController.abort(), userTimeout)
|
||||
: null;
|
||||
|
||||
let result: WorkerResult;
|
||||
try {
|
||||
result = await worker.execute({
|
||||
eventBus: { publish: async () => {} },
|
||||
memory: {
|
||||
get: async () => undefined,
|
||||
set: async () => {},
|
||||
search: async () => [],
|
||||
},
|
||||
logger: {
|
||||
debug: () => {},
|
||||
info: () => {},
|
||||
warn: () => {},
|
||||
error: () => {},
|
||||
},
|
||||
domains: {
|
||||
getDomainAPI: () => undefined,
|
||||
getDomainHealth: () => ({ status: 'healthy', errors: [] }),
|
||||
},
|
||||
signal: abortController.signal,
|
||||
});
|
||||
} finally {
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle);
|
||||
}
|
||||
|
||||
const dm = result.metrics.domainMetrics;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
success: result.success,
|
||||
durationMs: result.durationMs,
|
||||
healthScore: result.metrics.healthScore,
|
||||
trend: result.metrics.trend,
|
||||
promoted: Number(dm.promoted ?? 0),
|
||||
deprecated: Number(dm.deprecated ?? 0),
|
||||
decayed: Number(dm.decayed ?? 0),
|
||||
pendingExperiences: Number(dm.pendingExperiences ?? 0),
|
||||
avgConfidence: Number(dm.avgConfidence ?? 0),
|
||||
findingsCount: result.findings.length,
|
||||
recommendationsCount: result.recommendations.length,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
const msg = toErrorMessage(error);
|
||||
const isTimeout = msg.includes('aborted') || msg.includes('abort') || msg.includes('timed out');
|
||||
return {
|
||||
success: false,
|
||||
error: isTimeout
|
||||
? `Heartbeat timed out after ${params.timeout ?? 60000}ms`
|
||||
: `Failed to trigger heartbeat: ${msg}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the daily log for a given date (defaults to today).
|
||||
*/
|
||||
export async function handleHeartbeatLog(
|
||||
params: HeartbeatLogParams,
|
||||
): Promise<ToolResult<HeartbeatLogResult>> {
|
||||
try {
|
||||
const targetDate = params.date || new Date().toISOString().split('T')[0];
|
||||
|
||||
// Validate date format
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(targetDate)) {
|
||||
return {
|
||||
success: false,
|
||||
error: "Invalid date format. Use YYYY-MM-DD (e.g., '2026-03-27').",
|
||||
};
|
||||
}
|
||||
|
||||
const logDir = path.join(findProjectRoot(), '.agentic-qe', 'logs');
|
||||
const logPath = path.join(logDir, `${targetDate}.md`);
|
||||
|
||||
if (!fs.existsSync(logPath)) {
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
date: targetDate,
|
||||
exists: false,
|
||||
content: '',
|
||||
lineCount: 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(logPath, 'utf-8');
|
||||
const lineCount = content.split('\n').filter((l) => l.trim()).length;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
date: targetDate,
|
||||
exists: true,
|
||||
content,
|
||||
lineCount,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to read heartbeat log: ${toErrorMessage(error)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,8 @@ export {
|
||||
// ADR-051: Model routing handlers
|
||||
handleModelRoute,
|
||||
handleRoutingMetrics,
|
||||
// Imp-18: Economic routing handler
|
||||
handleRoutingEconomics,
|
||||
type TaskOrchestrateResult,
|
||||
type ModelRouteParams,
|
||||
type ModelRouteResult,
|
||||
@@ -115,6 +117,19 @@ export {
|
||||
type HypergraphQueryResult,
|
||||
} from './hypergraph-handler.js';
|
||||
|
||||
// Heartbeat handlers (Imp-10: Token-Free Heartbeat Scheduler)
|
||||
export {
|
||||
handleHeartbeatStatus,
|
||||
handleHeartbeatTrigger,
|
||||
handleHeartbeatLog,
|
||||
type HeartbeatStatusParams,
|
||||
type HeartbeatStatusResult,
|
||||
type HeartbeatTriggerParams,
|
||||
type HeartbeatTriggerResult,
|
||||
type HeartbeatLogParams,
|
||||
type HeartbeatLogResult,
|
||||
} from './heartbeat-handlers.js';
|
||||
|
||||
// Cross-phase handlers
|
||||
export {
|
||||
handleCrossPhaseStore,
|
||||
|
||||
@@ -679,6 +679,78 @@ export async function handleRoutingMetrics(
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Economic Routing Handler (Imp-18, Issue #334)
|
||||
// ============================================================================
|
||||
|
||||
export interface RoutingEconomicsParams {
|
||||
/** Task complexity score 0-1 for tier scoring (default: 0.5) */
|
||||
taskComplexity?: number;
|
||||
}
|
||||
|
||||
export interface RoutingEconomicsResult {
|
||||
tierEfficiency: Array<{
|
||||
tier: string;
|
||||
qualityScore: number;
|
||||
estimatedCostUsd: number;
|
||||
qualityPerDollar: number | string;
|
||||
economicScore: number;
|
||||
}>;
|
||||
currentHourlyCostUsd: number;
|
||||
currentDailyCostUsd: number;
|
||||
budgetRemaining: { hourly: number | null; daily: number | null };
|
||||
recommendation: string;
|
||||
savingsOpportunity: { usd: number; description: string } | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle economic routing report query
|
||||
*/
|
||||
export async function handleRoutingEconomics(
|
||||
params: RoutingEconomicsParams,
|
||||
): Promise<ToolResult<RoutingEconomicsResult>> {
|
||||
try {
|
||||
const { createRoutingFeedbackCollector } = await import('../../routing/routing-feedback.js');
|
||||
const { getGlobalCostTracker } = await import('../../shared/llm/cost-tracker.js');
|
||||
|
||||
// Create collector, initialize to load persisted state from DB, then enable economic routing
|
||||
const collector = createRoutingFeedbackCollector(100);
|
||||
await collector.initialize();
|
||||
collector.enableEconomicRouting(
|
||||
{ ...(params.taskComplexity != null ? {} : {}) },
|
||||
getGlobalCostTracker(),
|
||||
);
|
||||
|
||||
const report = collector.getEconomicReport();
|
||||
if (!report) {
|
||||
return { success: false, error: 'Economic routing is not available' };
|
||||
}
|
||||
|
||||
// Convert Infinity to string for JSON serialization
|
||||
const tierEfficiency = report.tierEfficiency.map(t => ({
|
||||
...t,
|
||||
qualityPerDollar: isFinite(t.qualityPerDollar) ? t.qualityPerDollar : 'Infinity',
|
||||
}));
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
tierEfficiency,
|
||||
currentHourlyCostUsd: report.currentHourlyCostUsd,
|
||||
currentDailyCostUsd: report.currentDailyCostUsd,
|
||||
budgetRemaining: report.budgetRemaining,
|
||||
recommendation: report.recommendation,
|
||||
savingsOpportunity: report.savingsOpportunity,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to get economic routing report: ${toErrorMessage(error)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Functions
|
||||
// ============================================================================
|
||||
|
||||
@@ -37,6 +37,8 @@ import {
|
||||
// ADR-051: Model routing handlers
|
||||
handleModelRoute,
|
||||
handleRoutingMetrics,
|
||||
// Imp-18: Economic routing handler
|
||||
handleRoutingEconomics,
|
||||
handleAgentList,
|
||||
handleAgentSpawn,
|
||||
handleAgentMetrics,
|
||||
@@ -1023,6 +1025,18 @@ export class MCPProtocolServer {
|
||||
handler: (params) => handleRoutingMetrics(params as unknown as Parameters<typeof handleRoutingMetrics>[0]),
|
||||
});
|
||||
|
||||
this.registerTool({
|
||||
definition: {
|
||||
name: 'routing_economics',
|
||||
description: 'Get economic routing report: tier efficiency, budget status, cost-per-quality analysis, and savings opportunities. Example: routing_economics({ taskComplexity: 0.5 })',
|
||||
category: 'routing',
|
||||
parameters: [
|
||||
{ name: 'taskComplexity', type: 'number', description: 'Task complexity score 0-1 for tier scoring (default: 0.5)', default: 0.5 },
|
||||
],
|
||||
},
|
||||
handler: (params) => handleRoutingEconomics(params as unknown as Parameters<typeof handleRoutingEconomics>[0]),
|
||||
});
|
||||
|
||||
// ADR-057: Infrastructure self-healing tools
|
||||
this.registerTool({
|
||||
definition: {
|
||||
|
||||
@@ -67,6 +67,10 @@ import {
|
||||
handlePipelineValidate,
|
||||
// Validation Pipeline handler (BMAD-003)
|
||||
handleValidationPipeline,
|
||||
// Heartbeat handlers (Imp-10: Token-Free Heartbeat Scheduler)
|
||||
handleHeartbeatStatus,
|
||||
handleHeartbeatTrigger,
|
||||
handleHeartbeatLog,
|
||||
} from './handlers';
|
||||
|
||||
// ============================================================================
|
||||
@@ -760,6 +764,44 @@ const PIPELINE_TOOLS: ToolEntry[] = [
|
||||
},
|
||||
];
|
||||
|
||||
// Imp-10: Token-Free Heartbeat Scheduler
|
||||
const HEARTBEAT_TOOLS: ToolEntry[] = [
|
||||
// Heartbeat Status
|
||||
{
|
||||
definition: {
|
||||
name: 'mcp__agentic_qe__heartbeat_status',
|
||||
description: 'Get heartbeat scheduler status, health metrics, and last result',
|
||||
category: 'core',
|
||||
parameters: [],
|
||||
},
|
||||
handler: handleHeartbeatStatus,
|
||||
},
|
||||
// Heartbeat Trigger
|
||||
{
|
||||
definition: {
|
||||
name: 'mcp__agentic_qe__heartbeat_trigger',
|
||||
description: 'Trigger an immediate heartbeat maintenance cycle (SQL-only, zero tokens)',
|
||||
category: 'core',
|
||||
parameters: [
|
||||
{ name: 'timeout', type: 'number', description: 'Timeout in milliseconds (default: 60000)' },
|
||||
],
|
||||
},
|
||||
handler: handleHeartbeatTrigger,
|
||||
},
|
||||
// Heartbeat Log
|
||||
{
|
||||
definition: {
|
||||
name: 'mcp__agentic_qe__heartbeat_log',
|
||||
description: 'Read the daily heartbeat log for a specific date',
|
||||
category: 'core',
|
||||
parameters: [
|
||||
{ name: 'date', type: 'string', description: 'Date in YYYY-MM-DD format (defaults to today)' },
|
||||
],
|
||||
},
|
||||
handler: handleHeartbeatLog,
|
||||
},
|
||||
];
|
||||
|
||||
// ============================================================================
|
||||
|
||||
export class MCPServer {
|
||||
@@ -787,6 +829,7 @@ export class MCPServer {
|
||||
...MEMORY_TOOLS,
|
||||
...CROSS_PHASE_TOOLS,
|
||||
...PIPELINE_TOOLS,
|
||||
...HEARTBEAT_TOOLS,
|
||||
];
|
||||
this.registry.registerAll(allTools);
|
||||
|
||||
|
||||
@@ -0,0 +1,390 @@
|
||||
/**
|
||||
* Economic Routing Model — Imp-18 (Issue #334)
|
||||
*
|
||||
* Quality-weighted cost optimization for the routing system.
|
||||
* Scores tiers by quality-per-dollar efficiency, respects budget limits,
|
||||
* and produces cost-adjusted rewards so the neural router learns to
|
||||
* prefer cost-efficient tiers.
|
||||
*
|
||||
* @module routing/economic-routing
|
||||
*/
|
||||
|
||||
import { CostTracker } from '../shared/llm/cost-tracker.js';
|
||||
import type { AgentTier } from './routing-config.js';
|
||||
import type { RoutingOutcome } from './types.js';
|
||||
|
||||
// ============================================================================
|
||||
// Constants & Types
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Tier cost estimates (per typical QE task, in USD).
|
||||
* Based on average token usage per task type.
|
||||
*/
|
||||
export const TIER_COST_ESTIMATES: Record<
|
||||
AgentTier,
|
||||
{ avgInputTokens: number; avgOutputTokens: number; costPerTask: number }
|
||||
> = {
|
||||
booster: { avgInputTokens: 0, avgOutputTokens: 0, costPerTask: 0 },
|
||||
haiku: { avgInputTokens: 2000, avgOutputTokens: 1000, costPerTask: 0.0035 },
|
||||
sonnet: { avgInputTokens: 2000, avgOutputTokens: 1000, costPerTask: 0.021 },
|
||||
opus: { avgInputTokens: 2000, avgOutputTokens: 1000, costPerTask: 0.105 },
|
||||
};
|
||||
|
||||
/** All tiers ordered cheapest to most expensive */
|
||||
const TIER_ORDER: AgentTier[] = ['booster', 'haiku', 'sonnet', 'opus'];
|
||||
|
||||
/** Maximum cost per task across all tiers (used for normalization) */
|
||||
const MAX_TIER_COST = TIER_COST_ESTIMATES.opus.costPerTask;
|
||||
|
||||
export interface EconomicScore {
|
||||
tier: AgentTier;
|
||||
/** Expected quality (0-1) */
|
||||
qualityScore: number;
|
||||
/** Estimated cost per task in USD */
|
||||
estimatedCostUsd: number;
|
||||
/** quality / cost (higher = more efficient). Infinity for zero-cost tiers. */
|
||||
qualityPerDollar: number;
|
||||
/** Combined score factoring quality + cost (higher = better) */
|
||||
economicScore: number;
|
||||
}
|
||||
|
||||
export interface EconomicRoutingConfig {
|
||||
/** Weight for quality in combined score (0-1, default 0.6) */
|
||||
qualityWeight: number;
|
||||
/** Weight for cost efficiency in combined score (0-1, default 0.4) */
|
||||
costWeight: number;
|
||||
/** Budget limit per hour in USD (0 = unlimited) */
|
||||
budgetPerHourUsd: number;
|
||||
/** Budget limit per day in USD (0 = unlimited) */
|
||||
budgetPerDayUsd: number;
|
||||
/** Minimum quality threshold -- never route to cheaper tier below this (0-1) */
|
||||
minQualityThreshold: number;
|
||||
/** Enable economic routing (default: true) */
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export const DEFAULT_ECONOMIC_CONFIG: EconomicRoutingConfig = {
|
||||
qualityWeight: 0.6,
|
||||
costWeight: 0.4,
|
||||
budgetPerHourUsd: 0,
|
||||
budgetPerDayUsd: 0,
|
||||
minQualityThreshold: 0.5,
|
||||
enabled: true,
|
||||
};
|
||||
|
||||
export interface EconomicReport {
|
||||
tierEfficiency: EconomicScore[];
|
||||
currentHourlyCostUsd: number;
|
||||
currentDailyCostUsd: number;
|
||||
budgetRemaining: { hourly: number | null; daily: number | null };
|
||||
recommendation: string;
|
||||
savingsOpportunity: { usd: number; description: string } | null;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EconomicRoutingModel
|
||||
// ============================================================================
|
||||
|
||||
/** EMA smoothing factor for quality estimate updates */
|
||||
const EMA_ALPHA = 0.15;
|
||||
|
||||
export class EconomicRoutingModel {
|
||||
private config: EconomicRoutingConfig;
|
||||
private costTracker: CostTracker;
|
||||
private tierQualityEstimates: Map<AgentTier, number> = new Map();
|
||||
private tierOutcomeCounts: Map<AgentTier, number> = new Map();
|
||||
|
||||
constructor(costTracker: CostTracker, config?: Partial<EconomicRoutingConfig>) {
|
||||
const merged = { ...DEFAULT_ECONOMIC_CONFIG, ...config };
|
||||
|
||||
// Validate and clamp config values to safe ranges
|
||||
merged.qualityWeight = Math.max(0, Math.min(1, merged.qualityWeight));
|
||||
merged.costWeight = Math.max(0, Math.min(1, merged.costWeight));
|
||||
merged.minQualityThreshold = Math.max(0, Math.min(1, merged.minQualityThreshold));
|
||||
merged.budgetPerHourUsd = Math.max(0, merged.budgetPerHourUsd);
|
||||
merged.budgetPerDayUsd = Math.max(0, merged.budgetPerDayUsd);
|
||||
|
||||
// Normalize weights so they sum to 1.0
|
||||
const weightSum = merged.qualityWeight + merged.costWeight;
|
||||
if (weightSum > 0 && weightSum !== 1) {
|
||||
merged.qualityWeight /= weightSum;
|
||||
merged.costWeight /= weightSum;
|
||||
}
|
||||
|
||||
this.config = merged;
|
||||
this.costTracker = costTracker;
|
||||
// Initialize with prior assumptions
|
||||
this.tierQualityEstimates.set('booster', 0.3);
|
||||
this.tierQualityEstimates.set('haiku', 0.55);
|
||||
this.tierQualityEstimates.set('sonnet', 0.75);
|
||||
this.tierQualityEstimates.set('opus', 0.90);
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Core Methods
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Score each tier by quality-per-dollar efficiency.
|
||||
* Returns all tiers sorted by economicScore descending.
|
||||
*/
|
||||
scoreTiers(taskComplexity: number): EconomicScore[] {
|
||||
const scores: EconomicScore[] = TIER_ORDER.map(tier => {
|
||||
const qualityScore = this.getQualityEstimate(tier, taskComplexity);
|
||||
const estimatedCostUsd = TIER_COST_ESTIMATES[tier].costPerTask;
|
||||
|
||||
// quality / cost (handle zero-cost booster)
|
||||
const qualityPerDollar = estimatedCostUsd > 0
|
||||
? qualityScore / estimatedCostUsd
|
||||
: qualityScore > 0 ? Infinity : 0;
|
||||
|
||||
// Normalized cost efficiency: 1 - (cost / maxCost)
|
||||
const costEfficiency = MAX_TIER_COST > 0
|
||||
? 1 - estimatedCostUsd / MAX_TIER_COST
|
||||
: 1;
|
||||
|
||||
const economicScore =
|
||||
this.config.qualityWeight * qualityScore +
|
||||
this.config.costWeight * costEfficiency;
|
||||
|
||||
return { tier, qualityScore, estimatedCostUsd, qualityPerDollar, economicScore };
|
||||
});
|
||||
|
||||
scores.sort((a, b) => b.economicScore - a.economicScore);
|
||||
return scores;
|
||||
}
|
||||
|
||||
/**
|
||||
* Select the best tier considering quality AND cost.
|
||||
* Respects budget limits and minimum quality thresholds.
|
||||
*/
|
||||
selectTier(
|
||||
taskComplexity: number,
|
||||
): { tier: AgentTier; reason: string; scores: EconomicScore[] } {
|
||||
const scores = this.scoreTiers(taskComplexity);
|
||||
|
||||
for (const score of scores) {
|
||||
// Skip tiers below minimum quality threshold
|
||||
if (score.qualityScore < this.config.minQualityThreshold) {
|
||||
continue;
|
||||
}
|
||||
// Skip tiers that would exceed budget
|
||||
if (this.wouldExceedBudget(score.tier)) {
|
||||
continue;
|
||||
}
|
||||
return {
|
||||
tier: score.tier,
|
||||
reason: `Best economic score (${score.economicScore.toFixed(3)}): ` +
|
||||
`quality=${score.qualityScore.toFixed(2)}, cost=$${score.estimatedCostUsd.toFixed(4)}`,
|
||||
scores,
|
||||
};
|
||||
}
|
||||
|
||||
// Fallback: pick the cheapest tier that meets quality, ignoring budget
|
||||
// Use spread copies to avoid mutating the original scores array
|
||||
const qualityFiltered = scores.filter(
|
||||
s => s.qualityScore >= this.config.minQualityThreshold,
|
||||
);
|
||||
if (qualityFiltered.length > 0) {
|
||||
const cheapest = [...qualityFiltered].sort(
|
||||
(a, b) => a.estimatedCostUsd - b.estimatedCostUsd,
|
||||
)[0];
|
||||
return {
|
||||
tier: cheapest.tier,
|
||||
reason: `Budget constrained fallback to ${cheapest.tier}`,
|
||||
scores,
|
||||
};
|
||||
}
|
||||
|
||||
// Final fallback: pick the best quality regardless
|
||||
const bestQuality = [...scores].sort((a, b) => b.qualityScore - a.qualityScore)[0];
|
||||
return {
|
||||
tier: bestQuality.tier,
|
||||
reason: `No tier meets quality threshold ${this.config.minQualityThreshold}; ` +
|
||||
`using best quality: ${bestQuality.tier}`,
|
||||
scores,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tier would exceed the budget.
|
||||
*/
|
||||
wouldExceedBudget(tier: AgentTier): boolean {
|
||||
const cost = TIER_COST_ESTIMATES[tier].costPerTask;
|
||||
if (cost === 0) return false;
|
||||
|
||||
const hourlyCost = this.costTracker.getCurrentCost('hour');
|
||||
const dailyCost = this.costTracker.getCurrentCost('day');
|
||||
|
||||
if (this.config.budgetPerHourUsd > 0 &&
|
||||
hourlyCost + cost > this.config.budgetPerHourUsd) {
|
||||
return true;
|
||||
}
|
||||
if (this.config.budgetPerDayUsd > 0 &&
|
||||
dailyCost + cost > this.config.budgetPerDayUsd) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update quality estimates from observed outcomes.
|
||||
* Uses EMA to smooth estimates.
|
||||
*/
|
||||
updateFromOutcome(outcome: RoutingOutcome, tier: AgentTier): void {
|
||||
const observedQuality = outcome.outcome.qualityScore;
|
||||
const current = this.tierQualityEstimates.get(tier) ?? 0.5;
|
||||
const count = this.tierOutcomeCounts.get(tier) ?? 0;
|
||||
|
||||
// Use EMA; for the first few observations, use a higher alpha
|
||||
const alpha = count < 5 ? 0.4 : EMA_ALPHA;
|
||||
const updated = current * (1 - alpha) + observedQuality * alpha;
|
||||
this.tierQualityEstimates.set(tier, updated);
|
||||
this.tierOutcomeCounts.set(tier, count + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get economic efficiency report.
|
||||
*/
|
||||
getEconomicReport(): EconomicReport {
|
||||
const tierEfficiency = this.scoreTiers(0.5); // mid-complexity as baseline
|
||||
const currentHourlyCostUsd = this.costTracker.getCurrentCost('hour');
|
||||
const currentDailyCostUsd = this.costTracker.getCurrentCost('day');
|
||||
|
||||
const budgetRemaining = {
|
||||
hourly: this.config.budgetPerHourUsd > 0
|
||||
? Math.max(0, this.config.budgetPerHourUsd - currentHourlyCostUsd)
|
||||
: null,
|
||||
daily: this.config.budgetPerDayUsd > 0
|
||||
? Math.max(0, this.config.budgetPerDayUsd - currentDailyCostUsd)
|
||||
: null,
|
||||
};
|
||||
|
||||
// Find savings opportunity: compare most-used expensive tier vs cheaper alternative
|
||||
let savingsOpportunity: EconomicReport['savingsOpportunity'] = null;
|
||||
const sorted = [...tierEfficiency].sort((a, b) => b.qualityPerDollar - a.qualityPerDollar);
|
||||
const mostEfficient = sorted.find(s => s.estimatedCostUsd > 0 && isFinite(s.qualityPerDollar));
|
||||
const leastEfficient = [...sorted].reverse().find(
|
||||
(s: EconomicScore) => s.estimatedCostUsd > 0 && isFinite(s.qualityPerDollar),
|
||||
);
|
||||
if (mostEfficient && leastEfficient && mostEfficient.tier !== leastEfficient.tier) {
|
||||
const potentialSavings = leastEfficient.estimatedCostUsd - mostEfficient.estimatedCostUsd;
|
||||
if (potentialSavings > 0) {
|
||||
savingsOpportunity = {
|
||||
usd: potentialSavings,
|
||||
description:
|
||||
`Switch from ${leastEfficient.tier} ($${leastEfficient.estimatedCostUsd.toFixed(4)}/task) ` +
|
||||
`to ${mostEfficient.tier} ($${mostEfficient.estimatedCostUsd.toFixed(4)}/task) ` +
|
||||
`for comparable tasks to save ~$${potentialSavings.toFixed(4)}/task`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const recommendation = this.generateRecommendation(tierEfficiency, budgetRemaining);
|
||||
|
||||
return {
|
||||
tierEfficiency,
|
||||
currentHourlyCostUsd,
|
||||
currentDailyCostUsd,
|
||||
budgetRemaining,
|
||||
recommendation,
|
||||
savingsOpportunity,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute cost-adjusted reward for the neural router.
|
||||
* Penalizes expensive tiers that don't deliver proportionally higher quality.
|
||||
*/
|
||||
computeCostAdjustedReward(
|
||||
baseReward: number,
|
||||
tier: AgentTier,
|
||||
qualityScore: number,
|
||||
): number {
|
||||
const tierCost = TIER_COST_ESTIMATES[tier].costPerTask;
|
||||
if (MAX_TIER_COST === 0) return baseReward;
|
||||
|
||||
const costRatio = tierCost / MAX_TIER_COST; // 0-1, where opus=1.0
|
||||
const qualityGain = Math.max(0, qualityScore - this.config.minQualityThreshold);
|
||||
const costPenalty = costRatio * (1 - qualityGain);
|
||||
const adjusted = baseReward - costPenalty * this.config.costWeight;
|
||||
|
||||
// Clamp to [-1, 1]
|
||||
return Math.max(-1, Math.min(1, adjusted));
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Persistence helpers
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Serialize quality estimates for persistence.
|
||||
*/
|
||||
serializeEstimates(): Record<string, { quality: number; count: number }> {
|
||||
const result: Record<string, { quality: number; count: number }> = {};
|
||||
for (const tier of TIER_ORDER) {
|
||||
result[tier] = {
|
||||
quality: this.tierQualityEstimates.get(tier) ?? 0.5,
|
||||
count: this.tierOutcomeCounts.get(tier) ?? 0,
|
||||
};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deserialize quality estimates from persistence.
|
||||
*/
|
||||
deserializeEstimates(data: Record<string, { quality: number; count: number }>): void {
|
||||
for (const tier of TIER_ORDER) {
|
||||
if (data[tier]) {
|
||||
this.tierQualityEstimates.set(tier, data[tier].quality);
|
||||
this.tierOutcomeCounts.set(tier, data[tier].count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current config (read-only copy).
|
||||
*/
|
||||
getConfig(): Readonly<EconomicRoutingConfig> {
|
||||
return { ...this.config };
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Private helpers
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Get quality estimate for a tier, adjusted by task complexity.
|
||||
* Higher complexity tasks benefit more from higher-tier models.
|
||||
*/
|
||||
private getQualityEstimate(tier: AgentTier, taskComplexity: number): number {
|
||||
const baseQuality = this.tierQualityEstimates.get(tier) ?? 0.5;
|
||||
// For complex tasks, cheaper tiers degrade more
|
||||
const complexityPenalty = tier === 'booster'
|
||||
? taskComplexity * 0.4
|
||||
: tier === 'haiku'
|
||||
? taskComplexity * 0.2
|
||||
: 0;
|
||||
return Math.max(0, Math.min(1, baseQuality - complexityPenalty));
|
||||
}
|
||||
|
||||
private generateRecommendation(
|
||||
tierEfficiency: EconomicScore[],
|
||||
budgetRemaining: { hourly: number | null; daily: number | null },
|
||||
): string {
|
||||
if (budgetRemaining.hourly !== null && budgetRemaining.hourly < 0.01) {
|
||||
return 'Hourly budget nearly exhausted. Consider increasing budget or routing to cheaper tiers.';
|
||||
}
|
||||
if (budgetRemaining.daily !== null && budgetRemaining.daily < 0.1) {
|
||||
return 'Daily budget nearly exhausted. Only critical tasks should use expensive tiers.';
|
||||
}
|
||||
const best = tierEfficiency[0];
|
||||
if (best) {
|
||||
return `Most cost-efficient tier: ${best.tier} ` +
|
||||
`(score=${best.economicScore.toFixed(3)}, quality=${best.qualityScore.toFixed(2)})`;
|
||||
}
|
||||
return 'No economic data available yet.';
|
||||
}
|
||||
}
|
||||
@@ -75,6 +75,19 @@ export {
|
||||
createRoutingFeedbackCollector,
|
||||
} from './routing-feedback.js';
|
||||
|
||||
// Economic Routing (Imp-18, Issue #334)
|
||||
export {
|
||||
EconomicRoutingModel,
|
||||
TIER_COST_ESTIMATES,
|
||||
DEFAULT_ECONOMIC_CONFIG,
|
||||
} from './economic-routing.js';
|
||||
|
||||
export type {
|
||||
EconomicScore,
|
||||
EconomicRoutingConfig,
|
||||
EconomicReport,
|
||||
} from './economic-routing.js';
|
||||
|
||||
// Task Classifier (TD-002)
|
||||
export {
|
||||
classifyTask,
|
||||
|
||||
@@ -20,7 +20,14 @@ import { safeJsonParse } from '../shared/safe-json.js';
|
||||
import { toErrorMessage } from '../shared/error-utils.js';
|
||||
import { EMACalibrator, type EMAConfig } from './calibration/index.js';
|
||||
import { AutoEscalationTracker, type EscalationConfig, type EscalationState } from './escalation/index.js';
|
||||
import type { RoutingConfig } from './routing-config.js';
|
||||
import type { RoutingConfig, AgentTier } from './routing-config.js';
|
||||
import {
|
||||
EconomicRoutingModel,
|
||||
type EconomicRoutingConfig,
|
||||
type EconomicScore,
|
||||
type EconomicReport,
|
||||
} from './economic-routing.js';
|
||||
import { CostTracker, getGlobalCostTracker } from '../shared/llm/cost-tracker.js';
|
||||
|
||||
// ============================================================================
|
||||
// Database Row Types
|
||||
@@ -103,6 +110,9 @@ export class RoutingFeedbackCollector {
|
||||
private db: UnifiedMemoryManager | null = null;
|
||||
private calibrator: EMACalibrator | null = null;
|
||||
private escalationTracker: AutoEscalationTracker | null = null;
|
||||
private economicModel: EconomicRoutingModel | null = null;
|
||||
private economicPersistCounter = 0;
|
||||
private static readonly ECONOMIC_PERSIST_INTERVAL = 10;
|
||||
private persistCount = 0;
|
||||
private readonly maxOutcomes: number;
|
||||
private static readonly RETENTION_CLEANUP_INTERVAL = 100;
|
||||
@@ -283,6 +293,44 @@ export class RoutingFeedbackCollector {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load persisted economic routing state from the database
|
||||
*/
|
||||
private loadEconomicState(): void {
|
||||
if (!this.db || !this.economicModel) return;
|
||||
try {
|
||||
const database = this.db.getDatabase();
|
||||
const row = database.prepare(
|
||||
`SELECT value FROM kv_store WHERE key = 'routing:economic_quality_estimates'`
|
||||
).get() as { value: string } | undefined;
|
||||
if (row) {
|
||||
const data = safeJsonParse(row.value);
|
||||
if (data && typeof data === 'object') {
|
||||
this.economicModel.deserializeEstimates(data as Record<string, { quality: number; count: number }>);
|
||||
console.log('[RoutingFeedbackCollector] Loaded economic quality estimates from DB');
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('[RoutingFeedbackCollector] Failed to load economic state:', toErrorMessage(error));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist economic routing state to the database
|
||||
*/
|
||||
private persistEconomicState(): void {
|
||||
if (!this.db || !this.economicModel) return;
|
||||
try {
|
||||
const database = this.db.getDatabase();
|
||||
const serialized = JSON.stringify(this.economicModel.serializeEstimates());
|
||||
database.prepare(
|
||||
`INSERT OR REPLACE INTO kv_store (key, value, updated_at) VALUES (?, ?, datetime('now'))`
|
||||
).run('routing:economic_quality_estimates', serialized);
|
||||
} catch (error) {
|
||||
console.warn('[RoutingFeedbackCollector] Failed to persist economic state:', toErrorMessage(error));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to router for automatic performance updates
|
||||
*/
|
||||
@@ -322,6 +370,36 @@ export class RoutingFeedbackCollector {
|
||||
return this.escalationTracker?.getState(agentId) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable economic routing model for quality-weighted cost optimization.
|
||||
* Once enabled, every recorded outcome feeds economic quality estimates
|
||||
* and the neural router receives cost-adjusted rewards.
|
||||
*/
|
||||
enableEconomicRouting(
|
||||
config?: Partial<EconomicRoutingConfig>,
|
||||
costTracker?: CostTracker,
|
||||
): void {
|
||||
const tracker = costTracker ?? getGlobalCostTracker();
|
||||
this.economicModel = new EconomicRoutingModel(tracker, config);
|
||||
this.loadEconomicState();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the economic routing report.
|
||||
* Returns null if economic routing is not enabled.
|
||||
*/
|
||||
getEconomicReport(): EconomicReport | null {
|
||||
return this.economicModel?.getEconomicReport() ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get economic scores for a given task complexity.
|
||||
* Returns null if economic routing is not enabled.
|
||||
*/
|
||||
getEconomicScore(taskComplexity: number): EconomicScore[] | null {
|
||||
return this.economicModel?.scoreTiers(taskComplexity) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a routing outcome
|
||||
*/
|
||||
@@ -383,6 +461,18 @@ export class RoutingFeedbackCollector {
|
||||
}
|
||||
}
|
||||
|
||||
// Update economic model if enabled
|
||||
if (this.economicModel) {
|
||||
const tier = this.inferTier(usedAgent) as AgentTier;
|
||||
this.economicModel.updateFromOutcome(routingOutcome, tier);
|
||||
|
||||
// Persist economic state periodically
|
||||
this.economicPersistCounter++;
|
||||
if (this.economicPersistCounter % RoutingFeedbackCollector.ECONOMIC_PERSIST_INTERVAL === 0) {
|
||||
this.persistEconomicState();
|
||||
}
|
||||
}
|
||||
|
||||
return routingOutcome;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
/**
|
||||
* Agentic QE v3 - Heartbeat CLI Handler Tests
|
||||
* Imp-10: Token-Free Heartbeat Scheduler CLI Integration
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
// Use vi.hoisted() so mock fns are available when vi.mock factory runs (hoisted above imports)
|
||||
const mocks = vi.hoisted(() => ({
|
||||
existsSync: vi.fn(),
|
||||
readFileSync: vi.fn(),
|
||||
writeFileSync: vi.fn(),
|
||||
mkdirSync: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('fs', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('fs')>();
|
||||
return {
|
||||
...actual,
|
||||
existsSync: mocks.existsSync,
|
||||
readFileSync: mocks.readFileSync,
|
||||
writeFileSync: mocks.writeFileSync,
|
||||
mkdirSync: mocks.mkdirSync,
|
||||
};
|
||||
});
|
||||
|
||||
// Mock the worker before importing the handler
|
||||
vi.mock('../../../src/workers/workers/heartbeat-scheduler.js', () => {
|
||||
return {
|
||||
HeartbeatSchedulerWorker: class MockHeartbeatWorker {
|
||||
config = {
|
||||
id: 'heartbeat-scheduler',
|
||||
name: 'Heartbeat Scheduler',
|
||||
intervalMs: 30 * 60 * 1000,
|
||||
enabled: true,
|
||||
};
|
||||
status = 'idle';
|
||||
lastResult = {
|
||||
workerId: 'heartbeat-scheduler',
|
||||
timestamp: new Date('2026-03-27T14:32:15Z'),
|
||||
durationMs: 245,
|
||||
success: true,
|
||||
metrics: {
|
||||
itemsAnalyzed: 50,
|
||||
issuesFound: 3,
|
||||
healthScore: 85,
|
||||
trend: 'stable' as const,
|
||||
domainMetrics: {
|
||||
promoted: 2,
|
||||
deprecated: 1,
|
||||
decayed: 15,
|
||||
pendingExperiences: 8,
|
||||
avgConfidence: 0.72,
|
||||
},
|
||||
},
|
||||
findings: [],
|
||||
recommendations: [],
|
||||
};
|
||||
lastRunAt = new Date('2026-03-27T14:32:15Z');
|
||||
nextRunAt = new Date('2026-03-27T15:02:15Z');
|
||||
|
||||
initialize = vi.fn().mockResolvedValue(undefined);
|
||||
execute = vi.fn().mockResolvedValue({
|
||||
workerId: 'heartbeat-scheduler',
|
||||
timestamp: new Date(),
|
||||
durationMs: 312,
|
||||
success: true,
|
||||
metrics: {
|
||||
itemsAnalyzed: 42,
|
||||
issuesFound: 1,
|
||||
healthScore: 88,
|
||||
trend: 'improving',
|
||||
domainMetrics: {
|
||||
promoted: 3,
|
||||
deprecated: 0,
|
||||
decayed: 10,
|
||||
pendingExperiences: 5,
|
||||
avgConfidence: 0.75,
|
||||
},
|
||||
},
|
||||
findings: [{ type: 'heartbeat-promotion', severity: 'info', domain: 'learning-optimization', title: 'Promoted', description: '3 promoted' }],
|
||||
recommendations: [],
|
||||
});
|
||||
getHealth = vi.fn().mockReturnValue({
|
||||
status: 'idle',
|
||||
healthScore: 85,
|
||||
totalExecutions: 42,
|
||||
successfulExecutions: 41,
|
||||
failedExecutions: 1,
|
||||
avgDurationMs: 280,
|
||||
recentResults: [],
|
||||
});
|
||||
pause = vi.fn();
|
||||
resume = vi.fn();
|
||||
stop = vi.fn().mockResolvedValue(undefined);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
// Mock unified-memory for findProjectRoot
|
||||
vi.mock('../../../src/kernel/unified-memory.js', () => ({
|
||||
findProjectRoot: () => '/tmp/test-project',
|
||||
getUnifiedMemory: vi.fn(),
|
||||
}));
|
||||
|
||||
import { HeartbeatHandler } from '../../../src/cli/handlers/heartbeat-handler.js';
|
||||
|
||||
describe('HeartbeatHandler', () => {
|
||||
let handler: HeartbeatHandler;
|
||||
let cleanupAndExit: ReturnType<typeof vi.fn>;
|
||||
let consoleSpy: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
cleanupAndExit = vi.fn().mockResolvedValue(undefined);
|
||||
handler = new HeartbeatHandler(cleanupAndExit);
|
||||
consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
function buildContext() {
|
||||
return {
|
||||
kernel: null,
|
||||
queen: null,
|
||||
router: null,
|
||||
workflowOrchestrator: null,
|
||||
scheduledWorkflows: new Map(),
|
||||
persistentScheduler: null,
|
||||
initialized: false,
|
||||
};
|
||||
}
|
||||
|
||||
function allOutput(): string {
|
||||
return consoleSpy.mock.calls.map(c => c.join(' ')).join('\n');
|
||||
}
|
||||
|
||||
describe('metadata', () => {
|
||||
it('should have correct name and description', () => {
|
||||
expect(handler.name).toBe('heartbeat');
|
||||
expect(handler.description).toBe('Manage the token-free heartbeat scheduler');
|
||||
});
|
||||
|
||||
it('should return help text with subcommands', () => {
|
||||
const help = handler.getHelp();
|
||||
expect(help).toContain('status');
|
||||
expect(help).toContain('run-now');
|
||||
expect(help).toContain('history');
|
||||
expect(help).toContain('log');
|
||||
expect(help).toContain('pause');
|
||||
expect(help).toContain('resume');
|
||||
});
|
||||
});
|
||||
|
||||
describe('status', () => {
|
||||
it('should display formatted health data', async () => {
|
||||
const { Command } = await import('commander');
|
||||
const program = new Command();
|
||||
handler.register(program, buildContext());
|
||||
|
||||
await program.parseAsync(['node', 'aqe', 'heartbeat', 'status']);
|
||||
|
||||
expect(cleanupAndExit).toHaveBeenCalledWith(0);
|
||||
|
||||
const output = allOutput();
|
||||
expect(output).toContain('Heartbeat Scheduler Status');
|
||||
expect(output).toContain('85');
|
||||
expect(output).toContain('42');
|
||||
});
|
||||
});
|
||||
|
||||
describe('run-now', () => {
|
||||
it('should execute heartbeat and return results', async () => {
|
||||
// storeHistoryEntry uses fs
|
||||
mocks.existsSync.mockReturnValue(false);
|
||||
|
||||
const { Command } = await import('commander');
|
||||
const program = new Command();
|
||||
handler.register(program, buildContext());
|
||||
|
||||
await program.parseAsync(['node', 'aqe', 'heartbeat', 'run-now']);
|
||||
|
||||
expect(cleanupAndExit).toHaveBeenCalledWith(0);
|
||||
|
||||
const output = allOutput();
|
||||
expect(output).toContain('Triggering heartbeat cycle');
|
||||
expect(output).toContain('Heartbeat cycle complete');
|
||||
});
|
||||
});
|
||||
|
||||
describe('log', () => {
|
||||
it('should read daily log file for today', async () => {
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
|
||||
mocks.existsSync.mockReturnValue(true);
|
||||
mocks.readFileSync.mockReturnValue(
|
||||
`# AQE Daily Log \u2014 ${today}\n| Time | Event | Summary |\n|------|-------|--------|\n| 14:32:15 | pattern-promoted | Heartbeat: 2 promoted |\n`
|
||||
);
|
||||
|
||||
const { Command } = await import('commander');
|
||||
const program = new Command();
|
||||
handler.register(program, buildContext());
|
||||
|
||||
await program.parseAsync(['node', 'aqe', 'heartbeat', 'log']);
|
||||
|
||||
expect(cleanupAndExit).toHaveBeenCalledWith(0);
|
||||
expect(allOutput()).toContain('Daily Log');
|
||||
});
|
||||
|
||||
it('should read log for a specific date', async () => {
|
||||
mocks.existsSync.mockReturnValue(true);
|
||||
mocks.readFileSync.mockReturnValue(
|
||||
`# AQE Daily Log \u2014 2026-03-25\n| Time | Event | Summary |\n`
|
||||
);
|
||||
|
||||
const { Command } = await import('commander');
|
||||
const program = new Command();
|
||||
handler.register(program, buildContext());
|
||||
|
||||
await program.parseAsync(['node', 'aqe', 'heartbeat', 'log', '--date', '2026-03-25']);
|
||||
|
||||
expect(cleanupAndExit).toHaveBeenCalledWith(0);
|
||||
expect(allOutput()).toContain('2026-03-25');
|
||||
});
|
||||
|
||||
it('should handle missing log file gracefully', async () => {
|
||||
mocks.existsSync.mockReturnValue(false);
|
||||
|
||||
const { Command } = await import('commander');
|
||||
const program = new Command();
|
||||
handler.register(program, buildContext());
|
||||
|
||||
await program.parseAsync(['node', 'aqe', 'heartbeat', 'log', '--date', '2020-01-01']);
|
||||
|
||||
expect(cleanupAndExit).toHaveBeenCalledWith(0);
|
||||
expect(allOutput()).toContain('No daily log found');
|
||||
});
|
||||
});
|
||||
|
||||
describe('pause/resume', () => {
|
||||
it('should pause the worker', async () => {
|
||||
const { Command } = await import('commander');
|
||||
const program = new Command();
|
||||
handler.register(program, buildContext());
|
||||
|
||||
await program.parseAsync(['node', 'aqe', 'heartbeat', 'pause']);
|
||||
|
||||
expect(cleanupAndExit).toHaveBeenCalledWith(0);
|
||||
expect(allOutput()).toContain('paused');
|
||||
});
|
||||
|
||||
it('should resume the worker', async () => {
|
||||
const { Command } = await import('commander');
|
||||
const program = new Command();
|
||||
handler.register(program, buildContext());
|
||||
|
||||
await program.parseAsync(['node', 'aqe', 'heartbeat', 'resume']);
|
||||
|
||||
expect(cleanupAndExit).toHaveBeenCalledWith(0);
|
||||
expect(allOutput()).toContain('resumed');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,248 @@
|
||||
/**
|
||||
* Unit tests for approval gate step type (Imp-9)
|
||||
*
|
||||
* Tests the approval gate functionality in the WorkflowOrchestrator:
|
||||
* - Step pauses at approval gate
|
||||
* - approveStep() resumes execution
|
||||
* - rejectStep() fails the step
|
||||
* - Auto-approve works after timeout
|
||||
* - Timeout rejection works
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
||||
import { WorkflowOrchestrator } from '../../../src/coordination/workflow-orchestrator.js';
|
||||
import type {
|
||||
WorkflowDefinition,
|
||||
WorkflowOrchestratorConfig,
|
||||
} from '../../../src/coordination/workflow-types.js';
|
||||
import { ok } from '../../../src/shared/types/index.js';
|
||||
import type { EventBus, MemoryBackend, AgentCoordinator, Subscription } from '../../../src/kernel/interfaces.js';
|
||||
|
||||
// ============================================================================
|
||||
// Mock Infrastructure
|
||||
// ============================================================================
|
||||
|
||||
function createMockEventBus(): EventBus {
|
||||
return {
|
||||
publish: vi.fn().mockResolvedValue(undefined),
|
||||
subscribe: vi.fn().mockReturnValue({ unsubscribe: vi.fn() } as Subscription),
|
||||
subscribeOnce: vi.fn().mockReturnValue({ unsubscribe: vi.fn() } as Subscription),
|
||||
} as unknown as EventBus;
|
||||
}
|
||||
|
||||
function createMockMemory(): MemoryBackend {
|
||||
return {
|
||||
get: vi.fn().mockResolvedValue(undefined),
|
||||
set: vi.fn().mockResolvedValue(undefined),
|
||||
delete: vi.fn().mockResolvedValue(true),
|
||||
search: vi.fn().mockResolvedValue([]),
|
||||
has: vi.fn().mockResolvedValue(false),
|
||||
keys: vi.fn().mockResolvedValue([]),
|
||||
} as unknown as MemoryBackend;
|
||||
}
|
||||
|
||||
function createMockCoordinator(): AgentCoordinator {
|
||||
return {
|
||||
canSpawn: vi.fn().mockReturnValue(true),
|
||||
spawn: vi.fn().mockResolvedValue(ok('agent-1')),
|
||||
stop: vi.fn().mockResolvedValue(ok(undefined)),
|
||||
list: vi.fn().mockReturnValue([]),
|
||||
getAgent: vi.fn().mockReturnValue(undefined),
|
||||
} as unknown as AgentCoordinator;
|
||||
}
|
||||
|
||||
const TEST_CONFIG: Partial<WorkflowOrchestratorConfig> = {
|
||||
maxConcurrentWorkflows: 5,
|
||||
defaultStepTimeout: 60000,
|
||||
defaultWorkflowTimeout: 300000,
|
||||
enableEventTriggers: false,
|
||||
persistExecutions: false,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Test Workflows
|
||||
// ============================================================================
|
||||
|
||||
function makeApprovalWorkflow(
|
||||
approval: WorkflowDefinition['steps'][0]['approval'],
|
||||
): WorkflowDefinition {
|
||||
return {
|
||||
id: 'approval-test',
|
||||
name: 'Approval Test Workflow',
|
||||
description: 'Test workflow with approval gate',
|
||||
version: '1.0.0',
|
||||
steps: [
|
||||
{
|
||||
id: 'gate-check',
|
||||
name: 'Quality Gate',
|
||||
domain: 'quality-assessment',
|
||||
action: 'gate-check',
|
||||
inputMapping: {},
|
||||
},
|
||||
{
|
||||
id: 'approval-step',
|
||||
name: 'Approval Gate',
|
||||
domain: 'quality-assessment',
|
||||
action: 'gate-check',
|
||||
dependsOn: ['gate-check'],
|
||||
approval,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('approval gate', () => {
|
||||
let orchestrator: WorkflowOrchestrator;
|
||||
let eventBus: EventBus;
|
||||
|
||||
beforeEach(async () => {
|
||||
eventBus = createMockEventBus();
|
||||
orchestrator = new WorkflowOrchestrator(
|
||||
eventBus,
|
||||
createMockMemory(),
|
||||
createMockCoordinator(),
|
||||
TEST_CONFIG,
|
||||
);
|
||||
await orchestrator.initialize();
|
||||
});
|
||||
|
||||
it('should pause at approval gate and resume on approve', async () => {
|
||||
const workflow = makeApprovalWorkflow({
|
||||
autoApproveAfter: 0, // never auto-approve
|
||||
message: 'Please approve',
|
||||
});
|
||||
orchestrator.registerWorkflow(workflow);
|
||||
|
||||
const execResult = await orchestrator.executeWorkflow('approval-test', {});
|
||||
expect(execResult.success).toBe(true);
|
||||
const executionId = execResult.value;
|
||||
|
||||
// Give the workflow time to reach the approval gate
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
|
||||
// The step should be awaiting approval
|
||||
const status = orchestrator.getWorkflowStatus(executionId);
|
||||
expect(status).toBeDefined();
|
||||
// Workflow should still be running (not completed yet)
|
||||
expect(status!.status).toBe('running');
|
||||
|
||||
// Approve the step
|
||||
const approved = orchestrator.approveStep(executionId, 'approval-step');
|
||||
expect(approved).toBe(true);
|
||||
|
||||
// Wait for workflow to complete
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
|
||||
const finalStatus = orchestrator.getWorkflowStatus(executionId);
|
||||
expect(finalStatus).toBeDefined();
|
||||
expect(finalStatus!.status).toBe('completed');
|
||||
expect(finalStatus!.completedSteps).toContain('approval-step');
|
||||
});
|
||||
|
||||
it('should fail step on reject', async () => {
|
||||
const workflow = makeApprovalWorkflow({
|
||||
autoApproveAfter: 0,
|
||||
message: 'Please approve',
|
||||
});
|
||||
orchestrator.registerWorkflow(workflow);
|
||||
|
||||
const execResult = await orchestrator.executeWorkflow('approval-test', {});
|
||||
expect(execResult.success).toBe(true);
|
||||
const executionId = execResult.value;
|
||||
|
||||
// Wait for approval gate
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
|
||||
// Reject the step
|
||||
const rejected = orchestrator.rejectStep(executionId, 'approval-step', 'Not ready');
|
||||
expect(rejected).toBe(true);
|
||||
|
||||
// Wait for workflow to finish
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
|
||||
const finalStatus = orchestrator.getWorkflowStatus(executionId);
|
||||
expect(finalStatus).toBeDefined();
|
||||
expect(finalStatus!.status).toBe('failed');
|
||||
expect(finalStatus!.failedSteps).toContain('approval-step');
|
||||
});
|
||||
|
||||
it('should auto-approve after timeout', async () => {
|
||||
const workflow = makeApprovalWorkflow({
|
||||
autoApproveAfter: 50, // 50ms
|
||||
message: 'Auto-approve test',
|
||||
});
|
||||
orchestrator.registerWorkflow(workflow);
|
||||
|
||||
const execResult = await orchestrator.executeWorkflow('approval-test', {});
|
||||
expect(execResult.success).toBe(true);
|
||||
const executionId = execResult.value;
|
||||
|
||||
// Wait for auto-approve + execution
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
|
||||
const finalStatus = orchestrator.getWorkflowStatus(executionId);
|
||||
expect(finalStatus).toBeDefined();
|
||||
expect(finalStatus!.status).toBe('completed');
|
||||
expect(finalStatus!.completedSteps).toContain('approval-step');
|
||||
});
|
||||
|
||||
it('should return false for approve on unknown execution', () => {
|
||||
const result = orchestrator.approveStep('nonexistent', 'step1');
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false for reject on unknown execution', () => {
|
||||
const result = orchestrator.rejectStep('nonexistent', 'step1');
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it('should work with simple boolean approval (auto-approve default 5min)', async () => {
|
||||
// With approval: true, auto-approve timeout is 300000ms (5 min default).
|
||||
// We just verify the workflow enters the gate. We approve manually
|
||||
// to avoid a long wait.
|
||||
const workflow = makeApprovalWorkflow(true);
|
||||
orchestrator.registerWorkflow(workflow);
|
||||
|
||||
const execResult = await orchestrator.executeWorkflow('approval-test', {});
|
||||
expect(execResult.success).toBe(true);
|
||||
const executionId = execResult.value;
|
||||
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
|
||||
// Approve manually
|
||||
orchestrator.approveStep(executionId, 'approval-step');
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
|
||||
const finalStatus = orchestrator.getWorkflowStatus(executionId);
|
||||
expect(finalStatus).toBeDefined();
|
||||
expect(finalStatus!.status).toBe('completed');
|
||||
});
|
||||
|
||||
it('should emit StepAwaitingApproval event', async () => {
|
||||
const workflow = makeApprovalWorkflow({
|
||||
autoApproveAfter: 0,
|
||||
message: 'Event test',
|
||||
});
|
||||
orchestrator.registerWorkflow(workflow);
|
||||
|
||||
const execResult = await orchestrator.executeWorkflow('approval-test', {});
|
||||
const executionId = execResult.value;
|
||||
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
|
||||
// Check that publish was called with the approval event
|
||||
const publishCalls = (eventBus.publish as ReturnType<typeof vi.fn>).mock.calls;
|
||||
const approvalEvents = publishCalls.filter(
|
||||
(call: unknown[]) => (call[0] as { type: string }).type === 'workflow.StepAwaitingApproval',
|
||||
);
|
||||
expect(approvalEvents.length).toBeGreaterThan(0);
|
||||
|
||||
// Clean up
|
||||
orchestrator.approveStep(executionId, 'approval-step');
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* Unit tests for deterministic actions (Imp-9)
|
||||
*
|
||||
* Verifies that each built-in action:
|
||||
* 1. Works with explicit inputs (source: 'input')
|
||||
* 2. Queries the database when inputs are omitted (source: 'database')
|
||||
* 3. Degrades gracefully when DB is unavailable
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
findDeterministicAction,
|
||||
getAllDeterministicActions,
|
||||
} from '../../../src/coordination/deterministic-actions.js';
|
||||
import type { WorkflowContext } from '../../../src/coordination/workflow-types.js';
|
||||
|
||||
// ============================================================================
|
||||
// Fixtures
|
||||
// ============================================================================
|
||||
|
||||
function makeContext(overrides?: Partial<WorkflowContext>): WorkflowContext {
|
||||
return {
|
||||
input: {},
|
||||
results: {},
|
||||
metadata: {
|
||||
executionId: 'test-exec-1',
|
||||
workflowId: 'test-workflow-1',
|
||||
startedAt: new Date(),
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// DB Mock Helpers
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Mock the unified memory module so deterministic actions see a fake DB.
|
||||
* Call restore() in afterEach.
|
||||
*/
|
||||
function mockUnifiedMemory(rows: Record<string, unknown>) {
|
||||
const mockPrepare = vi.fn().mockImplementation((sql: string) => ({
|
||||
get: vi.fn().mockImplementation(() => {
|
||||
// Return different rows based on the table being queried
|
||||
if (sql.includes('coverage_sessions')) return rows.coverage ?? undefined;
|
||||
if (sql.includes('test_outcomes') && sql.includes('SUM')) return rows.testOutcomes ?? undefined;
|
||||
if (sql.includes('test_outcomes') && sql.includes('bugs')) return rows.bugs ?? undefined;
|
||||
if (sql.includes('qe_patterns')) return rows.patterns ?? undefined;
|
||||
if (sql.includes('routing_outcomes')) return rows.routing ?? undefined;
|
||||
return undefined;
|
||||
}),
|
||||
all: vi.fn().mockReturnValue([]),
|
||||
}));
|
||||
|
||||
const mockDb = { prepare: mockPrepare };
|
||||
const mockManager = {
|
||||
isInitialized: vi.fn().mockReturnValue(true),
|
||||
getDatabase: vi.fn().mockReturnValue(mockDb),
|
||||
};
|
||||
|
||||
// Mock the require() call inside tryGetDb
|
||||
vi.doMock('../../../src/kernel/unified-memory.js', () => ({
|
||||
getUnifiedMemory: () => mockManager,
|
||||
}));
|
||||
|
||||
return { mockDb, mockPrepare, mockManager };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Registry Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('deterministic-actions registry', () => {
|
||||
it('should have all four built-in actions', () => {
|
||||
const actions = getAllDeterministicActions();
|
||||
expect(actions.length).toBe(4);
|
||||
|
||||
const ids = actions.map((a) => a.id);
|
||||
expect(ids).toContain('quality-gate-check');
|
||||
expect(ids).toContain('coverage-threshold');
|
||||
expect(ids).toContain('pattern-health');
|
||||
expect(ids).toContain('routing-accuracy');
|
||||
});
|
||||
|
||||
it('should find action by domain + action', () => {
|
||||
const action = findDeterministicAction('quality-assessment', 'gate-check');
|
||||
expect(action).toBeDefined();
|
||||
expect(action!.id).toBe('quality-gate-check');
|
||||
});
|
||||
|
||||
it('should return undefined for unknown domain/action', () => {
|
||||
const action = findDeterministicAction('quality-assessment', 'nonexistent');
|
||||
expect(action).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Quality Gate Check
|
||||
// ============================================================================
|
||||
|
||||
describe('quality-gate-check action', () => {
|
||||
it('should pass when all thresholds are met (explicit input)', async () => {
|
||||
const action = findDeterministicAction('quality-assessment', 'gate-check')!;
|
||||
const result = await action.execute(
|
||||
{
|
||||
coverageMin: 80,
|
||||
testsPassingMin: 90,
|
||||
maxBugs: 5,
|
||||
currentCoverage: 85,
|
||||
currentTestsPassingRate: 95,
|
||||
currentBugs: 2,
|
||||
},
|
||||
makeContext(),
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const data = result.value;
|
||||
expect(data.passed).toBe(true);
|
||||
expect(data.score).toBeGreaterThan(0);
|
||||
expect(data.source).toBe('input');
|
||||
expect(data.details).toBeDefined();
|
||||
});
|
||||
|
||||
it('should fail when coverage is below threshold', async () => {
|
||||
const action = findDeterministicAction('quality-assessment', 'gate-check')!;
|
||||
const result = await action.execute(
|
||||
{
|
||||
coverageMin: 80,
|
||||
testsPassingMin: 90,
|
||||
maxBugs: 5,
|
||||
currentCoverage: 50,
|
||||
currentTestsPassingRate: 95,
|
||||
currentBugs: 2,
|
||||
},
|
||||
makeContext(),
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const data = result.value;
|
||||
expect(data.passed).toBe(false);
|
||||
expect((data.details as Record<string, unknown> & { coverage: { passed: boolean } }).coverage.passed).toBe(false);
|
||||
});
|
||||
|
||||
it('should use defaults and report database source when no input provided', async () => {
|
||||
const action = findDeterministicAction('quality-assessment', 'gate-check')!;
|
||||
const result = await action.execute({}, makeContext());
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const data = result.value;
|
||||
// With no DB available and no inputs, defaults to 0 → fails
|
||||
expect(data.passed).toBe(false);
|
||||
expect(typeof data.score).toBe('number');
|
||||
expect(data.source).toBe('database');
|
||||
});
|
||||
|
||||
it('should handle maxBugs=0 edge case', async () => {
|
||||
const action = findDeterministicAction('quality-assessment', 'gate-check')!;
|
||||
const result = await action.execute(
|
||||
{ maxBugs: 0, currentBugs: 0, currentCoverage: 100, currentTestsPassingRate: 100 },
|
||||
makeContext(),
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.value.passed).toBe(true);
|
||||
|
||||
// maxBugs=0 with bugs>0 should fail
|
||||
const result2 = await action.execute(
|
||||
{ maxBugs: 0, currentBugs: 1, currentCoverage: 100, currentTestsPassingRate: 100 },
|
||||
makeContext(),
|
||||
);
|
||||
expect(result2.success).toBe(true);
|
||||
expect(result2.value.passed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Coverage Threshold Check
|
||||
// ============================================================================
|
||||
|
||||
describe('coverage-threshold action', () => {
|
||||
it('should pass when coverage meets threshold (explicit input)', async () => {
|
||||
const action = findDeterministicAction('coverage-analysis', 'threshold-check')!;
|
||||
const result = await action.execute(
|
||||
{ minCoverage: 80, currentCoverage: 90 },
|
||||
makeContext(),
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const data = result.value;
|
||||
expect(data.passed).toBe(true);
|
||||
expect(data.gap).toBe(0);
|
||||
expect(data.currentCoverage).toBe(90);
|
||||
expect(data.source).toBe('input');
|
||||
});
|
||||
|
||||
it('should fail and report gap when below threshold', async () => {
|
||||
const action = findDeterministicAction('coverage-analysis', 'threshold-check')!;
|
||||
const result = await action.execute(
|
||||
{ minCoverage: 80, currentCoverage: 65 },
|
||||
makeContext(),
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const data = result.value;
|
||||
expect(data.passed).toBe(false);
|
||||
expect(data.gap).toBe(15);
|
||||
});
|
||||
|
||||
it('should report database source when inputs omitted', async () => {
|
||||
const action = findDeterministicAction('coverage-analysis', 'threshold-check')!;
|
||||
const result = await action.execute({}, makeContext());
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const data = result.value;
|
||||
expect(data.currentCoverage).toBe(0); // no DB available → 0
|
||||
expect(data.passed).toBe(false);
|
||||
expect(data.source).toBe('database');
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Pattern Health Check
|
||||
// ============================================================================
|
||||
|
||||
describe('pattern-health action', () => {
|
||||
it('should compute a health score from explicit stats', async () => {
|
||||
const action = findDeterministicAction('learning-optimization', 'health-check')!;
|
||||
const result = await action.execute(
|
||||
{ totalPatterns: 100, activePatterns: 80, avgConfidence: 0.9 },
|
||||
makeContext(),
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const data = result.value;
|
||||
expect(data.healthScore).toBeGreaterThan(0);
|
||||
expect(data.healthScore).toBeLessThanOrEqual(1);
|
||||
expect(data.totalPatterns).toBe(100);
|
||||
expect(data.activePatterns).toBe(80);
|
||||
expect(data.source).toBe('input');
|
||||
});
|
||||
|
||||
it('should return zero health for empty state and database source', async () => {
|
||||
const action = findDeterministicAction('learning-optimization', 'health-check')!;
|
||||
const result = await action.execute({}, makeContext());
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const data = result.value;
|
||||
expect(data.healthScore).toBe(0);
|
||||
expect(data.totalPatterns).toBe(0);
|
||||
expect(data.source).toBe('database');
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Routing Accuracy Check
|
||||
// ============================================================================
|
||||
|
||||
describe('routing-accuracy action', () => {
|
||||
it('should compute success rate from explicit outcomes', async () => {
|
||||
const action = findDeterministicAction('learning-optimization', 'routing-check')!;
|
||||
const result = await action.execute(
|
||||
{ totalOutcomes: 200, successfulOutcomes: 180, confidenceCorrelation: 0.85 },
|
||||
makeContext(),
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const data = result.value;
|
||||
expect(data.successRate).toBe(90);
|
||||
expect(data.totalOutcomes).toBe(200);
|
||||
expect(data.confidenceCorrelation).toBe(0.85);
|
||||
expect(data.source).toBe('input');
|
||||
});
|
||||
|
||||
it('should handle zero outcomes gracefully and report database source', async () => {
|
||||
const action = findDeterministicAction('learning-optimization', 'routing-check')!;
|
||||
const result = await action.execute({}, makeContext());
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const data = result.value;
|
||||
expect(data.successRate).toBe(0);
|
||||
expect(data.totalOutcomes).toBe(0);
|
||||
expect(data.source).toBe('database');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,219 @@
|
||||
/**
|
||||
* Agentic QE v3 - Heartbeat MCP Handler Tests
|
||||
* Imp-10: Token-Free Heartbeat Scheduler MCP Integration
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
// Use vi.hoisted() so mock fns are available when vi.mock factory runs (hoisted above imports)
|
||||
const mocks = vi.hoisted(() => ({
|
||||
existsSync: vi.fn(),
|
||||
readFileSync: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('fs', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('fs')>();
|
||||
return {
|
||||
...actual,
|
||||
existsSync: mocks.existsSync,
|
||||
readFileSync: mocks.readFileSync,
|
||||
};
|
||||
});
|
||||
|
||||
// Mock the worker before importing handlers
|
||||
vi.mock('../../../src/workers/workers/heartbeat-scheduler.js', () => {
|
||||
return {
|
||||
HeartbeatSchedulerWorker: class MockHeartbeatWorker {
|
||||
config = {
|
||||
id: 'heartbeat-scheduler',
|
||||
name: 'Heartbeat Scheduler',
|
||||
intervalMs: 30 * 60 * 1000,
|
||||
enabled: true,
|
||||
};
|
||||
status = 'idle';
|
||||
lastResult = {
|
||||
workerId: 'heartbeat-scheduler',
|
||||
timestamp: new Date('2026-03-27T14:32:15Z'),
|
||||
durationMs: 245,
|
||||
success: true,
|
||||
metrics: {
|
||||
itemsAnalyzed: 50,
|
||||
issuesFound: 3,
|
||||
healthScore: 85,
|
||||
trend: 'stable' as const,
|
||||
domainMetrics: {
|
||||
promoted: 2,
|
||||
deprecated: 1,
|
||||
decayed: 15,
|
||||
pendingExperiences: 8,
|
||||
avgConfidence: 0.72,
|
||||
},
|
||||
},
|
||||
findings: [],
|
||||
recommendations: [],
|
||||
};
|
||||
lastRunAt = new Date('2026-03-27T14:32:15Z');
|
||||
nextRunAt = new Date('2026-03-27T15:02:15Z');
|
||||
|
||||
initialize = vi.fn().mockResolvedValue(undefined);
|
||||
execute = vi.fn().mockResolvedValue({
|
||||
workerId: 'heartbeat-scheduler',
|
||||
timestamp: new Date(),
|
||||
durationMs: 312,
|
||||
success: true,
|
||||
metrics: {
|
||||
itemsAnalyzed: 42,
|
||||
issuesFound: 1,
|
||||
healthScore: 88,
|
||||
trend: 'improving',
|
||||
domainMetrics: {
|
||||
promoted: 3,
|
||||
deprecated: 0,
|
||||
decayed: 10,
|
||||
pendingExperiences: 5,
|
||||
avgConfidence: 0.75,
|
||||
},
|
||||
},
|
||||
findings: [{ type: 'heartbeat-promotion', severity: 'info', domain: 'learning-optimization', title: 'Promoted', description: '3 promoted' }],
|
||||
recommendations: [],
|
||||
});
|
||||
getHealth = vi.fn().mockReturnValue({
|
||||
status: 'idle',
|
||||
healthScore: 85,
|
||||
totalExecutions: 42,
|
||||
successfulExecutions: 41,
|
||||
failedExecutions: 1,
|
||||
avgDurationMs: 280,
|
||||
recentResults: [],
|
||||
});
|
||||
pause = vi.fn();
|
||||
resume = vi.fn();
|
||||
stop = vi.fn().mockResolvedValue(undefined);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
// Mock unified-memory for findProjectRoot
|
||||
vi.mock('../../../src/kernel/unified-memory.js', () => ({
|
||||
findProjectRoot: () => '/tmp/test-project',
|
||||
getUnifiedMemory: vi.fn(),
|
||||
}));
|
||||
|
||||
import {
|
||||
handleHeartbeatStatus,
|
||||
handleHeartbeatTrigger,
|
||||
handleHeartbeatLog,
|
||||
} from '../../../src/mcp/handlers/heartbeat-handlers.js';
|
||||
|
||||
describe('Heartbeat MCP Handlers', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('handleHeartbeatStatus', () => {
|
||||
it('should return health JSON with correct structure', async () => {
|
||||
const result = await handleHeartbeatStatus({});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toBeDefined();
|
||||
expect(result.data!.status).toBe('idle');
|
||||
expect(result.data!.healthScore).toBe(85);
|
||||
expect(result.data!.totalExecutions).toBe(42);
|
||||
expect(result.data!.successfulExecutions).toBe(41);
|
||||
expect(result.data!.failedExecutions).toBe(1);
|
||||
expect(result.data!.avgDurationMs).toBe(280);
|
||||
});
|
||||
|
||||
it('should include last run timestamps', async () => {
|
||||
const result = await handleHeartbeatStatus({});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data!.lastRunAt).toBe('2026-03-27T14:32:15.000Z');
|
||||
expect(result.data!.nextRunAt).toBe('2026-03-27T15:02:15.000Z');
|
||||
});
|
||||
|
||||
it('should include last result details', async () => {
|
||||
const result = await handleHeartbeatStatus({});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data!.lastResult).toBeDefined();
|
||||
expect(result.data!.lastResult!.healthScore).toBe(85);
|
||||
expect(result.data!.lastResult!.trend).toBe('stable');
|
||||
expect(result.data!.lastResult!.domainMetrics.promoted).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleHeartbeatTrigger', () => {
|
||||
it('should execute heartbeat and return result', async () => {
|
||||
const result = await handleHeartbeatTrigger({});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toBeDefined();
|
||||
expect(result.data!.success).toBe(true);
|
||||
expect(result.data!.healthScore).toBe(88);
|
||||
expect(result.data!.trend).toBe('improving');
|
||||
expect(result.data!.promoted).toBe(3);
|
||||
expect(result.data!.deprecated).toBe(0);
|
||||
expect(result.data!.decayed).toBe(10);
|
||||
expect(result.data!.pendingExperiences).toBe(5);
|
||||
expect(result.data!.avgConfidence).toBe(0.75);
|
||||
expect(result.data!.findingsCount).toBe(1);
|
||||
expect(result.data!.recommendationsCount).toBe(0);
|
||||
});
|
||||
|
||||
it('should return numeric duration', async () => {
|
||||
const result = await handleHeartbeatTrigger({});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(typeof result.data!.durationMs).toBe('number');
|
||||
expect(result.data!.durationMs).toBe(312);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleHeartbeatLog', () => {
|
||||
it('should return log content when file exists', async () => {
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
const logContent = `# AQE Daily Log \u2014 ${today}\n| Time | Event | Summary |\n`;
|
||||
|
||||
mocks.existsSync.mockReturnValue(true);
|
||||
mocks.readFileSync.mockReturnValue(logContent);
|
||||
|
||||
const result = await handleHeartbeatLog({});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data!.date).toBe(today);
|
||||
expect(result.data!.exists).toBe(true);
|
||||
expect(result.data!.content).toContain('AQE Daily Log');
|
||||
expect(result.data!.lineCount).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should accept a specific date parameter', async () => {
|
||||
mocks.existsSync.mockReturnValue(true);
|
||||
mocks.readFileSync.mockReturnValue('# AQE Daily Log \u2014 2026-03-25\n');
|
||||
|
||||
const result = await handleHeartbeatLog({ date: '2026-03-25' });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data!.date).toBe('2026-03-25');
|
||||
expect(result.data!.exists).toBe(true);
|
||||
});
|
||||
|
||||
it('should report missing log files gracefully', async () => {
|
||||
mocks.existsSync.mockReturnValue(false);
|
||||
|
||||
const result = await handleHeartbeatLog({ date: '2020-01-01' });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data!.exists).toBe(false);
|
||||
expect(result.data!.content).toBe('');
|
||||
expect(result.data!.lineCount).toBe(0);
|
||||
});
|
||||
|
||||
it('should reject invalid date formats', async () => {
|
||||
const result = await handleHeartbeatLog({ date: 'not-a-date' });
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('Invalid date format');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Unit tests for the routing_economics MCP handler (Imp-18)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
// Mock unified memory — keep all real exports but stub the singleton
|
||||
vi.mock('../../../src/kernel/unified-memory.js', async (importOriginal) => {
|
||||
const actual = await importOriginal<Record<string, unknown>>();
|
||||
const mockDb = {
|
||||
prepare: vi.fn().mockReturnValue({
|
||||
get: vi.fn().mockReturnValue(undefined),
|
||||
all: vi.fn().mockReturnValue([]),
|
||||
run: vi.fn(),
|
||||
}),
|
||||
};
|
||||
return {
|
||||
...actual,
|
||||
getUnifiedMemory: vi.fn().mockReturnValue({
|
||||
isInitialized: vi.fn().mockReturnValue(true),
|
||||
initialize: vi.fn().mockResolvedValue(undefined),
|
||||
getDatabase: vi.fn().mockReturnValue(mockDb),
|
||||
}),
|
||||
initializeUnifiedMemory: vi.fn().mockResolvedValue({
|
||||
isInitialized: () => true,
|
||||
initialize: async () => {},
|
||||
getDatabase: () => mockDb,
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
// Mock cost tracker — keep real exports but stub the global singleton
|
||||
vi.mock('../../../src/shared/llm/cost-tracker.js', async (importOriginal) => {
|
||||
const actual = await importOriginal<Record<string, unknown>>();
|
||||
return {
|
||||
...actual,
|
||||
getGlobalCostTracker: vi.fn().mockReturnValue({
|
||||
getCurrentCost: vi.fn().mockReturnValue(0),
|
||||
getSummary: vi.fn().mockReturnValue({
|
||||
totalCost: 0, totalTokens: 0, totalRequests: 0,
|
||||
period: 'all', periodStart: new Date(0), periodEnd: new Date(),
|
||||
byProvider: {}, byModel: {},
|
||||
}),
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
import { handleRoutingEconomics } from '../../../src/mcp/handlers/task-handlers.js';
|
||||
|
||||
describe('handleRoutingEconomics MCP handler', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should return a successful economic report', async () => {
|
||||
const result = await handleRoutingEconomics({ taskComplexity: 0.5 });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toBeDefined();
|
||||
expect(result.data!.tierEfficiency).toBeInstanceOf(Array);
|
||||
expect(result.data!.tierEfficiency.length).toBe(4);
|
||||
expect(typeof result.data!.currentHourlyCostUsd).toBe('number');
|
||||
expect(typeof result.data!.currentDailyCostUsd).toBe('number');
|
||||
expect(typeof result.data!.recommendation).toBe('string');
|
||||
});
|
||||
|
||||
it('should include budget remaining as null when no limits set', async () => {
|
||||
const result = await handleRoutingEconomics({ taskComplexity: 0.3 });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data!.budgetRemaining).toBeDefined();
|
||||
expect(result.data!.budgetRemaining.hourly).toBeNull();
|
||||
expect(result.data!.budgetRemaining.daily).toBeNull();
|
||||
});
|
||||
|
||||
it('should serialize Infinity qualityPerDollar for booster tier', async () => {
|
||||
const result = await handleRoutingEconomics({ taskComplexity: 0.1 });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const booster = result.data!.tierEfficiency.find(
|
||||
(t: Record<string, unknown>) => t.tier === 'booster',
|
||||
);
|
||||
expect(booster).toBeDefined();
|
||||
// Infinity should be serialized as string for JSON safety
|
||||
expect(booster!.qualityPerDollar).toBe('Infinity');
|
||||
});
|
||||
|
||||
it('should return tiers sorted by economic score descending', async () => {
|
||||
const result = await handleRoutingEconomics({ taskComplexity: 0.5 });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const scores = result.data!.tierEfficiency.map(
|
||||
(t: Record<string, unknown>) => t.economicScore as number,
|
||||
);
|
||||
for (let i = 1; i < scores.length; i++) {
|
||||
expect(scores[i - 1]).toBeGreaterThanOrEqual(scores[i]);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,408 @@
|
||||
/**
|
||||
* Economic Routing Model Tests — Imp-18, Issue #334
|
||||
*
|
||||
* Tests quality-weighted cost optimization for the routing system.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import {
|
||||
EconomicRoutingModel,
|
||||
TIER_COST_ESTIMATES,
|
||||
DEFAULT_ECONOMIC_CONFIG,
|
||||
type EconomicRoutingConfig,
|
||||
} from '../../../src/routing/economic-routing.js';
|
||||
import { CostTracker } from '../../../src/shared/llm/cost-tracker.js';
|
||||
import type { RoutingOutcome } from '../../../src/routing/types.js';
|
||||
import type { AgentTier } from '../../../src/routing/routing-config.js';
|
||||
import {
|
||||
RoutingFeedbackCollector,
|
||||
createRoutingFeedbackCollector,
|
||||
} from '../../../src/routing/routing-feedback.js';
|
||||
import type { QETask, QERoutingDecision } from '../../../src/routing/types.js';
|
||||
|
||||
// ============================================================================
|
||||
// Helpers
|
||||
// ============================================================================
|
||||
|
||||
function createModel(
|
||||
config?: Partial<EconomicRoutingConfig>,
|
||||
costTracker?: CostTracker,
|
||||
): EconomicRoutingModel {
|
||||
return new EconomicRoutingModel(costTracker ?? new CostTracker(), config);
|
||||
}
|
||||
|
||||
function makeOutcome(overrides?: Partial<{
|
||||
success: boolean;
|
||||
qualityScore: number;
|
||||
durationMs: number;
|
||||
}>): RoutingOutcome {
|
||||
return {
|
||||
id: `test-${Date.now()}`,
|
||||
task: { description: 'test task' },
|
||||
decision: {
|
||||
recommended: 'agent-1',
|
||||
confidence: 0.8,
|
||||
alternatives: [],
|
||||
reasoning: 'test',
|
||||
scores: { similarity: 0.8, performance: 0.7, capabilities: 0.9, combined: 0.8 },
|
||||
latencyMs: 10,
|
||||
timestamp: new Date(),
|
||||
},
|
||||
usedAgent: 'agent-1',
|
||||
followedRecommendation: true,
|
||||
outcome: {
|
||||
success: overrides?.success ?? true,
|
||||
qualityScore: overrides?.qualityScore ?? 0.8,
|
||||
durationMs: overrides?.durationMs ?? 3000,
|
||||
},
|
||||
timestamp: new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockTask(description: string): QETask {
|
||||
return { description };
|
||||
}
|
||||
|
||||
function createMockDecision(recommended: string, confidence: number): QERoutingDecision {
|
||||
return {
|
||||
recommended,
|
||||
confidence,
|
||||
alternatives: [],
|
||||
reasoning: `Selected ${recommended}`,
|
||||
scores: { similarity: 0.8, performance: 0.7, capabilities: 0.9, combined: confidence },
|
||||
latencyMs: 50,
|
||||
timestamp: new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('EconomicRoutingModel', () => {
|
||||
let model: EconomicRoutingModel;
|
||||
|
||||
beforeEach(() => {
|
||||
model = createModel();
|
||||
});
|
||||
|
||||
describe('scoreTiers()', () => {
|
||||
it('should return all four tiers sorted by economic score descending', () => {
|
||||
const scores = model.scoreTiers(0.5);
|
||||
|
||||
expect(scores).toHaveLength(4);
|
||||
// Verify sorted descending by economicScore
|
||||
for (let i = 1; i < scores.length; i++) {
|
||||
expect(scores[i - 1].economicScore).toBeGreaterThanOrEqual(scores[i].economicScore);
|
||||
}
|
||||
});
|
||||
|
||||
it('should include all expected properties in each score', () => {
|
||||
const scores = model.scoreTiers(0.3);
|
||||
for (const score of scores) {
|
||||
expect(score).toHaveProperty('tier');
|
||||
expect(score).toHaveProperty('qualityScore');
|
||||
expect(score).toHaveProperty('estimatedCostUsd');
|
||||
expect(score).toHaveProperty('qualityPerDollar');
|
||||
expect(score).toHaveProperty('economicScore');
|
||||
expect(score.qualityScore).toBeGreaterThanOrEqual(0);
|
||||
expect(score.qualityScore).toBeLessThanOrEqual(1);
|
||||
expect(score.estimatedCostUsd).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('should give booster infinite qualityPerDollar when cost is zero', () => {
|
||||
const scores = model.scoreTiers(0);
|
||||
const booster = scores.find(s => s.tier === 'booster');
|
||||
expect(booster).toBeDefined();
|
||||
expect(booster!.estimatedCostUsd).toBe(0);
|
||||
// booster quality > 0 and cost = 0 => Infinity
|
||||
expect(booster!.qualityPerDollar).toBe(Infinity);
|
||||
});
|
||||
|
||||
it('should penalize cheaper tiers more for complex tasks', () => {
|
||||
const simpleScores = model.scoreTiers(0.1);
|
||||
const complexScores = model.scoreTiers(0.9);
|
||||
|
||||
const haikuSimple = simpleScores.find(s => s.tier === 'haiku')!;
|
||||
const haikuComplex = complexScores.find(s => s.tier === 'haiku')!;
|
||||
|
||||
// Haiku quality should be lower for complex tasks
|
||||
expect(haikuComplex.qualityScore).toBeLessThan(haikuSimple.qualityScore);
|
||||
});
|
||||
});
|
||||
|
||||
describe('selectTier()', () => {
|
||||
it('should select a tier that meets quality threshold', () => {
|
||||
const result = model.selectTier(0.5);
|
||||
expect(result.tier).toBeDefined();
|
||||
expect(result.reason).toBeTruthy();
|
||||
expect(result.scores).toHaveLength(4);
|
||||
});
|
||||
|
||||
it('should respect budget limits by falling back to cheaper tier', () => {
|
||||
// Create a cost tracker that has already spent near the limit
|
||||
const costTracker = new CostTracker();
|
||||
// Record usage to simulate high spend
|
||||
costTracker.recordUsage('claude', 'claude-3-opus-20240229', {
|
||||
promptTokens: 100000,
|
||||
completionTokens: 50000,
|
||||
totalTokens: 150000,
|
||||
}, 'req-1');
|
||||
|
||||
const budgetModel = createModel({
|
||||
budgetPerHourUsd: 0.001, // Very tight budget
|
||||
budgetPerDayUsd: 0.001,
|
||||
minQualityThreshold: 0.1, // Lower threshold to allow booster as fallback
|
||||
}, costTracker);
|
||||
|
||||
const result = budgetModel.selectTier(0.1); // Low complexity so booster quality is adequate
|
||||
// Should fall back to booster (free) since budget is exceeded for paid tiers
|
||||
expect(result.tier).toBe('booster');
|
||||
});
|
||||
|
||||
it('should respect minimum quality threshold', () => {
|
||||
const result = model.selectTier(0.95); // Very complex task
|
||||
// Booster quality at 0.95 complexity: 0.3 - 0.95*0.4 = -0.08 (clamped to 0)
|
||||
// Should not pick booster for this complexity
|
||||
expect(result.tier).not.toBe('booster');
|
||||
});
|
||||
|
||||
it('should provide a reason with the selection', () => {
|
||||
const result = model.selectTier(0.5);
|
||||
expect(result.reason.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('wouldExceedBudget()', () => {
|
||||
it('should return false when no budget limits are set', () => {
|
||||
expect(model.wouldExceedBudget('opus')).toBe(false);
|
||||
expect(model.wouldExceedBudget('haiku')).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false for booster (zero cost)', () => {
|
||||
const budgetModel = createModel({ budgetPerHourUsd: 0.001 });
|
||||
expect(budgetModel.wouldExceedBudget('booster')).toBe(false);
|
||||
});
|
||||
|
||||
it('should return true when hourly budget would be exceeded', () => {
|
||||
const costTracker = new CostTracker();
|
||||
costTracker.recordUsage('claude', 'claude-3-opus-20240229', {
|
||||
promptTokens: 500000,
|
||||
completionTokens: 200000,
|
||||
totalTokens: 700000,
|
||||
}, 'req-1');
|
||||
|
||||
const budgetModel = createModel({
|
||||
budgetPerHourUsd: 0.001,
|
||||
}, costTracker);
|
||||
|
||||
expect(budgetModel.wouldExceedBudget('opus')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return true when daily budget would be exceeded', () => {
|
||||
const costTracker = new CostTracker();
|
||||
costTracker.recordUsage('claude', 'claude-3-opus-20240229', {
|
||||
promptTokens: 500000,
|
||||
completionTokens: 200000,
|
||||
totalTokens: 700000,
|
||||
}, 'req-1');
|
||||
|
||||
const budgetModel = createModel({
|
||||
budgetPerDayUsd: 0.001,
|
||||
}, costTracker);
|
||||
|
||||
expect(budgetModel.wouldExceedBudget('sonnet')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('updateFromOutcome()', () => {
|
||||
it('should adjust quality estimates via EMA', () => {
|
||||
const initialScores = model.scoreTiers(0.5);
|
||||
const initialHaiku = initialScores.find(s => s.tier === 'haiku')!.qualityScore;
|
||||
|
||||
// Record several high-quality haiku outcomes
|
||||
for (let i = 0; i < 10; i++) {
|
||||
model.updateFromOutcome(makeOutcome({ qualityScore: 0.95 }), 'haiku');
|
||||
}
|
||||
|
||||
const updatedScores = model.scoreTiers(0.5);
|
||||
const updatedHaiku = updatedScores.find(s => s.tier === 'haiku')!.qualityScore;
|
||||
|
||||
// Quality should have increased toward 0.95
|
||||
expect(updatedHaiku).toBeGreaterThan(initialHaiku);
|
||||
});
|
||||
|
||||
it('should decrease quality estimate after low-quality outcomes', () => {
|
||||
const initialScores = model.scoreTiers(0.3);
|
||||
const initialOpus = initialScores.find(s => s.tier === 'opus')!.qualityScore;
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
model.updateFromOutcome(makeOutcome({ qualityScore: 0.2 }), 'opus');
|
||||
}
|
||||
|
||||
const updatedScores = model.scoreTiers(0.3);
|
||||
const updatedOpus = updatedScores.find(s => s.tier === 'opus')!.qualityScore;
|
||||
|
||||
expect(updatedOpus).toBeLessThan(initialOpus);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeCostAdjustedReward()', () => {
|
||||
it('should return the base reward for booster (zero cost)', () => {
|
||||
const adjusted = model.computeCostAdjustedReward(0.5, 'booster', 0.8);
|
||||
// booster costRatio = 0 => costPenalty = 0 => adjusted = baseReward
|
||||
expect(adjusted).toBe(0.5);
|
||||
});
|
||||
|
||||
it('should penalize expensive tiers with low quality', () => {
|
||||
const adjusted = model.computeCostAdjustedReward(0.5, 'opus', 0.3);
|
||||
// opus costRatio=1.0, qualityGain = max(0, 0.3-0.5) = 0
|
||||
// costPenalty = 1.0 * (1 - 0) = 1.0
|
||||
// adjusted = 0.5 - 1.0 * 0.4 = 0.1
|
||||
expect(adjusted).toBeCloseTo(0.1, 1);
|
||||
});
|
||||
|
||||
it('should not penalize expensive tiers that deliver high quality', () => {
|
||||
const adjusted = model.computeCostAdjustedReward(0.5, 'opus', 0.9);
|
||||
// qualityGain = 0.9 - 0.5 = 0.4
|
||||
// costPenalty = 1.0 * (1 - 0.4) = 0.6
|
||||
// adjusted = 0.5 - 0.6 * 0.4 = 0.26
|
||||
// Still penalized somewhat, but less than low-quality
|
||||
const lowQualityAdjusted = model.computeCostAdjustedReward(0.5, 'opus', 0.3);
|
||||
expect(adjusted).toBeGreaterThan(lowQualityAdjusted);
|
||||
});
|
||||
|
||||
it('should clamp the result to [-1, 1]', () => {
|
||||
const adjusted = model.computeCostAdjustedReward(-0.9, 'opus', 0.1);
|
||||
expect(adjusted).toBeGreaterThanOrEqual(-1);
|
||||
expect(adjusted).toBeLessThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getEconomicReport()', () => {
|
||||
it('should return a well-formed report', () => {
|
||||
const report = model.getEconomicReport();
|
||||
|
||||
expect(report).toHaveProperty('tierEfficiency');
|
||||
expect(report).toHaveProperty('currentHourlyCostUsd');
|
||||
expect(report).toHaveProperty('currentDailyCostUsd');
|
||||
expect(report).toHaveProperty('budgetRemaining');
|
||||
expect(report).toHaveProperty('recommendation');
|
||||
expect(report.tierEfficiency).toHaveLength(4);
|
||||
expect(typeof report.recommendation).toBe('string');
|
||||
expect(report.recommendation.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should report null budget remaining when no limits set', () => {
|
||||
const report = model.getEconomicReport();
|
||||
expect(report.budgetRemaining.hourly).toBeNull();
|
||||
expect(report.budgetRemaining.daily).toBeNull();
|
||||
});
|
||||
|
||||
it('should report budget remaining when limits are set', () => {
|
||||
const budgetModel = createModel({
|
||||
budgetPerHourUsd: 1.0,
|
||||
budgetPerDayUsd: 10.0,
|
||||
});
|
||||
const report = budgetModel.getEconomicReport();
|
||||
expect(report.budgetRemaining.hourly).toBe(1.0);
|
||||
expect(report.budgetRemaining.daily).toBe(10.0);
|
||||
});
|
||||
|
||||
it('should include savings opportunity', () => {
|
||||
const report = model.getEconomicReport();
|
||||
// There should be a savings opportunity comparing opus vs haiku
|
||||
expect(report.savingsOpportunity).not.toBeNull();
|
||||
expect(report.savingsOpportunity!.usd).toBeGreaterThan(0);
|
||||
expect(report.savingsOpportunity!.description.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('serialization', () => {
|
||||
it('should serialize and deserialize quality estimates', () => {
|
||||
// Update some quality estimates
|
||||
model.updateFromOutcome(makeOutcome({ qualityScore: 0.9 }), 'haiku');
|
||||
model.updateFromOutcome(makeOutcome({ qualityScore: 0.6 }), 'opus');
|
||||
|
||||
const serialized = model.serializeEstimates();
|
||||
expect(serialized).toHaveProperty('haiku');
|
||||
expect(serialized).toHaveProperty('opus');
|
||||
|
||||
// Create a new model and deserialize
|
||||
const newModel = createModel();
|
||||
newModel.deserializeEstimates(serialized);
|
||||
|
||||
// Scores should match after deserialization
|
||||
const originalScores = model.scoreTiers(0.5);
|
||||
const restoredScores = newModel.scoreTiers(0.5);
|
||||
|
||||
for (const tier of ['booster', 'haiku', 'sonnet', 'opus'] as AgentTier[]) {
|
||||
const orig = originalScores.find(s => s.tier === tier)!;
|
||||
const restored = restoredScores.find(s => s.tier === tier)!;
|
||||
expect(restored.qualityScore).toBeCloseTo(orig.qualityScore, 4);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('RoutingFeedbackCollector — Economic Integration', () => {
|
||||
let collector: RoutingFeedbackCollector;
|
||||
|
||||
beforeEach(() => {
|
||||
collector = createRoutingFeedbackCollector(100);
|
||||
});
|
||||
|
||||
it('should return null economic report when not enabled', () => {
|
||||
expect(collector.getEconomicReport()).toBeNull();
|
||||
expect(collector.getEconomicScore(0.5)).toBeNull();
|
||||
});
|
||||
|
||||
it('should enable economic routing and return a report', () => {
|
||||
collector.enableEconomicRouting();
|
||||
const report = collector.getEconomicReport();
|
||||
expect(report).not.toBeNull();
|
||||
expect(report!.tierEfficiency).toHaveLength(4);
|
||||
});
|
||||
|
||||
it('should return economic scores for a given complexity', () => {
|
||||
collector.enableEconomicRouting();
|
||||
const scores = collector.getEconomicScore(0.3);
|
||||
expect(scores).not.toBeNull();
|
||||
expect(scores!).toHaveLength(4);
|
||||
});
|
||||
|
||||
it('should update economic model when recording outcomes', () => {
|
||||
collector.enableEconomicRouting();
|
||||
|
||||
const task = createMockTask('Generate tests');
|
||||
const decision = createMockDecision('haiku-agent', 0.85);
|
||||
|
||||
// Record several outcomes
|
||||
for (let i = 0; i < 5; i++) {
|
||||
collector.recordOutcome(task, decision, 'haiku-agent', {
|
||||
success: true,
|
||||
qualityScore: 0.9,
|
||||
durationMs: 2000,
|
||||
});
|
||||
}
|
||||
|
||||
// Economic scores should now reflect the observed quality
|
||||
const scores = collector.getEconomicScore(0.3);
|
||||
expect(scores).not.toBeNull();
|
||||
// haiku-agent maps to 'haiku' tier by default inference (contains 'haiku')
|
||||
// but the default inferTier maps most qe-* to sonnet; 'haiku-agent' contains 'haiku'
|
||||
// so it should update haiku tier quality
|
||||
});
|
||||
|
||||
it('should accept custom config for economic routing', () => {
|
||||
collector.enableEconomicRouting({
|
||||
qualityWeight: 0.8,
|
||||
costWeight: 0.2,
|
||||
budgetPerDayUsd: 5.0,
|
||||
});
|
||||
const report = collector.getEconomicReport();
|
||||
expect(report).not.toBeNull();
|
||||
expect(report!.budgetRemaining.daily).toBe(5.0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user