diff --git a/examples/qe-pipeline.yaml b/examples/qe-pipeline.yaml index 26041aea..719cd8b3 100644 --- a/examples/qe-pipeline.yaml +++ b/examples/qe-pipeline.yaml @@ -1,55 +1,58 @@ # Agentic QE v3 - Daily QE Pipeline Example -# Per ADR-041: V3 QE CLI Enhancement +# Deterministic pipeline: zero LLM tokens (Imp-9) -name: daily-qe-pipeline -description: Comprehensive daily quality engineering pipeline +id: daily-qe-pipeline +name: Daily QE Pipeline +description: Deterministic daily quality engineering pipeline (zero LLM tokens) version: "1.0.0" -schedule: 0 2 * * * +defaultMode: sequential tags: - - testing - quality - daily + - token-free -stages: - - name: test-generation - command: aqe test generate - params: - target: src/ - coverage-goal: 90 - ai-enhanced: true - timeout: 120 +steps: + - id: coverage-check + name: Coverage Threshold Check + domain: coverage-analysis + action: threshold-check + inputMapping: + minCoverage: "input.coverageMin" + timeout: 30000 - - name: parallel-execution - command: aqe test execute - depends_on: - - test-generation - params: - parallel: 8 - retry: 3 - timeout: 300 + - id: pattern-health + name: Pattern Health Check + domain: learning-optimization + action: health-check + timeout: 30000 - - name: coverage-analysis - command: aqe coverage analyze - depends_on: - - parallel-execution - params: - gap-detection: true - report-format: html + - id: quality-gate + name: Quality Gate + domain: quality-assessment + action: gate-check + dependsOn: [coverage-check] + inputMapping: + coverageMin: "input.coverageMin" + currentCoverage: "results.coverage-check.currentCoverage" + continueOnFailure: false + timeout: 30000 - - name: quality-gate - command: aqe quality gate - depends_on: - - coverage-analysis - params: - fail-threshold: 80 - coverage-min: 85 + - id: routing-check + name: Routing Accuracy Check + domain: learning-optimization + action: routing-check + timeout: 30000 -triggers: - - event: push - branches: - - main - - develop - - event: pull_request - types: - - opened - - synchronize + - id: deploy-approval + name: Deploy Approval + domain: quality-assessment + action: gate-check + dependsOn: [quality-gate] + approval: + autoApproveAfter: 300000 + message: "All quality checks passed. Approve deployment?" + condition: + path: "results.quality-gate.passed" + operator: eq + value: true + timeout: 600000 diff --git a/src/cli/command-registry.ts b/src/cli/command-registry.ts index 5ba4734a..eb3a6fca 100644 --- a/src/cli/command-registry.ts +++ b/src/cli/command-registry.ts @@ -18,6 +18,8 @@ import { createProtocolHandler, createBrainHandler, createHypergraphHandler, + createHeartbeatHandler, + createRoutingHandler, } from './handlers/index.js'; // ============================================================================ @@ -117,6 +119,12 @@ export class CommandRegistry { // Hypergraph queries this.register(createHypergraphHandler(this.cleanupAndExit, this.ensureInitialized)); + + // Heartbeat scheduler (Imp-10) + this.register(createHeartbeatHandler(this.cleanupAndExit)); + + // Routing economics & accuracy (Imp-18) + this.register(createRoutingHandler(this.cleanupAndExit)); } /** diff --git a/src/cli/commands/pipeline.ts b/src/cli/commands/pipeline.ts new file mode 100644 index 00000000..89d7942c --- /dev/null +++ b/src/cli/commands/pipeline.ts @@ -0,0 +1,342 @@ +/** + * Agentic QE v3 - Pipeline Command (Imp-9) + * + * CLI commands for YAML deterministic pipeline management: + * aqe pipeline load [--vars key=value] + * aqe pipeline validate + * aqe pipeline run [--input key=value] [--wait] + * aqe pipeline list + * aqe pipeline status + * aqe pipeline approve + * aqe pipeline reject [--reason "..."] + */ + +import { Command } from 'commander'; +import { resolve } from 'path'; +import chalk from 'chalk'; +import type { CLIContext } from '../handlers/interfaces.js'; +import { YamlPipelineLoader } from '../../coordination/yaml-pipeline-loader.js'; +import type { WorkflowOrchestrator } from '../../coordination/workflow-orchestrator.js'; + +// ============================================================================ +// Helpers +// ============================================================================ + +/** + * Parse `--vars key1=value1 key2=value2` into a Record. + */ +function parseVars(rawVars: string[]): Record { + const vars: Record = {}; + for (const entry of rawVars) { + const eqIndex = entry.indexOf('='); + if (eqIndex === -1) { + vars[entry] = true; + } else { + const key = entry.slice(0, eqIndex); + const value = entry.slice(eqIndex + 1); + // Try to parse as number or boolean + if (value === 'true') vars[key] = true; + else if (value === 'false') vars[key] = false; + else if (!isNaN(Number(value)) && value !== '') vars[key] = Number(value); + else vars[key] = value; + } + } + return vars; +} + +/** + * Parse `--input key1=value1 key2=value2` into a Record. + */ +function parseInput(rawInput: string[]): Record { + return parseVars(rawInput); +} + +function getOrchestrator(context: CLIContext): WorkflowOrchestrator | null { + return context.workflowOrchestrator; +} + +// ============================================================================ +// Command Factory +// ============================================================================ + +export function createPipelineCommand( + context: CLIContext, + cleanupAndExit: (code: number) => Promise, + ensureInitialized: () => Promise, +): Command { + const loader = new YamlPipelineLoader(); + + const pipelineCmd = new Command('pipeline') + .description('Manage YAML deterministic pipelines (Imp-9)'); + + // --------------------------------------------------------------- + // pipeline load + // --------------------------------------------------------------- + pipelineCmd + .command('load ') + .description('Load and register a pipeline from a YAML file') + .option('--vars ', 'Variable substitutions (key=value)') + .action(async (file: string, options: { vars?: string[] }) => { + if (!await ensureInitialized()) return; + const orchestrator = getOrchestrator(context); + if (!orchestrator) { + console.error(chalk.red(' Workflow orchestrator not available. Run "aqe fleet init" first.')); + await cleanupAndExit(1); + return; + } + + const filePath = resolve(file); + const vars = options.vars ? parseVars(options.vars) : undefined; + + console.log(chalk.blue(`\n Loading pipeline from: ${file}\n`)); + + const result = await loader.loadFromFile(filePath, vars); + if (!result.success) { + console.error(chalk.red(` Parse error: ${result.error.message}`)); + await cleanupAndExit(1); + return; + } + + const definition = result.value; + const registerResult = orchestrator.registerWorkflow(definition); + if (!registerResult.success) { + console.error(chalk.red(` Registration error: ${registerResult.error.message}`)); + await cleanupAndExit(1); + return; + } + + console.log(chalk.green(' Pipeline loaded successfully.')); + console.log(` ID: ${chalk.cyan(definition.id)}`); + console.log(` Name: ${chalk.cyan(definition.name)}`); + console.log(` Steps: ${chalk.cyan(definition.steps.length)}`); + console.log(` Version: ${chalk.cyan(definition.version)}`); + if (definition.tags?.length) { + console.log(` Tags: ${chalk.cyan(definition.tags.join(', '))}`); + } + console.log(''); + await cleanupAndExit(0); + }); + + // --------------------------------------------------------------- + // pipeline validate + // --------------------------------------------------------------- + pipelineCmd + .command('validate ') + .description('Validate a YAML pipeline without registering it') + .option('--vars ', 'Variable substitutions (key=value)') + .action(async (file: string, options: { vars?: string[] }) => { + const filePath = resolve(file); + const vars = options.vars ? parseVars(options.vars) : undefined; + + console.log(chalk.blue(`\n Validating pipeline: ${file}\n`)); + + const result = await loader.loadFromFile(filePath, vars); + if (!result.success) { + console.log(chalk.red(` Invalid: ${result.error.message}`)); + await cleanupAndExit(1); + return; + } + + const def = result.value; + console.log(chalk.green(' Valid pipeline.')); + console.log(` ID: ${chalk.cyan(def.id)}`); + console.log(` Name: ${chalk.cyan(def.name)}`); + console.log(` Steps: ${chalk.cyan(def.steps.length)}`); + console.log(''); + await cleanupAndExit(0); + }); + + // --------------------------------------------------------------- + // pipeline run + // --------------------------------------------------------------- + pipelineCmd + .command('run ') + .description('Execute a registered pipeline') + .option('--input ', 'Input parameters (key=value)') + .option('--wait', 'Wait for execution to complete') + .action(async (pipelineId: string, options: { input?: string[]; wait?: boolean }) => { + if (!await ensureInitialized()) return; + const orchestrator = getOrchestrator(context); + if (!orchestrator) { + console.error(chalk.red(' Workflow orchestrator not available. Run "aqe fleet init" first.')); + await cleanupAndExit(1); + return; + } + + const input = options.input ? parseInput(options.input) : {}; + + console.log(chalk.blue(`\n Running pipeline: ${pipelineId}\n`)); + + const result = await orchestrator.executeWorkflow(pipelineId, input); + if (!result.success) { + console.error(chalk.red(` Failed: ${result.error.message}`)); + await cleanupAndExit(1); + return; + } + + const executionId = result.value; + console.log(chalk.green(' Pipeline started.')); + console.log(` Execution ID: ${chalk.cyan(executionId)}`); + + if (options.wait) { + console.log(chalk.gray(' Waiting for completion...')); + let status = orchestrator.getWorkflowStatus(executionId); + while (status && (status.status === 'running' || status.status === 'paused')) { + await new Promise((r) => setTimeout(r, 500)); + status = orchestrator.getWorkflowStatus(executionId); + } + if (status) { + const statusColor = status.status === 'completed' ? chalk.green : chalk.red; + console.log(` Status: ${statusColor(status.status)}`); + if (status.duration) console.log(` Duration: ${chalk.cyan(`${status.duration}ms`)}`); + if (status.error) console.log(` Error: ${chalk.red(status.error)}`); + } + } + + console.log(''); + await cleanupAndExit(0); + }); + + // --------------------------------------------------------------- + // pipeline list + // --------------------------------------------------------------- + pipelineCmd + .command('list') + .description('List all registered pipelines') + .action(async () => { + if (!await ensureInitialized()) return; + const orchestrator = getOrchestrator(context); + if (!orchestrator) { + console.error(chalk.red(' Workflow orchestrator not available. Run "aqe fleet init" first.')); + await cleanupAndExit(1); + return; + } + + const workflows = orchestrator.listWorkflows(); + console.log(chalk.blue(`\n Registered Pipelines (${workflows.length})\n`)); + + if (workflows.length === 0) { + console.log(chalk.gray(' No pipelines registered.')); + } else { + for (const wf of workflows) { + console.log(` ${chalk.cyan(wf.id)} — ${wf.name} (${wf.stepCount} steps, v${wf.version})`); + if (wf.tags?.length) { + console.log(` Tags: ${chalk.gray(wf.tags.join(', '))}`); + } + } + } + + console.log(''); + await cleanupAndExit(0); + }); + + // --------------------------------------------------------------- + // pipeline status + // --------------------------------------------------------------- + pipelineCmd + .command('status ') + .description('Show the status of a pipeline execution') + .action(async (executionId: string) => { + if (!await ensureInitialized()) return; + const orchestrator = getOrchestrator(context); + if (!orchestrator) { + console.error(chalk.red(' Workflow orchestrator not available. Run "aqe fleet init" first.')); + await cleanupAndExit(1); + return; + } + + const status = orchestrator.getWorkflowStatus(executionId); + if (!status) { + console.error(chalk.red(` Execution not found: ${executionId}`)); + await cleanupAndExit(1); + return; + } + + const statusColor = + status.status === 'completed' ? chalk.green : + status.status === 'failed' ? chalk.red : + status.status === 'running' ? chalk.yellow : chalk.gray; + + console.log(chalk.blue(`\n Pipeline Execution Status\n`)); + console.log(` Execution: ${chalk.cyan(executionId)}`); + console.log(` Pipeline: ${chalk.cyan(status.workflowName)} (${status.workflowId})`); + console.log(` Status: ${statusColor(status.status)}`); + console.log(` Progress: ${chalk.cyan(`${status.progress}%`)}`); + console.log(` Completed: ${chalk.cyan(status.completedSteps.join(', ') || 'none')}`); + if (status.failedSteps.length > 0) { + console.log(` Failed: ${chalk.red(status.failedSteps.join(', '))}`); + } + if (status.skippedSteps.length > 0) { + console.log(` Skipped: ${chalk.gray(status.skippedSteps.join(', '))}`); + } + if (status.currentSteps.length > 0) { + console.log(` Running: ${chalk.yellow(status.currentSteps.join(', '))}`); + } + if (status.duration) { + console.log(` Duration: ${chalk.cyan(`${status.duration}ms`)}`); + } + if (status.error) { + console.log(` Error: ${chalk.red(status.error)}`); + } + + console.log(''); + await cleanupAndExit(0); + }); + + // --------------------------------------------------------------- + // pipeline approve + // --------------------------------------------------------------- + pipelineCmd + .command('approve ') + .description('Approve a step that is awaiting approval') + .action(async (executionId: string, stepId: string) => { + if (!await ensureInitialized()) return; + const orchestrator = getOrchestrator(context); + if (!orchestrator) { + console.error(chalk.red(' Workflow orchestrator not available.')); + await cleanupAndExit(1); + return; + } + + const success = orchestrator.approveStep(executionId, stepId); + if (success) { + console.log(chalk.green(`\n Step '${stepId}' approved.\n`)); + } else { + console.error(chalk.red(`\n No pending approval found for step '${stepId}' in execution '${executionId}'.\n`)); + await cleanupAndExit(1); + return; + } + await cleanupAndExit(0); + }); + + // --------------------------------------------------------------- + // pipeline reject + // --------------------------------------------------------------- + pipelineCmd + .command('reject ') + .description('Reject a step that is awaiting approval') + .option('--reason ', 'Rejection reason') + .action(async (executionId: string, stepId: string, options: { reason?: string }) => { + if (!await ensureInitialized()) return; + const orchestrator = getOrchestrator(context); + if (!orchestrator) { + console.error(chalk.red(' Workflow orchestrator not available.')); + await cleanupAndExit(1); + return; + } + + const success = orchestrator.rejectStep(executionId, stepId, options.reason); + if (success) { + console.log(chalk.green(`\n Step '${stepId}' rejected.`)); + if (options.reason) console.log(` Reason: ${chalk.gray(options.reason)}`); + console.log(''); + } else { + console.error(chalk.red(`\n No pending approval found for step '${stepId}' in execution '${executionId}'.\n`)); + await cleanupAndExit(1); + return; + } + await cleanupAndExit(0); + }); + + return pipelineCmd; +} diff --git a/src/cli/handlers/heartbeat-handler.ts b/src/cli/handlers/heartbeat-handler.ts new file mode 100644 index 00000000..cf460419 --- /dev/null +++ b/src/cli/handlers/heartbeat-handler.ts @@ -0,0 +1,440 @@ +/** + * Agentic QE v3 - Heartbeat Command Handler + * Imp-10: Token-Free Heartbeat Scheduler CLI Integration + * + * Handles the 'aqe heartbeat' command with subcommands: + * status, run-now, history, log, pause, resume + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { Command } from 'commander'; +import chalk from 'chalk'; +import { ICommandHandler, CLIContext, formatDuration } from './interfaces.js'; +import { HeartbeatSchedulerWorker } from '../../workers/workers/heartbeat-scheduler.js'; +import type { WorkerResult } from '../../workers/interfaces.js'; +import { toErrorMessage } from '../../shared/error-utils.js'; +import { findProjectRoot } from '../../kernel/unified-memory.js'; + +// ============================================================================ +// Heartbeat Handler +// ============================================================================ + +export class HeartbeatHandler implements ICommandHandler { + readonly name = 'heartbeat'; + readonly description = 'Manage the token-free heartbeat scheduler'; + + private cleanupAndExit: (code: number) => Promise; + private worker: HeartbeatSchedulerWorker; + + constructor(cleanupAndExit: (code: number) => Promise) { + this.cleanupAndExit = cleanupAndExit; + this.worker = new HeartbeatSchedulerWorker(); + } + + register(program: Command, _context: CLIContext): void { + const heartbeat = program + .command('heartbeat') + .description(this.description); + + heartbeat + .command('status') + .description('Show heartbeat worker status, health, and schedule') + .action(async () => { + await this.executeStatus(); + }); + + heartbeat + .command('run-now') + .description('Trigger an immediate heartbeat cycle') + .option('-t, --timeout ', 'Timeout in milliseconds (default: worker built-in 60s)') + .action(async (options: { timeout?: string }) => { + const timeout = options.timeout ? parseInt(options.timeout, 10) : undefined; + await this.executeRunNow(timeout); + }); + + heartbeat + .command('history') + .description('Show recent heartbeat results') + .option('-n, --count ', 'Number of entries to show', '10') + .action(async (options: { count: string }) => { + await this.executeHistory(parseInt(options.count, 10) || 10); + }); + + heartbeat + .command('log') + .description("Show today's daily log entries") + .option('-d, --date ', 'Show log for specific date (YYYY-MM-DD)') + .action(async (options: { date?: string }) => { + await this.executeLog(options.date); + }); + + heartbeat + .command('pause') + .description('Pause the heartbeat worker') + .action(async () => { + await this.executePause(); + }); + + heartbeat + .command('resume') + .description('Resume the heartbeat worker') + .action(async () => { + await this.executeResume(); + }); + } + + // -------------------------------------------------------------------------- + // Subcommand Implementations + // -------------------------------------------------------------------------- + + private async executeStatus(): Promise { + try { + await this.worker.initialize(); + const health = this.worker.getHealth(); + const lastResult = this.worker.lastResult; + + console.log(chalk.blue('\n Heartbeat Scheduler Status')); + console.log(chalk.gray(' ' + '\u2500'.repeat(35))); + + console.log(` Status: ${statusColor(health.status)}`); + console.log(` Health Score: ${scoreColor(health.healthScore)}${chalk.gray('/100')}`); + + if (this.worker.lastRunAt) { + const ago = formatRelativeTime(this.worker.lastRunAt); + console.log(` Last Run: ${chalk.cyan(this.worker.lastRunAt.toISOString().replace('T', ' ').slice(0, 19))} ${chalk.gray(`(${ago})`)}`); + } else { + console.log(` Last Run: ${chalk.gray('never')}`); + } + + if (this.worker.nextRunAt) { + const until = formatRelativeTime(this.worker.nextRunAt, true); + console.log(` Next Run: ${chalk.cyan(this.worker.nextRunAt.toISOString().replace('T', ' ').slice(0, 19))} ${chalk.gray(`(${until})`)}`); + } + + console.log(` Total Runs: ${chalk.cyan(String(health.totalExecutions))}`); + const successRate = health.totalExecutions > 0 + ? ((health.successfulExecutions / health.totalExecutions) * 100).toFixed(1) + : '100.0'; + console.log(` Success Rate: ${chalk.cyan(successRate + '%')}`); + + if (lastResult?.metrics?.domainMetrics) { + const dm = lastResult.metrics.domainMetrics; + console.log(''); + console.log(chalk.blue(' Last Result:')); + console.log(` Promoted: ${chalk.cyan(String(dm.promoted ?? 0))} patterns`); + console.log(` Deprecated: ${chalk.cyan(String(dm.deprecated ?? 0))} patterns`); + console.log(` Decayed: ${chalk.cyan(String(dm.decayed ?? 0))} patterns`); + console.log(` Pending Exp: ${chalk.cyan(String(dm.pendingExperiences ?? 0))}`); + console.log(` Avg Conf: ${chalk.cyan(String(dm.avgConfidence ?? 0))}`); + } + + console.log(''); + await this.cleanupAndExit(0); + } catch (error) { + console.error(chalk.red('\n Failed to get heartbeat status:'), toErrorMessage(error)); + await this.cleanupAndExit(1); + } + } + + private async executeRunNow(timeoutMs?: number): Promise { + try { + console.log(chalk.blue('\n Triggering heartbeat cycle...\n')); + + await this.worker.initialize(); + + const abortController = new AbortController(); + + // Only add external timeout if the user explicitly requests one shorter + // than the worker's built-in 60s timeout. + const timeoutHandle = timeoutMs && timeoutMs > 0 + ? setTimeout(() => abortController.abort(), timeoutMs) + : null; + + // Use a real logger that outputs to the console, and a lightweight + // event bus / memory — the heartbeat worker only needs DB access + // (which it gets via getUnifiedMemory() internally). + const result: WorkerResult = await this.worker.execute({ + eventBus: { publish: async () => {} }, + memory: { + get: async () => undefined, + set: async () => {}, + search: async () => [], + }, + logger: { + debug: () => {}, + info: (...args: unknown[]) => console.log(chalk.gray(' [heartbeat]'), ...args), + warn: (...args: unknown[]) => console.warn(chalk.yellow(' [heartbeat]'), ...args), + error: (...args: unknown[]) => console.error(chalk.red(' [heartbeat]'), ...args), + }, + domains: { + getDomainAPI: () => undefined, + getDomainHealth: () => ({ status: 'healthy', errors: [] }), + }, + signal: abortController.signal, + }); + + if (timeoutHandle) clearTimeout(timeoutHandle); + + if (result.success) { + const dm = result.metrics.domainMetrics; + console.log(chalk.green(' Heartbeat cycle complete.')); + console.log(` Duration: ${chalk.cyan(formatDuration(result.durationMs))}`); + console.log(` Health Score: ${scoreColor(result.metrics.healthScore)}${chalk.gray('/100')}`); + console.log(` Trend: ${trendColor(result.metrics.trend)}`); + console.log(` Promoted: ${chalk.cyan(String(dm.promoted ?? 0))}`); + console.log(` Deprecated: ${chalk.cyan(String(dm.deprecated ?? 0))}`); + console.log(` Decayed: ${chalk.cyan(String(dm.decayed ?? 0))}`); + console.log(` Findings: ${chalk.cyan(String(result.findings.length))}`); + } else { + console.error(chalk.red(' Heartbeat cycle failed:'), result.error); + } + + // Store history entry + storeHistoryEntry(result); + + console.log(''); + await this.cleanupAndExit(result.success ? 0 : 1); + } catch (error) { + console.error(chalk.red('\n Failed to run heartbeat:'), toErrorMessage(error)); + await this.cleanupAndExit(1); + } + } + + private async executeHistory(count: number): Promise { + try { + const entries = loadHistoryEntries(count); + + if (entries.length === 0) { + console.log(chalk.yellow('\n No heartbeat history found. Run `aqe heartbeat run-now` first.\n')); + await this.cleanupAndExit(0); + return; + } + + console.log(chalk.blue(`\n Heartbeat History (last ${entries.length})`)); + console.log(chalk.gray(' ' + '\u2500'.repeat(60))); + + for (const entry of entries) { + const status = entry.success ? chalk.green('OK') : chalk.red('FAIL'); + const ts = entry.timestamp.slice(0, 19).replace('T', ' '); + const dm = entry.domainMetrics || {}; + console.log( + ` ${chalk.gray(ts)} ${status} ` + + `score:${chalk.cyan(String(entry.healthScore))} ` + + `+${dm.promoted ?? 0}/-${dm.deprecated ?? 0} ` + + `${chalk.gray(formatDuration(entry.durationMs))}` + ); + } + + console.log(''); + await this.cleanupAndExit(0); + } catch (error) { + console.error(chalk.red('\n Failed to load history:'), toErrorMessage(error)); + await this.cleanupAndExit(1); + } + } + + private async executeLog(date?: string): Promise { + try { + const targetDate = date || new Date().toISOString().split('T')[0]; + + // Validate date format to prevent path traversal (CLI-MCP parity with heartbeat-handlers.ts) + if (!/^\d{4}-\d{2}-\d{2}$/.test(targetDate)) { + console.error(chalk.red(`\n Invalid date format: "${targetDate}". Use YYYY-MM-DD.\n`)); + await this.cleanupAndExit(1); + return; + } + + const logDir = path.join(findProjectRoot(), '.agentic-qe', 'logs'); + const logPath = path.join(logDir, `${targetDate}.md`); + + if (!fs.existsSync(logPath)) { + console.log(chalk.yellow(`\n No daily log found for ${targetDate}.\n`)); + await this.cleanupAndExit(0); + return; + } + + const content = fs.readFileSync(logPath, 'utf-8'); + console.log(chalk.blue(`\n Daily Log \u2014 ${targetDate}`)); + console.log(chalk.gray(' ' + '\u2500'.repeat(40))); + // Indent and display each line + for (const line of content.split('\n')) { + if (line.trim()) { + console.log(` ${line}`); + } + } + console.log(''); + await this.cleanupAndExit(0); + } catch (error) { + console.error(chalk.red('\n Failed to read daily log:'), toErrorMessage(error)); + await this.cleanupAndExit(1); + } + } + + private async executePause(): Promise { + try { + this.worker.pause(); + console.log(chalk.yellow('\n Heartbeat worker paused.\n')); + await this.cleanupAndExit(0); + } catch (error) { + console.error(chalk.red('\n Failed to pause heartbeat:'), toErrorMessage(error)); + await this.cleanupAndExit(1); + } + } + + private async executeResume(): Promise { + try { + this.worker.resume(); + console.log(chalk.green('\n Heartbeat worker resumed.\n')); + await this.cleanupAndExit(0); + } catch (error) { + console.error(chalk.red('\n Failed to resume heartbeat:'), toErrorMessage(error)); + await this.cleanupAndExit(1); + } + } + + getHelp(): string { + return ` +Manage the token-free heartbeat scheduler (Imp-10). + +The heartbeat runs every 30 minutes performing SQL-only maintenance: + - Pattern promotion checks + - Stale pattern deprecation + - Confidence decay application + - Experience buffer monitoring + - Daily Markdown log entries + +Subcommands: + status Show heartbeat worker status, health, and schedule + run-now Trigger an immediate heartbeat cycle + history Show recent heartbeat results (last 10) + log Show today's daily log entries + pause Pause the heartbeat worker + resume Resume the heartbeat worker + +Examples: + aqe heartbeat status + aqe heartbeat run-now + aqe heartbeat history -n 5 + aqe heartbeat log + aqe heartbeat log --date 2026-03-25 + aqe heartbeat pause + aqe heartbeat resume +`; + } +} + +// ============================================================================ +// History Persistence +// ============================================================================ + +interface HeartbeatHistoryEntry { + timestamp: string; + success: boolean; + durationMs: number; + healthScore: number; + domainMetrics: Record; +} + +const MAX_HISTORY_ENTRIES = 100; + +function getHistoryPath(): string { + return path.join(findProjectRoot(), '.agentic-qe', 'heartbeat-history.json'); +} + +function storeHistoryEntry(result: WorkerResult): void { + try { + const historyPath = getHistoryPath(); + const dir = path.dirname(historyPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + let entries: HeartbeatHistoryEntry[] = []; + if (fs.existsSync(historyPath)) { + try { + entries = JSON.parse(fs.readFileSync(historyPath, 'utf-8')); + } catch { + entries = []; + } + } + + entries.unshift({ + timestamp: result.timestamp.toISOString(), + success: result.success, + durationMs: result.durationMs, + healthScore: result.metrics.healthScore, + domainMetrics: result.metrics.domainMetrics, + }); + + // Prune to max entries + if (entries.length > MAX_HISTORY_ENTRIES) { + entries = entries.slice(0, MAX_HISTORY_ENTRIES); + } + + fs.writeFileSync(historyPath, JSON.stringify(entries, null, 2)); + } catch { + // Non-critical: don't fail the command if history persistence fails + } +} + +function loadHistoryEntries(count: number): HeartbeatHistoryEntry[] { + try { + const historyPath = getHistoryPath(); + if (!fs.existsSync(historyPath)) { + return []; + } + const entries: HeartbeatHistoryEntry[] = JSON.parse(fs.readFileSync(historyPath, 'utf-8')); + return entries.slice(0, count); + } catch { + return []; + } +} + +// ============================================================================ +// Display Helpers +// ============================================================================ + +function statusColor(status: string): string { + switch (status) { + case 'idle': return chalk.cyan(status); + case 'running': return chalk.yellow(status); + case 'paused': return chalk.yellow(status); + case 'stopped': return chalk.gray(status); + case 'error': return chalk.red(status); + default: return chalk.white(status); + } +} + +function scoreColor(score: number): string { + if (score >= 80) return chalk.green(String(score)); + if (score >= 50) return chalk.yellow(String(score)); + return chalk.red(String(score)); +} + +function trendColor(trend: string): string { + switch (trend) { + case 'improving': return chalk.green(trend); + case 'stable': return chalk.cyan(trend); + case 'degrading': return chalk.red(trend); + default: return chalk.gray(trend); + } +} + +function formatRelativeTime(date: Date, future = false): string { + const diffMs = future ? date.getTime() - Date.now() : Date.now() - date.getTime(); + if (diffMs < 0) return future ? 'now' : 'just now'; + if (diffMs < 60_000) return `${Math.floor(diffMs / 1000)}s ${future ? 'from now' : 'ago'}`; + if (diffMs < 3_600_000) return `${Math.floor(diffMs / 60_000)}m ${future ? 'from now' : 'ago'}`; + return `${Math.floor(diffMs / 3_600_000)}h ${future ? 'from now' : 'ago'}`; +} + +// ============================================================================ +// Factory +// ============================================================================ + +export function createHeartbeatHandler( + cleanupAndExit: (code: number) => Promise +): HeartbeatHandler { + return new HeartbeatHandler(cleanupAndExit); +} diff --git a/src/cli/handlers/index.ts b/src/cli/handlers/index.ts index 5a5598d7..ad43bdcd 100644 --- a/src/cli/handlers/index.ts +++ b/src/cli/handlers/index.ts @@ -16,3 +16,5 @@ export { DomainHandler, createDomainHandler } from './domain-handler.js'; export { ProtocolHandler, createProtocolHandler } from './protocol-handler.js'; export { BrainHandler, createBrainHandler } from './brain-handler.js'; export { HypergraphHandler, createHypergraphHandler } from './hypergraph-handler.js'; +export { HeartbeatHandler, createHeartbeatHandler } from './heartbeat-handler.js'; +export { RoutingHandler, createRoutingHandler } from './routing-handler.js'; diff --git a/src/cli/handlers/routing-handler.ts b/src/cli/handlers/routing-handler.ts new file mode 100644 index 00000000..f166a69d --- /dev/null +++ b/src/cli/handlers/routing-handler.ts @@ -0,0 +1,271 @@ +/** + * Agentic QE v3 - Routing Command Handler + * Imp-18: Economic Routing Model CLI Integration + * + * Handles the 'aqe routing' command with subcommands: + * economics, accuracy, metrics + */ + +import { Command } from 'commander'; +import chalk from 'chalk'; +import { ICommandHandler, CLIContext } from './interfaces.js'; +import { toErrorMessage } from '../../shared/error-utils.js'; +import { createRoutingFeedbackCollector } from '../../routing/routing-feedback.js'; +import { getGlobalCostTracker } from '../../shared/llm/cost-tracker.js'; +import type { EconomicReport, EconomicScore } from '../../routing/economic-routing.js'; + +// ============================================================================ +// Routing Handler +// ============================================================================ + +export class RoutingHandler implements ICommandHandler { + readonly name = 'routing'; + readonly description = 'View routing performance, economics, and accuracy'; + + private cleanupAndExit: (code: number) => Promise; + + constructor(cleanupAndExit: (code: number) => Promise) { + this.cleanupAndExit = cleanupAndExit; + } + + getHelp(): string { + return [ + 'aqe routing economics [--complexity <0-1>] [--json] Show tier efficiency & budget', + 'aqe routing accuracy [--json] Show routing accuracy analysis', + 'aqe routing metrics [--json] Show per-agent performance', + ].join('\n'); + } + + register(program: Command, _context: CLIContext): void { + const routing = program + .command('routing') + .description(this.description); + + routing + .command('economics') + .description('Show economic routing report: tier efficiency, budget, savings') + .option('-c, --complexity ', 'Task complexity for scoring (0-1)', '0.5') + .option('--json', 'Output as JSON') + .action(async (options: { complexity: string; json?: boolean }) => { + await this.executeEconomics(parseFloat(options.complexity) || 0.5, !!options.json); + }); + + routing + .command('accuracy') + .description('Show routing accuracy analysis') + .option('--json', 'Output as JSON') + .action(async (options: { json?: boolean }) => { + await this.executeAccuracy(!!options.json); + }); + + routing + .command('metrics') + .description('Show per-agent performance metrics') + .option('--json', 'Output as JSON') + .action(async (options: { json?: boolean }) => { + await this.executeMetrics(!!options.json); + }); + } + + // -------------------------------------------------------------------------- + // Economics + // -------------------------------------------------------------------------- + + private async executeEconomics(complexity: number, json: boolean): Promise { + try { + const collector = createRoutingFeedbackCollector(100); + await collector.initialize(); + collector.enableEconomicRouting({}, getGlobalCostTracker()); + + const report = collector.getEconomicReport(); + if (!report) { + console.error(chalk.red('\n Economic routing is not available.\n')); + await this.cleanupAndExit(1); + return; + } + + if (json) { + console.log(JSON.stringify(report, (_k, v) => (v === Infinity ? 'Infinity' : v), 2)); + await this.cleanupAndExit(0); + return; + } + + console.log(chalk.blue('\n Economic Routing Report')); + console.log(chalk.gray(' ' + '\u2500'.repeat(50))); + + // Tier efficiency table + console.log(chalk.white('\n Tier Efficiency (complexity=' + complexity.toFixed(1) + '):\n')); + console.log(chalk.gray(' Tier Quality Cost/Task Q/$ Score')); + console.log(chalk.gray(' ' + '\u2500'.repeat(50))); + + const scores = collector.getEconomicScore(complexity) ?? report.tierEfficiency; + for (const s of scores) { + const qpd = isFinite(s.qualityPerDollar) ? s.qualityPerDollar.toFixed(1) : '\u221E'; + console.log( + ` ${padRight(s.tier, 10)}` + + `${chalk.cyan(s.qualityScore.toFixed(2))} ` + + `$${s.estimatedCostUsd.toFixed(4)} ` + + `${chalk.yellow(padLeft(qpd, 8))} ` + + `${scoreColor(s.economicScore)}`, + ); + } + + // Budget + console.log(chalk.white('\n Budget:')); + console.log(` Hourly cost: $${report.currentHourlyCostUsd.toFixed(4)}`); + console.log(` Daily cost: $${report.currentDailyCostUsd.toFixed(4)}`); + if (report.budgetRemaining.hourly !== null) { + console.log(` Hourly left: $${report.budgetRemaining.hourly.toFixed(4)}`); + } + if (report.budgetRemaining.daily !== null) { + console.log(` Daily left: $${report.budgetRemaining.daily.toFixed(4)}`); + } + + // Recommendation + console.log(chalk.white('\n Recommendation:')); + console.log(` ${chalk.green(report.recommendation)}`); + + if (report.savingsOpportunity) { + console.log(chalk.white('\n Savings Opportunity:')); + console.log(` ${chalk.yellow(report.savingsOpportunity.description)}`); + } + + console.log(''); + await this.cleanupAndExit(0); + } catch (error) { + console.error(chalk.red('\n Failed to get economic report:'), toErrorMessage(error)); + await this.cleanupAndExit(1); + } + } + + // -------------------------------------------------------------------------- + // Accuracy + // -------------------------------------------------------------------------- + + private async executeAccuracy(json: boolean): Promise { + try { + const collector = createRoutingFeedbackCollector(10000); + await collector.initialize(); + + const accuracy = collector.analyzeRoutingAccuracy(); + + if (json) { + console.log(JSON.stringify(accuracy, null, 2)); + await this.cleanupAndExit(0); + return; + } + + console.log(chalk.blue('\n Routing Accuracy Analysis')); + console.log(chalk.gray(' ' + '\u2500'.repeat(40))); + console.log(` Total outcomes: ${chalk.cyan(String(accuracy.totalOutcomes))}`); + console.log(` Followed recs: ${chalk.cyan(String(accuracy.followedRecommendations))}`); + console.log(` Override rate: ${chalk.yellow((accuracy.overrideRate * 100).toFixed(1) + '%')}`); + console.log(` Rec success rate: ${scoreColor100(accuracy.recommendationSuccessRate * 100)}`); + console.log(` Override success rate: ${scoreColor100(accuracy.overrideSuccessRate * 100)}`); + console.log(` Confidence correlation: ${chalk.cyan(accuracy.confidenceCorrelation.toFixed(3))}`); + + // Recommendations + const recs = collector.getImprovementRecommendations(); + if (recs.length > 0) { + console.log(chalk.white('\n Recommendations:')); + for (const rec of recs) { + console.log(` ${chalk.gray('\u2022')} ${rec}`); + } + } + + console.log(''); + await this.cleanupAndExit(0); + } catch (error) { + console.error(chalk.red('\n Failed to analyze routing accuracy:'), toErrorMessage(error)); + await this.cleanupAndExit(1); + } + } + + // -------------------------------------------------------------------------- + // Metrics + // -------------------------------------------------------------------------- + + private async executeMetrics(json: boolean): Promise { + try { + const collector = createRoutingFeedbackCollector(10000); + await collector.initialize(); + + const metrics = collector.getAllAgentMetrics(); + + if (json) { + console.log(JSON.stringify(metrics, null, 2)); + await this.cleanupAndExit(0); + return; + } + + if (metrics.length === 0) { + console.log(chalk.yellow('\n No routing metrics available yet. Run some QE tasks first.\n')); + await this.cleanupAndExit(0); + return; + } + + console.log(chalk.blue('\n Agent Routing Metrics')); + console.log(chalk.gray(' ' + '\u2500'.repeat(60))); + console.log(chalk.gray(' Agent Tasks Success Quality Trend')); + console.log(chalk.gray(' ' + '\u2500'.repeat(60))); + + for (const m of metrics.slice(0, 20)) { + const trend = m.trend === 'improving' ? chalk.green('\u2191') + : m.trend === 'declining' ? chalk.red('\u2193') + : chalk.gray('\u2192'); + console.log( + ` ${padRight(m.agentId, 24)}` + + `${padLeft(String(m.totalTasks), 5)} ` + + `${scoreColor100(m.successRate * 100)} ` + + `${chalk.cyan(m.avgQualityScore.toFixed(2))} ` + + `${trend} ${m.trend}`, + ); + } + + const stats = collector.getStats(); + console.log(chalk.gray(`\n ${stats.totalOutcomes} total outcomes, ${stats.uniqueAgentsUsed} agents`)); + console.log(''); + await this.cleanupAndExit(0); + } catch (error) { + console.error(chalk.red('\n Failed to get agent metrics:'), toErrorMessage(error)); + await this.cleanupAndExit(1); + } + } +} + +// ============================================================================ +// Helpers +// ============================================================================ + +function padRight(str: string, len: number): string { + return str.length >= len ? str : str + ' '.repeat(len - str.length); +} + +function padLeft(str: string, len: number): string { + return str.length >= len ? str : ' '.repeat(len - str.length) + str; +} + +function scoreColor(score: number): string { + const pct = score * 100; + const str = score.toFixed(3); + if (pct >= 70) return chalk.green(str); + if (pct >= 40) return chalk.yellow(str); + return chalk.red(str); +} + +function scoreColor100(pct: number): string { + const str = pct.toFixed(1) + '%'; + if (pct >= 70) return chalk.green(str); + if (pct >= 40) return chalk.yellow(str); + return chalk.red(str); +} + +// ============================================================================ +// Factory +// ============================================================================ + +export function createRoutingHandler( + cleanupAndExit: (code: number) => Promise, +): RoutingHandler { + return new RoutingHandler(cleanupAndExit); +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 8b6bf31b..9ad74b78 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -991,6 +991,7 @@ import { createPlatformCommand } from './commands/platform.js'; import { createProveCommand } from './commands/prove.js'; import { createRuVectorCommand } from './commands/ruvector-commands.js'; import { createAuditCommand } from './commands/audit.js'; +import { createPipelineCommand } from './commands/pipeline.js'; program.addCommand(createTokenUsageCommand()); program.addCommand(createLLMRouterCommand()); @@ -1003,6 +1004,7 @@ program.addCommand(createPlatformCommand()); program.addCommand(createProveCommand(context, cleanupAndExit, ensureInitialized)); program.addCommand(createRuVectorCommand()); program.addCommand(createAuditCommand(context, cleanupAndExit, ensureInitialized)); +program.addCommand(createPipelineCommand(context, cleanupAndExit, ensureInitialized)); // ============================================================================ // Shutdown Handlers diff --git a/src/coordination/deterministic-actions.ts b/src/coordination/deterministic-actions.ts new file mode 100644 index 00000000..e075a0ae --- /dev/null +++ b/src/coordination/deterministic-actions.ts @@ -0,0 +1,331 @@ +/** + * Deterministic Step Actions (Imp-9) + * + * Built-in actions that execute WITHOUT LLM tokens. Each action maps to a + * domain + action pair and is automatically wired into the WorkflowOrchestrator + * as a fallback before domain service delegation. + * + * When the caller supplies explicit input values they are used directly. + * When inputs are omitted, the actions query the unified SQLite database + * for live metrics — keeping execution fully deterministic (SQL-only). + */ + +import { Result, ok, err, DomainName } from '../shared/types/index.js'; +import type { WorkflowContext } from './workflow-types.js'; +import { toErrorMessage } from '../shared/error-utils.js'; + +// ============================================================================ +// Types +// ============================================================================ + +/** + * A deterministic action that runs without any LLM tokens. + */ +export interface DeterministicAction { + /** Unique action identifier */ + id: string; + /** Target domain */ + domain: DomainName; + /** Action name within the domain */ + action: string; + /** Execute the action with the given input */ + execute( + input: Record, + context: WorkflowContext, + ): Promise, Error>>; +} + +// ============================================================================ +// DB helpers (fail-safe: return null when DB unavailable) +// ============================================================================ + +interface DbAccessor { + prepare(sql: string): { get(...params: unknown[]): unknown; all(...params: unknown[]): unknown[] }; +} + +function tryGetDb(): DbAccessor | null { + try { + // Dynamic import to avoid hard dependency — the module may not be initialised + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { getUnifiedMemory } = require('../kernel/unified-memory.js'); + const um = getUnifiedMemory(); + if (!um.isInitialized()) return null; + return um.getDatabase() as DbAccessor; + } catch { + return null; + } +} + +// ============================================================================ +// Quality Gate Check +// ============================================================================ + +const qualityGateCheck: DeterministicAction = { + id: 'quality-gate-check', + domain: 'quality-assessment', + action: 'gate-check', + async execute(input) { + const coverageMin = typeof input.coverageMin === 'number' ? input.coverageMin : 80; + const testsPassingMin = typeof input.testsPassingMin === 'number' ? input.testsPassingMin : 90; + const maxBugs = typeof input.maxBugs === 'number' ? input.maxBugs : 5; + + // ----- Fetch live data from DB when not supplied by caller ----- + let currentCoverage = typeof input.currentCoverage === 'number' ? input.currentCoverage : null; + let currentTestsPassingRate = typeof input.currentTestsPassingRate === 'number' ? input.currentTestsPassingRate : null; + let currentBugs = typeof input.currentBugs === 'number' ? input.currentBugs : null; + + if (currentCoverage === null || currentTestsPassingRate === null || currentBugs === null) { + const db = tryGetDb(); + if (db) { + try { + // Latest coverage from coverage_sessions + if (currentCoverage === null) { + const row = db.prepare( + `SELECT after_lines FROM coverage_sessions ORDER BY created_at DESC LIMIT 1`, + ).get() as { after_lines: number } | undefined; + currentCoverage = row?.after_lines ?? 0; + } + + // Test pass rate from recent test_outcomes + if (currentTestsPassingRate === null) { + const row = db.prepare( + `SELECT COUNT(*) as total, + SUM(CASE WHEN passed = 1 THEN 1 ELSE 0 END) as passed + FROM test_outcomes + WHERE created_at > datetime('now', '-7 days')`, + ).get() as { total: number; passed: number } | undefined; + currentTestsPassingRate = + row && row.total > 0 ? (row.passed / row.total) * 100 : 0; + } + + // Bug count: failed non-flaky tests in the last 7 days + if (currentBugs === null) { + const row = db.prepare( + `SELECT COUNT(*) as bugs FROM test_outcomes + WHERE passed = 0 AND flaky = 0 + AND created_at > datetime('now', '-7 days')`, + ).get() as { bugs: number } | undefined; + currentBugs = row?.bugs ?? 0; + } + } catch { + // Graceful degradation — fall through to defaults + } + } + } + + // Apply defaults for anything still null + currentCoverage = currentCoverage ?? 0; + currentTestsPassingRate = currentTestsPassingRate ?? 0; + currentBugs = currentBugs ?? 0; + + const coveragePassed = currentCoverage >= coverageMin; + const testsPassed = currentTestsPassingRate >= testsPassingMin; + const bugsPassed = currentBugs <= maxBugs; + const passed = coveragePassed && testsPassed && bugsPassed; + + // Score is a normalized 0-1 value + const coverageScore = Math.min(currentCoverage / coverageMin, 1); + const testsScore = Math.min(currentTestsPassingRate / testsPassingMin, 1); + const bugsScore = maxBugs > 0 ? Math.max(0, 1 - currentBugs / maxBugs) : (currentBugs === 0 ? 1 : 0); + const score = (coverageScore + testsScore + bugsScore) / 3; + + return ok({ + passed, + score: Math.round(score * 100) / 100, + source: typeof input.currentCoverage === 'number' ? 'input' : 'database', + details: { + coverage: { current: currentCoverage, threshold: coverageMin, passed: coveragePassed }, + testsPassing: { current: currentTestsPassingRate, threshold: testsPassingMin, passed: testsPassed }, + bugs: { current: currentBugs, threshold: maxBugs, passed: bugsPassed }, + }, + }); + }, +}; + +// ============================================================================ +// Coverage Threshold Check +// ============================================================================ + +const coverageThresholdCheck: DeterministicAction = { + id: 'coverage-threshold', + domain: 'coverage-analysis', + action: 'threshold-check', + async execute(input) { + const minCoverage = typeof input.minCoverage === 'number' ? input.minCoverage : 80; + + // ----- Fetch from DB when not supplied ----- + let currentCoverage = typeof input.currentCoverage === 'number' ? input.currentCoverage : null; + let source: 'input' | 'database' = 'input'; + + if (currentCoverage === null) { + source = 'database'; + const db = tryGetDb(); + if (db) { + try { + const row = db.prepare( + `SELECT after_lines FROM coverage_sessions ORDER BY created_at DESC LIMIT 1`, + ).get() as { after_lines: number } | undefined; + currentCoverage = row?.after_lines ?? 0; + } catch { + currentCoverage = 0; + } + } else { + currentCoverage = 0; + } + } + + const passed = currentCoverage >= minCoverage; + const gap = passed ? 0 : Math.round((minCoverage - currentCoverage) * 100) / 100; + + return ok({ + currentCoverage, + passed, + gap, + minCoverage, + source, + }); + }, +}; + +// ============================================================================ +// Pattern Health Check +// ============================================================================ + +const patternHealthCheck: DeterministicAction = { + id: 'pattern-health', + domain: 'learning-optimization', + action: 'health-check', + async execute(input) { + // ----- Fetch from DB when not supplied ----- + let totalPatterns = typeof input.totalPatterns === 'number' ? input.totalPatterns : null; + let activePatterns = typeof input.activePatterns === 'number' ? input.activePatterns : null; + let avgConfidence = typeof input.avgConfidence === 'number' ? input.avgConfidence : null; + let source: 'input' | 'database' = 'input'; + + if (totalPatterns === null || activePatterns === null || avgConfidence === null) { + source = 'database'; + const db = tryGetDb(); + if (db) { + try { + const row = db.prepare(` + SELECT COUNT(*) as total, + SUM(CASE WHEN deprecated_at IS NULL AND confidence >= 0.3 THEN 1 ELSE 0 END) as active, + AVG(confidence) as avg_conf + FROM qe_patterns + `).get() as { total: number; active: number; avg_conf: number | null } | undefined; + + totalPatterns = totalPatterns ?? (row?.total ?? 0); + activePatterns = activePatterns ?? (row?.active ?? 0); + avgConfidence = avgConfidence ?? (row?.avg_conf ?? 0); + } catch { + // Graceful degradation + } + } + } + + // Apply defaults for anything still null + totalPatterns = totalPatterns ?? 0; + activePatterns = activePatterns ?? 0; + avgConfidence = avgConfidence ?? 0; + + // Health score: weighted combination of volume, activity ratio, and confidence + const volumeScore = Math.min(totalPatterns / 100, 1); // 100 patterns = max volume + const activityRatio = totalPatterns > 0 ? activePatterns / totalPatterns : 0; + const healthScore = Math.round( + (volumeScore * 0.3 + activityRatio * 0.3 + avgConfidence * 0.4) * 100, + ) / 100; + + return ok({ + totalPatterns, + activePatterns, + avgConfidence, + healthScore, + source, + }); + }, +}; + +// ============================================================================ +// Routing Accuracy Check +// ============================================================================ + +const routingAccuracyCheck: DeterministicAction = { + id: 'routing-accuracy', + domain: 'learning-optimization', + action: 'routing-check', + async execute(input) { + // ----- Fetch from DB when not supplied ----- + let totalOutcomes = typeof input.totalOutcomes === 'number' ? input.totalOutcomes : null; + let successfulOutcomes = typeof input.successfulOutcomes === 'number' ? input.successfulOutcomes : null; + let confidenceCorrelation = typeof input.confidenceCorrelation === 'number' ? input.confidenceCorrelation : null; + let source: 'input' | 'database' = 'input'; + + if (totalOutcomes === null || successfulOutcomes === null) { + source = 'database'; + const db = tryGetDb(); + if (db) { + try { + const row = db.prepare(` + SELECT COUNT(*) as total, + SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful + FROM routing_outcomes + `).get() as { total: number; successful: number } | undefined; + + totalOutcomes = totalOutcomes ?? (row?.total ?? 0); + successfulOutcomes = successfulOutcomes ?? (row?.successful ?? 0); + } catch { + // Graceful degradation + } + } + } + + // Apply defaults + totalOutcomes = totalOutcomes ?? 0; + successfulOutcomes = successfulOutcomes ?? 0; + confidenceCorrelation = confidenceCorrelation ?? 0; + + const successRate = totalOutcomes > 0 + ? Math.round((successfulOutcomes / totalOutcomes) * 10000) / 100 + : 0; + + return ok({ + successRate, + totalOutcomes, + successfulOutcomes, + confidenceCorrelation, + source, + }); + }, +}; + +// ============================================================================ +// Registry +// ============================================================================ + +/** All built-in deterministic actions */ +const DETERMINISTIC_ACTIONS: DeterministicAction[] = [ + qualityGateCheck, + coverageThresholdCheck, + patternHealthCheck, + routingAccuracyCheck, +]; + +/** + * Look up a deterministic action by domain + action. + * Returns undefined if no built-in action matches. + */ +export function findDeterministicAction( + domain: DomainName, + action: string, +): DeterministicAction | undefined { + return DETERMINISTIC_ACTIONS.find( + (a) => a.domain === domain && a.action === action, + ); +} + +/** + * Get all registered deterministic actions. + */ +export function getAllDeterministicActions(): readonly DeterministicAction[] { + return DETERMINISTIC_ACTIONS; +} diff --git a/src/coordination/workflow-orchestrator.ts b/src/coordination/workflow-orchestrator.ts index f006f6ae..421d0c67 100644 --- a/src/coordination/workflow-orchestrator.ts +++ b/src/coordination/workflow-orchestrator.ts @@ -45,6 +45,7 @@ export type { WorkflowCompletedPayload, WorkflowFailedPayload, StepEventPayload, + StepAwaitingApprovalPayload, IWorkflowOrchestrator, DomainAction, DomainActionRegistry, @@ -73,10 +74,14 @@ import type { WorkflowCompletedPayload, WorkflowFailedPayload, StepEventPayload, + StepAwaitingApprovalPayload, } from './workflow-types.js'; import { WorkflowEvents, DEFAULT_WORKFLOW_CONFIG } from './workflow-types.js'; +// Import deterministic actions +import { findDeterministicAction } from './deterministic-actions.js'; + // Import built-in workflows import { getBuiltInWorkflows, BUILTIN_WORKFLOW_IDS } from './workflow-builtin.js'; @@ -90,6 +95,11 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator { private readonly executions: Map = new Map(); private readonly actionRegistry: DomainActionRegistry = {}; private readonly eventSubscriptions: Subscription[] = []; + /** Pending approval gates: Map<`${executionId}:${stepId}`, gate> */ + private readonly approvalGates: Map void; + }> = new Map(); + private initialized = false; constructor( @@ -225,6 +235,14 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator { execution.completedAt = new Date(); execution.duration = execution.completedAt.getTime() - execution.startedAt.getTime(); + // Clean up any pending approval gates for this execution to prevent timer leaks + for (const [key, gate] of this.approvalGates.entries()) { + if (key.startsWith(`${executionId}:`)) { + gate.resolve({ approved: false, reason: 'Workflow cancelled' }); + this.approvalGates.delete(key); + } + } + await this.publishEvent(WorkflowEvents.WorkflowCancelled, { executionId, workflowId: execution.workflowId, workflowName: execution.workflowName, }, execution.context.metadata.correlationId); @@ -277,6 +295,38 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator { return this.workflows.get(workflowId); } + // ============================================================================ + // Approval Gate Methods + // ============================================================================ + + /** + * Approve a step that is awaiting approval. Returns true if the step + * was found and approved, false otherwise. + */ + approveStep(executionId: string, stepId: string): boolean { + const key = `${executionId}:${stepId}`; + const gate = this.approvalGates.get(key); + if (!gate) return false; + + gate.resolve({ approved: true }); + this.approvalGates.delete(key); + return true; + } + + /** + * Reject a step that is awaiting approval. Returns true if the step + * was found and rejected, false otherwise. + */ + rejectStep(executionId: string, stepId: string, reason?: string): boolean { + const key = `${executionId}:${stepId}`; + const gate = this.approvalGates.get(key); + if (!gate) return false; + + gate.resolve({ approved: false, reason }); + this.approvalGates.delete(key); + return true; + } + // ============================================================================ // Public Utility Methods // ============================================================================ @@ -521,6 +571,21 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator { this.mapStepOutput(step, output, execution.context); + // Handle approval gate if configured + if (step.approval) { + const approvalResult = await this.waitForApproval(step, execution); + if (!approvalResult.approved) { + result.status = 'failed'; + result.error = approvalResult.reason || 'Step rejected at approval gate'; + result.output = output; + result.completedAt = new Date(); + result.duration = result.completedAt.getTime() - startedAt.getTime(); + execution.stepResults.set(step.id, result); + await this.publishStepFailed(execution, step, result.error); + return result; + } + } + result.status = 'completed'; result.output = output; result.completedAt = new Date(); result.duration = result.completedAt.getTime() - startedAt.getTime(); execution.stepResults.set(step.id, result); @@ -554,12 +619,21 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator { private async executeStepAction( step: WorkflowStepDefinition, input: Record, context: WorkflowContext, timeout: number ): Promise { + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error(`Step timeout after ${timeout}ms`)), timeout); + }); + + // Check for a deterministic action first (zero LLM tokens) + const deterministicAction = findDeterministicAction(step.domain, step.action); + if (deterministicAction) { + const actionResult = await Promise.race([deterministicAction.execute(input, context), timeoutPromise]); + if (!actionResult.success) throw actionResult.error; + return actionResult.value; + } + + // Fall back to registered domain actions const domainActions = this.actionRegistry[step.domain]; if (domainActions?.[step.action]) { - const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => reject(new Error(`Step timeout after ${timeout}ms`)), timeout); - }); - const actionResult = await Promise.race([domainActions[step.action](input, context), timeoutPromise]); if (!actionResult.success) throw actionResult.error; return actionResult.value; @@ -887,6 +961,67 @@ export class WorkflowOrchestrator implements IWorkflowOrchestrator { } } + // ============================================================================ + // Private Methods - Approval Gates + // ============================================================================ + + /** + * Wait for external approval (or auto-approve after timeout). + * Returns an object with approved status and optional rejection reason. + */ + private async waitForApproval( + step: WorkflowStepDefinition, + execution: WorkflowExecutionStatus, + ): Promise<{ approved: boolean; reason?: string }> { + const approvalConfig = typeof step.approval === 'object' ? step.approval : {}; + const autoApproveAfter = approvalConfig.autoApproveAfter ?? 300000; // 5 min default + const message = approvalConfig.message ?? `Awaiting approval for step: ${step.name}`; + + // Update step result status + const stepResult = execution.stepResults.get(step.id); + if (stepResult) stepResult.status = 'awaiting_approval'; + + // Emit StepAwaitingApproval event + await this.publishEvent( + WorkflowEvents.StepAwaitingApproval, + { + executionId: execution.executionId, + workflowId: execution.workflowId, + stepId: step.id, + stepName: step.name, + domain: step.domain, + message, + autoApproveAfter: autoApproveAfter > 0 ? autoApproveAfter : undefined, + }, + execution.context.metadata.correlationId, + ); + + const key = `${execution.executionId}:${step.id}`; + + return new Promise<{ approved: boolean; reason?: string }>((resolve) => { + // Store the gate so approveStep/rejectStep can resolve it + const gate: { resolve: (result: { approved: boolean; reason?: string }) => void } = { resolve }; + this.approvalGates.set(key, gate); + + // Auto-approve timer (0 = never auto-approve) + if (autoApproveAfter > 0) { + const timer = setTimeout(() => { + if (this.approvalGates.has(key)) { + this.approvalGates.delete(key); + resolve({ approved: true }); // auto-approve on timeout + } + }, autoApproveAfter); + + // Wrap resolve to also clear the timer + const originalResolve = gate.resolve; + gate.resolve = (result) => { + clearTimeout(timer); + originalResolve(result); + }; + } + }); + } + // ============================================================================ // Private Methods - Utilities // ============================================================================ diff --git a/src/coordination/workflow-types.ts b/src/coordination/workflow-types.ts index 9457b4f9..918613ed 100644 --- a/src/coordination/workflow-types.ts +++ b/src/coordination/workflow-types.ts @@ -22,7 +22,7 @@ export type StepExecutionMode = 'sequential' | 'parallel'; /** * Step status */ -export type StepStatus = 'pending' | 'running' | 'completed' | 'failed' | 'skipped'; +export type StepStatus = 'pending' | 'running' | 'completed' | 'failed' | 'skipped' | 'awaiting_approval'; /** * Workflow status @@ -84,6 +84,13 @@ export interface WorkflowStepDefinition { }; /** Continue workflow on failure */ continueOnFailure?: boolean; + /** Approval gate configuration */ + approval?: boolean | { + /** Auto-approve after this many ms (0 = never auto-approve) */ + autoApproveAfter?: number; + /** Approval prompt message */ + message?: string; + }; } /** @@ -203,6 +210,9 @@ export const WorkflowEvents = { StepCompleted: 'workflow.StepCompleted', StepFailed: 'workflow.StepFailed', StepSkipped: 'workflow.StepSkipped', + StepAwaitingApproval: 'workflow.StepAwaitingApproval', + StepApproved: 'workflow.StepApproved', + StepRejected: 'workflow.StepRejected', } as const; export interface WorkflowStartedPayload { @@ -237,6 +247,11 @@ export interface StepEventPayload { domain: DomainName; } +export interface StepAwaitingApprovalPayload extends StepEventPayload { + message?: string; + autoApproveAfter?: number; +} + // ============================================================================ // Workflow Orchestrator Interface // ============================================================================ @@ -270,6 +285,10 @@ export interface IWorkflowOrchestrator { getActiveExecutions(): WorkflowExecutionStatus[]; /** Get workflow definition */ getWorkflow(workflowId: string): WorkflowDefinition | undefined; + /** Approve a step that is awaiting approval */ + approveStep(executionId: string, stepId: string): boolean; + /** Reject a step that is awaiting approval */ + rejectStep(executionId: string, stepId: string, reason?: string): boolean; } // ============================================================================ diff --git a/src/coordination/yaml-pipeline-loader.ts b/src/coordination/yaml-pipeline-loader.ts index 0f60c171..7ccdb915 100644 --- a/src/coordination/yaml-pipeline-loader.ts +++ b/src/coordination/yaml-pipeline-loader.ts @@ -257,6 +257,14 @@ export class YamlPipelineLoader { ? s.continueOnFailure : undefined; + // Validate approval gate configuration + let approval: WorkflowStepDefinition['approval']; + if (s.approval !== undefined) { + const approvalResult = this.validateApproval(s.approval, s.id as string); + if (!approvalResult.success) return approvalResult; + approval = approvalResult.value; + } + const step: WorkflowStepDefinition = { id: s.id, name: s.name, @@ -271,6 +279,7 @@ export class YamlPipelineLoader { ...(retry !== undefined && { retry }), ...(rollback !== undefined && { rollback }), ...(continueOnFailure !== undefined && { continueOnFailure }), + ...(approval !== undefined && { approval }), }; steps.push(step); @@ -404,6 +413,40 @@ export class YamlPipelineLoader { }); } + private validateApproval( + raw: unknown, + stepId: string, + ): Result, Error> { + // Simple boolean form: approval: true + if (typeof raw === 'boolean') { + return ok(raw); + } + + // Object form: approval: { autoApproveAfter, message } + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + return err(new Error(`Step '${stepId}': 'approval' must be a boolean or object`)); + } + + const a = raw as Record; + const result: { autoApproveAfter?: number; message?: string } = {}; + + if (a.autoApproveAfter !== undefined) { + if (typeof a.autoApproveAfter !== 'number' || a.autoApproveAfter < 0) { + return err(new Error(`Step '${stepId}': approval.autoApproveAfter must be a non-negative number`)); + } + result.autoApproveAfter = a.autoApproveAfter; + } + + if (a.message !== undefined) { + if (typeof a.message !== 'string') { + return err(new Error(`Step '${stepId}': approval.message must be a string`)); + } + result.message = a.message; + } + + return ok(result); + } + /** Validate that a value is Record, undefined, or return an Error. */ private validateStringRecord( value: unknown, diff --git a/src/mcp/handlers/heartbeat-handlers.ts b/src/mcp/handlers/heartbeat-handlers.ts new file mode 100644 index 00000000..2fdd0c7f --- /dev/null +++ b/src/mcp/handlers/heartbeat-handlers.ts @@ -0,0 +1,256 @@ +/** + * Heartbeat MCP Handlers (Imp-10) + * + * Exposes the token-free heartbeat scheduler through MCP tools: + * - heartbeat_status: Returns current heartbeat health and metrics + * - heartbeat_trigger: Triggers an immediate heartbeat cycle + * - heartbeat_log: Returns daily log entries for a given date + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import type { ToolResult } from '../types.js'; +import { HeartbeatSchedulerWorker } from '../../workers/workers/heartbeat-scheduler.js'; +import type { WorkerResult } from '../../workers/interfaces.js'; +import { toErrorMessage } from '../../shared/error-utils.js'; +import { findProjectRoot } from '../../kernel/unified-memory.js'; + +// ============================================================================ +// Types +// ============================================================================ + +export type HeartbeatStatusParams = Record; + +export interface HeartbeatStatusResult { + status: string; + healthScore: number; + totalExecutions: number; + successfulExecutions: number; + failedExecutions: number; + avgDurationMs: number; + lastRunAt: string | null; + nextRunAt: string | null; + lastResult: { + success: boolean; + durationMs: number; + healthScore: number; + trend: string; + domainMetrics: Record; + } | null; +} + +export interface HeartbeatTriggerParams { + /** Optional: timeout in milliseconds (default: 60000) */ + timeout?: number; +} + +export interface HeartbeatTriggerResult { + success: boolean; + durationMs: number; + healthScore: number; + trend: string; + promoted: number; + deprecated: number; + decayed: number; + pendingExperiences: number; + avgConfidence: number; + findingsCount: number; + recommendationsCount: number; +} + +export interface HeartbeatLogParams { + /** Date in YYYY-MM-DD format (defaults to today) */ + date?: string; +} + +export interface HeartbeatLogResult { + date: string; + exists: boolean; + content: string; + lineCount: number; +} + +// ============================================================================ +// Shared State +// ============================================================================ + +let sharedWorker: HeartbeatSchedulerWorker | null = null; + +function getWorker(): HeartbeatSchedulerWorker { + if (!sharedWorker) { + sharedWorker = new HeartbeatSchedulerWorker(); + } + return sharedWorker; +} + +// ============================================================================ +// Handlers +// ============================================================================ + +/** + * Get the current heartbeat scheduler status and health metrics. + */ +export async function handleHeartbeatStatus( + _params: HeartbeatStatusParams, +): Promise> { + try { + const worker = getWorker(); + await worker.initialize(); + const health = worker.getHealth(); + const lastResult = worker.lastResult; + + return { + success: true, + data: { + status: health.status, + healthScore: health.healthScore, + totalExecutions: health.totalExecutions, + successfulExecutions: health.successfulExecutions, + failedExecutions: health.failedExecutions, + avgDurationMs: health.avgDurationMs, + lastRunAt: worker.lastRunAt?.toISOString() ?? null, + nextRunAt: worker.nextRunAt?.toISOString() ?? null, + lastResult: lastResult + ? { + success: lastResult.success, + durationMs: lastResult.durationMs, + healthScore: lastResult.metrics.healthScore, + trend: lastResult.metrics.trend, + domainMetrics: lastResult.metrics.domainMetrics, + } + : null, + }, + }; + } catch (error) { + return { + success: false, + error: `Failed to get heartbeat status: ${toErrorMessage(error)}`, + }; + } +} + +/** + * Trigger an immediate heartbeat cycle and return the result. + */ +export async function handleHeartbeatTrigger( + params: HeartbeatTriggerParams, +): Promise> { + try { + const worker = getWorker(); + await worker.initialize(); + + const abortController = new AbortController(); + + // The worker already enforces its own 60s timeout via BaseWorker.executeWithTimeout(). + // Only add an external abort if the caller explicitly requests a *shorter* timeout. + const userTimeout = params.timeout && params.timeout > 0 ? params.timeout : null; + const timeoutHandle = userTimeout + ? setTimeout(() => abortController.abort(), userTimeout) + : null; + + let result: WorkerResult; + try { + result = await worker.execute({ + eventBus: { publish: async () => {} }, + memory: { + get: async () => undefined, + set: async () => {}, + search: async () => [], + }, + logger: { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + }, + domains: { + getDomainAPI: () => undefined, + getDomainHealth: () => ({ status: 'healthy', errors: [] }), + }, + signal: abortController.signal, + }); + } finally { + if (timeoutHandle) clearTimeout(timeoutHandle); + } + + const dm = result.metrics.domainMetrics; + + return { + success: true, + data: { + success: result.success, + durationMs: result.durationMs, + healthScore: result.metrics.healthScore, + trend: result.metrics.trend, + promoted: Number(dm.promoted ?? 0), + deprecated: Number(dm.deprecated ?? 0), + decayed: Number(dm.decayed ?? 0), + pendingExperiences: Number(dm.pendingExperiences ?? 0), + avgConfidence: Number(dm.avgConfidence ?? 0), + findingsCount: result.findings.length, + recommendationsCount: result.recommendations.length, + }, + }; + } catch (error) { + const msg = toErrorMessage(error); + const isTimeout = msg.includes('aborted') || msg.includes('abort') || msg.includes('timed out'); + return { + success: false, + error: isTimeout + ? `Heartbeat timed out after ${params.timeout ?? 60000}ms` + : `Failed to trigger heartbeat: ${msg}`, + }; + } +} + +/** + * Read the daily log for a given date (defaults to today). + */ +export async function handleHeartbeatLog( + params: HeartbeatLogParams, +): Promise> { + try { + const targetDate = params.date || new Date().toISOString().split('T')[0]; + + // Validate date format + if (!/^\d{4}-\d{2}-\d{2}$/.test(targetDate)) { + return { + success: false, + error: "Invalid date format. Use YYYY-MM-DD (e.g., '2026-03-27').", + }; + } + + const logDir = path.join(findProjectRoot(), '.agentic-qe', 'logs'); + const logPath = path.join(logDir, `${targetDate}.md`); + + if (!fs.existsSync(logPath)) { + return { + success: true, + data: { + date: targetDate, + exists: false, + content: '', + lineCount: 0, + }, + }; + } + + const content = fs.readFileSync(logPath, 'utf-8'); + const lineCount = content.split('\n').filter((l) => l.trim()).length; + + return { + success: true, + data: { + date: targetDate, + exists: true, + content, + lineCount, + }, + }; + } catch (error) { + return { + success: false, + error: `Failed to read heartbeat log: ${toErrorMessage(error)}`, + }; + } +} diff --git a/src/mcp/handlers/index.ts b/src/mcp/handlers/index.ts index d73a4e21..6c5c4f1e 100644 --- a/src/mcp/handlers/index.ts +++ b/src/mcp/handlers/index.ts @@ -24,6 +24,8 @@ export { // ADR-051: Model routing handlers handleModelRoute, handleRoutingMetrics, + // Imp-18: Economic routing handler + handleRoutingEconomics, type TaskOrchestrateResult, type ModelRouteParams, type ModelRouteResult, @@ -115,6 +117,19 @@ export { type HypergraphQueryResult, } from './hypergraph-handler.js'; +// Heartbeat handlers (Imp-10: Token-Free Heartbeat Scheduler) +export { + handleHeartbeatStatus, + handleHeartbeatTrigger, + handleHeartbeatLog, + type HeartbeatStatusParams, + type HeartbeatStatusResult, + type HeartbeatTriggerParams, + type HeartbeatTriggerResult, + type HeartbeatLogParams, + type HeartbeatLogResult, +} from './heartbeat-handlers.js'; + // Cross-phase handlers export { handleCrossPhaseStore, diff --git a/src/mcp/handlers/task-handlers.ts b/src/mcp/handlers/task-handlers.ts index 0dc36e78..3a8fd4ee 100644 --- a/src/mcp/handlers/task-handlers.ts +++ b/src/mcp/handlers/task-handlers.ts @@ -679,6 +679,78 @@ export async function handleRoutingMetrics( } } +// ============================================================================ +// Economic Routing Handler (Imp-18, Issue #334) +// ============================================================================ + +export interface RoutingEconomicsParams { + /** Task complexity score 0-1 for tier scoring (default: 0.5) */ + taskComplexity?: number; +} + +export interface RoutingEconomicsResult { + tierEfficiency: Array<{ + tier: string; + qualityScore: number; + estimatedCostUsd: number; + qualityPerDollar: number | string; + economicScore: number; + }>; + currentHourlyCostUsd: number; + currentDailyCostUsd: number; + budgetRemaining: { hourly: number | null; daily: number | null }; + recommendation: string; + savingsOpportunity: { usd: number; description: string } | null; +} + +/** + * Handle economic routing report query + */ +export async function handleRoutingEconomics( + params: RoutingEconomicsParams, +): Promise> { + try { + const { createRoutingFeedbackCollector } = await import('../../routing/routing-feedback.js'); + const { getGlobalCostTracker } = await import('../../shared/llm/cost-tracker.js'); + + // Create collector, initialize to load persisted state from DB, then enable economic routing + const collector = createRoutingFeedbackCollector(100); + await collector.initialize(); + collector.enableEconomicRouting( + { ...(params.taskComplexity != null ? {} : {}) }, + getGlobalCostTracker(), + ); + + const report = collector.getEconomicReport(); + if (!report) { + return { success: false, error: 'Economic routing is not available' }; + } + + // Convert Infinity to string for JSON serialization + const tierEfficiency = report.tierEfficiency.map(t => ({ + ...t, + qualityPerDollar: isFinite(t.qualityPerDollar) ? t.qualityPerDollar : 'Infinity', + })); + + return { + success: true, + data: { + tierEfficiency, + currentHourlyCostUsd: report.currentHourlyCostUsd, + currentDailyCostUsd: report.currentDailyCostUsd, + budgetRemaining: report.budgetRemaining, + recommendation: report.recommendation, + savingsOpportunity: report.savingsOpportunity, + }, + }; + } catch (error) { + return { + success: false, + error: `Failed to get economic routing report: ${toErrorMessage(error)}`, + }; + } +} + // ============================================================================ // Helper Functions // ============================================================================ diff --git a/src/mcp/protocol-server.ts b/src/mcp/protocol-server.ts index e8fe438b..04a6e899 100644 --- a/src/mcp/protocol-server.ts +++ b/src/mcp/protocol-server.ts @@ -37,6 +37,8 @@ import { // ADR-051: Model routing handlers handleModelRoute, handleRoutingMetrics, + // Imp-18: Economic routing handler + handleRoutingEconomics, handleAgentList, handleAgentSpawn, handleAgentMetrics, @@ -1023,6 +1025,18 @@ export class MCPProtocolServer { handler: (params) => handleRoutingMetrics(params as unknown as Parameters[0]), }); + this.registerTool({ + definition: { + name: 'routing_economics', + description: 'Get economic routing report: tier efficiency, budget status, cost-per-quality analysis, and savings opportunities. Example: routing_economics({ taskComplexity: 0.5 })', + category: 'routing', + parameters: [ + { name: 'taskComplexity', type: 'number', description: 'Task complexity score 0-1 for tier scoring (default: 0.5)', default: 0.5 }, + ], + }, + handler: (params) => handleRoutingEconomics(params as unknown as Parameters[0]), + }); + // ADR-057: Infrastructure self-healing tools this.registerTool({ definition: { diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 10acc3fc..6e0ff3fb 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -67,6 +67,10 @@ import { handlePipelineValidate, // Validation Pipeline handler (BMAD-003) handleValidationPipeline, + // Heartbeat handlers (Imp-10: Token-Free Heartbeat Scheduler) + handleHeartbeatStatus, + handleHeartbeatTrigger, + handleHeartbeatLog, } from './handlers'; // ============================================================================ @@ -760,6 +764,44 @@ const PIPELINE_TOOLS: ToolEntry[] = [ }, ]; +// Imp-10: Token-Free Heartbeat Scheduler +const HEARTBEAT_TOOLS: ToolEntry[] = [ + // Heartbeat Status + { + definition: { + name: 'mcp__agentic_qe__heartbeat_status', + description: 'Get heartbeat scheduler status, health metrics, and last result', + category: 'core', + parameters: [], + }, + handler: handleHeartbeatStatus, + }, + // Heartbeat Trigger + { + definition: { + name: 'mcp__agentic_qe__heartbeat_trigger', + description: 'Trigger an immediate heartbeat maintenance cycle (SQL-only, zero tokens)', + category: 'core', + parameters: [ + { name: 'timeout', type: 'number', description: 'Timeout in milliseconds (default: 60000)' }, + ], + }, + handler: handleHeartbeatTrigger, + }, + // Heartbeat Log + { + definition: { + name: 'mcp__agentic_qe__heartbeat_log', + description: 'Read the daily heartbeat log for a specific date', + category: 'core', + parameters: [ + { name: 'date', type: 'string', description: 'Date in YYYY-MM-DD format (defaults to today)' }, + ], + }, + handler: handleHeartbeatLog, + }, +]; + // ============================================================================ export class MCPServer { @@ -787,6 +829,7 @@ export class MCPServer { ...MEMORY_TOOLS, ...CROSS_PHASE_TOOLS, ...PIPELINE_TOOLS, + ...HEARTBEAT_TOOLS, ]; this.registry.registerAll(allTools); diff --git a/src/routing/economic-routing.ts b/src/routing/economic-routing.ts new file mode 100644 index 00000000..fba84bc4 --- /dev/null +++ b/src/routing/economic-routing.ts @@ -0,0 +1,390 @@ +/** + * Economic Routing Model — Imp-18 (Issue #334) + * + * Quality-weighted cost optimization for the routing system. + * Scores tiers by quality-per-dollar efficiency, respects budget limits, + * and produces cost-adjusted rewards so the neural router learns to + * prefer cost-efficient tiers. + * + * @module routing/economic-routing + */ + +import { CostTracker } from '../shared/llm/cost-tracker.js'; +import type { AgentTier } from './routing-config.js'; +import type { RoutingOutcome } from './types.js'; + +// ============================================================================ +// Constants & Types +// ============================================================================ + +/** + * Tier cost estimates (per typical QE task, in USD). + * Based on average token usage per task type. + */ +export const TIER_COST_ESTIMATES: Record< + AgentTier, + { avgInputTokens: number; avgOutputTokens: number; costPerTask: number } +> = { + booster: { avgInputTokens: 0, avgOutputTokens: 0, costPerTask: 0 }, + haiku: { avgInputTokens: 2000, avgOutputTokens: 1000, costPerTask: 0.0035 }, + sonnet: { avgInputTokens: 2000, avgOutputTokens: 1000, costPerTask: 0.021 }, + opus: { avgInputTokens: 2000, avgOutputTokens: 1000, costPerTask: 0.105 }, +}; + +/** All tiers ordered cheapest to most expensive */ +const TIER_ORDER: AgentTier[] = ['booster', 'haiku', 'sonnet', 'opus']; + +/** Maximum cost per task across all tiers (used for normalization) */ +const MAX_TIER_COST = TIER_COST_ESTIMATES.opus.costPerTask; + +export interface EconomicScore { + tier: AgentTier; + /** Expected quality (0-1) */ + qualityScore: number; + /** Estimated cost per task in USD */ + estimatedCostUsd: number; + /** quality / cost (higher = more efficient). Infinity for zero-cost tiers. */ + qualityPerDollar: number; + /** Combined score factoring quality + cost (higher = better) */ + economicScore: number; +} + +export interface EconomicRoutingConfig { + /** Weight for quality in combined score (0-1, default 0.6) */ + qualityWeight: number; + /** Weight for cost efficiency in combined score (0-1, default 0.4) */ + costWeight: number; + /** Budget limit per hour in USD (0 = unlimited) */ + budgetPerHourUsd: number; + /** Budget limit per day in USD (0 = unlimited) */ + budgetPerDayUsd: number; + /** Minimum quality threshold -- never route to cheaper tier below this (0-1) */ + minQualityThreshold: number; + /** Enable economic routing (default: true) */ + enabled: boolean; +} + +export const DEFAULT_ECONOMIC_CONFIG: EconomicRoutingConfig = { + qualityWeight: 0.6, + costWeight: 0.4, + budgetPerHourUsd: 0, + budgetPerDayUsd: 0, + minQualityThreshold: 0.5, + enabled: true, +}; + +export interface EconomicReport { + tierEfficiency: EconomicScore[]; + currentHourlyCostUsd: number; + currentDailyCostUsd: number; + budgetRemaining: { hourly: number | null; daily: number | null }; + recommendation: string; + savingsOpportunity: { usd: number; description: string } | null; +} + +// ============================================================================ +// EconomicRoutingModel +// ============================================================================ + +/** EMA smoothing factor for quality estimate updates */ +const EMA_ALPHA = 0.15; + +export class EconomicRoutingModel { + private config: EconomicRoutingConfig; + private costTracker: CostTracker; + private tierQualityEstimates: Map = new Map(); + private tierOutcomeCounts: Map = new Map(); + + constructor(costTracker: CostTracker, config?: Partial) { + const merged = { ...DEFAULT_ECONOMIC_CONFIG, ...config }; + + // Validate and clamp config values to safe ranges + merged.qualityWeight = Math.max(0, Math.min(1, merged.qualityWeight)); + merged.costWeight = Math.max(0, Math.min(1, merged.costWeight)); + merged.minQualityThreshold = Math.max(0, Math.min(1, merged.minQualityThreshold)); + merged.budgetPerHourUsd = Math.max(0, merged.budgetPerHourUsd); + merged.budgetPerDayUsd = Math.max(0, merged.budgetPerDayUsd); + + // Normalize weights so they sum to 1.0 + const weightSum = merged.qualityWeight + merged.costWeight; + if (weightSum > 0 && weightSum !== 1) { + merged.qualityWeight /= weightSum; + merged.costWeight /= weightSum; + } + + this.config = merged; + this.costTracker = costTracker; + // Initialize with prior assumptions + this.tierQualityEstimates.set('booster', 0.3); + this.tierQualityEstimates.set('haiku', 0.55); + this.tierQualityEstimates.set('sonnet', 0.75); + this.tierQualityEstimates.set('opus', 0.90); + } + + // -------------------------------------------------------------------------- + // Core Methods + // -------------------------------------------------------------------------- + + /** + * Score each tier by quality-per-dollar efficiency. + * Returns all tiers sorted by economicScore descending. + */ + scoreTiers(taskComplexity: number): EconomicScore[] { + const scores: EconomicScore[] = TIER_ORDER.map(tier => { + const qualityScore = this.getQualityEstimate(tier, taskComplexity); + const estimatedCostUsd = TIER_COST_ESTIMATES[tier].costPerTask; + + // quality / cost (handle zero-cost booster) + const qualityPerDollar = estimatedCostUsd > 0 + ? qualityScore / estimatedCostUsd + : qualityScore > 0 ? Infinity : 0; + + // Normalized cost efficiency: 1 - (cost / maxCost) + const costEfficiency = MAX_TIER_COST > 0 + ? 1 - estimatedCostUsd / MAX_TIER_COST + : 1; + + const economicScore = + this.config.qualityWeight * qualityScore + + this.config.costWeight * costEfficiency; + + return { tier, qualityScore, estimatedCostUsd, qualityPerDollar, economicScore }; + }); + + scores.sort((a, b) => b.economicScore - a.economicScore); + return scores; + } + + /** + * Select the best tier considering quality AND cost. + * Respects budget limits and minimum quality thresholds. + */ + selectTier( + taskComplexity: number, + ): { tier: AgentTier; reason: string; scores: EconomicScore[] } { + const scores = this.scoreTiers(taskComplexity); + + for (const score of scores) { + // Skip tiers below minimum quality threshold + if (score.qualityScore < this.config.minQualityThreshold) { + continue; + } + // Skip tiers that would exceed budget + if (this.wouldExceedBudget(score.tier)) { + continue; + } + return { + tier: score.tier, + reason: `Best economic score (${score.economicScore.toFixed(3)}): ` + + `quality=${score.qualityScore.toFixed(2)}, cost=$${score.estimatedCostUsd.toFixed(4)}`, + scores, + }; + } + + // Fallback: pick the cheapest tier that meets quality, ignoring budget + // Use spread copies to avoid mutating the original scores array + const qualityFiltered = scores.filter( + s => s.qualityScore >= this.config.minQualityThreshold, + ); + if (qualityFiltered.length > 0) { + const cheapest = [...qualityFiltered].sort( + (a, b) => a.estimatedCostUsd - b.estimatedCostUsd, + )[0]; + return { + tier: cheapest.tier, + reason: `Budget constrained fallback to ${cheapest.tier}`, + scores, + }; + } + + // Final fallback: pick the best quality regardless + const bestQuality = [...scores].sort((a, b) => b.qualityScore - a.qualityScore)[0]; + return { + tier: bestQuality.tier, + reason: `No tier meets quality threshold ${this.config.minQualityThreshold}; ` + + `using best quality: ${bestQuality.tier}`, + scores, + }; + } + + /** + * Check if a tier would exceed the budget. + */ + wouldExceedBudget(tier: AgentTier): boolean { + const cost = TIER_COST_ESTIMATES[tier].costPerTask; + if (cost === 0) return false; + + const hourlyCost = this.costTracker.getCurrentCost('hour'); + const dailyCost = this.costTracker.getCurrentCost('day'); + + if (this.config.budgetPerHourUsd > 0 && + hourlyCost + cost > this.config.budgetPerHourUsd) { + return true; + } + if (this.config.budgetPerDayUsd > 0 && + dailyCost + cost > this.config.budgetPerDayUsd) { + return true; + } + return false; + } + + /** + * Update quality estimates from observed outcomes. + * Uses EMA to smooth estimates. + */ + updateFromOutcome(outcome: RoutingOutcome, tier: AgentTier): void { + const observedQuality = outcome.outcome.qualityScore; + const current = this.tierQualityEstimates.get(tier) ?? 0.5; + const count = this.tierOutcomeCounts.get(tier) ?? 0; + + // Use EMA; for the first few observations, use a higher alpha + const alpha = count < 5 ? 0.4 : EMA_ALPHA; + const updated = current * (1 - alpha) + observedQuality * alpha; + this.tierQualityEstimates.set(tier, updated); + this.tierOutcomeCounts.set(tier, count + 1); + } + + /** + * Get economic efficiency report. + */ + getEconomicReport(): EconomicReport { + const tierEfficiency = this.scoreTiers(0.5); // mid-complexity as baseline + const currentHourlyCostUsd = this.costTracker.getCurrentCost('hour'); + const currentDailyCostUsd = this.costTracker.getCurrentCost('day'); + + const budgetRemaining = { + hourly: this.config.budgetPerHourUsd > 0 + ? Math.max(0, this.config.budgetPerHourUsd - currentHourlyCostUsd) + : null, + daily: this.config.budgetPerDayUsd > 0 + ? Math.max(0, this.config.budgetPerDayUsd - currentDailyCostUsd) + : null, + }; + + // Find savings opportunity: compare most-used expensive tier vs cheaper alternative + let savingsOpportunity: EconomicReport['savingsOpportunity'] = null; + const sorted = [...tierEfficiency].sort((a, b) => b.qualityPerDollar - a.qualityPerDollar); + const mostEfficient = sorted.find(s => s.estimatedCostUsd > 0 && isFinite(s.qualityPerDollar)); + const leastEfficient = [...sorted].reverse().find( + (s: EconomicScore) => s.estimatedCostUsd > 0 && isFinite(s.qualityPerDollar), + ); + if (mostEfficient && leastEfficient && mostEfficient.tier !== leastEfficient.tier) { + const potentialSavings = leastEfficient.estimatedCostUsd - mostEfficient.estimatedCostUsd; + if (potentialSavings > 0) { + savingsOpportunity = { + usd: potentialSavings, + description: + `Switch from ${leastEfficient.tier} ($${leastEfficient.estimatedCostUsd.toFixed(4)}/task) ` + + `to ${mostEfficient.tier} ($${mostEfficient.estimatedCostUsd.toFixed(4)}/task) ` + + `for comparable tasks to save ~$${potentialSavings.toFixed(4)}/task`, + }; + } + } + + const recommendation = this.generateRecommendation(tierEfficiency, budgetRemaining); + + return { + tierEfficiency, + currentHourlyCostUsd, + currentDailyCostUsd, + budgetRemaining, + recommendation, + savingsOpportunity, + }; + } + + /** + * Compute cost-adjusted reward for the neural router. + * Penalizes expensive tiers that don't deliver proportionally higher quality. + */ + computeCostAdjustedReward( + baseReward: number, + tier: AgentTier, + qualityScore: number, + ): number { + const tierCost = TIER_COST_ESTIMATES[tier].costPerTask; + if (MAX_TIER_COST === 0) return baseReward; + + const costRatio = tierCost / MAX_TIER_COST; // 0-1, where opus=1.0 + const qualityGain = Math.max(0, qualityScore - this.config.minQualityThreshold); + const costPenalty = costRatio * (1 - qualityGain); + const adjusted = baseReward - costPenalty * this.config.costWeight; + + // Clamp to [-1, 1] + return Math.max(-1, Math.min(1, adjusted)); + } + + // -------------------------------------------------------------------------- + // Persistence helpers + // -------------------------------------------------------------------------- + + /** + * Serialize quality estimates for persistence. + */ + serializeEstimates(): Record { + const result: Record = {}; + for (const tier of TIER_ORDER) { + result[tier] = { + quality: this.tierQualityEstimates.get(tier) ?? 0.5, + count: this.tierOutcomeCounts.get(tier) ?? 0, + }; + } + return result; + } + + /** + * Deserialize quality estimates from persistence. + */ + deserializeEstimates(data: Record): void { + for (const tier of TIER_ORDER) { + if (data[tier]) { + this.tierQualityEstimates.set(tier, data[tier].quality); + this.tierOutcomeCounts.set(tier, data[tier].count); + } + } + } + + /** + * Get the current config (read-only copy). + */ + getConfig(): Readonly { + return { ...this.config }; + } + + // -------------------------------------------------------------------------- + // Private helpers + // -------------------------------------------------------------------------- + + /** + * Get quality estimate for a tier, adjusted by task complexity. + * Higher complexity tasks benefit more from higher-tier models. + */ + private getQualityEstimate(tier: AgentTier, taskComplexity: number): number { + const baseQuality = this.tierQualityEstimates.get(tier) ?? 0.5; + // For complex tasks, cheaper tiers degrade more + const complexityPenalty = tier === 'booster' + ? taskComplexity * 0.4 + : tier === 'haiku' + ? taskComplexity * 0.2 + : 0; + return Math.max(0, Math.min(1, baseQuality - complexityPenalty)); + } + + private generateRecommendation( + tierEfficiency: EconomicScore[], + budgetRemaining: { hourly: number | null; daily: number | null }, + ): string { + if (budgetRemaining.hourly !== null && budgetRemaining.hourly < 0.01) { + return 'Hourly budget nearly exhausted. Consider increasing budget or routing to cheaper tiers.'; + } + if (budgetRemaining.daily !== null && budgetRemaining.daily < 0.1) { + return 'Daily budget nearly exhausted. Only critical tasks should use expensive tiers.'; + } + const best = tierEfficiency[0]; + if (best) { + return `Most cost-efficient tier: ${best.tier} ` + + `(score=${best.economicScore.toFixed(3)}, quality=${best.qualityScore.toFixed(2)})`; + } + return 'No economic data available yet.'; + } +} diff --git a/src/routing/index.ts b/src/routing/index.ts index 5f0c8aa8..70ac68a1 100644 --- a/src/routing/index.ts +++ b/src/routing/index.ts @@ -75,6 +75,19 @@ export { createRoutingFeedbackCollector, } from './routing-feedback.js'; +// Economic Routing (Imp-18, Issue #334) +export { + EconomicRoutingModel, + TIER_COST_ESTIMATES, + DEFAULT_ECONOMIC_CONFIG, +} from './economic-routing.js'; + +export type { + EconomicScore, + EconomicRoutingConfig, + EconomicReport, +} from './economic-routing.js'; + // Task Classifier (TD-002) export { classifyTask, diff --git a/src/routing/routing-feedback.ts b/src/routing/routing-feedback.ts index ffaa888e..f0aab857 100644 --- a/src/routing/routing-feedback.ts +++ b/src/routing/routing-feedback.ts @@ -20,7 +20,14 @@ import { safeJsonParse } from '../shared/safe-json.js'; import { toErrorMessage } from '../shared/error-utils.js'; import { EMACalibrator, type EMAConfig } from './calibration/index.js'; import { AutoEscalationTracker, type EscalationConfig, type EscalationState } from './escalation/index.js'; -import type { RoutingConfig } from './routing-config.js'; +import type { RoutingConfig, AgentTier } from './routing-config.js'; +import { + EconomicRoutingModel, + type EconomicRoutingConfig, + type EconomicScore, + type EconomicReport, +} from './economic-routing.js'; +import { CostTracker, getGlobalCostTracker } from '../shared/llm/cost-tracker.js'; // ============================================================================ // Database Row Types @@ -103,6 +110,9 @@ export class RoutingFeedbackCollector { private db: UnifiedMemoryManager | null = null; private calibrator: EMACalibrator | null = null; private escalationTracker: AutoEscalationTracker | null = null; + private economicModel: EconomicRoutingModel | null = null; + private economicPersistCounter = 0; + private static readonly ECONOMIC_PERSIST_INTERVAL = 10; private persistCount = 0; private readonly maxOutcomes: number; private static readonly RETENTION_CLEANUP_INTERVAL = 100; @@ -283,6 +293,44 @@ export class RoutingFeedbackCollector { } } + /** + * Load persisted economic routing state from the database + */ + private loadEconomicState(): void { + if (!this.db || !this.economicModel) return; + try { + const database = this.db.getDatabase(); + const row = database.prepare( + `SELECT value FROM kv_store WHERE key = 'routing:economic_quality_estimates'` + ).get() as { value: string } | undefined; + if (row) { + const data = safeJsonParse(row.value); + if (data && typeof data === 'object') { + this.economicModel.deserializeEstimates(data as Record); + console.log('[RoutingFeedbackCollector] Loaded economic quality estimates from DB'); + } + } + } catch (error) { + console.warn('[RoutingFeedbackCollector] Failed to load economic state:', toErrorMessage(error)); + } + } + + /** + * Persist economic routing state to the database + */ + private persistEconomicState(): void { + if (!this.db || !this.economicModel) return; + try { + const database = this.db.getDatabase(); + const serialized = JSON.stringify(this.economicModel.serializeEstimates()); + database.prepare( + `INSERT OR REPLACE INTO kv_store (key, value, updated_at) VALUES (?, ?, datetime('now'))` + ).run('routing:economic_quality_estimates', serialized); + } catch (error) { + console.warn('[RoutingFeedbackCollector] Failed to persist economic state:', toErrorMessage(error)); + } + } + /** * Connect to router for automatic performance updates */ @@ -322,6 +370,36 @@ export class RoutingFeedbackCollector { return this.escalationTracker?.getState(agentId) ?? null; } + /** + * Enable economic routing model for quality-weighted cost optimization. + * Once enabled, every recorded outcome feeds economic quality estimates + * and the neural router receives cost-adjusted rewards. + */ + enableEconomicRouting( + config?: Partial, + costTracker?: CostTracker, + ): void { + const tracker = costTracker ?? getGlobalCostTracker(); + this.economicModel = new EconomicRoutingModel(tracker, config); + this.loadEconomicState(); + } + + /** + * Get the economic routing report. + * Returns null if economic routing is not enabled. + */ + getEconomicReport(): EconomicReport | null { + return this.economicModel?.getEconomicReport() ?? null; + } + + /** + * Get economic scores for a given task complexity. + * Returns null if economic routing is not enabled. + */ + getEconomicScore(taskComplexity: number): EconomicScore[] | null { + return this.economicModel?.scoreTiers(taskComplexity) ?? null; + } + /** * Record a routing outcome */ @@ -383,6 +461,18 @@ export class RoutingFeedbackCollector { } } + // Update economic model if enabled + if (this.economicModel) { + const tier = this.inferTier(usedAgent) as AgentTier; + this.economicModel.updateFromOutcome(routingOutcome, tier); + + // Persist economic state periodically + this.economicPersistCounter++; + if (this.economicPersistCounter % RoutingFeedbackCollector.ECONOMIC_PERSIST_INTERVAL === 0) { + this.persistEconomicState(); + } + } + return routingOutcome; } diff --git a/tests/unit/cli/heartbeat-handler.test.ts b/tests/unit/cli/heartbeat-handler.test.ts new file mode 100644 index 00000000..46a07d1c --- /dev/null +++ b/tests/unit/cli/heartbeat-handler.test.ts @@ -0,0 +1,263 @@ +/** + * Agentic QE v3 - Heartbeat CLI Handler Tests + * Imp-10: Token-Free Heartbeat Scheduler CLI Integration + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Use vi.hoisted() so mock fns are available when vi.mock factory runs (hoisted above imports) +const mocks = vi.hoisted(() => ({ + existsSync: vi.fn(), + readFileSync: vi.fn(), + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), +})); + +vi.mock('fs', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + existsSync: mocks.existsSync, + readFileSync: mocks.readFileSync, + writeFileSync: mocks.writeFileSync, + mkdirSync: mocks.mkdirSync, + }; +}); + +// Mock the worker before importing the handler +vi.mock('../../../src/workers/workers/heartbeat-scheduler.js', () => { + return { + HeartbeatSchedulerWorker: class MockHeartbeatWorker { + config = { + id: 'heartbeat-scheduler', + name: 'Heartbeat Scheduler', + intervalMs: 30 * 60 * 1000, + enabled: true, + }; + status = 'idle'; + lastResult = { + workerId: 'heartbeat-scheduler', + timestamp: new Date('2026-03-27T14:32:15Z'), + durationMs: 245, + success: true, + metrics: { + itemsAnalyzed: 50, + issuesFound: 3, + healthScore: 85, + trend: 'stable' as const, + domainMetrics: { + promoted: 2, + deprecated: 1, + decayed: 15, + pendingExperiences: 8, + avgConfidence: 0.72, + }, + }, + findings: [], + recommendations: [], + }; + lastRunAt = new Date('2026-03-27T14:32:15Z'); + nextRunAt = new Date('2026-03-27T15:02:15Z'); + + initialize = vi.fn().mockResolvedValue(undefined); + execute = vi.fn().mockResolvedValue({ + workerId: 'heartbeat-scheduler', + timestamp: new Date(), + durationMs: 312, + success: true, + metrics: { + itemsAnalyzed: 42, + issuesFound: 1, + healthScore: 88, + trend: 'improving', + domainMetrics: { + promoted: 3, + deprecated: 0, + decayed: 10, + pendingExperiences: 5, + avgConfidence: 0.75, + }, + }, + findings: [{ type: 'heartbeat-promotion', severity: 'info', domain: 'learning-optimization', title: 'Promoted', description: '3 promoted' }], + recommendations: [], + }); + getHealth = vi.fn().mockReturnValue({ + status: 'idle', + healthScore: 85, + totalExecutions: 42, + successfulExecutions: 41, + failedExecutions: 1, + avgDurationMs: 280, + recentResults: [], + }); + pause = vi.fn(); + resume = vi.fn(); + stop = vi.fn().mockResolvedValue(undefined); + }, + }; +}); + +// Mock unified-memory for findProjectRoot +vi.mock('../../../src/kernel/unified-memory.js', () => ({ + findProjectRoot: () => '/tmp/test-project', + getUnifiedMemory: vi.fn(), +})); + +import { HeartbeatHandler } from '../../../src/cli/handlers/heartbeat-handler.js'; + +describe('HeartbeatHandler', () => { + let handler: HeartbeatHandler; + let cleanupAndExit: ReturnType; + let consoleSpy: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + cleanupAndExit = vi.fn().mockResolvedValue(undefined); + handler = new HeartbeatHandler(cleanupAndExit); + consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + }); + + function buildContext() { + return { + kernel: null, + queen: null, + router: null, + workflowOrchestrator: null, + scheduledWorkflows: new Map(), + persistentScheduler: null, + initialized: false, + }; + } + + function allOutput(): string { + return consoleSpy.mock.calls.map(c => c.join(' ')).join('\n'); + } + + describe('metadata', () => { + it('should have correct name and description', () => { + expect(handler.name).toBe('heartbeat'); + expect(handler.description).toBe('Manage the token-free heartbeat scheduler'); + }); + + it('should return help text with subcommands', () => { + const help = handler.getHelp(); + expect(help).toContain('status'); + expect(help).toContain('run-now'); + expect(help).toContain('history'); + expect(help).toContain('log'); + expect(help).toContain('pause'); + expect(help).toContain('resume'); + }); + }); + + describe('status', () => { + it('should display formatted health data', async () => { + const { Command } = await import('commander'); + const program = new Command(); + handler.register(program, buildContext()); + + await program.parseAsync(['node', 'aqe', 'heartbeat', 'status']); + + expect(cleanupAndExit).toHaveBeenCalledWith(0); + + const output = allOutput(); + expect(output).toContain('Heartbeat Scheduler Status'); + expect(output).toContain('85'); + expect(output).toContain('42'); + }); + }); + + describe('run-now', () => { + it('should execute heartbeat and return results', async () => { + // storeHistoryEntry uses fs + mocks.existsSync.mockReturnValue(false); + + const { Command } = await import('commander'); + const program = new Command(); + handler.register(program, buildContext()); + + await program.parseAsync(['node', 'aqe', 'heartbeat', 'run-now']); + + expect(cleanupAndExit).toHaveBeenCalledWith(0); + + const output = allOutput(); + expect(output).toContain('Triggering heartbeat cycle'); + expect(output).toContain('Heartbeat cycle complete'); + }); + }); + + describe('log', () => { + it('should read daily log file for today', async () => { + const today = new Date().toISOString().split('T')[0]; + + mocks.existsSync.mockReturnValue(true); + mocks.readFileSync.mockReturnValue( + `# AQE Daily Log \u2014 ${today}\n| Time | Event | Summary |\n|------|-------|--------|\n| 14:32:15 | pattern-promoted | Heartbeat: 2 promoted |\n` + ); + + const { Command } = await import('commander'); + const program = new Command(); + handler.register(program, buildContext()); + + await program.parseAsync(['node', 'aqe', 'heartbeat', 'log']); + + expect(cleanupAndExit).toHaveBeenCalledWith(0); + expect(allOutput()).toContain('Daily Log'); + }); + + it('should read log for a specific date', async () => { + mocks.existsSync.mockReturnValue(true); + mocks.readFileSync.mockReturnValue( + `# AQE Daily Log \u2014 2026-03-25\n| Time | Event | Summary |\n` + ); + + const { Command } = await import('commander'); + const program = new Command(); + handler.register(program, buildContext()); + + await program.parseAsync(['node', 'aqe', 'heartbeat', 'log', '--date', '2026-03-25']); + + expect(cleanupAndExit).toHaveBeenCalledWith(0); + expect(allOutput()).toContain('2026-03-25'); + }); + + it('should handle missing log file gracefully', async () => { + mocks.existsSync.mockReturnValue(false); + + const { Command } = await import('commander'); + const program = new Command(); + handler.register(program, buildContext()); + + await program.parseAsync(['node', 'aqe', 'heartbeat', 'log', '--date', '2020-01-01']); + + expect(cleanupAndExit).toHaveBeenCalledWith(0); + expect(allOutput()).toContain('No daily log found'); + }); + }); + + describe('pause/resume', () => { + it('should pause the worker', async () => { + const { Command } = await import('commander'); + const program = new Command(); + handler.register(program, buildContext()); + + await program.parseAsync(['node', 'aqe', 'heartbeat', 'pause']); + + expect(cleanupAndExit).toHaveBeenCalledWith(0); + expect(allOutput()).toContain('paused'); + }); + + it('should resume the worker', async () => { + const { Command } = await import('commander'); + const program = new Command(); + handler.register(program, buildContext()); + + await program.parseAsync(['node', 'aqe', 'heartbeat', 'resume']); + + expect(cleanupAndExit).toHaveBeenCalledWith(0); + expect(allOutput()).toContain('resumed'); + }); + }); +}); diff --git a/tests/unit/coordination/approval-gate.test.ts b/tests/unit/coordination/approval-gate.test.ts new file mode 100644 index 00000000..8cc916c9 --- /dev/null +++ b/tests/unit/coordination/approval-gate.test.ts @@ -0,0 +1,248 @@ +/** + * Unit tests for approval gate step type (Imp-9) + * + * Tests the approval gate functionality in the WorkflowOrchestrator: + * - Step pauses at approval gate + * - approveStep() resumes execution + * - rejectStep() fails the step + * - Auto-approve works after timeout + * - Timeout rejection works + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { WorkflowOrchestrator } from '../../../src/coordination/workflow-orchestrator.js'; +import type { + WorkflowDefinition, + WorkflowOrchestratorConfig, +} from '../../../src/coordination/workflow-types.js'; +import { ok } from '../../../src/shared/types/index.js'; +import type { EventBus, MemoryBackend, AgentCoordinator, Subscription } from '../../../src/kernel/interfaces.js'; + +// ============================================================================ +// Mock Infrastructure +// ============================================================================ + +function createMockEventBus(): EventBus { + return { + publish: vi.fn().mockResolvedValue(undefined), + subscribe: vi.fn().mockReturnValue({ unsubscribe: vi.fn() } as Subscription), + subscribeOnce: vi.fn().mockReturnValue({ unsubscribe: vi.fn() } as Subscription), + } as unknown as EventBus; +} + +function createMockMemory(): MemoryBackend { + return { + get: vi.fn().mockResolvedValue(undefined), + set: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(true), + search: vi.fn().mockResolvedValue([]), + has: vi.fn().mockResolvedValue(false), + keys: vi.fn().mockResolvedValue([]), + } as unknown as MemoryBackend; +} + +function createMockCoordinator(): AgentCoordinator { + return { + canSpawn: vi.fn().mockReturnValue(true), + spawn: vi.fn().mockResolvedValue(ok('agent-1')), + stop: vi.fn().mockResolvedValue(ok(undefined)), + list: vi.fn().mockReturnValue([]), + getAgent: vi.fn().mockReturnValue(undefined), + } as unknown as AgentCoordinator; +} + +const TEST_CONFIG: Partial = { + maxConcurrentWorkflows: 5, + defaultStepTimeout: 60000, + defaultWorkflowTimeout: 300000, + enableEventTriggers: false, + persistExecutions: false, +}; + +// ============================================================================ +// Test Workflows +// ============================================================================ + +function makeApprovalWorkflow( + approval: WorkflowDefinition['steps'][0]['approval'], +): WorkflowDefinition { + return { + id: 'approval-test', + name: 'Approval Test Workflow', + description: 'Test workflow with approval gate', + version: '1.0.0', + steps: [ + { + id: 'gate-check', + name: 'Quality Gate', + domain: 'quality-assessment', + action: 'gate-check', + inputMapping: {}, + }, + { + id: 'approval-step', + name: 'Approval Gate', + domain: 'quality-assessment', + action: 'gate-check', + dependsOn: ['gate-check'], + approval, + }, + ], + }; +} + +// ============================================================================ +// Tests +// ============================================================================ + +describe('approval gate', () => { + let orchestrator: WorkflowOrchestrator; + let eventBus: EventBus; + + beforeEach(async () => { + eventBus = createMockEventBus(); + orchestrator = new WorkflowOrchestrator( + eventBus, + createMockMemory(), + createMockCoordinator(), + TEST_CONFIG, + ); + await orchestrator.initialize(); + }); + + it('should pause at approval gate and resume on approve', async () => { + const workflow = makeApprovalWorkflow({ + autoApproveAfter: 0, // never auto-approve + message: 'Please approve', + }); + orchestrator.registerWorkflow(workflow); + + const execResult = await orchestrator.executeWorkflow('approval-test', {}); + expect(execResult.success).toBe(true); + const executionId = execResult.value; + + // Give the workflow time to reach the approval gate + await new Promise((r) => setTimeout(r, 100)); + + // The step should be awaiting approval + const status = orchestrator.getWorkflowStatus(executionId); + expect(status).toBeDefined(); + // Workflow should still be running (not completed yet) + expect(status!.status).toBe('running'); + + // Approve the step + const approved = orchestrator.approveStep(executionId, 'approval-step'); + expect(approved).toBe(true); + + // Wait for workflow to complete + await new Promise((r) => setTimeout(r, 100)); + + const finalStatus = orchestrator.getWorkflowStatus(executionId); + expect(finalStatus).toBeDefined(); + expect(finalStatus!.status).toBe('completed'); + expect(finalStatus!.completedSteps).toContain('approval-step'); + }); + + it('should fail step on reject', async () => { + const workflow = makeApprovalWorkflow({ + autoApproveAfter: 0, + message: 'Please approve', + }); + orchestrator.registerWorkflow(workflow); + + const execResult = await orchestrator.executeWorkflow('approval-test', {}); + expect(execResult.success).toBe(true); + const executionId = execResult.value; + + // Wait for approval gate + await new Promise((r) => setTimeout(r, 100)); + + // Reject the step + const rejected = orchestrator.rejectStep(executionId, 'approval-step', 'Not ready'); + expect(rejected).toBe(true); + + // Wait for workflow to finish + await new Promise((r) => setTimeout(r, 100)); + + const finalStatus = orchestrator.getWorkflowStatus(executionId); + expect(finalStatus).toBeDefined(); + expect(finalStatus!.status).toBe('failed'); + expect(finalStatus!.failedSteps).toContain('approval-step'); + }); + + it('should auto-approve after timeout', async () => { + const workflow = makeApprovalWorkflow({ + autoApproveAfter: 50, // 50ms + message: 'Auto-approve test', + }); + orchestrator.registerWorkflow(workflow); + + const execResult = await orchestrator.executeWorkflow('approval-test', {}); + expect(execResult.success).toBe(true); + const executionId = execResult.value; + + // Wait for auto-approve + execution + await new Promise((r) => setTimeout(r, 300)); + + const finalStatus = orchestrator.getWorkflowStatus(executionId); + expect(finalStatus).toBeDefined(); + expect(finalStatus!.status).toBe('completed'); + expect(finalStatus!.completedSteps).toContain('approval-step'); + }); + + it('should return false for approve on unknown execution', () => { + const result = orchestrator.approveStep('nonexistent', 'step1'); + expect(result).toBe(false); + }); + + it('should return false for reject on unknown execution', () => { + const result = orchestrator.rejectStep('nonexistent', 'step1'); + expect(result).toBe(false); + }); + + it('should work with simple boolean approval (auto-approve default 5min)', async () => { + // With approval: true, auto-approve timeout is 300000ms (5 min default). + // We just verify the workflow enters the gate. We approve manually + // to avoid a long wait. + const workflow = makeApprovalWorkflow(true); + orchestrator.registerWorkflow(workflow); + + const execResult = await orchestrator.executeWorkflow('approval-test', {}); + expect(execResult.success).toBe(true); + const executionId = execResult.value; + + await new Promise((r) => setTimeout(r, 100)); + + // Approve manually + orchestrator.approveStep(executionId, 'approval-step'); + await new Promise((r) => setTimeout(r, 100)); + + const finalStatus = orchestrator.getWorkflowStatus(executionId); + expect(finalStatus).toBeDefined(); + expect(finalStatus!.status).toBe('completed'); + }); + + it('should emit StepAwaitingApproval event', async () => { + const workflow = makeApprovalWorkflow({ + autoApproveAfter: 0, + message: 'Event test', + }); + orchestrator.registerWorkflow(workflow); + + const execResult = await orchestrator.executeWorkflow('approval-test', {}); + const executionId = execResult.value; + + await new Promise((r) => setTimeout(r, 100)); + + // Check that publish was called with the approval event + const publishCalls = (eventBus.publish as ReturnType).mock.calls; + const approvalEvents = publishCalls.filter( + (call: unknown[]) => (call[0] as { type: string }).type === 'workflow.StepAwaitingApproval', + ); + expect(approvalEvents.length).toBeGreaterThan(0); + + // Clean up + orchestrator.approveStep(executionId, 'approval-step'); + await new Promise((r) => setTimeout(r, 50)); + }); +}); diff --git a/tests/unit/coordination/deterministic-actions.test.ts b/tests/unit/coordination/deterministic-actions.test.ts new file mode 100644 index 00000000..b48e3878 --- /dev/null +++ b/tests/unit/coordination/deterministic-actions.test.ts @@ -0,0 +1,284 @@ +/** + * Unit tests for deterministic actions (Imp-9) + * + * Verifies that each built-in action: + * 1. Works with explicit inputs (source: 'input') + * 2. Queries the database when inputs are omitted (source: 'database') + * 3. Degrades gracefully when DB is unavailable + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { + findDeterministicAction, + getAllDeterministicActions, +} from '../../../src/coordination/deterministic-actions.js'; +import type { WorkflowContext } from '../../../src/coordination/workflow-types.js'; + +// ============================================================================ +// Fixtures +// ============================================================================ + +function makeContext(overrides?: Partial): WorkflowContext { + return { + input: {}, + results: {}, + metadata: { + executionId: 'test-exec-1', + workflowId: 'test-workflow-1', + startedAt: new Date(), + }, + ...overrides, + }; +} + +// ============================================================================ +// DB Mock Helpers +// ============================================================================ + +/** + * Mock the unified memory module so deterministic actions see a fake DB. + * Call restore() in afterEach. + */ +function mockUnifiedMemory(rows: Record) { + const mockPrepare = vi.fn().mockImplementation((sql: string) => ({ + get: vi.fn().mockImplementation(() => { + // Return different rows based on the table being queried + if (sql.includes('coverage_sessions')) return rows.coverage ?? undefined; + if (sql.includes('test_outcomes') && sql.includes('SUM')) return rows.testOutcomes ?? undefined; + if (sql.includes('test_outcomes') && sql.includes('bugs')) return rows.bugs ?? undefined; + if (sql.includes('qe_patterns')) return rows.patterns ?? undefined; + if (sql.includes('routing_outcomes')) return rows.routing ?? undefined; + return undefined; + }), + all: vi.fn().mockReturnValue([]), + })); + + const mockDb = { prepare: mockPrepare }; + const mockManager = { + isInitialized: vi.fn().mockReturnValue(true), + getDatabase: vi.fn().mockReturnValue(mockDb), + }; + + // Mock the require() call inside tryGetDb + vi.doMock('../../../src/kernel/unified-memory.js', () => ({ + getUnifiedMemory: () => mockManager, + })); + + return { mockDb, mockPrepare, mockManager }; +} + +// ============================================================================ +// Registry Tests +// ============================================================================ + +describe('deterministic-actions registry', () => { + it('should have all four built-in actions', () => { + const actions = getAllDeterministicActions(); + expect(actions.length).toBe(4); + + const ids = actions.map((a) => a.id); + expect(ids).toContain('quality-gate-check'); + expect(ids).toContain('coverage-threshold'); + expect(ids).toContain('pattern-health'); + expect(ids).toContain('routing-accuracy'); + }); + + it('should find action by domain + action', () => { + const action = findDeterministicAction('quality-assessment', 'gate-check'); + expect(action).toBeDefined(); + expect(action!.id).toBe('quality-gate-check'); + }); + + it('should return undefined for unknown domain/action', () => { + const action = findDeterministicAction('quality-assessment', 'nonexistent'); + expect(action).toBeUndefined(); + }); +}); + +// ============================================================================ +// Quality Gate Check +// ============================================================================ + +describe('quality-gate-check action', () => { + it('should pass when all thresholds are met (explicit input)', async () => { + const action = findDeterministicAction('quality-assessment', 'gate-check')!; + const result = await action.execute( + { + coverageMin: 80, + testsPassingMin: 90, + maxBugs: 5, + currentCoverage: 85, + currentTestsPassingRate: 95, + currentBugs: 2, + }, + makeContext(), + ); + + expect(result.success).toBe(true); + const data = result.value; + expect(data.passed).toBe(true); + expect(data.score).toBeGreaterThan(0); + expect(data.source).toBe('input'); + expect(data.details).toBeDefined(); + }); + + it('should fail when coverage is below threshold', async () => { + const action = findDeterministicAction('quality-assessment', 'gate-check')!; + const result = await action.execute( + { + coverageMin: 80, + testsPassingMin: 90, + maxBugs: 5, + currentCoverage: 50, + currentTestsPassingRate: 95, + currentBugs: 2, + }, + makeContext(), + ); + + expect(result.success).toBe(true); + const data = result.value; + expect(data.passed).toBe(false); + expect((data.details as Record & { coverage: { passed: boolean } }).coverage.passed).toBe(false); + }); + + it('should use defaults and report database source when no input provided', async () => { + const action = findDeterministicAction('quality-assessment', 'gate-check')!; + const result = await action.execute({}, makeContext()); + + expect(result.success).toBe(true); + const data = result.value; + // With no DB available and no inputs, defaults to 0 → fails + expect(data.passed).toBe(false); + expect(typeof data.score).toBe('number'); + expect(data.source).toBe('database'); + }); + + it('should handle maxBugs=0 edge case', async () => { + const action = findDeterministicAction('quality-assessment', 'gate-check')!; + const result = await action.execute( + { maxBugs: 0, currentBugs: 0, currentCoverage: 100, currentTestsPassingRate: 100 }, + makeContext(), + ); + expect(result.success).toBe(true); + expect(result.value.passed).toBe(true); + + // maxBugs=0 with bugs>0 should fail + const result2 = await action.execute( + { maxBugs: 0, currentBugs: 1, currentCoverage: 100, currentTestsPassingRate: 100 }, + makeContext(), + ); + expect(result2.success).toBe(true); + expect(result2.value.passed).toBe(false); + }); +}); + +// ============================================================================ +// Coverage Threshold Check +// ============================================================================ + +describe('coverage-threshold action', () => { + it('should pass when coverage meets threshold (explicit input)', async () => { + const action = findDeterministicAction('coverage-analysis', 'threshold-check')!; + const result = await action.execute( + { minCoverage: 80, currentCoverage: 90 }, + makeContext(), + ); + + expect(result.success).toBe(true); + const data = result.value; + expect(data.passed).toBe(true); + expect(data.gap).toBe(0); + expect(data.currentCoverage).toBe(90); + expect(data.source).toBe('input'); + }); + + it('should fail and report gap when below threshold', async () => { + const action = findDeterministicAction('coverage-analysis', 'threshold-check')!; + const result = await action.execute( + { minCoverage: 80, currentCoverage: 65 }, + makeContext(), + ); + + expect(result.success).toBe(true); + const data = result.value; + expect(data.passed).toBe(false); + expect(data.gap).toBe(15); + }); + + it('should report database source when inputs omitted', async () => { + const action = findDeterministicAction('coverage-analysis', 'threshold-check')!; + const result = await action.execute({}, makeContext()); + + expect(result.success).toBe(true); + const data = result.value; + expect(data.currentCoverage).toBe(0); // no DB available → 0 + expect(data.passed).toBe(false); + expect(data.source).toBe('database'); + }); +}); + +// ============================================================================ +// Pattern Health Check +// ============================================================================ + +describe('pattern-health action', () => { + it('should compute a health score from explicit stats', async () => { + const action = findDeterministicAction('learning-optimization', 'health-check')!; + const result = await action.execute( + { totalPatterns: 100, activePatterns: 80, avgConfidence: 0.9 }, + makeContext(), + ); + + expect(result.success).toBe(true); + const data = result.value; + expect(data.healthScore).toBeGreaterThan(0); + expect(data.healthScore).toBeLessThanOrEqual(1); + expect(data.totalPatterns).toBe(100); + expect(data.activePatterns).toBe(80); + expect(data.source).toBe('input'); + }); + + it('should return zero health for empty state and database source', async () => { + const action = findDeterministicAction('learning-optimization', 'health-check')!; + const result = await action.execute({}, makeContext()); + + expect(result.success).toBe(true); + const data = result.value; + expect(data.healthScore).toBe(0); + expect(data.totalPatterns).toBe(0); + expect(data.source).toBe('database'); + }); +}); + +// ============================================================================ +// Routing Accuracy Check +// ============================================================================ + +describe('routing-accuracy action', () => { + it('should compute success rate from explicit outcomes', async () => { + const action = findDeterministicAction('learning-optimization', 'routing-check')!; + const result = await action.execute( + { totalOutcomes: 200, successfulOutcomes: 180, confidenceCorrelation: 0.85 }, + makeContext(), + ); + + expect(result.success).toBe(true); + const data = result.value; + expect(data.successRate).toBe(90); + expect(data.totalOutcomes).toBe(200); + expect(data.confidenceCorrelation).toBe(0.85); + expect(data.source).toBe('input'); + }); + + it('should handle zero outcomes gracefully and report database source', async () => { + const action = findDeterministicAction('learning-optimization', 'routing-check')!; + const result = await action.execute({}, makeContext()); + + expect(result.success).toBe(true); + const data = result.value; + expect(data.successRate).toBe(0); + expect(data.totalOutcomes).toBe(0); + expect(data.source).toBe('database'); + }); +}); diff --git a/tests/unit/mcp/heartbeat-handlers.test.ts b/tests/unit/mcp/heartbeat-handlers.test.ts new file mode 100644 index 00000000..e5fa985f --- /dev/null +++ b/tests/unit/mcp/heartbeat-handlers.test.ts @@ -0,0 +1,219 @@ +/** + * Agentic QE v3 - Heartbeat MCP Handler Tests + * Imp-10: Token-Free Heartbeat Scheduler MCP Integration + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Use vi.hoisted() so mock fns are available when vi.mock factory runs (hoisted above imports) +const mocks = vi.hoisted(() => ({ + existsSync: vi.fn(), + readFileSync: vi.fn(), +})); + +vi.mock('fs', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + existsSync: mocks.existsSync, + readFileSync: mocks.readFileSync, + }; +}); + +// Mock the worker before importing handlers +vi.mock('../../../src/workers/workers/heartbeat-scheduler.js', () => { + return { + HeartbeatSchedulerWorker: class MockHeartbeatWorker { + config = { + id: 'heartbeat-scheduler', + name: 'Heartbeat Scheduler', + intervalMs: 30 * 60 * 1000, + enabled: true, + }; + status = 'idle'; + lastResult = { + workerId: 'heartbeat-scheduler', + timestamp: new Date('2026-03-27T14:32:15Z'), + durationMs: 245, + success: true, + metrics: { + itemsAnalyzed: 50, + issuesFound: 3, + healthScore: 85, + trend: 'stable' as const, + domainMetrics: { + promoted: 2, + deprecated: 1, + decayed: 15, + pendingExperiences: 8, + avgConfidence: 0.72, + }, + }, + findings: [], + recommendations: [], + }; + lastRunAt = new Date('2026-03-27T14:32:15Z'); + nextRunAt = new Date('2026-03-27T15:02:15Z'); + + initialize = vi.fn().mockResolvedValue(undefined); + execute = vi.fn().mockResolvedValue({ + workerId: 'heartbeat-scheduler', + timestamp: new Date(), + durationMs: 312, + success: true, + metrics: { + itemsAnalyzed: 42, + issuesFound: 1, + healthScore: 88, + trend: 'improving', + domainMetrics: { + promoted: 3, + deprecated: 0, + decayed: 10, + pendingExperiences: 5, + avgConfidence: 0.75, + }, + }, + findings: [{ type: 'heartbeat-promotion', severity: 'info', domain: 'learning-optimization', title: 'Promoted', description: '3 promoted' }], + recommendations: [], + }); + getHealth = vi.fn().mockReturnValue({ + status: 'idle', + healthScore: 85, + totalExecutions: 42, + successfulExecutions: 41, + failedExecutions: 1, + avgDurationMs: 280, + recentResults: [], + }); + pause = vi.fn(); + resume = vi.fn(); + stop = vi.fn().mockResolvedValue(undefined); + }, + }; +}); + +// Mock unified-memory for findProjectRoot +vi.mock('../../../src/kernel/unified-memory.js', () => ({ + findProjectRoot: () => '/tmp/test-project', + getUnifiedMemory: vi.fn(), +})); + +import { + handleHeartbeatStatus, + handleHeartbeatTrigger, + handleHeartbeatLog, +} from '../../../src/mcp/handlers/heartbeat-handlers.js'; + +describe('Heartbeat MCP Handlers', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('handleHeartbeatStatus', () => { + it('should return health JSON with correct structure', async () => { + const result = await handleHeartbeatStatus({}); + + expect(result.success).toBe(true); + expect(result.data).toBeDefined(); + expect(result.data!.status).toBe('idle'); + expect(result.data!.healthScore).toBe(85); + expect(result.data!.totalExecutions).toBe(42); + expect(result.data!.successfulExecutions).toBe(41); + expect(result.data!.failedExecutions).toBe(1); + expect(result.data!.avgDurationMs).toBe(280); + }); + + it('should include last run timestamps', async () => { + const result = await handleHeartbeatStatus({}); + + expect(result.success).toBe(true); + expect(result.data!.lastRunAt).toBe('2026-03-27T14:32:15.000Z'); + expect(result.data!.nextRunAt).toBe('2026-03-27T15:02:15.000Z'); + }); + + it('should include last result details', async () => { + const result = await handleHeartbeatStatus({}); + + expect(result.success).toBe(true); + expect(result.data!.lastResult).toBeDefined(); + expect(result.data!.lastResult!.healthScore).toBe(85); + expect(result.data!.lastResult!.trend).toBe('stable'); + expect(result.data!.lastResult!.domainMetrics.promoted).toBe(2); + }); + }); + + describe('handleHeartbeatTrigger', () => { + it('should execute heartbeat and return result', async () => { + const result = await handleHeartbeatTrigger({}); + + expect(result.success).toBe(true); + expect(result.data).toBeDefined(); + expect(result.data!.success).toBe(true); + expect(result.data!.healthScore).toBe(88); + expect(result.data!.trend).toBe('improving'); + expect(result.data!.promoted).toBe(3); + expect(result.data!.deprecated).toBe(0); + expect(result.data!.decayed).toBe(10); + expect(result.data!.pendingExperiences).toBe(5); + expect(result.data!.avgConfidence).toBe(0.75); + expect(result.data!.findingsCount).toBe(1); + expect(result.data!.recommendationsCount).toBe(0); + }); + + it('should return numeric duration', async () => { + const result = await handleHeartbeatTrigger({}); + + expect(result.success).toBe(true); + expect(typeof result.data!.durationMs).toBe('number'); + expect(result.data!.durationMs).toBe(312); + }); + }); + + describe('handleHeartbeatLog', () => { + it('should return log content when file exists', async () => { + const today = new Date().toISOString().split('T')[0]; + const logContent = `# AQE Daily Log \u2014 ${today}\n| Time | Event | Summary |\n`; + + mocks.existsSync.mockReturnValue(true); + mocks.readFileSync.mockReturnValue(logContent); + + const result = await handleHeartbeatLog({}); + + expect(result.success).toBe(true); + expect(result.data!.date).toBe(today); + expect(result.data!.exists).toBe(true); + expect(result.data!.content).toContain('AQE Daily Log'); + expect(result.data!.lineCount).toBeGreaterThan(0); + }); + + it('should accept a specific date parameter', async () => { + mocks.existsSync.mockReturnValue(true); + mocks.readFileSync.mockReturnValue('# AQE Daily Log \u2014 2026-03-25\n'); + + const result = await handleHeartbeatLog({ date: '2026-03-25' }); + + expect(result.success).toBe(true); + expect(result.data!.date).toBe('2026-03-25'); + expect(result.data!.exists).toBe(true); + }); + + it('should report missing log files gracefully', async () => { + mocks.existsSync.mockReturnValue(false); + + const result = await handleHeartbeatLog({ date: '2020-01-01' }); + + expect(result.success).toBe(true); + expect(result.data!.exists).toBe(false); + expect(result.data!.content).toBe(''); + expect(result.data!.lineCount).toBe(0); + }); + + it('should reject invalid date formats', async () => { + const result = await handleHeartbeatLog({ date: 'not-a-date' }); + + expect(result.success).toBe(false); + expect(result.error).toContain('Invalid date format'); + }); + }); +}); diff --git a/tests/unit/mcp/routing-economics-handler.test.ts b/tests/unit/mcp/routing-economics-handler.test.ts new file mode 100644 index 00000000..f08ea006 --- /dev/null +++ b/tests/unit/mcp/routing-economics-handler.test.ts @@ -0,0 +1,99 @@ +/** + * Unit tests for the routing_economics MCP handler (Imp-18) + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Mock unified memory — keep all real exports but stub the singleton +vi.mock('../../../src/kernel/unified-memory.js', async (importOriginal) => { + const actual = await importOriginal>(); + const mockDb = { + prepare: vi.fn().mockReturnValue({ + get: vi.fn().mockReturnValue(undefined), + all: vi.fn().mockReturnValue([]), + run: vi.fn(), + }), + }; + return { + ...actual, + getUnifiedMemory: vi.fn().mockReturnValue({ + isInitialized: vi.fn().mockReturnValue(true), + initialize: vi.fn().mockResolvedValue(undefined), + getDatabase: vi.fn().mockReturnValue(mockDb), + }), + initializeUnifiedMemory: vi.fn().mockResolvedValue({ + isInitialized: () => true, + initialize: async () => {}, + getDatabase: () => mockDb, + }), + }; +}); + +// Mock cost tracker — keep real exports but stub the global singleton +vi.mock('../../../src/shared/llm/cost-tracker.js', async (importOriginal) => { + const actual = await importOriginal>(); + return { + ...actual, + getGlobalCostTracker: vi.fn().mockReturnValue({ + getCurrentCost: vi.fn().mockReturnValue(0), + getSummary: vi.fn().mockReturnValue({ + totalCost: 0, totalTokens: 0, totalRequests: 0, + period: 'all', periodStart: new Date(0), periodEnd: new Date(), + byProvider: {}, byModel: {}, + }), + }), + }; +}); + +import { handleRoutingEconomics } from '../../../src/mcp/handlers/task-handlers.js'; + +describe('handleRoutingEconomics MCP handler', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should return a successful economic report', async () => { + const result = await handleRoutingEconomics({ taskComplexity: 0.5 }); + + expect(result.success).toBe(true); + expect(result.data).toBeDefined(); + expect(result.data!.tierEfficiency).toBeInstanceOf(Array); + expect(result.data!.tierEfficiency.length).toBe(4); + expect(typeof result.data!.currentHourlyCostUsd).toBe('number'); + expect(typeof result.data!.currentDailyCostUsd).toBe('number'); + expect(typeof result.data!.recommendation).toBe('string'); + }); + + it('should include budget remaining as null when no limits set', async () => { + const result = await handleRoutingEconomics({ taskComplexity: 0.3 }); + + expect(result.success).toBe(true); + expect(result.data!.budgetRemaining).toBeDefined(); + expect(result.data!.budgetRemaining.hourly).toBeNull(); + expect(result.data!.budgetRemaining.daily).toBeNull(); + }); + + it('should serialize Infinity qualityPerDollar for booster tier', async () => { + const result = await handleRoutingEconomics({ taskComplexity: 0.1 }); + + expect(result.success).toBe(true); + const booster = result.data!.tierEfficiency.find( + (t: Record) => t.tier === 'booster', + ); + expect(booster).toBeDefined(); + // Infinity should be serialized as string for JSON safety + expect(booster!.qualityPerDollar).toBe('Infinity'); + }); + + it('should return tiers sorted by economic score descending', async () => { + const result = await handleRoutingEconomics({ taskComplexity: 0.5 }); + + expect(result.success).toBe(true); + const scores = result.data!.tierEfficiency.map( + (t: Record) => t.economicScore as number, + ); + for (let i = 1; i < scores.length; i++) { + expect(scores[i - 1]).toBeGreaterThanOrEqual(scores[i]); + } + }); +}); diff --git a/tests/unit/routing/economic-routing.test.ts b/tests/unit/routing/economic-routing.test.ts new file mode 100644 index 00000000..e7b3f3f0 --- /dev/null +++ b/tests/unit/routing/economic-routing.test.ts @@ -0,0 +1,408 @@ +/** + * Economic Routing Model Tests — Imp-18, Issue #334 + * + * Tests quality-weighted cost optimization for the routing system. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { + EconomicRoutingModel, + TIER_COST_ESTIMATES, + DEFAULT_ECONOMIC_CONFIG, + type EconomicRoutingConfig, +} from '../../../src/routing/economic-routing.js'; +import { CostTracker } from '../../../src/shared/llm/cost-tracker.js'; +import type { RoutingOutcome } from '../../../src/routing/types.js'; +import type { AgentTier } from '../../../src/routing/routing-config.js'; +import { + RoutingFeedbackCollector, + createRoutingFeedbackCollector, +} from '../../../src/routing/routing-feedback.js'; +import type { QETask, QERoutingDecision } from '../../../src/routing/types.js'; + +// ============================================================================ +// Helpers +// ============================================================================ + +function createModel( + config?: Partial, + costTracker?: CostTracker, +): EconomicRoutingModel { + return new EconomicRoutingModel(costTracker ?? new CostTracker(), config); +} + +function makeOutcome(overrides?: Partial<{ + success: boolean; + qualityScore: number; + durationMs: number; +}>): RoutingOutcome { + return { + id: `test-${Date.now()}`, + task: { description: 'test task' }, + decision: { + recommended: 'agent-1', + confidence: 0.8, + alternatives: [], + reasoning: 'test', + scores: { similarity: 0.8, performance: 0.7, capabilities: 0.9, combined: 0.8 }, + latencyMs: 10, + timestamp: new Date(), + }, + usedAgent: 'agent-1', + followedRecommendation: true, + outcome: { + success: overrides?.success ?? true, + qualityScore: overrides?.qualityScore ?? 0.8, + durationMs: overrides?.durationMs ?? 3000, + }, + timestamp: new Date(), + }; +} + +function createMockTask(description: string): QETask { + return { description }; +} + +function createMockDecision(recommended: string, confidence: number): QERoutingDecision { + return { + recommended, + confidence, + alternatives: [], + reasoning: `Selected ${recommended}`, + scores: { similarity: 0.8, performance: 0.7, capabilities: 0.9, combined: confidence }, + latencyMs: 50, + timestamp: new Date(), + }; +} + +// ============================================================================ +// Tests +// ============================================================================ + +describe('EconomicRoutingModel', () => { + let model: EconomicRoutingModel; + + beforeEach(() => { + model = createModel(); + }); + + describe('scoreTiers()', () => { + it('should return all four tiers sorted by economic score descending', () => { + const scores = model.scoreTiers(0.5); + + expect(scores).toHaveLength(4); + // Verify sorted descending by economicScore + for (let i = 1; i < scores.length; i++) { + expect(scores[i - 1].economicScore).toBeGreaterThanOrEqual(scores[i].economicScore); + } + }); + + it('should include all expected properties in each score', () => { + const scores = model.scoreTiers(0.3); + for (const score of scores) { + expect(score).toHaveProperty('tier'); + expect(score).toHaveProperty('qualityScore'); + expect(score).toHaveProperty('estimatedCostUsd'); + expect(score).toHaveProperty('qualityPerDollar'); + expect(score).toHaveProperty('economicScore'); + expect(score.qualityScore).toBeGreaterThanOrEqual(0); + expect(score.qualityScore).toBeLessThanOrEqual(1); + expect(score.estimatedCostUsd).toBeGreaterThanOrEqual(0); + } + }); + + it('should give booster infinite qualityPerDollar when cost is zero', () => { + const scores = model.scoreTiers(0); + const booster = scores.find(s => s.tier === 'booster'); + expect(booster).toBeDefined(); + expect(booster!.estimatedCostUsd).toBe(0); + // booster quality > 0 and cost = 0 => Infinity + expect(booster!.qualityPerDollar).toBe(Infinity); + }); + + it('should penalize cheaper tiers more for complex tasks', () => { + const simpleScores = model.scoreTiers(0.1); + const complexScores = model.scoreTiers(0.9); + + const haikuSimple = simpleScores.find(s => s.tier === 'haiku')!; + const haikuComplex = complexScores.find(s => s.tier === 'haiku')!; + + // Haiku quality should be lower for complex tasks + expect(haikuComplex.qualityScore).toBeLessThan(haikuSimple.qualityScore); + }); + }); + + describe('selectTier()', () => { + it('should select a tier that meets quality threshold', () => { + const result = model.selectTier(0.5); + expect(result.tier).toBeDefined(); + expect(result.reason).toBeTruthy(); + expect(result.scores).toHaveLength(4); + }); + + it('should respect budget limits by falling back to cheaper tier', () => { + // Create a cost tracker that has already spent near the limit + const costTracker = new CostTracker(); + // Record usage to simulate high spend + costTracker.recordUsage('claude', 'claude-3-opus-20240229', { + promptTokens: 100000, + completionTokens: 50000, + totalTokens: 150000, + }, 'req-1'); + + const budgetModel = createModel({ + budgetPerHourUsd: 0.001, // Very tight budget + budgetPerDayUsd: 0.001, + minQualityThreshold: 0.1, // Lower threshold to allow booster as fallback + }, costTracker); + + const result = budgetModel.selectTier(0.1); // Low complexity so booster quality is adequate + // Should fall back to booster (free) since budget is exceeded for paid tiers + expect(result.tier).toBe('booster'); + }); + + it('should respect minimum quality threshold', () => { + const result = model.selectTier(0.95); // Very complex task + // Booster quality at 0.95 complexity: 0.3 - 0.95*0.4 = -0.08 (clamped to 0) + // Should not pick booster for this complexity + expect(result.tier).not.toBe('booster'); + }); + + it('should provide a reason with the selection', () => { + const result = model.selectTier(0.5); + expect(result.reason.length).toBeGreaterThan(0); + }); + }); + + describe('wouldExceedBudget()', () => { + it('should return false when no budget limits are set', () => { + expect(model.wouldExceedBudget('opus')).toBe(false); + expect(model.wouldExceedBudget('haiku')).toBe(false); + }); + + it('should return false for booster (zero cost)', () => { + const budgetModel = createModel({ budgetPerHourUsd: 0.001 }); + expect(budgetModel.wouldExceedBudget('booster')).toBe(false); + }); + + it('should return true when hourly budget would be exceeded', () => { + const costTracker = new CostTracker(); + costTracker.recordUsage('claude', 'claude-3-opus-20240229', { + promptTokens: 500000, + completionTokens: 200000, + totalTokens: 700000, + }, 'req-1'); + + const budgetModel = createModel({ + budgetPerHourUsd: 0.001, + }, costTracker); + + expect(budgetModel.wouldExceedBudget('opus')).toBe(true); + }); + + it('should return true when daily budget would be exceeded', () => { + const costTracker = new CostTracker(); + costTracker.recordUsage('claude', 'claude-3-opus-20240229', { + promptTokens: 500000, + completionTokens: 200000, + totalTokens: 700000, + }, 'req-1'); + + const budgetModel = createModel({ + budgetPerDayUsd: 0.001, + }, costTracker); + + expect(budgetModel.wouldExceedBudget('sonnet')).toBe(true); + }); + }); + + describe('updateFromOutcome()', () => { + it('should adjust quality estimates via EMA', () => { + const initialScores = model.scoreTiers(0.5); + const initialHaiku = initialScores.find(s => s.tier === 'haiku')!.qualityScore; + + // Record several high-quality haiku outcomes + for (let i = 0; i < 10; i++) { + model.updateFromOutcome(makeOutcome({ qualityScore: 0.95 }), 'haiku'); + } + + const updatedScores = model.scoreTiers(0.5); + const updatedHaiku = updatedScores.find(s => s.tier === 'haiku')!.qualityScore; + + // Quality should have increased toward 0.95 + expect(updatedHaiku).toBeGreaterThan(initialHaiku); + }); + + it('should decrease quality estimate after low-quality outcomes', () => { + const initialScores = model.scoreTiers(0.3); + const initialOpus = initialScores.find(s => s.tier === 'opus')!.qualityScore; + + for (let i = 0; i < 10; i++) { + model.updateFromOutcome(makeOutcome({ qualityScore: 0.2 }), 'opus'); + } + + const updatedScores = model.scoreTiers(0.3); + const updatedOpus = updatedScores.find(s => s.tier === 'opus')!.qualityScore; + + expect(updatedOpus).toBeLessThan(initialOpus); + }); + }); + + describe('computeCostAdjustedReward()', () => { + it('should return the base reward for booster (zero cost)', () => { + const adjusted = model.computeCostAdjustedReward(0.5, 'booster', 0.8); + // booster costRatio = 0 => costPenalty = 0 => adjusted = baseReward + expect(adjusted).toBe(0.5); + }); + + it('should penalize expensive tiers with low quality', () => { + const adjusted = model.computeCostAdjustedReward(0.5, 'opus', 0.3); + // opus costRatio=1.0, qualityGain = max(0, 0.3-0.5) = 0 + // costPenalty = 1.0 * (1 - 0) = 1.0 + // adjusted = 0.5 - 1.0 * 0.4 = 0.1 + expect(adjusted).toBeCloseTo(0.1, 1); + }); + + it('should not penalize expensive tiers that deliver high quality', () => { + const adjusted = model.computeCostAdjustedReward(0.5, 'opus', 0.9); + // qualityGain = 0.9 - 0.5 = 0.4 + // costPenalty = 1.0 * (1 - 0.4) = 0.6 + // adjusted = 0.5 - 0.6 * 0.4 = 0.26 + // Still penalized somewhat, but less than low-quality + const lowQualityAdjusted = model.computeCostAdjustedReward(0.5, 'opus', 0.3); + expect(adjusted).toBeGreaterThan(lowQualityAdjusted); + }); + + it('should clamp the result to [-1, 1]', () => { + const adjusted = model.computeCostAdjustedReward(-0.9, 'opus', 0.1); + expect(adjusted).toBeGreaterThanOrEqual(-1); + expect(adjusted).toBeLessThanOrEqual(1); + }); + }); + + describe('getEconomicReport()', () => { + it('should return a well-formed report', () => { + const report = model.getEconomicReport(); + + expect(report).toHaveProperty('tierEfficiency'); + expect(report).toHaveProperty('currentHourlyCostUsd'); + expect(report).toHaveProperty('currentDailyCostUsd'); + expect(report).toHaveProperty('budgetRemaining'); + expect(report).toHaveProperty('recommendation'); + expect(report.tierEfficiency).toHaveLength(4); + expect(typeof report.recommendation).toBe('string'); + expect(report.recommendation.length).toBeGreaterThan(0); + }); + + it('should report null budget remaining when no limits set', () => { + const report = model.getEconomicReport(); + expect(report.budgetRemaining.hourly).toBeNull(); + expect(report.budgetRemaining.daily).toBeNull(); + }); + + it('should report budget remaining when limits are set', () => { + const budgetModel = createModel({ + budgetPerHourUsd: 1.0, + budgetPerDayUsd: 10.0, + }); + const report = budgetModel.getEconomicReport(); + expect(report.budgetRemaining.hourly).toBe(1.0); + expect(report.budgetRemaining.daily).toBe(10.0); + }); + + it('should include savings opportunity', () => { + const report = model.getEconomicReport(); + // There should be a savings opportunity comparing opus vs haiku + expect(report.savingsOpportunity).not.toBeNull(); + expect(report.savingsOpportunity!.usd).toBeGreaterThan(0); + expect(report.savingsOpportunity!.description.length).toBeGreaterThan(0); + }); + }); + + describe('serialization', () => { + it('should serialize and deserialize quality estimates', () => { + // Update some quality estimates + model.updateFromOutcome(makeOutcome({ qualityScore: 0.9 }), 'haiku'); + model.updateFromOutcome(makeOutcome({ qualityScore: 0.6 }), 'opus'); + + const serialized = model.serializeEstimates(); + expect(serialized).toHaveProperty('haiku'); + expect(serialized).toHaveProperty('opus'); + + // Create a new model and deserialize + const newModel = createModel(); + newModel.deserializeEstimates(serialized); + + // Scores should match after deserialization + const originalScores = model.scoreTiers(0.5); + const restoredScores = newModel.scoreTiers(0.5); + + for (const tier of ['booster', 'haiku', 'sonnet', 'opus'] as AgentTier[]) { + const orig = originalScores.find(s => s.tier === tier)!; + const restored = restoredScores.find(s => s.tier === tier)!; + expect(restored.qualityScore).toBeCloseTo(orig.qualityScore, 4); + } + }); + }); +}); + +describe('RoutingFeedbackCollector — Economic Integration', () => { + let collector: RoutingFeedbackCollector; + + beforeEach(() => { + collector = createRoutingFeedbackCollector(100); + }); + + it('should return null economic report when not enabled', () => { + expect(collector.getEconomicReport()).toBeNull(); + expect(collector.getEconomicScore(0.5)).toBeNull(); + }); + + it('should enable economic routing and return a report', () => { + collector.enableEconomicRouting(); + const report = collector.getEconomicReport(); + expect(report).not.toBeNull(); + expect(report!.tierEfficiency).toHaveLength(4); + }); + + it('should return economic scores for a given complexity', () => { + collector.enableEconomicRouting(); + const scores = collector.getEconomicScore(0.3); + expect(scores).not.toBeNull(); + expect(scores!).toHaveLength(4); + }); + + it('should update economic model when recording outcomes', () => { + collector.enableEconomicRouting(); + + const task = createMockTask('Generate tests'); + const decision = createMockDecision('haiku-agent', 0.85); + + // Record several outcomes + for (let i = 0; i < 5; i++) { + collector.recordOutcome(task, decision, 'haiku-agent', { + success: true, + qualityScore: 0.9, + durationMs: 2000, + }); + } + + // Economic scores should now reflect the observed quality + const scores = collector.getEconomicScore(0.3); + expect(scores).not.toBeNull(); + // haiku-agent maps to 'haiku' tier by default inference (contains 'haiku') + // but the default inferTier maps most qe-* to sonnet; 'haiku-agent' contains 'haiku' + // so it should update haiku tier quality + }); + + it('should accept custom config for economic routing', () => { + collector.enableEconomicRouting({ + qualityWeight: 0.8, + costWeight: 0.2, + budgetPerDayUsd: 5.0, + }); + const report = collector.getEconomicReport(); + expect(report).not.toBeNull(); + expect(report!.budgetRemaining.daily).toBe(5.0); + }); +});