mirror of
https://github.com/thedotmack/claude-mem.git
synced 2026-09-20 04:23:02 +08:00
feat(worker): alert the user at session start when observations stop flowing (#3538)
* feat(worker): surface observer outages at session start via observer-health ledger Generator failures now record to a file-backed ledger (~/.claude-mem/observer-health.json); successful observation stores reset it. Session-start context injection prepends a loud outage warning (with the provider's last error, which often embeds the remedy URL) once 3+ consecutive failures are newer than the last success — including on the empty-state and missing-DB paths. Prevents a repeat of the silent 17-hour 2026-08-09 provider-quota outage. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(observer-health): lock the ledger, widen credential redaction, plain-language warning Greptile P1s on #3538: - Credential redaction leaked provider secrets into injected context. scrubErrorMessage only caught `sk-…` and `Bearer …`, so `api_key=…`, JSON `"apiKey": "…"`, `Authorization: Basic …` and query-string secrets were persisted in observer-health.json and rendered verbatim at session start. Redaction now covers the common key/value, JSON, authorization and query-string shapes, skips purely numeric values so `max_tokens: 200000` style diagnostics survive, and runs again at render time so a ledger written by an older build cannot leak. - Concurrent failure writes could suppress the outage warning. The ledger's read-modify-write was unlocked, so simultaneous writers overwrote each other's increments and consecutiveFailures stayed under the threshold. Serialized with a `wx` lockfile (same idiom as worker-spawn-gate.ts), stale-breaking at 5s and failing open after a 2s wait — a racy count beats no count. Regression test drives three synchronized writer processes and requires the threshold to be reached; it fails ~2 in 3 runs without the lock. Also rewrites the warning for humans (it is read by the user, not just the agent) with a describeDuration helper, and rebuilds the plugin artifacts — the committed context-generator.cjs was stale and shipped without the observer-health injection. * test(worker): restore the real ModeManager after session-manager-project mocks it bun's mock.module is process-global and mock.restore() does not undo it, so this file's partial ModeManager stub (no class prototype, no loadMode) leaked into every later test file in the run. When readdir order put it before tests/server/server-boot.test.ts and the server-runtime smoke, loadServerMode() hit `modeManager.loadMode is not a function` and CI went red — which is exactly what happened on this PR while main stayed green. Applies the snapshot + afterAll re-register pattern already used in agent-formatter, SearchManager.timeline-anchor, and response-processor tests; session-manager-project was the last ModeManager mock without it. --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+323
-180
File diff suppressed because one or more lines are too long
+438
-414
File diff suppressed because one or more lines are too long
+12
-12
File diff suppressed because one or more lines are too long
@@ -26,6 +26,11 @@ import { shouldShowSummary, renderSummaryFields } from './sections/SummaryRender
|
||||
import { renderPreviouslySection, renderFooter } from './sections/FooterRenderer.js';
|
||||
import { renderAgentEmptyState } from './formatters/AgentFormatter.js';
|
||||
import { renderHumanEmptyState } from './formatters/HumanFormatter.js';
|
||||
import {
|
||||
readObserverHealth,
|
||||
isObserverUnhealthy,
|
||||
renderObserverHealthWarning,
|
||||
} from '../../shared/observer-health.js';
|
||||
|
||||
const VERSION_MARKER_PATH = path.join(
|
||||
homedir(),
|
||||
@@ -168,6 +173,20 @@ function buildInjectStats(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepend the observer-health outage warning when the observer is failing.
|
||||
* Applied to EVERY context path (including empty-state and missing-DB) so the
|
||||
* outage is surfaced even when there is nothing else to render.
|
||||
*/
|
||||
function withObserverHealthWarning(text: string): string {
|
||||
const health = readObserverHealth();
|
||||
if (!isObserverUnhealthy(health)) {
|
||||
return text;
|
||||
}
|
||||
const warning = renderObserverHealthWarning(health);
|
||||
return text ? `${warning}\n\n${text}` : warning;
|
||||
}
|
||||
|
||||
export async function generateContextWithStats(
|
||||
input?: ContextInput,
|
||||
forHuman: boolean = false
|
||||
@@ -186,7 +205,7 @@ export async function generateContextWithStats(
|
||||
|
||||
const rawDb = initializeDatabase();
|
||||
if (!rawDb) {
|
||||
return { text: '', stats: null };
|
||||
return { text: withObserverHealthWarning(''), stats: null };
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -199,7 +218,7 @@ export async function generateContextWithStats(
|
||||
const summaries = querySummariesMulti(db, queryProjects, config, platformSource);
|
||||
|
||||
if (observations.length === 0 && summaries.length === 0) {
|
||||
return { text: renderEmptyState(project, forHuman), stats: null };
|
||||
return { text: withObserverHealthWarning(renderEmptyState(project, forHuman)), stats: null };
|
||||
}
|
||||
|
||||
const output = buildContextOutput(
|
||||
@@ -212,7 +231,10 @@ export async function generateContextWithStats(
|
||||
forHuman
|
||||
);
|
||||
|
||||
return { text: output, stats: buildInjectStats(observations, summaries, Boolean(input?.full)) };
|
||||
return {
|
||||
text: withObserverHealthWarning(output),
|
||||
stats: buildInjectStats(observations, summaries, Boolean(input?.full)),
|
||||
};
|
||||
} finally {
|
||||
rawDb.close();
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import { updateCursorContextForProject } from '../../integrations/CursorHooksIns
|
||||
import { notifyTelegram } from '../../integrations/TelegramNotifier.js';
|
||||
import { updateFolderClaudeMdFiles } from '../../../utils/claude-md-utils.js';
|
||||
import { getWorkerPort } from '../../../shared/worker-utils.js';
|
||||
import { recordObserverSuccess } from '../../../shared/observer-health.js';
|
||||
import { SettingsDefaultsManager } from '../../../shared/SettingsDefaultsManager.js';
|
||||
import { USER_SETTINGS_PATH } from '../../../shared/paths.js';
|
||||
import type { ActiveSession, PendingMessage } from '../../worker-types.js';
|
||||
@@ -424,6 +425,10 @@ export async function processAgentResponse(
|
||||
|
||||
session.lastSummaryStored = result.summaryId !== null;
|
||||
|
||||
// A completed store proves the observer pipeline works end-to-end — clear
|
||||
// the failure streak in the observer-health ledger.
|
||||
recordObserverSuccess();
|
||||
|
||||
// Telemetry: counts, enums, and REAL usage only (lastUsage is never an
|
||||
// estimate — providers leave it null when the API gave no usage split).
|
||||
const typeCounts: Record<string, number> = { bugfix: 0, discovery: 0, decision: 0, refactor: 0, other: 0 };
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
recordClaudeCliSetupRequired,
|
||||
} from '../../../../shared/dependency-health.js';
|
||||
import { findClaudeExecutable } from '../../../../shared/find-claude-executable.js';
|
||||
import { recordObserverFailure } from '../../../../shared/observer-health.js';
|
||||
import { isClassified } from '../../provider-errors.js';
|
||||
import { classifyClaudeError } from '../../ClaudeProvider.js';
|
||||
|
||||
@@ -213,6 +214,9 @@ export class SessionRoutes extends BaseRouteHandler {
|
||||
provider,
|
||||
error: errorMsg,
|
||||
}, error);
|
||||
// Observer-health ledger: repeated generator failures mean observations
|
||||
// are being dropped — session-start context warns the user via this.
|
||||
recordObserverFailure(provider, errorMsg);
|
||||
telemetryBuffer.record('session_compressed', session.sessionDbId, {
|
||||
outcome: 'error',
|
||||
provider,
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
/**
|
||||
* File-backed observer pipeline health, so "observations are not flowing" is
|
||||
* never silent (the 2026-08-09 provider-quota outage ran 17 hours unnoticed).
|
||||
*
|
||||
* The worker records generator failures (SessionRoutes generator catch) and
|
||||
* successful stores (ResponseProcessor). Session-start context assembly
|
||||
* (ContextBuilder) reads the file and prepends a loud warning when the
|
||||
* observer is unhealthy. A file — not the in-memory dependency-health map —
|
||||
* because the state must survive worker restarts and be readable from any
|
||||
* process without the worker HTTP API (same pattern as the oauth-stale
|
||||
* marker in oauth-token.ts).
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, statSync, unlinkSync, writeFileSync } from 'fs';
|
||||
import { dirname, join } from 'path';
|
||||
import { paths } from './paths.js';
|
||||
import { logger } from '../utils/logger.js';
|
||||
|
||||
export interface ObserverHealthState {
|
||||
/** Failures since the last successful store. */
|
||||
consecutiveFailures: number;
|
||||
/** Epoch ms of the first failure in the current streak. */
|
||||
failingSinceAt: number | null;
|
||||
/** Epoch ms of the most recent failure. */
|
||||
lastErrorAt: number | null;
|
||||
/** Scrubbed + truncated message of the most recent failure. */
|
||||
lastErrorMessage: string | null;
|
||||
/** Provider whose generator failed most recently. */
|
||||
lastErrorProvider: string | null;
|
||||
/** Epoch ms of the most recent successful observation/summary store. */
|
||||
lastSuccessAt: number | null;
|
||||
}
|
||||
|
||||
export const OBSERVER_HEALTH_FILENAME = 'observer-health.json';
|
||||
|
||||
/** Warn only after repeated failures — a single blip self-heals on retry. */
|
||||
export const OBSERVER_UNHEALTHY_FAILURE_THRESHOLD = 3;
|
||||
|
||||
const MAX_ERROR_MESSAGE_LENGTH = 600;
|
||||
|
||||
const EMPTY_STATE: ObserverHealthState = {
|
||||
consecutiveFailures: 0,
|
||||
failingSinceAt: null,
|
||||
lastErrorAt: null,
|
||||
lastErrorMessage: null,
|
||||
lastErrorProvider: null,
|
||||
lastSuccessAt: null,
|
||||
};
|
||||
|
||||
function defaultHealthFilePath(): string {
|
||||
return join(paths.dataDir(), OBSERVER_HEALTH_FILENAME);
|
||||
}
|
||||
|
||||
/**
|
||||
* Names that carry a secret when they appear as `name=value`, `name: value`,
|
||||
* or `"name": "value"` — the shapes provider errors echo back from request
|
||||
* bodies, query strings, and header dumps.
|
||||
*/
|
||||
const CREDENTIAL_ASSIGNMENT_PATTERN =
|
||||
/(["']?\b(?:api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|id[_-]?token|session[_-]?token|client[_-]?secret|secret[_-]?key|secret|password|passwd|pwd|authorization|auth|token|key)\b["']?\s*[=:]\s*)(["']?)([^\s"'&,;}\]]+)\2/gi;
|
||||
|
||||
/**
|
||||
* Keep the message useful (provider errors often embed the remedy, e.g. an
|
||||
* OpenRouter manage-key URL) while dropping anything credential-shaped.
|
||||
*
|
||||
* Applied before persistence AND again at render time, so a ledger written by
|
||||
* an older build cannot inject a secret into session-start context.
|
||||
*/
|
||||
export function scrubErrorMessage(message: string): string {
|
||||
return message
|
||||
.replace(/\bsk-[A-Za-z0-9_-]{8,}/g, 'sk-…')
|
||||
.replace(/\b(Bearer|Basic|Digest|Token)\s+[^\s"']+/gi, '$1 …')
|
||||
// Numbers are limits/counts, not credentials — `max_tokens: 200000` and
|
||||
// `key limit exceeded` style diagnostics survive intact.
|
||||
.replace(CREDENTIAL_ASSIGNMENT_PATTERN, (match, prefix: string, quote: string, value: string) =>
|
||||
/^\d+$/.test(value) ? match : `${prefix}${quote}…${quote}`)
|
||||
.slice(0, MAX_ERROR_MESSAGE_LENGTH);
|
||||
}
|
||||
|
||||
export function readObserverHealth(filePath: string = defaultHealthFilePath()): ObserverHealthState | null {
|
||||
try {
|
||||
if (!existsSync(filePath)) return null;
|
||||
const parsed: unknown = JSON.parse(readFileSync(filePath, 'utf-8'));
|
||||
if (typeof parsed !== 'object' || parsed === null) return null;
|
||||
return { ...EMPTY_STATE, ...(parsed as Partial<ObserverHealthState>) };
|
||||
} catch (error) {
|
||||
logger.warn('SESSION', 'Failed to read observer-health file', { filePath },
|
||||
error instanceof Error ? error : new Error(String(error)));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function writeObserverHealth(state: ObserverHealthState, filePath: string): void {
|
||||
try {
|
||||
const dir = join(filePath, '..');
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
writeFileSync(filePath, JSON.stringify(state, null, 2), { encoding: 'utf-8', mode: 0o600 });
|
||||
} catch (error) {
|
||||
logger.warn('SESSION', 'Failed to write observer-health file', { filePath },
|
||||
error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}
|
||||
|
||||
/** A holder that has not released within this window is presumed dead. */
|
||||
const LEDGER_LOCK_STALE_MS = 5_000;
|
||||
/** Give up waiting and update unlocked rather than lose the failure entirely. */
|
||||
const LEDGER_LOCK_MAX_WAIT_MS = 2_000;
|
||||
const LEDGER_LOCK_RETRY_MS = 10;
|
||||
|
||||
function sleepSync(ms: number): void {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the ledger's read-modify-write across processes. Every worker and
|
||||
* hook process shares one file, and lost increments would hold
|
||||
* consecutiveFailures below the threshold — suppressing the very warning this
|
||||
* module exists to raise.
|
||||
*
|
||||
* `wx` (O_CREAT|O_EXCL) makes the create itself the atomicity, matching
|
||||
* worker-spawn-gate.ts. The lock fails OPEN (unlocked update) when the
|
||||
* filesystem refuses it or the wait times out: a racy count beats no count.
|
||||
*/
|
||||
function withLedgerLock<T>(filePath: string, mutate: () => T): T {
|
||||
const lockPath = `${filePath}.lock`;
|
||||
const deadline = Date.now() + LEDGER_LOCK_MAX_WAIT_MS;
|
||||
let held = false;
|
||||
|
||||
while (!held && Date.now() < deadline) {
|
||||
try {
|
||||
mkdirSync(dirname(lockPath), { recursive: true, mode: 0o700 });
|
||||
writeFileSync(lockPath, String(process.pid), { flag: 'wx', mode: 0o600 });
|
||||
held = true;
|
||||
} catch (error) {
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
if (code !== 'EEXIST') {
|
||||
logger.warn('SESSION', 'Observer-health lock unavailable; updating unlocked', { lockPath, code },
|
||||
error instanceof Error ? error : new Error(String(error)));
|
||||
break;
|
||||
}
|
||||
let mtimeMs: number;
|
||||
try {
|
||||
mtimeMs = statSync(lockPath).mtimeMs;
|
||||
} catch {
|
||||
// Holder released between our failed create and the stat — retry.
|
||||
continue;
|
||||
}
|
||||
if (Date.now() - mtimeMs > LEDGER_LOCK_STALE_MS) {
|
||||
try {
|
||||
unlinkSync(lockPath);
|
||||
} catch {
|
||||
// A competing breaker won, or the fs refused the delete; the retry
|
||||
// loop re-evaluates either way.
|
||||
}
|
||||
continue;
|
||||
}
|
||||
sleepSync(LEDGER_LOCK_RETRY_MS);
|
||||
}
|
||||
}
|
||||
|
||||
if (!held && Date.now() >= deadline) {
|
||||
logger.warn('SESSION', 'Timed out waiting for the observer-health lock; updating unlocked', { lockPath });
|
||||
}
|
||||
|
||||
try {
|
||||
return mutate();
|
||||
} finally {
|
||||
if (held) {
|
||||
try {
|
||||
unlinkSync(lockPath);
|
||||
} catch {
|
||||
// Already broken as stale by a waiter — nothing to release.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function recordObserverFailure(
|
||||
provider: string,
|
||||
errorMessage: string,
|
||||
filePath: string = defaultHealthFilePath(),
|
||||
): void {
|
||||
withLedgerLock(filePath, () => {
|
||||
const prior = readObserverHealth(filePath) ?? EMPTY_STATE;
|
||||
const now = Date.now();
|
||||
writeObserverHealth({
|
||||
...prior,
|
||||
consecutiveFailures: prior.consecutiveFailures + 1,
|
||||
failingSinceAt: prior.consecutiveFailures > 0 ? prior.failingSinceAt : now,
|
||||
lastErrorAt: now,
|
||||
lastErrorMessage: scrubErrorMessage(errorMessage),
|
||||
lastErrorProvider: provider,
|
||||
}, filePath);
|
||||
});
|
||||
}
|
||||
|
||||
export function recordObserverSuccess(filePath: string = defaultHealthFilePath()): void {
|
||||
withLedgerLock(filePath, () => {
|
||||
const prior = readObserverHealth(filePath) ?? EMPTY_STATE;
|
||||
writeObserverHealth({
|
||||
...prior,
|
||||
consecutiveFailures: 0,
|
||||
failingSinceAt: null,
|
||||
lastSuccessAt: Date.now(),
|
||||
}, filePath);
|
||||
});
|
||||
}
|
||||
|
||||
export function isObserverUnhealthy(state: ObserverHealthState | null): state is ObserverHealthState {
|
||||
return state !== null
|
||||
&& state.consecutiveFailures >= OBSERVER_UNHEALTHY_FAILURE_THRESHOLD
|
||||
&& (state.lastErrorAt ?? 0) > (state.lastSuccessAt ?? 0);
|
||||
}
|
||||
|
||||
/** "3 minutes" / "about 2 hours" / "about 3 days" for outage durations. */
|
||||
export function describeDuration(ms: number): string {
|
||||
const minutes = Math.max(1, Math.round(ms / 60_000));
|
||||
if (minutes < 60) return `${minutes} minute${minutes === 1 ? '' : 's'}`;
|
||||
const hours = Math.round(minutes / 60);
|
||||
if (hours < 48) return `about ${hours} hour${hours === 1 ? '' : 's'}`;
|
||||
return `about ${Math.round(hours / 24)} days`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The block prepended to session-start context when unhealthy. Read by both
|
||||
* the user and the agent, so it stays calm and plain-spoken — but the agent
|
||||
* must still relay the outage (and the remedy embedded in the provider's
|
||||
* error message) to the user immediately.
|
||||
*/
|
||||
export function renderObserverHealthWarning(state: ObserverHealthState, nowMs: number = Date.now()): string {
|
||||
const since = state.failingSinceAt ?? state.lastErrorAt;
|
||||
const sinceText = since
|
||||
? `for ${describeDuration(nowMs - since)} (since ${new Date(since).toISOString()})`
|
||||
: 'for an unknown amount of time';
|
||||
const provider = state.lastErrorProvider ?? 'unknown provider';
|
||||
const count = state.consecutiveFailures;
|
||||
const lines = [
|
||||
"⚠️ Heads up: claude-mem can't save memories right now.",
|
||||
'',
|
||||
`The memory observer (provider: ${provider}) has failed ${count} time${count === 1 ? '' : 's'} in a row ${sinceText}.`,
|
||||
'',
|
||||
`Latest error: ${state.lastErrorMessage ? scrubErrorMessage(state.lastErrorMessage) : 'unknown'}`,
|
||||
'',
|
||||
"Until it's fixed, nothing from this session — or any other — will be remembered.",
|
||||
"To fix it, check the observer provider's API key, spend limit, and base URL in",
|
||||
'~/.claude-mem/settings.json. The error above may include a management link with the remedy.',
|
||||
'',
|
||||
'(Assistant: please tell the user about this outage at the very start of your first reply,',
|
||||
'quoting the error above.)',
|
||||
];
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
import { describe, expect, it, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import {
|
||||
readObserverHealth,
|
||||
recordObserverFailure,
|
||||
recordObserverSuccess,
|
||||
isObserverUnhealthy,
|
||||
renderObserverHealthWarning,
|
||||
describeDuration,
|
||||
scrubErrorMessage,
|
||||
OBSERVER_UNHEALTHY_FAILURE_THRESHOLD,
|
||||
type ObserverHealthState,
|
||||
} from '../src/shared/observer-health.ts';
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
|
||||
let dataDir: string;
|
||||
let healthPath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dataDir = mkdtempSync(join(tmpdir(), 'claude-mem-observer-health-'));
|
||||
healthPath = join(dataDir, 'observer-health.json');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function unhealthyState(overrides: Partial<ObserverHealthState> = {}): ObserverHealthState {
|
||||
return {
|
||||
consecutiveFailures: OBSERVER_UNHEALTHY_FAILURE_THRESHOLD,
|
||||
failingSinceAt: 1_754_700_000_000,
|
||||
lastErrorAt: 1_754_700_100_000,
|
||||
lastErrorMessage: 'Key limit exceeded (monthly limit). Manage it using https://openrouter.ai/keys/abc',
|
||||
lastErrorProvider: 'openrouter',
|
||||
lastSuccessAt: 1_754_600_000_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('observer-health ledger', () => {
|
||||
it('returns null when no health file exists', () => {
|
||||
expect(readObserverHealth(healthPath)).toBeNull();
|
||||
});
|
||||
|
||||
it('records a failure streak: increments count, pins failingSinceAt to the first failure', () => {
|
||||
recordObserverFailure('openrouter', 'boom one', healthPath);
|
||||
const first = readObserverHealth(healthPath)!;
|
||||
expect(first.consecutiveFailures).toBe(1);
|
||||
expect(first.failingSinceAt).toBe(first.lastErrorAt);
|
||||
expect(first.lastErrorProvider).toBe('openrouter');
|
||||
|
||||
recordObserverFailure('openrouter', 'boom two', healthPath);
|
||||
const second = readObserverHealth(healthPath)!;
|
||||
expect(second.consecutiveFailures).toBe(2);
|
||||
expect(second.failingSinceAt).toBe(first.failingSinceAt);
|
||||
expect(second.lastErrorMessage).toBe('boom two');
|
||||
});
|
||||
|
||||
it('success resets the streak but keeps last error details for diagnostics', () => {
|
||||
recordObserverFailure('openrouter', 'boom', healthPath);
|
||||
recordObserverSuccess(healthPath);
|
||||
const state = readObserverHealth(healthPath)!;
|
||||
expect(state.consecutiveFailures).toBe(0);
|
||||
expect(state.failingSinceAt).toBeNull();
|
||||
expect(state.lastSuccessAt).toBeGreaterThan(0);
|
||||
expect(state.lastErrorMessage).toBe('boom');
|
||||
});
|
||||
|
||||
it('tolerates a corrupt health file by returning null', () => {
|
||||
writeFileSync(healthPath, 'not json{{{');
|
||||
expect(readObserverHealth(healthPath)).toBeNull();
|
||||
});
|
||||
|
||||
it('scrubs credential-shaped content but keeps remedy URLs, and truncates', () => {
|
||||
const scrubbed = scrubErrorMessage(
|
||||
'auth sk-or-v1-3b5aaaaaaaaaaaaaaaa failed, Bearer abc.def.ghi rejected; manage at https://openrouter.ai/keys/2101b95e'
|
||||
);
|
||||
expect(scrubbed).not.toContain('sk-or-v1-3b5');
|
||||
expect(scrubbed).not.toContain('abc.def.ghi');
|
||||
expect(scrubbed).toContain('https://openrouter.ai/keys/2101b95e');
|
||||
expect(scrubErrorMessage('x'.repeat(10_000)).length).toBeLessThanOrEqual(600);
|
||||
});
|
||||
|
||||
it('scrubs key/value, JSON, authorization, and query-string credential shapes', () => {
|
||||
const scrubbed = scrubErrorMessage(
|
||||
'POST https://api.example.com/v1/chat?api_key=SUPERSECRETONE&token=SUPERSECRETTWO failed: '
|
||||
+ '{"apiKey": "SUPERSECRETTHREE", "client_secret":"SUPERSECRETFOUR"} '
|
||||
+ 'Authorization: Basic SUPERSECRETFIVE; password=SUPERSECRETSIX'
|
||||
);
|
||||
for (const secret of ['SUPERSECRETONE', 'SUPERSECRETTWO', 'SUPERSECRETTHREE', 'SUPERSECRETFOUR', 'SUPERSECRETFIVE', 'SUPERSECRETSIX']) {
|
||||
expect(scrubbed).not.toContain(secret);
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps numeric diagnostics and remedy URLs that merely look credential-adjacent', () => {
|
||||
const scrubbed = scrubErrorMessage(
|
||||
'max_tokens: 200000 exceeded; Key limit exceeded (monthly limit). Manage it using https://openrouter.ai/keys/2101b95e'
|
||||
);
|
||||
expect(scrubbed).toContain('200000');
|
||||
expect(scrubbed).toContain('Key limit exceeded');
|
||||
expect(scrubbed).toContain('https://openrouter.ai/keys/2101b95e');
|
||||
});
|
||||
|
||||
it('failure records pass the raw message through the scrubber', () => {
|
||||
recordObserverFailure('openrouter', 'key sk-or-v1-deadbeefdeadbeef died', healthPath);
|
||||
expect(readObserverHealth(healthPath)!.lastErrorMessage).not.toContain('deadbeef');
|
||||
expect(readFileSync(healthPath, 'utf-8')).not.toContain('deadbeef');
|
||||
});
|
||||
|
||||
it('serializes concurrent failure writes so the warning threshold is never lost', async () => {
|
||||
const gatePath = join(dataDir, 'writers.gate');
|
||||
const writers = Array.from({ length: 3 }, () => Bun.spawn(['bun', '-e', `
|
||||
import { existsSync } from 'fs';
|
||||
import { recordObserverFailure } from ${JSON.stringify(join(repoRoot, 'src/shared/observer-health.ts'))};
|
||||
while (!existsSync(${JSON.stringify(gatePath)})) Bun.sleepSync(1);
|
||||
recordObserverFailure('openrouter', 'concurrent boom', ${JSON.stringify(healthPath)});
|
||||
`], { cwd: repoRoot, stderr: 'pipe' }));
|
||||
|
||||
await Bun.sleep(500);
|
||||
writeFileSync(gatePath, 'go');
|
||||
const exitCodes = await Promise.all(writers.map((writer) => writer.exited));
|
||||
expect(exitCodes).toEqual([0, 0, 0]);
|
||||
|
||||
const state = readObserverHealth(healthPath)!;
|
||||
expect(state.consecutiveFailures).toBe(3);
|
||||
expect(isObserverUnhealthy(state)).toBe(true);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
describe('isObserverUnhealthy', () => {
|
||||
it('requires the failure threshold AND failures newer than the last success', () => {
|
||||
expect(isObserverUnhealthy(null)).toBe(false);
|
||||
expect(isObserverUnhealthy(unhealthyState())).toBe(true);
|
||||
expect(isObserverUnhealthy(unhealthyState({ consecutiveFailures: OBSERVER_UNHEALTHY_FAILURE_THRESHOLD - 1 }))).toBe(false);
|
||||
expect(isObserverUnhealthy(unhealthyState({ lastSuccessAt: Date.now() + 60_000 }))).toBe(false);
|
||||
expect(isObserverUnhealthy(unhealthyState({ lastSuccessAt: null }))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderObserverHealthWarning', () => {
|
||||
it('includes count, provider, since-time, last error, and the tell-the-user instruction', () => {
|
||||
const nowMs = 1_754_700_000_000 + 2 * 60 * 60_000;
|
||||
const warning = renderObserverHealthWarning(unhealthyState({ consecutiveFailures: 4245 }), nowMs);
|
||||
expect(warning).toContain("can't save memories");
|
||||
expect(warning).toContain('4245 times in a row');
|
||||
expect(warning).toContain('for about 2 hours');
|
||||
expect(warning).toContain('openrouter');
|
||||
expect(warning).toContain(new Date(1_754_700_000_000).toISOString());
|
||||
expect(warning).toContain('https://openrouter.ai/keys/abc');
|
||||
expect(warning).toContain('tell the user');
|
||||
});
|
||||
|
||||
it('re-scrubs the stored error, so a ledger written by an older build cannot leak a secret', () => {
|
||||
const warning = renderObserverHealthWarning(
|
||||
unhealthyState({ lastErrorMessage: 'rejected: api_key=SUPERSECRETLEDGER' })
|
||||
);
|
||||
expect(warning).not.toContain('SUPERSECRETLEDGER');
|
||||
});
|
||||
});
|
||||
|
||||
describe('describeDuration', () => {
|
||||
it('renders minutes, hours, and days at human granularity', () => {
|
||||
expect(describeDuration(30_000)).toBe('1 minute');
|
||||
expect(describeDuration(57 * 60_000)).toBe('57 minutes');
|
||||
expect(describeDuration(3 * 60 * 60_000)).toBe('about 3 hours');
|
||||
expect(describeDuration(72 * 60 * 60_000)).toBe('about 3 days');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ContextBuilder observer-health injection', () => {
|
||||
function runContextChild(childDataDir: string): { emptyDbText: string } {
|
||||
const result = Bun.spawnSync(['bun', '-e', `
|
||||
import { generateContext } from './src/services/context/ContextBuilder.ts';
|
||||
import { ModeManager } from './src/services/domain/ModeManager.ts';
|
||||
ModeManager.getInstance().loadMode('code');
|
||||
const emptyDbText = await generateContext({ projects: ['observer-health-test'] });
|
||||
console.log(JSON.stringify({ emptyDbText }));
|
||||
`], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
CLAUDE_MEM_DATA_DIR: childDataDir,
|
||||
CLAUDE_CONFIG_DIR: childDataDir,
|
||||
CLAUDE_MEM_MODES_DIR: join(repoRoot, 'plugin', 'modes'),
|
||||
},
|
||||
});
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(new TextDecoder().decode(result.stderr));
|
||||
}
|
||||
return JSON.parse(new TextDecoder().decode(result.stdout).trim());
|
||||
}
|
||||
|
||||
it('prepends the outage warning even when there is no database to render', () => {
|
||||
writeFileSync(join(dataDir, 'observer-health.json'), JSON.stringify(unhealthyState()));
|
||||
const { emptyDbText } = runContextChild(dataDir);
|
||||
expect(emptyDbText).toContain("can't save memories");
|
||||
expect(emptyDbText).toContain('openrouter');
|
||||
});
|
||||
|
||||
it('stays silent when the observer is healthy', () => {
|
||||
writeFileSync(
|
||||
join(dataDir, 'observer-health.json'),
|
||||
JSON.stringify(unhealthyState({ consecutiveFailures: 0, lastSuccessAt: Date.now() }))
|
||||
);
|
||||
const { emptyDbText } = runContextChild(dataDir);
|
||||
expect(emptyDbText).not.toContain("can't save memories");
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, it, expect, mock, beforeEach, afterEach, spyOn } from 'bun:test';
|
||||
import { describe, it, expect, mock, beforeEach, afterEach, afterAll, spyOn } from 'bun:test';
|
||||
import { logger } from '../../src/utils/logger.js';
|
||||
import { SessionManager } from '../../src/services/worker/SessionManager.js';
|
||||
import { processAgentResponse } from '../../src/services/worker/agents/ResponseProcessor.js';
|
||||
@@ -14,6 +14,21 @@ mock.module('../../src/shared/worker-utils.js', () => ({
|
||||
getWorkerPort: () => 37777,
|
||||
}));
|
||||
|
||||
// Capture the real exports before mock.module mutates the live namespace, then
|
||||
// re-register the snapshot in afterAll. bun's mock.module is process-global and
|
||||
// mock.restore() does NOT undo it, so without this the partial ModeManager stub
|
||||
// below (no class prototype, no loadMode) leaks into later test files and
|
||||
// breaks tests/server/server-boot.test.ts and server-runtime-smoke whenever the
|
||||
// readdir-dependent file order runs them after this file. Same pattern as
|
||||
// tests/context/formatters/agent-formatter.test.ts.
|
||||
import * as realModeManagerModule from '../../src/services/domain/ModeManager.js';
|
||||
|
||||
const realModeManagerSnapshot = { ...realModeManagerModule };
|
||||
|
||||
afterAll(() => {
|
||||
mock.module('../../src/services/domain/ModeManager.js', () => realModeManagerSnapshot);
|
||||
});
|
||||
|
||||
mock.module('../../src/services/domain/ModeManager.js', () => ({
|
||||
ModeManager: {
|
||||
getInstance: () => ({
|
||||
|
||||
Reference in New Issue
Block a user