mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
fix(learning): stop brain-export hook leak + harden DB backups on virtiofs mount
Real-time hook learning-capture was failing on this macOS Docker bind mount with `Failed to initialize UnifiedMemoryManager: database is locked`, and a brain-checkpoint hook was leaking deadlocked `brain export` subprocesses (4 found, ~1.5GB RSS). Root cause is SQLite concurrency on a WAL-hostile virtiofs mount, NOT a native-binary mismatch (better-sqlite3 loads fine). - brain-checkpoint.cjs: add AQE_DISABLE_BRAIN_CHECKPOINT env / marker-file kill switch (opt-in; shipped installs unaffected). The export spawns a native RVF writer that deadlocks in a futex and ignores the SIGTERM the timeout sends, so orphaned exporters piled up. Disabling stops the leak at the source. - aqe-hook.cjs: make the FATAL health-log fix-hint conditional. Only suggest `npm rebuild better-sqlite3` for real native markers; lock-contention failures now point at the actual cause instead of sending users down the wrong path. - feature-flags.ts: add the missing RUVECTOR_USE_RVF_PATTERN_STORE env override (the only RuVector flag lacking one) so an RVF FsyncFailed can fall back to the SQLite HNSW PatternStore. Activates after build. - scripts/aqe-db-backup.sh: consistent VACUUM INTO snapshots (safe with concurrent writers), integrity-verified before promotion, rotating + daily, to the host mount so they survive container loss. Restore round-trip tested. - devcontainer.json: auto-restart the backup loop (postStartCommand) and set bind-mount env mitigations (remoteEnv: AQE_DISABLE_WAL, RUVECTOR_USE_RVF_PATTERN_STORE=false, AQE_DISABLE_BRAIN_CHECKPOINT). - learning-config.json: clear the stale rebuildMode flag (no src reader; on since 2025-12). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dragan Spiridonov
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"enabled": true,
|
||||
"rebuildMode": true,
|
||||
"rebuildModeNote": "ACCELERATED CONFIG - Set rebuildMode to false after learning data is rebuilt",
|
||||
"rebuildMode": false,
|
||||
"rebuildModeNote": "Rebuild completed; accelerated config cleared 2026-07-08 (data rebuilt since 2025-12).",
|
||||
"scheduler": {
|
||||
"mode": "continuous",
|
||||
"schedule": {
|
||||
@@ -30,6 +30,6 @@
|
||||
}
|
||||
},
|
||||
"createdAt": "2025-12-29T13:03:38.969Z",
|
||||
"updatedAt": "2025-12-29T19:15:00.000Z",
|
||||
"updatedAt": "2026-07-08T00:00:00.000Z",
|
||||
"rebuildStartedAt": "2025-12-29T19:15:00.000Z"
|
||||
}
|
||||
|
||||
@@ -18,6 +18,16 @@ const RVF_PATH = path.join(AQE_DIR, 'aqe.rvf');
|
||||
const DB_PATH = path.join(AQE_DIR, 'memory.db');
|
||||
const MAX_AGE_HOURS = 24;
|
||||
|
||||
// Mount-local kill switch. `brain export` (below) spawns a native RVF writer
|
||||
// that, on a macOS Docker virtiofs bind mount, deadlocks in a futex and IGNORES
|
||||
// the SIGTERM that execFileSync's timeout sends — so the 60s timeout can't reap
|
||||
// it and orphaned exporters pile up (~1.5GB RSS observed 2026-07-08). Disable
|
||||
// via env `AQE_DISABLE_BRAIN_CHECKPOINT=1` or a `.agentic-qe/DISABLE_BRAIN_CHECKPOINT`
|
||||
// marker file. Opt-in: shipped installs without the flag behave exactly as before.
|
||||
const CHECKPOINT_DISABLED =
|
||||
process.env.AQE_DISABLE_BRAIN_CHECKPOINT === '1' ||
|
||||
fs.existsSync(path.join(AQE_DIR, 'DISABLE_BRAIN_CHECKPOINT'));
|
||||
|
||||
function log(msg) { process.stderr.write('[brain-checkpoint] ' + msg + '\n'); }
|
||||
|
||||
function exportBrain() {
|
||||
@@ -54,5 +64,10 @@ function verifyBrain() {
|
||||
}
|
||||
|
||||
const cmd = process.argv[2] || 'verify';
|
||||
const result = cmd === 'export' ? exportBrain() : verifyBrain();
|
||||
let result;
|
||||
if (CHECKPOINT_DISABLED) {
|
||||
result = { disabled: true, reason: 'brain-checkpoint disabled on this mount' };
|
||||
} else {
|
||||
result = cmd === 'export' ? exportBrain() : verifyBrain();
|
||||
}
|
||||
if (process.argv.includes('--json')) process.stdout.write(JSON.stringify(result));
|
||||
|
||||
@@ -129,9 +129,22 @@ try {
|
||||
const hit = FATAL_MARKERS.find((m) => stderr.includes(m));
|
||||
const ts = () => new Date().toISOString();
|
||||
if (hit) {
|
||||
// The fix depends on WHICH failure hit. A native-binary marker (invalid ELF
|
||||
// header / ERR_DLOPEN_FAILED / wrong Node version) is a real rebuild case.
|
||||
// But "Failed to initialize UnifiedMemoryManager" is almost always LOCK
|
||||
// CONTENTION — another AQE process (MCP server, daemon, hooks) holding
|
||||
// memory.db, often amplified by a WAL-hostile bind mount — NOT a bad binary.
|
||||
// Suggesting `npm rebuild` there sends people down the wrong path (verified
|
||||
// 2026-07-08: better-sqlite3 loaded fine while this marker still fired).
|
||||
const nativeMismatch = /invalid ELF header|ERR_DLOPEN_FAILED|different Node\.js version/.test(hit);
|
||||
const fix = nativeMismatch
|
||||
? '`npm rebuild better-sqlite3` (host/container native-binary mismatch).'
|
||||
: 'lock contention — check for concurrent AQE processes holding memory.db '
|
||||
+ '(MCP server / daemon / other hooks) and WAL safety on bind mounts '
|
||||
+ '(AQE_DISABLE_WAL); NOT a native-binary rebuild.';
|
||||
recordHookHealth(`[${ts()}] FATAL hook persistence failure `
|
||||
+ `(cmd=${subcmd}): "${hit}". Learning is NOT being captured. `
|
||||
+ `Fix: \`npm rebuild better-sqlite3\` (host/container native-binary mismatch).\n`);
|
||||
+ `Fix: ${fix}\n`);
|
||||
} else if (res && res.signal) {
|
||||
// Killed by our own SPAWN_TIMEOUT_MS (or another signal) before finishing.
|
||||
// No output means no persistence happened for this invocation.
|
||||
|
||||
@@ -11,6 +11,17 @@
|
||||
"--memory-swappiness=60"
|
||||
],
|
||||
"remoteUser": "vscode",
|
||||
// Environment mitigations for this macOS virtiofs bind mount (see the
|
||||
// postStartCommand backup loop and .agentic-qe/DISABLE_BRAIN_CHECKPOINT):
|
||||
// - AQE_DISABLE_WAL: rollback-journal instead of WAL (WAL mmap corrupts here).
|
||||
// - RUVECTOR_USE_RVF_PATTERN_STORE=false: RVF native init FsyncFails on this
|
||||
// mount; fall back to SQLite HNSW. (Takes effect after `npm run build`.)
|
||||
// - AQE_DISABLE_BRAIN_CHECKPOINT: stop the leaking `brain export` hook.
|
||||
"remoteEnv": {
|
||||
"AQE_DISABLE_WAL": "1",
|
||||
"RUVECTOR_USE_RVF_PATTERN_STORE": "false",
|
||||
"AQE_DISABLE_BRAIN_CHECKPOINT": "1"
|
||||
},
|
||||
"features": {
|
||||
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
|
||||
"ghcr.io/devcontainers/features/node:1": {},
|
||||
@@ -38,6 +49,12 @@
|
||||
// via the workspace bind mount. The host's node_modules is untouched.
|
||||
// The volume is root-owned on first creation, hence the chown before npm ci.
|
||||
"postCreateCommand": "sudo chown vscode:vscode node_modules && bash .devcontainer/install-tools.sh && npm ci",
|
||||
// Auto-restart the verified learning-DB backup loop on every container start.
|
||||
// memory.db lives on the macOS virtiofs bind mount (WAL-corruption prone), so
|
||||
// scripts/aqe-db-backup.sh takes consistent, integrity-checked snapshots to
|
||||
// .agentic-qe/backups/verified/ every 30 min — durable if the container dies.
|
||||
// Idempotent: skips if a loop is already running.
|
||||
"postStartCommand": "pgrep -f 'aqe-db-backup.sh loop' >/dev/null || (nohup bash scripts/aqe-db-backup.sh loop 1800 >/dev/null 2>&1 &)",
|
||||
"shutdownAction": "none",
|
||||
"mounts": [
|
||||
"source=agentic-qe-node-modules,target=${containerWorkspaceFolder}/node_modules,type=volume"
|
||||
|
||||
Executable
+120
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# AQE learning-DB backup + restore.
|
||||
#
|
||||
# WHY: .agentic-qe/memory.db holds 1K+ irreplaceable learning records. In this
|
||||
# dev container it lives on a macOS virtiofs bind mount that has corrupted the
|
||||
# DB twice (2026-06-08, 2026-07-07). This tool makes CONSISTENT, integrity-
|
||||
# verified snapshots to a durable location that survives a container crash or
|
||||
# rebuild.
|
||||
#
|
||||
# SAFE PRIMITIVE: uses SQLite `VACUUM INTO` from a READ-ONLY connection. Unlike
|
||||
# `cp`, this cannot capture a torn mid-checkpoint state while writers are active,
|
||||
# and it verifies the result before promoting it. Backups that fail integrity
|
||||
# are discarded and raise an alert marker instead of overwriting a good one.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/aqe-db-backup.sh backup # make one verified snapshot (default)
|
||||
# scripts/aqe-db-backup.sh list # list existing verified backups
|
||||
# scripts/aqe-db-backup.sh restore <file> # restore a backup (backs up current first)
|
||||
# scripts/aqe-db-backup.sh loop [SECONDS] # run backup every SECONDS (default 1800)
|
||||
#
|
||||
# Env:
|
||||
# AQE_PROJECT_DIR project root (default: /workspaces/agentic-qe)
|
||||
# AQE_BACKUP_DIR backup destination (default: $ROOT/.agentic-qe/backups/verified)
|
||||
# AQE_KEEP_ROTATING number of fine-grained snapshots to keep (default 12)
|
||||
# AQE_KEEP_DAILY number of daily snapshots to keep (default 14)
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${AQE_PROJECT_DIR:-/workspaces/agentic-qe}"
|
||||
DB="$ROOT/.agentic-qe/memory.db"
|
||||
DEST="${AQE_BACKUP_DIR:-$ROOT/.agentic-qe/backups/verified}"
|
||||
LOG="$DEST/backup.log"
|
||||
ALERT="$DEST/BACKUP.ALERT"
|
||||
KEEP_ROTATING="${AQE_KEEP_ROTATING:-12}"
|
||||
KEEP_DAILY="${AQE_KEEP_DAILY:-14}"
|
||||
|
||||
ts() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
logline() { mkdir -p "$DEST"; echo "[$(ts)] $*" >>"$LOG"; }
|
||||
|
||||
do_backup() {
|
||||
[ -f "$DB" ] || { logline "ERROR: source DB missing: $DB"; echo "ERROR: $DB missing" >&2; return 2; }
|
||||
mkdir -p "$DEST"
|
||||
local stamp inflight final day pat exp chk
|
||||
stamp="$(date +%Y%m%dT%H%M%S)"
|
||||
day="$(date +%Y%m%d)"
|
||||
inflight="$DEST/.inflight-$stamp.db"
|
||||
final="$DEST/memory-$stamp.db"
|
||||
|
||||
# Consistent snapshot from a read-only connection (safe with concurrent writers).
|
||||
if ! sqlite3 "file:$DB?mode=ro" "VACUUM INTO '$inflight';" 2>>"$LOG"; then
|
||||
logline "ERROR: VACUUM INTO failed"; rm -f "$inflight"; return 3
|
||||
fi
|
||||
|
||||
# Verify BEFORE promoting — never let a bad backup replace a good one.
|
||||
chk="$(sqlite3 "$inflight" "PRAGMA integrity_check;" 2>>"$LOG" | head -1)"
|
||||
if [ "$chk" != "ok" ]; then
|
||||
logline "ALERT: backup integrity FAILED ($chk) — discarding, keeping prior good backups"
|
||||
rm -f "$inflight"; : >"$ALERT"; return 11
|
||||
fi
|
||||
pat="$(sqlite3 "$inflight" "SELECT COUNT(*) FROM qe_patterns;" 2>/dev/null || echo '?')"
|
||||
exp="$(sqlite3 "$inflight" "SELECT COUNT(*) FROM captured_experiences;" 2>/dev/null || echo '?')"
|
||||
|
||||
mv "$inflight" "$final"
|
||||
# Newest-of-day snapshot (hardlink: shares storage, survives rotation of memory-*).
|
||||
ln -f "$final" "$DEST/daily-$day.db"
|
||||
# Clear any stale alert once a good backup lands.
|
||||
rm -f "$ALERT"
|
||||
|
||||
# Rotation.
|
||||
ls -1t "$DEST"/memory-*.db 2>/dev/null | tail -n +$((KEEP_ROTATING + 1)) | xargs -r rm -f
|
||||
ls -1t "$DEST"/daily-*.db 2>/dev/null | tail -n +$((KEEP_DAILY + 1)) | xargs -r rm -f
|
||||
|
||||
logline "OK memory-$stamp.db ($(du -h "$final" | cut -f1)) patterns=$pat experiences=$exp"
|
||||
echo "OK: $final (patterns=$pat experiences=$exp, integrity=ok)"
|
||||
}
|
||||
|
||||
do_list() {
|
||||
mkdir -p "$DEST"
|
||||
echo "Backups in $DEST:"
|
||||
ls -1t "$DEST"/memory-*.db "$DEST"/daily-*.db 2>/dev/null | while read -r f; do
|
||||
printf " %-40s %6s %s\n" "$(basename "$f")" "$(du -h "$f" | cut -f1)" "$(date -r "$f" '+%Y-%m-%d %H:%M')"
|
||||
done || echo " (none)"
|
||||
}
|
||||
|
||||
do_restore() {
|
||||
local src="$1"
|
||||
[ -f "$src" ] || { echo "ERROR: backup not found: $src" >&2; return 2; }
|
||||
local chk; chk="$(sqlite3 "$src" "PRAGMA integrity_check;" 2>/dev/null | head -1)"
|
||||
[ "$chk" = "ok" ] || { echo "ERROR: backup fails integrity_check ($chk); refusing to restore" >&2; return 3; }
|
||||
# Back up the CURRENT db before overwriting (data-protection rule).
|
||||
if [ -f "$DB" ]; then
|
||||
local pre="$DB.pre-restore-$(date +%s)"
|
||||
cp "$DB" "$pre"; echo "Current DB saved to: $pre"
|
||||
fi
|
||||
# Remove stale WAL/SHM belonging to the old file, then copy the verified backup in.
|
||||
rm -f "$DB-wal" "$DB-shm"
|
||||
cp "$src" "$DB"
|
||||
echo "Restored $src -> $DB"
|
||||
echo "Post-restore: patterns=$(sqlite3 "file:$DB?mode=ro" 'SELECT COUNT(*) FROM qe_patterns;') experiences=$(sqlite3 "file:$DB?mode=ro" 'SELECT COUNT(*) FROM captured_experiences;')"
|
||||
logline "RESTORE from $(basename "$src")"
|
||||
}
|
||||
|
||||
do_loop() {
|
||||
local interval="${1:-1800}"
|
||||
logline "loop started (interval=${interval}s)"
|
||||
while true; do
|
||||
do_backup || logline "backup cycle returned non-zero"
|
||||
sleep "$interval"
|
||||
done
|
||||
}
|
||||
|
||||
cmd="${1:-backup}"
|
||||
case "$cmd" in
|
||||
backup) do_backup ;;
|
||||
list) do_list ;;
|
||||
restore) shift; do_restore "${1:?usage: restore <backup-file>}" ;;
|
||||
loop) shift; do_loop "${1:-1800}" ;;
|
||||
*) echo "usage: $0 {backup|list|restore <file>|loop [seconds]}" >&2; exit 1 ;;
|
||||
esac
|
||||
@@ -922,6 +922,14 @@ export function isHyperbolicHnswEnabled(): boolean {
|
||||
export function initFeatureFlagsFromEnv(): void {
|
||||
const envFlags: Partial<RuVectorFeatureFlags> = {};
|
||||
|
||||
// RVF-backed PatternStore (ADR-066). Was the only flag WITHOUT an env override,
|
||||
// so an environment where RVF native init fails (e.g. `RVF error 0x0303:
|
||||
// FsyncFailed` on a macOS virtiofs bind mount) had no runtime way to fall back
|
||||
// to the SQLite HNSW PatternStore. Set RUVECTOR_USE_RVF_PATTERN_STORE=false there.
|
||||
if (process.env.RUVECTOR_USE_RVF_PATTERN_STORE !== undefined) {
|
||||
envFlags.useRVFPatternStore = process.env.RUVECTOR_USE_RVF_PATTERN_STORE === 'true';
|
||||
}
|
||||
|
||||
if (process.env.RUVECTOR_USE_SONA !== undefined) {
|
||||
envFlags.useQESONA = process.env.RUVECTOR_USE_SONA === 'true';
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user