fix(daemon): reduce memory footprint, fix LLM crash, fix DB lock contention

package.json

@@ -1,6 +1,6 @@
 {
   "name": "smriti",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "description": "Smriti - Unified memory layer across all AI agents",
   "type": "module",
   "bin": {
@@ -17,7 +17,8 @@
     "bench:scorecard": "bun run scripts/bench-scorecard.ts --baseline bench/baseline.ci-small.json --profile ci-small --threshold-pct 20",
     "bench:ingest-hotpaths": "bun run scripts/bench-ingest-hotpaths.ts",
     "bench:ingest-pipeline": "bun run scripts/bench-ingest-pipeline.ts --sessions 120 --messages 12",
-    "release:meta": "bun run scripts/release-meta.ts"
+    "release:meta": "bun run scripts/release-meta.ts",
+    "monitor": "bun run scripts/monitor-resources.ts"
   },
   "dependencies": {
     "fast-glob": "3.3.3",

scripts/monitor-resources.ts

@@ -0,0 +1,221 @@
+/**
+ * monitor-resources.ts - Sample RSS/CPU for Smriti-related processes
+ * (daemon, ingest, any bun src/index.ts invocation) so resource usage
+ * can be inspected without babysitting btop.
+ *
+ *   bun run monitor watch [--pattern <regex>] [--interval-ms 1000] [--out <path>]
+ *   bun run monitor exec [--pattern <regex>] [--interval-ms 1000] [--out <path>] -- <command...>
+ *
+ * `watch` samples until Ctrl+C. `exec` runs the given command to
+ * completion, sampling the whole time, then prints a peak-usage summary
+ * for every matching process seen during the run (e.g. compare the
+ * daemon's steady baseline against a one-off `smriti ingest claude`).
+ *
+ * Samples are always appended as CSV to --out (default .tmp/, gitignored)
+ * so a run can be inspected after the fact.
+ */
+import { appendFileSync, mkdirSync } from "fs";
+import { dirname } from "path";
+
+type Sample = {
+  ts: number;
+  pid: number;
+  ppid: number;
+  rssKb: number;
+  cpuPct: number;
+  etime: string;
+  command: string;
+};
+
+type ProcSummary = {
+  pid: number;
+  command: string;
+  firstSeen: number;
+  lastSeen: number;
+  peakRssKb: number;
+  peakCpuPct: number;
+  sampleCount: number;
+};
+
+const args = process.argv.slice(2);
+const flag = (name: string, fallback: string) => {
+  const i = args.indexOf(`--${name}`);
+  return i >= 0 && args[i + 1] ? args[i + 1] : fallback;
+};
+
+const mode = args[0] === "exec" ? "exec" : "watch";
+// Matches actual `bun ... index.ts` invocations (daemon/ingest/recall/etc),
+// not just any process whose path happens to contain "smriti" (e.g. an
+// editor's tsserver running out of this repo's node_modules).
+const pattern = new RegExp(flag("pattern", "bun\\b.*index\\.ts"), "i");
+const intervalMs = Number(flag("interval-ms", "1000"));
+const outPath = flag("out", `.tmp/resource-monitor-${Date.now()}.csv`);
+
+const dashIdx = args.indexOf("--");
+const execCommand = mode === "exec" ? args.slice(dashIdx + 1) : [];
+if (mode === "exec" && execCommand.length === 0) {
+  console.error("exec mode requires a command after --, e.g.:");
+  console.error("  bun run monitor exec -- smriti ingest claude");
+  process.exit(1);
+}
+
+mkdirSync(dirname(outPath), { recursive: true });
+let wroteHeader = false;
+
+async function sampleOnce(): Promise<Sample[]> {
+  const proc = Bun.spawn(["ps", "-eo", "pid,ppid,rss,pcpu,etime,args"], {
+    stdout: "pipe",
+  });
+  const text = await new Response(proc.stdout).text();
+  await proc.exited;
+
+  const ts = Date.now();
+  const samples: Sample[] = [];
+  for (const line of text.split("\n").slice(1)) {
+    const m = line.match(/^\s*(\d+)\s+(\d+)\s+(\d+)\s+([\d.]+)\s+(\S+)\s+(.*)$/);
+    if (!m) continue;
+    const [, pid, ppid, rss, cpu, etime, command] = m;
+    if (!pattern.test(command)) continue;
+    if (command.includes("monitor-resources") || command.includes("ps -eo")) continue;
+    samples.push({
+      ts,
+      pid: Number(pid),
+      ppid: Number(ppid),
+      rssKb: Number(rss),
+      cpuPct: Number(cpu),
+      etime,
+      command: command.trim(),
+    });
+  }
+  return samples;
+}
+
+function writeCsv(samples: Sample[]) {
+  if (samples.length === 0) return;
+  if (!wroteHeader) {
+    appendFileSync(outPath, "ts,pid,ppid,rss_kb,cpu_pct,etime,command\n");
+    wroteHeader = true;
+  }
+  const rows = samples
+    .map((s) => `${s.ts},${s.pid},${s.ppid},${s.rssKb},${s.cpuPct},${s.etime},"${s.command.replace(/"/g, '""')}"`)
+    .join("\n");
+  appendFileSync(outPath, rows + "\n");
+}
+
+function summarize(all: Sample[]): ProcSummary[] {
+  const byPid = new Map<number, ProcSummary>();
+  for (const s of all) {
+    const existing = byPid.get(s.pid);
+    if (!existing) {
+      byPid.set(s.pid, {
+        pid: s.pid,
+        command: s.command,
+        firstSeen: s.ts,
+        lastSeen: s.ts,
+        peakRssKb: s.rssKb,
+        peakCpuPct: s.cpuPct,
+        sampleCount: 1,
+      });
+      continue;
+    }
+    existing.lastSeen = s.ts;
+    existing.peakRssKb = Math.max(existing.peakRssKb, s.rssKb);
+    existing.peakCpuPct = Math.max(existing.peakCpuPct, s.cpuPct);
+    existing.sampleCount++;
+  }
+  return [...byPid.values()].sort((a, b) => b.peakRssKb - a.peakRssKb);
+}
+
+function isPidAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Wait for a spawned child to exit. Races the real `exited` promise against
+ * periodic liveness checks, because `child.exited` has been observed to
+ * never resolve even after the child process has fully terminated (no
+ * zombie, no remaining process) — a real hang seen during a long `smriti
+ * ingest --force` run. Once we've seen the pid alive at least once and then
+ * can no longer find it, treat that as exit rather than waiting forever.
+ */
+async function waitForChildExit(child: Bun.Subprocess, pollMs: number): Promise<number | null> {
+  let sawAlive = false;
+  while (true) {
+    const raced = await Promise.race([
+      child.exited.then((code) => ({ done: true as const, code })),
+      Bun.sleep(pollMs).then(() => ({ done: false as const, code: null })),
+    ]);
+    if (raced.done) return raced.code;
+
+    if (isPidAlive(child.pid)) {
+      sawAlive = true;
+      continue;
+    }
+    if (sawAlive) {
+      console.error(
+        `[monitor] child.exited did not resolve after pid ${child.pid} was no longer running — treating as exited.`
+      );
+      return null;
+    }
+    // else: pid not observed yet (spawn race at startup) — keep waiting.
+  }
+}
+
+function printSummary(all: Sample[]) {
+  const procs = summarize(all);
+  if (procs.length === 0) {
+    console.log(`\nNo processes matched /${pattern.source}/ during this run.`);
+    return;
+  }
+  console.log(`\n${"PID".padEnd(8)}${"PEAK RSS".padEnd(12)}${"PEAK CPU".padEnd(10)}${"DURATION".padEnd(10)}COMMAND`);
+  for (const p of procs) {
+    const rssMb = (p.peakRssKb / 1024).toFixed(1) + " MB";
+    const durationS = ((p.lastSeen - p.firstSeen) / 1000).toFixed(1) + "s";
+    console.log(
+      `${String(p.pid).padEnd(8)}${rssMb.padEnd(12)}${(p.peakCpuPct.toFixed(1) + "%").padEnd(10)}${durationS.padEnd(10)}${p.command.slice(0, 80)}`
+    );
+  }
+  console.log(`\nRaw samples: ${outPath}`);
+}
+
+async function main() {
+  const allSamples: Sample[] = [];
+  let stopped = false;
+
+  const loop = (async () => {
+    while (!stopped) {
+      const samples = await sampleOnce();
+      allSamples.push(...samples);
+      writeCsv(samples);
+      await Bun.sleep(intervalMs);
+    }
+  })();
+
+  if (mode === "watch") {
+    console.log(`Watching /${pattern.source}/ every ${intervalMs}ms — Ctrl+C to stop and print summary.`);
+    process.on("SIGINT", () => {
+      stopped = true;
+      printSummary(allSamples);
+      process.exit(0);
+    });
+    await loop;
+  } else {
+    console.log(`Sampling /${pattern.source}/ every ${intervalMs}ms while running: ${execCommand.join(" ")}`);
+    const child = Bun.spawn(execCommand, { stdout: "inherit", stderr: "inherit" });
+    const exitCode = await waitForChildExit(child, Math.min(intervalMs, 500));
+    stopped = true;
+    await loop;
+    printSummary(allSamples);
+    process.exit(exitCode ?? 0);
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

scripts/release-meta.ts

@@ -117,7 +117,14 @@ function isConventional(c: Commit): boolean {
 
 function getCommits(rangeFrom: string | null, rangeTo: string): Commit[] {
   const range = rangeFrom ? `${rangeFrom}..${rangeTo}` : rangeTo;
-  const raw = run(`git log --no-merges --pretty=format:%H%x09%s%x09%b ${range}`);
+  // Not run()'s full .trim(): the oldest commit in range can have an empty
+  // body, leaving a trailing %x09 as the last character of the whole
+  // output. A full trim() strips that tab along with it, dropping that
+  // commit's field count below 3 and silently discarding it from
+  // getCommits()'s result (and thus from release notes / commit_count).
+  const raw = execSync(`git log --no-merges --pretty=format:%H%x09%s%x09%b ${range}`, {
+    encoding: "utf8",
+  }).replace(/\n+$/, "");
   if (!raw) return [];
   return raw
     .split("\n")

src/daemon/index.ts

@@ -16,8 +16,9 @@
  * real DB or spawning real agent log files.
  */
 
+import type { Database } from "bun:sqlite";
 import { QMD_DB_PATH } from "../config";
-import { initSmriti } from "../db";
+import { initSmriti, closeDb } from "../db";
 import { ingest } from "../ingest";
 import { startDaemon as startServer, type DaemonHandle } from "./server";
 import { watchRecursive, type WatcherHandle } from "./watcher";
@@ -41,6 +42,15 @@ export type RunDaemonOptions = {
   log?: (msg: string) => void;
   /** Override the per-project debounce window in ms. */
   debounceMs?: number;
+  /**
+   * Let routine daemon ingest trigger LLM query-expansion enrichment.
+   * Defaults to false: the daemon is parse-and-write only (per
+   * docs/internal/daemon-prd.md #93 — no embedding model in the daemon).
+   * Enrichment is a manual `smriti enrich` concern; leaving this on
+   * caused the recurring "LlamaGrammar ... different Llama instance"
+   * crash when combined with a fresh LlamaCpp per flush.
+   */
+  enrichOnIngest?: boolean;
 };
 
 export type RunningDaemon = {
@@ -52,30 +62,55 @@ export type RunningDaemon = {
   shutdown(): Promise<void>;
 };
 
+/**
+ * Chain every flush onto this promise so at most one is ever in flight.
+ * The debounce timers in queue.ts stay independent per project — this only
+ * serializes the actual open-DB/ingest/close body. Required because:
+ *   1. initSmriti()/closeDb() go through module-level singletons in
+ *      src/db.ts and src/store.ts shared across the whole process — two
+ *      concurrent flushes would stomp on each other's connection.
+ *   2. Disposing one flush's LlamaCpp/Llama backend (see closeDb below)
+ *      while another flush's is still loading/active can abort the
+ *      process with a native GGML_ASSERT in ggml-metal — reproduced when
+ *      multiple LlamaCpp instances were constructed/disposed concurrently
+ *      in one process during verification of this fix.
+ */
+let flushChain: Promise<void> = Promise.resolve();
+
 /**
  * Default per-flush behavior: open SQLite, call ingest(), close SQLite.
  * Errors are logged but not rethrown — one bad flush should not crash
  * the daemon.
  */
 async function defaultFlushAgent(agent: string, log: (m: string) => void): Promise<void> {
-  let db;
-  try {
-    db = await initSmriti(QMD_DB_PATH);
-  } catch (err) {
-    log(`[flush ${agent}] failed to open DB: ${(err as Error).message}`);
-    return;
-  }
-  try {
-    const r = await ingest(db, agent);
-    log(
-      `[flush ${agent}] ingested=${r.sessionsIngested}/${r.sessionsFound}, ` +
-      `msgs=${r.messagesIngested}, errs=${r.errors.length}`,
-    );
-  } catch (err) {
-    log(`[flush ${agent}] ingest failed: ${(err as Error).message}`);
-  } finally {
-    try { db.close(); } catch {}
-  }
+  const run = async (): Promise<void> => {
+    let db: Database;
+    try {
+      db = await initSmriti(QMD_DB_PATH);
+    } catch (err) {
+      log(`[flush ${agent}] failed to open DB: ${(err as Error).message}`);
+      return;
+    }
+    try {
+      const r = await ingest(db, agent);
+      log(
+        `[flush ${agent}] ingested=${r.sessionsIngested}/${r.sessionsFound}, ` +
+        `msgs=${r.messagesIngested}, errs=${r.errors.length}`,
+      );
+    } catch (err) {
+      log(`[flush ${agent}] ingest failed: ${(err as Error).message}`);
+    } finally {
+      // closeDb() (not a raw db.close()) so the QMD store's LlamaCpp/Llama
+      // backend is disposed too, not just the SQLite handle — otherwise
+      // each flush's LlamaCpp instance is orphaned to a 5-min inactivity
+      // timer instead of being freed immediately.
+      try { await closeDb(); } catch {}
+    }
+  };
+
+  const next = flushChain.then(run, run);
+  flushChain = next;
+  return next;
 }
 
 /**
@@ -89,6 +124,16 @@ async function defaultFlushAgent(agent: string, log: (m: string) => void): Promi
  */
 export async function runDaemon(opts: RunDaemonOptions = {}): Promise<RunningDaemon> {
   const log = opts.log ?? ((msg: string) => console.error(`[smriti] ${msg}`));
+
+  // The daemon does parse-and-write only — LLM enrichment (query expansion)
+  // is a manual `smriti enrich` concern, not something ingest should trigger
+  // as a side effect of every flush. See docs/internal/daemon-prd.md #93 and
+  // the recurring "LlamaGrammar ... different Llama instance" crash this
+  // caused when combined with a fresh LlamaCpp per flush.
+  if (opts.enrichOnIngest !== true) {
+    process.env.SMRITI_INGEST_NO_ENRICH = "1";
+  }
+
   const agentRoots = opts.agentRoots ?? getDefaultAgentRoots();
 
   if (agentRoots.length === 0) {

src/db.ts

@@ -27,10 +27,10 @@ export function getDb(): Database {
   return _db;
 }
 
-/** Close the database and release the QMD store. */
-export function closeDb(): void {
+/** Close the database and release the QMD store (including its LLM backend). */
+export async function closeDb(): Promise<void> {
   _db = null;
-  closeQmdStore();
+  await closeQmdStore();
 }
 
 // =============================================================================
@@ -647,6 +647,11 @@ export async function initSmriti(dbPath?: string): Promise<Database> {
   setQmdStore(store);
   const db = store.internal.db as unknown as Database;
   _db = db;
+  // busy_timeout is per-connection, not persisted in the DB file — set it on
+  // every open. Without it, two processes opening the same SQLite file at
+  // once (e.g. the daemon's flush and a manual `smriti ingest --force`) fail
+  // immediately with "database is locked" instead of retrying briefly.
+  db.exec("PRAGMA busy_timeout = 5000");
   initializeMemoryTables(db as any);
   initializeSmritiTables(db);
   seedDefaults(db);