diff --git a/package.json b/package.json index 512bcbf..f76c6be 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "smriti", - "version": "0.8.0", + "version": "0.8.2", "description": "Smriti - Unified memory layer across all AI agents", "type": "module", "bin": { @@ -17,7 +17,8 @@ "bench:scorecard": "bun run scripts/bench-scorecard.ts --baseline bench/baseline.ci-small.json --profile ci-small --threshold-pct 20", "bench:ingest-hotpaths": "bun run scripts/bench-ingest-hotpaths.ts", "bench:ingest-pipeline": "bun run scripts/bench-ingest-pipeline.ts --sessions 120 --messages 12", - "release:meta": "bun run scripts/release-meta.ts" + "release:meta": "bun run scripts/release-meta.ts", + "monitor": "bun run scripts/monitor-resources.ts" }, "dependencies": { "fast-glob": "3.3.3", diff --git a/scripts/monitor-resources.ts b/scripts/monitor-resources.ts new file mode 100644 index 0000000..b9628a0 --- /dev/null +++ b/scripts/monitor-resources.ts @@ -0,0 +1,221 @@ +/** + * monitor-resources.ts - Sample RSS/CPU for Smriti-related processes + * (daemon, ingest, any bun src/index.ts invocation) so resource usage + * can be inspected without babysitting btop. + * + * bun run monitor watch [--pattern ] [--interval-ms 1000] [--out ] + * bun run monitor exec [--pattern ] [--interval-ms 1000] [--out ] -- + * + * `watch` samples until Ctrl+C. `exec` runs the given command to + * completion, sampling the whole time, then prints a peak-usage summary + * for every matching process seen during the run (e.g. compare the + * daemon's steady baseline against a one-off `smriti ingest claude`). + * + * Samples are always appended as CSV to --out (default .tmp/, gitignored) + * so a run can be inspected after the fact. + */ +import { appendFileSync, mkdirSync } from "fs"; +import { dirname } from "path"; + +type Sample = { + ts: number; + pid: number; + ppid: number; + rssKb: number; + cpuPct: number; + etime: string; + command: string; +}; + +type ProcSummary = { + pid: number; + command: string; + firstSeen: number; + lastSeen: number; + peakRssKb: number; + peakCpuPct: number; + sampleCount: number; +}; + +const args = process.argv.slice(2); +const flag = (name: string, fallback: string) => { + const i = args.indexOf(`--${name}`); + return i >= 0 && args[i + 1] ? args[i + 1] : fallback; +}; + +const mode = args[0] === "exec" ? "exec" : "watch"; +// Matches actual `bun ... index.ts` invocations (daemon/ingest/recall/etc), +// not just any process whose path happens to contain "smriti" (e.g. an +// editor's tsserver running out of this repo's node_modules). +const pattern = new RegExp(flag("pattern", "bun\\b.*index\\.ts"), "i"); +const intervalMs = Number(flag("interval-ms", "1000")); +const outPath = flag("out", `.tmp/resource-monitor-${Date.now()}.csv`); + +const dashIdx = args.indexOf("--"); +const execCommand = mode === "exec" ? args.slice(dashIdx + 1) : []; +if (mode === "exec" && execCommand.length === 0) { + console.error("exec mode requires a command after --, e.g.:"); + console.error(" bun run monitor exec -- smriti ingest claude"); + process.exit(1); +} + +mkdirSync(dirname(outPath), { recursive: true }); +let wroteHeader = false; + +async function sampleOnce(): Promise { + const proc = Bun.spawn(["ps", "-eo", "pid,ppid,rss,pcpu,etime,args"], { + stdout: "pipe", + }); + const text = await new Response(proc.stdout).text(); + await proc.exited; + + const ts = Date.now(); + const samples: Sample[] = []; + for (const line of text.split("\n").slice(1)) { + const m = line.match(/^\s*(\d+)\s+(\d+)\s+(\d+)\s+([\d.]+)\s+(\S+)\s+(.*)$/); + if (!m) continue; + const [, pid, ppid, rss, cpu, etime, command] = m; + if (!pattern.test(command)) continue; + if (command.includes("monitor-resources") || command.includes("ps -eo")) continue; + samples.push({ + ts, + pid: Number(pid), + ppid: Number(ppid), + rssKb: Number(rss), + cpuPct: Number(cpu), + etime, + command: command.trim(), + }); + } + return samples; +} + +function writeCsv(samples: Sample[]) { + if (samples.length === 0) return; + if (!wroteHeader) { + appendFileSync(outPath, "ts,pid,ppid,rss_kb,cpu_pct,etime,command\n"); + wroteHeader = true; + } + const rows = samples + .map((s) => `${s.ts},${s.pid},${s.ppid},${s.rssKb},${s.cpuPct},${s.etime},"${s.command.replace(/"/g, '""')}"`) + .join("\n"); + appendFileSync(outPath, rows + "\n"); +} + +function summarize(all: Sample[]): ProcSummary[] { + const byPid = new Map(); + for (const s of all) { + const existing = byPid.get(s.pid); + if (!existing) { + byPid.set(s.pid, { + pid: s.pid, + command: s.command, + firstSeen: s.ts, + lastSeen: s.ts, + peakRssKb: s.rssKb, + peakCpuPct: s.cpuPct, + sampleCount: 1, + }); + continue; + } + existing.lastSeen = s.ts; + existing.peakRssKb = Math.max(existing.peakRssKb, s.rssKb); + existing.peakCpuPct = Math.max(existing.peakCpuPct, s.cpuPct); + existing.sampleCount++; + } + return [...byPid.values()].sort((a, b) => b.peakRssKb - a.peakRssKb); +} + +function isPidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** + * Wait for a spawned child to exit. Races the real `exited` promise against + * periodic liveness checks, because `child.exited` has been observed to + * never resolve even after the child process has fully terminated (no + * zombie, no remaining process) — a real hang seen during a long `smriti + * ingest --force` run. Once we've seen the pid alive at least once and then + * can no longer find it, treat that as exit rather than waiting forever. + */ +async function waitForChildExit(child: Bun.Subprocess, pollMs: number): Promise { + let sawAlive = false; + while (true) { + const raced = await Promise.race([ + child.exited.then((code) => ({ done: true as const, code })), + Bun.sleep(pollMs).then(() => ({ done: false as const, code: null })), + ]); + if (raced.done) return raced.code; + + if (isPidAlive(child.pid)) { + sawAlive = true; + continue; + } + if (sawAlive) { + console.error( + `[monitor] child.exited did not resolve after pid ${child.pid} was no longer running — treating as exited.` + ); + return null; + } + // else: pid not observed yet (spawn race at startup) — keep waiting. + } +} + +function printSummary(all: Sample[]) { + const procs = summarize(all); + if (procs.length === 0) { + console.log(`\nNo processes matched /${pattern.source}/ during this run.`); + return; + } + console.log(`\n${"PID".padEnd(8)}${"PEAK RSS".padEnd(12)}${"PEAK CPU".padEnd(10)}${"DURATION".padEnd(10)}COMMAND`); + for (const p of procs) { + const rssMb = (p.peakRssKb / 1024).toFixed(1) + " MB"; + const durationS = ((p.lastSeen - p.firstSeen) / 1000).toFixed(1) + "s"; + console.log( + `${String(p.pid).padEnd(8)}${rssMb.padEnd(12)}${(p.peakCpuPct.toFixed(1) + "%").padEnd(10)}${durationS.padEnd(10)}${p.command.slice(0, 80)}` + ); + } + console.log(`\nRaw samples: ${outPath}`); +} + +async function main() { + const allSamples: Sample[] = []; + let stopped = false; + + const loop = (async () => { + while (!stopped) { + const samples = await sampleOnce(); + allSamples.push(...samples); + writeCsv(samples); + await Bun.sleep(intervalMs); + } + })(); + + if (mode === "watch") { + console.log(`Watching /${pattern.source}/ every ${intervalMs}ms — Ctrl+C to stop and print summary.`); + process.on("SIGINT", () => { + stopped = true; + printSummary(allSamples); + process.exit(0); + }); + await loop; + } else { + console.log(`Sampling /${pattern.source}/ every ${intervalMs}ms while running: ${execCommand.join(" ")}`); + const child = Bun.spawn(execCommand, { stdout: "inherit", stderr: "inherit" }); + const exitCode = await waitForChildExit(child, Math.min(intervalMs, 500)); + stopped = true; + await loop; + printSummary(allSamples); + process.exit(exitCode ?? 0); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/release-meta.ts b/scripts/release-meta.ts index 3da67fb..dd8d52e 100644 --- a/scripts/release-meta.ts +++ b/scripts/release-meta.ts @@ -117,7 +117,14 @@ function isConventional(c: Commit): boolean { function getCommits(rangeFrom: string | null, rangeTo: string): Commit[] { const range = rangeFrom ? `${rangeFrom}..${rangeTo}` : rangeTo; - const raw = run(`git log --no-merges --pretty=format:%H%x09%s%x09%b ${range}`); + // Not run()'s full .trim(): the oldest commit in range can have an empty + // body, leaving a trailing %x09 as the last character of the whole + // output. A full trim() strips that tab along with it, dropping that + // commit's field count below 3 and silently discarding it from + // getCommits()'s result (and thus from release notes / commit_count). + const raw = execSync(`git log --no-merges --pretty=format:%H%x09%s%x09%b ${range}`, { + encoding: "utf8", + }).replace(/\n+$/, ""); if (!raw) return []; return raw .split("\n") diff --git a/src/daemon/index.ts b/src/daemon/index.ts index b1b24bc..52b809a 100644 --- a/src/daemon/index.ts +++ b/src/daemon/index.ts @@ -16,8 +16,9 @@ * real DB or spawning real agent log files. */ +import type { Database } from "bun:sqlite"; import { QMD_DB_PATH } from "../config"; -import { initSmriti } from "../db"; +import { initSmriti, closeDb } from "../db"; import { ingest } from "../ingest"; import { startDaemon as startServer, type DaemonHandle } from "./server"; import { watchRecursive, type WatcherHandle } from "./watcher"; @@ -41,6 +42,15 @@ export type RunDaemonOptions = { log?: (msg: string) => void; /** Override the per-project debounce window in ms. */ debounceMs?: number; + /** + * Let routine daemon ingest trigger LLM query-expansion enrichment. + * Defaults to false: the daemon is parse-and-write only (per + * docs/internal/daemon-prd.md #93 — no embedding model in the daemon). + * Enrichment is a manual `smriti enrich` concern; leaving this on + * caused the recurring "LlamaGrammar ... different Llama instance" + * crash when combined with a fresh LlamaCpp per flush. + */ + enrichOnIngest?: boolean; }; export type RunningDaemon = { @@ -52,30 +62,55 @@ export type RunningDaemon = { shutdown(): Promise; }; +/** + * Chain every flush onto this promise so at most one is ever in flight. + * The debounce timers in queue.ts stay independent per project — this only + * serializes the actual open-DB/ingest/close body. Required because: + * 1. initSmriti()/closeDb() go through module-level singletons in + * src/db.ts and src/store.ts shared across the whole process — two + * concurrent flushes would stomp on each other's connection. + * 2. Disposing one flush's LlamaCpp/Llama backend (see closeDb below) + * while another flush's is still loading/active can abort the + * process with a native GGML_ASSERT in ggml-metal — reproduced when + * multiple LlamaCpp instances were constructed/disposed concurrently + * in one process during verification of this fix. + */ +let flushChain: Promise = Promise.resolve(); + /** * Default per-flush behavior: open SQLite, call ingest(), close SQLite. * Errors are logged but not rethrown — one bad flush should not crash * the daemon. */ async function defaultFlushAgent(agent: string, log: (m: string) => void): Promise { - let db; - try { - db = await initSmriti(QMD_DB_PATH); - } catch (err) { - log(`[flush ${agent}] failed to open DB: ${(err as Error).message}`); - return; - } - try { - const r = await ingest(db, agent); - log( - `[flush ${agent}] ingested=${r.sessionsIngested}/${r.sessionsFound}, ` + - `msgs=${r.messagesIngested}, errs=${r.errors.length}`, - ); - } catch (err) { - log(`[flush ${agent}] ingest failed: ${(err as Error).message}`); - } finally { - try { db.close(); } catch {} - } + const run = async (): Promise => { + let db: Database; + try { + db = await initSmriti(QMD_DB_PATH); + } catch (err) { + log(`[flush ${agent}] failed to open DB: ${(err as Error).message}`); + return; + } + try { + const r = await ingest(db, agent); + log( + `[flush ${agent}] ingested=${r.sessionsIngested}/${r.sessionsFound}, ` + + `msgs=${r.messagesIngested}, errs=${r.errors.length}`, + ); + } catch (err) { + log(`[flush ${agent}] ingest failed: ${(err as Error).message}`); + } finally { + // closeDb() (not a raw db.close()) so the QMD store's LlamaCpp/Llama + // backend is disposed too, not just the SQLite handle — otherwise + // each flush's LlamaCpp instance is orphaned to a 5-min inactivity + // timer instead of being freed immediately. + try { await closeDb(); } catch {} + } + }; + + const next = flushChain.then(run, run); + flushChain = next; + return next; } /** @@ -89,6 +124,16 @@ async function defaultFlushAgent(agent: string, log: (m: string) => void): Promi */ export async function runDaemon(opts: RunDaemonOptions = {}): Promise { const log = opts.log ?? ((msg: string) => console.error(`[smriti] ${msg}`)); + + // The daemon does parse-and-write only — LLM enrichment (query expansion) + // is a manual `smriti enrich` concern, not something ingest should trigger + // as a side effect of every flush. See docs/internal/daemon-prd.md #93 and + // the recurring "LlamaGrammar ... different Llama instance" crash this + // caused when combined with a fresh LlamaCpp per flush. + if (opts.enrichOnIngest !== true) { + process.env.SMRITI_INGEST_NO_ENRICH = "1"; + } + const agentRoots = opts.agentRoots ?? getDefaultAgentRoots(); if (agentRoots.length === 0) { diff --git a/src/db.ts b/src/db.ts index 3be3823..59d8ba7 100644 --- a/src/db.ts +++ b/src/db.ts @@ -27,10 +27,10 @@ export function getDb(): Database { return _db; } -/** Close the database and release the QMD store. */ -export function closeDb(): void { +/** Close the database and release the QMD store (including its LLM backend). */ +export async function closeDb(): Promise { _db = null; - closeQmdStore(); + await closeQmdStore(); } // ============================================================================= @@ -647,6 +647,11 @@ export async function initSmriti(dbPath?: string): Promise { setQmdStore(store); const db = store.internal.db as unknown as Database; _db = db; + // busy_timeout is per-connection, not persisted in the DB file — set it on + // every open. Without it, two processes opening the same SQLite file at + // once (e.g. the daemon's flush and a manual `smriti ingest --force`) fail + // immediately with "database is locked" instead of retrying briefly. + db.exec("PRAGMA busy_timeout = 5000"); initializeMemoryTables(db as any); initializeSmritiTables(db); seedDefaults(db); diff --git a/src/index.ts b/src/index.ts index 5559e7c..bb186d5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1508,7 +1508,7 @@ async function main() { process.exit(1); } } finally { - closeDb(); + await closeDb(); } } diff --git a/src/store.ts b/src/store.ts index 1e76694..6c58c84 100644 --- a/src/store.ts +++ b/src/store.ts @@ -19,8 +19,13 @@ export function getQmdStore(): QMDStore { return _store; } -export function closeQmdStore(): void { +export async function closeQmdStore(): Promise { if (_store) { + // Dispose the LlamaCpp/Llama backend explicitly instead of leaving it + // to the 5-min inactivity timer — otherwise a fresh Store created on + // the next initSmriti() call (e.g. the daemon's per-flush open) can + // overlap with the still-loaded previous instance. + try { await _store.internal.llm?.dispose(); } catch { /* best-effort */ } _store.internal.close(); _store = null; } diff --git a/test/daemon-runner.test.ts b/test/daemon-runner.test.ts index d953302..4a6ffe0 100644 --- a/test/daemon-runner.test.ts +++ b/test/daemon-runner.test.ts @@ -161,4 +161,45 @@ describe("runDaemon", () => { await daemon.shutdown(); // should not throw expect(true).toBe(true); // assertion presence }); + + const ENRICH_ENV_KEY = "SMRITI_INGEST_NO_ENRICH"; + // Read via a non-literal key so TS doesn't (incorrectly) narrow this to + // `undefined` from the `delete` below — it can't see that runDaemon() + // mutates process.env internally. + const readEnrichFlag = () => process.env[ENRICH_ENV_KEY]; + + test("sets SMRITI_INGEST_NO_ENRICH by default so routine ingest never triggers LLM enrichment", async () => { + const original = readEnrichFlag(); + delete process.env[ENRICH_ENV_KEY]; + try { + const daemon = await runDaemon({ + agentRoots: [], + flushAgent: () => {}, + log: () => {}, + }); + expect(readEnrichFlag()).toBe("1"); + await daemon.shutdown(); + } finally { + if (original === undefined) delete process.env[ENRICH_ENV_KEY]; + else process.env[ENRICH_ENV_KEY] = original; + } + }); + + test("enrichOnIngest: true opts back into LLM enrichment during ingest", async () => { + const original = readEnrichFlag(); + delete process.env[ENRICH_ENV_KEY]; + try { + const daemon = await runDaemon({ + agentRoots: [], + flushAgent: () => {}, + log: () => {}, + enrichOnIngest: true, + }); + expect(readEnrichFlag()).toBeUndefined(); + await daemon.shutdown(); + } finally { + if (original === undefined) delete process.env[ENRICH_ENV_KEY]; + else process.env[ENRICH_ENV_KEY] = original; + } + }); }); diff --git a/test/store.test.ts b/test/store.test.ts new file mode 100644 index 0000000..ce0e8dd --- /dev/null +++ b/test/store.test.ts @@ -0,0 +1,33 @@ +/** + * test/store.test.ts + * + * Verifies closeDb()/closeQmdStore() dispose the QMD store's LlamaCpp/Llama + * backend, not just the SQLite handle — otherwise a fresh Store created on + * the next initSmriti() call (e.g. the daemon's per-flush open) can overlap + * with the still-loaded previous instance and leak memory between flushes. + */ +import { test, expect } from "bun:test"; +import { initSmriti, closeDb } from "../src/db"; +import { getQmdStore } from "../src/store"; + +test("closeDb disposes the QMD store's LLM backend, not just the SQLite handle", async () => { + await initSmriti(":memory:"); + const store = getQmdStore(); + + let disposeCalls = 0; + if (store.internal.llm) { + (store.internal.llm as any).dispose = async () => { disposeCalls++; }; + } + + await closeDb(); + + expect(disposeCalls).toBe(1); + expect(() => getQmdStore()).toThrow(); +}); + +test("closeDb is safe to call again after initSmriti() re-opens", async () => { + await initSmriti(":memory:"); + await closeDb(); + await initSmriti(":memory:"); + await closeDb(); // should not throw +}); diff --git a/test/team-segmented.test.ts b/test/team-segmented.test.ts index 2e3494c..cc1aeb1 100644 --- a/test/team-segmented.test.ts +++ b/test/team-segmented.test.ts @@ -21,8 +21,8 @@ beforeAll(async () => { db = await initSmriti(":memory:"); }); -afterAll(() => { - closeDb(); +afterAll(async () => { + await closeDb(); }); // ============================================================================= diff --git a/test/team.test.ts b/test/team.test.ts index a99fcbc..e1e82eb 100644 --- a/test/team.test.ts +++ b/test/team.test.ts @@ -25,8 +25,8 @@ beforeAll(async () => { mkdirSync(tmpDir, { recursive: true }); }); -afterAll(() => { - closeDb(); +afterAll(async () => { + await closeDb(); try { rmSync(tmpDir, { recursive: true }); } catch {} });