From 971c4efea0bcc0e90d6dfd8af5641b5d97e3aac1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Jul 2026 09:35:59 +0000 Subject: [PATCH] feat(learn): add continuous knowledge consolidation layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `smriti consolidate` and `smriti learnings`, applying Progressive Summarization: cheap Stage-1 segmentation runs broadly over dense sessions into a new smriti_knowledge_units table, and expensive Stage-2 polish (existing segmentSession/generateDocument pipeline) only runs once a unit proves reuse via recall or scored high relevance at extraction time. - src/db.ts: smriti_knowledge_units table + CRUD helpers (insertKnowledgeUnit, findUnsegmentedDenseSessions, findPromotableUnits, incrementRetrievalCount, promoteKnowledgeUnit, listKnowledgeUnits) - src/search/recall.ts: track retrieval_count on every recall() path - src/learn/consolidate.ts: segment + promote phases, reusing the existing 3-stage segmentation pipeline from src/team/segment.ts and document.ts - src/index.ts, src/format.ts: CLI wiring for `consolidate` and `learnings` CLI-only, not wired into the daemon — consolidation runs two LLM stages, which the daemon's flush path deliberately avoids (see the enrichOnIngest comment in src/daemon/index.ts). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01PnYgLUDwWALu1bUAgrFQDG --- src/db.ts | 209 ++++++++++++++++++++++++ src/format.ts | 57 +++++++ src/index.ts | 43 ++++- src/learn/consolidate.ts | 196 +++++++++++++++++++++++ src/search/recall.ts | 22 +++ src/team/share.ts | 2 +- test/learn-consolidate.test.ts | 283 +++++++++++++++++++++++++++++++++ 7 files changed, 810 insertions(+), 2 deletions(-) create mode 100644 src/learn/consolidate.ts create mode 100644 test/learn-consolidate.test.ts diff --git a/src/db.ts b/src/db.ts index 59d8ba7..b04bd3e 100644 --- a/src/db.ts +++ b/src/db.ts @@ -14,6 +14,7 @@ import { QMD_DB_PATH, SMRITI_SESSIONS_DIR } from "./config"; import { initializeMemoryTables } from "./qmd"; import { createStore } from "../qmd/src/index"; import { setQmdStore, closeQmdStore } from "./store"; +import type { KnowledgeUnit } from "./team/types"; // ============================================================================= // Connection @@ -204,6 +205,33 @@ export function initializeSmritiTables(db: Database): void { entities TEXT ); + -- Knowledge consolidation: raw Stage-1 extracts, promoted to canonical on reuse + CREATE TABLE IF NOT EXISTS smriti_knowledge_units ( + id TEXT PRIMARY KEY, -- KnowledgeUnit.id (uuid) + session_id TEXT NOT NULL, + project_id TEXT, + topic TEXT NOT NULL, + category TEXT NOT NULL, + relevance REAL NOT NULL DEFAULT 0, -- 0-10, from Stage 1 + entities TEXT, -- JSON array + files TEXT, -- JSON array + plain_text TEXT NOT NULL, -- raw Stage-1 extract + line_ranges TEXT, -- JSON array of {start,end} + content_hash TEXT NOT NULL, -- hashContent({topic,category,plainText}) — Stage-1 dedup key + tier TEXT NOT NULL DEFAULT 'segmented', -- 'segmented' | 'canonical' + retrieval_count INTEGER NOT NULL DEFAULT 0, + last_recalled_at TEXT, + promoted_at TEXT, + canonical_doc_path TEXT, -- relative path under .smriti/knowledge/, set on promotion + share_id TEXT, -- points at the smriti_shares row created on promotion + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_session ON smriti_knowledge_units(session_id); + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_hash ON smriti_knowledge_units(content_hash); + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_tier ON smriti_knowledge_units(tier); + -- Tool usage tracking CREATE TABLE IF NOT EXISTS smriti_tool_usage ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -1246,6 +1274,187 @@ export function getDensityScore(db: Database, sessionId: string): number { return row?.density_score ?? 0; } +// ============================================================================= +// Knowledge Consolidation (Progressive Summarization) +// ============================================================================= + +export interface StoredKnowledgeUnit { + id: string; + session_id: string; + project_id: string | null; + topic: string; + category: string; + relevance: number; + entities: string[]; + files: string[]; + plain_text: string; + line_ranges: Array<{ start: number; end: number }>; + content_hash: string; + tier: "segmented" | "canonical"; + retrieval_count: number; + last_recalled_at: string | null; + promoted_at: string | null; + canonical_doc_path: string | null; + share_id: string | null; +} + +type KnowledgeUnitRow = { + id: string; + session_id: string; + project_id: string | null; + topic: string; + category: string; + relevance: number; + entities: string | null; + files: string | null; + plain_text: string; + line_ranges: string | null; + content_hash: string; + tier: string; + retrieval_count: number; + last_recalled_at: string | null; + promoted_at: string | null; + canonical_doc_path: string | null; + share_id: string | null; +}; + +function deserializeKnowledgeUnit(row: KnowledgeUnitRow): StoredKnowledgeUnit { + return { + ...row, + entities: row.entities ? JSON.parse(row.entities) : [], + files: row.files ? JSON.parse(row.files) : [], + line_ranges: row.line_ranges ? JSON.parse(row.line_ranges) : [], + tier: row.tier as "segmented" | "canonical", + }; +} + +/** + * Insert a Stage-1 knowledge unit if its content hash isn't already stored. + * Returns true if inserted, false if it was a duplicate (caller distinguishes + * "stored" from "skipped" the same way shareSegmentedKnowledge does for shares). + */ +export function insertKnowledgeUnit( + db: Database, + unit: KnowledgeUnit, + sessionId: string, + projectId: string | null, + contentHash: string +): boolean { + const exists = db + .prepare(`SELECT 1 FROM smriti_knowledge_units WHERE content_hash = ?`) + .get(contentHash); + if (exists) return false; + + db.prepare( + `INSERT INTO smriti_knowledge_units + (id, session_id, project_id, topic, category, relevance, entities, files, plain_text, line_ranges, content_hash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + unit.id, + sessionId, + projectId, + unit.topic, + unit.category, + unit.relevance, + JSON.stringify(unit.entities || []), + JSON.stringify(unit.files || []), + unit.plainText, + JSON.stringify(unit.lineRanges || []), + contentHash + ); + return true; +} + +/** Dense sessions (by density_score) that haven't been segmented into knowledge units yet. */ +export function findUnsegmentedDenseSessions( + db: Database, + minDensity: number, + limit?: number +): Array<{ session_id: string; project_id: string | null; density_score: number }> { + const query = ` + SELECT sm.session_id, sm.project_id, sm.density_score + FROM smriti_session_meta sm + WHERE sm.density_score >= ? + AND NOT EXISTS (SELECT 1 FROM smriti_knowledge_units ku WHERE ku.session_id = sm.session_id) + ORDER BY sm.density_score DESC + ${limit ? "LIMIT ?" : ""} + `; + const rows = limit + ? db.prepare(query).all(minDensity, limit) + : db.prepare(query).all(minDensity); + return rows as Array<{ session_id: string; project_id: string | null; density_score: number }>; +} + +/** Segmented units that have proven reuse (via recall) or scored high relevance at extraction time. */ +export function findPromotableUnits( + db: Database, + minRetrievals: number, + minRelevance: number +): StoredKnowledgeUnit[] { + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units + WHERE tier = 'segmented' AND (retrieval_count >= ? OR relevance >= ?)` + ) + .all(minRetrievals, minRelevance) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + +/** Bump retrieval_count for any knowledge units belonging to a recalled session. No-op if none exist yet. */ +export function incrementRetrievalCount(db: Database, sessionId: string): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET retrieval_count = retrieval_count + 1, + last_recalled_at = datetime('now'), + updated_at = datetime('now') + WHERE session_id = ?` + ).run(sessionId); +} + +export function promoteKnowledgeUnit( + db: Database, + unitId: string, + canonicalDocPath: string, + shareId: string +): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET tier = 'canonical', promoted_at = datetime('now'), + canonical_doc_path = ?, share_id = ?, updated_at = datetime('now') + WHERE id = ?` + ).run(canonicalDocPath, shareId, unitId); +} + +export function listKnowledgeUnits( + db: Database, + options: { tier?: "segmented" | "canonical"; minRetrievals?: number; limit?: number } = {} +): StoredKnowledgeUnit[] { + const conditions: string[] = []; + const params: any[] = []; + + if (options.tier) { + conditions.push("tier = ?"); + params.push(options.tier); + } + if (options.minRetrievals !== undefined) { + conditions.push("retrieval_count >= ?"); + params.push(options.minRetrievals); + } + + const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; + const limitClause = options.limit ? "LIMIT ?" : ""; + if (options.limit) params.push(options.limit); + + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units ${where} + ORDER BY retrieval_count DESC, relevance DESC + ${limitClause}` + ) + .all(...params) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + // ============================================================================= // Session Query Labels (#60) // ============================================================================= diff --git a/src/format.ts b/src/format.ts index 023b0c3..e414186 100644 --- a/src/format.ts +++ b/src/format.ts @@ -268,6 +268,63 @@ export function formatShareResult(result: { return lines.join("\n"); } +// ============================================================================= +// Consolidate Result Formatting +// ============================================================================= + +export function formatConsolidateResult(result: { + sessionsSegmented: number; + unitsStored: number; + unitsSkipped: number; + unitsPromoted: number; + errors: string[]; +}): string { + const lines = [ + `Sessions segmented: ${result.sessionsSegmented}`, + `Units stored: ${result.unitsStored}`, + `Units skipped (dedup): ${result.unitsSkipped}`, + `Units promoted: ${result.unitsPromoted}`, + ]; + + if (result.errors.length > 0) { + lines.push(`Errors: ${result.errors.length}`); + for (const err of result.errors.slice(0, 5)) { + lines.push(` - ${err}`); + } + } + + return lines.join("\n"); +} + +// ============================================================================= +// Knowledge Units (Learnings) Formatting +// ============================================================================= + +export function formatLearnings( + units: Array<{ + tier: string; + topic: string; + category: string; + retrieval_count: number; + relevance: number; + canonical_doc_path: string | null; + }> +): string { + if (units.length === 0) return "No knowledge units found."; + + const headers = ["Tier", "Topic", "Category", "Retrievals", "Relevance", "Doc Path"]; + const rows = units.map((u) => [ + u.tier === "canonical" ? "✓ canonical" : "segmented", + u.topic, + u.category, + String(u.retrieval_count), + u.relevance.toFixed(1), + u.canonical_doc_path || "-", + ]); + + return table(headers, rows, [14, 40, 20, 10, 9, 40]); +} + // ============================================================================= // Sync Result Formatting // ============================================================================= diff --git a/src/index.ts b/src/index.ts index bb186d5..ecea67b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ * schema-based categorization, and team knowledge sharing. */ -import { initSmriti, closeDb, getCategories, getCategoryTree, addCategory, listProjects, tagSession, getProjectReport, getTagUsage, computeDensityScore, updateDensityScore, insertSessionQueries, getUnenrichedSessionIds } from "./db"; +import { initSmriti, closeDb, getCategories, getCategoryTree, addCategory, listProjects, tagSession, getProjectReport, getTagUsage, computeDensityScore, updateDensityScore, insertSessionQueries, getUnenrichedSessionIds, listKnowledgeUnits } from "./db"; import { getMessages, getSession, getMemoryStatus, embedMemoryMessages } from "./qmd"; import { ingest, ingestAll } from "./ingest/index"; import { categorizeUncategorized } from "./categorize/classifier"; @@ -16,6 +16,7 @@ import { searchFiltered, listSessions } from "./search/index"; import { recall } from "./search/recall"; import { shareKnowledge } from "./team/share"; import { syncTeamKnowledge, listTeamContributions } from "./team/sync"; +import { consolidateKnowledge } from "./learn/consolidate"; import { generateContext, compareSessions, @@ -53,6 +54,8 @@ import { formatTagUsage, formatDensityBreakdown, formatDigest, + formatConsolidateResult, + formatLearnings, json, } from "./format"; import { generateDigest } from "./digest"; @@ -213,6 +216,8 @@ Commands: compare Compare two sessions (tokens, tools, files) compare --last Compare last 2 sessions for current project share [filters] Export knowledge to .smriti/ + consolidate [options] Segment dense sessions, promote reused knowledge units + learnings [options] List extracted knowledge units (tier, retrievals, relevance) sync Import team knowledge from .smriti/ team View team contributions list [filters] List sessions @@ -812,6 +817,42 @@ async function main() { break; } + // ===================================================================== + // CONSOLIDATE + // ===================================================================== + case "consolidate": { + const result = await consolidateKnowledge(db, { + minDensity: Number(getArg(args, "--min-density")) || undefined, + minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, + minRelevance: Number(getArg(args, "--min-relevance")) || undefined, + model: getArg(args, "--model"), + outputDir: getArg(args, "--output"), + sessionLimit: Number(getArg(args, "--session-limit")) || undefined, + onProgress: (msg) => console.log(` ${msg}`), + }); + + console.log(formatConsolidateResult(result)); + break; + } + + // ===================================================================== + // LEARNINGS + // ===================================================================== + case "learnings": { + const units = listKnowledgeUnits(db, { + tier: getArg(args, "--tier") as "segmented" | "canonical" | undefined, + minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, + limit: Number(getArg(args, "--limit")) || 50, + }); + + if (hasFlag(args, "--json")) { + console.log(json(units)); + } else { + console.log(formatLearnings(units)); + } + break; + } + // ===================================================================== // SYNC // ===================================================================== diff --git a/src/learn/consolidate.ts b/src/learn/consolidate.ts new file mode 100644 index 0000000..b6d973b --- /dev/null +++ b/src/learn/consolidate.ts @@ -0,0 +1,196 @@ +/** + * learn/consolidate.ts - Continuous knowledge consolidation + * + * Progressive Summarization: cheap Stage-1 extraction runs broadly over dense + * sessions; expensive Stage-2 polish only runs once a unit proves it's reused + * (recalled repeatedly) or scored high relevance at extraction time. + * + * Two independent phases, run sequentially: + * - Segment: dense, not-yet-segmented sessions -> segmentSession() -> smriti_knowledge_units + * - Promote: knowledge units that cleared the reuse/relevance bar -> generateDocument() + * -> written to .smriti/knowledge/ + recorded in smriti_shares + * + * CLI-only, like `categorize`/`share` — never wired into the daemon (see + * src/daemon/index.ts's enrichOnIngest comment for why LLM work per-flush is unsafe). + */ + +import type { Database } from "bun:sqlite"; +import { mkdirSync } from "fs"; +import { join } from "path"; +import { SMRITI_DIR, AUTHOR } from "../config"; +import { hashContent } from "../qmd"; +import { + findUnsegmentedDenseSessions, + insertKnowledgeUnit, + findPromotableUnits, + promoteKnowledgeUnit, +} from "../db"; +import { getSessionMessages } from "../team/share"; +import { segmentSession } from "../team/segment"; +import { generateDocument, generateFrontmatter } from "../team/document"; +import { isSessionWorthSharing } from "../team/formatter"; +import type { RawMessage } from "../team/formatter"; +import type { KnowledgeUnit } from "../team/types"; + +// ============================================================================= +// Types +// ============================================================================= + +export type ConsolidateOptions = { + minDensity?: number; + minRetrievals?: number; + minRelevance?: number; + model?: string; + outputDir?: string; + author?: string; + sessionLimit?: number; + onProgress?: (msg: string) => void; +}; + +export type ConsolidateResult = { + sessionsSegmented: number; + unitsStored: number; + unitsSkipped: number; + unitsPromoted: number; + errors: string[]; +}; + +// ============================================================================= +// Consolidation +// ============================================================================= + +export async function consolidateKnowledge( + db: Database, + options: ConsolidateOptions = {} +): Promise { + const author = options.author || AUTHOR; + const outputDir = options.outputDir || join(process.cwd(), SMRITI_DIR); + + const result: ConsolidateResult = { + sessionsSegmented: 0, + unitsStored: 0, + unitsSkipped: 0, + unitsPromoted: 0, + errors: [], + }; + + // =========================================================================== + // Segment phase: cheap Stage-1 extraction over dense, unsegmented sessions + // =========================================================================== + + const sessions = findUnsegmentedDenseSessions( + db, + options.minDensity ?? 0.5, + options.sessionLimit ?? 20 + ); + + for (const s of sessions) { + try { + const messages = getSessionMessages(db, s.session_id); + if (messages.length === 0) continue; + + const rawMessages: RawMessage[] = messages.map((m) => ({ + role: m.role, + content: m.content, + })); + + if (!isSessionWorthSharing(rawMessages)) continue; + + const segmentationResult = await segmentSession(db, s.session_id, rawMessages, { + model: options.model, + }); + result.sessionsSegmented++; + + for (const unit of segmentationResult.units) { + const contentHash = await hashContent( + JSON.stringify({ topic: unit.topic, category: unit.category, plainText: unit.plainText }) + ); + const inserted = insertKnowledgeUnit(db, unit, s.session_id, s.project_id, contentHash); + inserted ? result.unitsStored++ : result.unitsSkipped++; + } + } catch (err: any) { + result.errors.push(`segment ${s.session_id}: ${err.message}`); + } + } + + options.onProgress?.( + `segment phase: ${result.sessionsSegmented} sessions, ${result.unitsStored} units stored, ${result.unitsSkipped} skipped` + ); + + // =========================================================================== + // Promote phase: expensive Stage-2 polish for units that proved reuse + // =========================================================================== + + const knowledgeDir = join(outputDir, "knowledge"); + mkdirSync(knowledgeDir, { recursive: true }); + + const promotable = findPromotableUnits( + db, + options.minRetrievals ?? 3, + options.minRelevance ?? 8 + ); + + for (const stored of promotable) { + try { + const unit: KnowledgeUnit = { + id: stored.id, + topic: stored.topic, + category: stored.category, + relevance: stored.relevance, + entities: stored.entities, + files: stored.files, + plainText: stored.plain_text, + lineRanges: stored.line_ranges, + }; + + const doc = await generateDocument(unit, stored.topic, { + model: options.model, + projectSmritiDir: outputDir, + author, + }); + + const categoryDir = join(knowledgeDir, doc.category.replaceAll("/", "-")); + mkdirSync(categoryDir, { recursive: true }); + const filePath = join(categoryDir, doc.filename); + + const fm = generateFrontmatter( + stored.session_id, + doc.unitId, + { ...doc.frontmatter, pipeline: "consolidated" }, + author, + stored.project_id || undefined + ); + await Bun.write(filePath, fm + "\n\n" + doc.markdown); + + const shareId = crypto.randomUUID().slice(0, 8); + const shareHash = await hashContent( + JSON.stringify({ content: doc.markdown, category: doc.category, entities: doc.frontmatter.entities }) + ); + + db.prepare( + `INSERT INTO smriti_shares (id, session_id, category_id, project_id, author, content_hash, unit_id, relevance_score, entities) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + shareId, + stored.session_id, + doc.category, + stored.project_id, + author, + shareHash, + doc.unitId, + stored.relevance, + JSON.stringify(stored.entities) + ); + + const relPath = `knowledge/${doc.category.replaceAll("/", "-")}/${doc.filename}`; + promoteKnowledgeUnit(db, stored.id, relPath, shareId); + result.unitsPromoted++; + } catch (err: any) { + result.errors.push(`promote ${stored.id}: ${err.message}`); + } + } + + options.onProgress?.(`promote phase: ${result.unitsPromoted} units promoted`); + + return result; +} diff --git a/src/search/recall.ts b/src/search/recall.ts index 1e219a4..f94fe7b 100644 --- a/src/search/recall.ts +++ b/src/search/recall.ts @@ -9,6 +9,7 @@ import { DEFAULT_RECALL_LIMIT, OLLAMA_HOST, OLLAMA_MODEL } from "../config"; import { recallMemories, ollamaRecall } from "../qmd"; import { searchFiltered, type SearchFilters, type SearchResult } from "./index"; import { getQmdStore } from "../store"; +import { incrementRetrievalCount } from "../db"; // ============================================================================= // Types @@ -27,6 +28,24 @@ export type RecallResult = { synthesis?: string; }; +// ============================================================================= +// Retrieval Tracking +// ============================================================================= + +/** + * Best-effort bump of retrieval_count for any consolidated knowledge units + * belonging to the recalled sessions. Never lets a tracking failure break recall. + */ +function trackRetrieval(db: Database, results: SearchResult[]): void { + try { + for (const sessionId of new Set(results.map((r) => r.session_id).filter(Boolean))) { + incrementRetrievalCount(db, sessionId); + } + } catch { + // Never let this break recall. + } +} + // ============================================================================= // Filtered Recall // ============================================================================= @@ -58,6 +77,7 @@ export async function recall( if (options.synthesize && storeResults.length > 0) { synthesis = await synthesizeResults(query, storeResults, options); } + trackRetrieval(db, storeResults); return { results: storeResults, synthesis }; } @@ -70,6 +90,7 @@ export async function recall( fast: options.fast, intent: rerankIntent, }); + trackRetrieval(db, qmdResult.results); return { results: qmdResult.results, synthesis: qmdResult.synthesis, @@ -102,6 +123,7 @@ export async function recall( synthesis = await synthesizeResults(query, deduped, options); } + trackRetrieval(db, deduped); return { results: deduped, synthesis }; } diff --git a/src/team/share.ts b/src/team/share.ts index 06c44e5..0d61733 100644 --- a/src/team/share.ts +++ b/src/team/share.ts @@ -140,7 +140,7 @@ function querySessions( } /** Get messages for a session */ -function getSessionMessages( +export function getSessionMessages( db: Database, sessionId: string ): Array<{ diff --git a/test/learn-consolidate.test.ts b/test/learn-consolidate.test.ts new file mode 100644 index 0000000..8748d8c --- /dev/null +++ b/test/learn-consolidate.test.ts @@ -0,0 +1,283 @@ +/** + * test/learn-consolidate.test.ts - Tests for continuous knowledge consolidation + * + * Mirrors test/team-segmented.test.ts's style: initSmriti(":memory:"), a + * mocked global.fetch standing in for Ollama, and a scratch tmpDir for + * filesystem output (never process.cwd() — consolidateKnowledge writes real + * files). + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync, writeFileSync, existsSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { + initSmriti, + closeDb, + upsertSessionMeta, + upsertProject, + updateDensityScore, + insertKnowledgeUnit, + listKnowledgeUnits, +} from "../src/db"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import { recall } from "../src/search/recall"; +import type { KnowledgeUnit } from "../src/team/types"; + +// ============================================================================= +// Setup +// ============================================================================= + +let db: Database; +let tmpDir: string; + +beforeAll(async () => { + db = await initSmriti(":memory:"); + tmpDir = join(tmpdir(), `smriti-consolidate-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(tmpDir, { recursive: true }); } catch {} +}); + +/** Insert a session with real message rows so getSessionMessages() finds content. */ +function seedSession(sessionId: string, projectId: string, messages: Array<{ role: string; content: string }>) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +const DENSE_CONVERSATION = [ + { role: "user", content: "I'm getting a JWT token expiry issue. Sessions timeout after 1 hour but tests expect 24 hours." }, + { role: "assistant", content: "Let me look at the auth middleware to understand the token expiry logic." }, + { role: "user", content: "Found it — src/auth.ts hardcodes 3600 seconds instead of reading JWT_TTL from the environment." }, + { role: "assistant", content: "Updated it to use process.env.JWT_TTL || 3600. Tests pass now." }, +]; + +/** Mock fetch that distinguishes Stage 1 (segmentation) vs Stage 2 (document) calls by prompt content. */ +function mockOllamaFetch(stage1Response: () => object) { + return mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const isStage1 = (body.prompt as string).includes("Knowledge Unit Segmentation"); + if (isStage1) { + return new Response( + JSON.stringify({ response: "```json\n" + JSON.stringify(stage1Response()) + "\n```" }), + { status: 200 } + ); + } + return new Response( + JSON.stringify({ response: "# Consolidated Doc\n\nPolished content." }), + { status: 200 } + ); + }); +} + +// ============================================================================= +// Segment-phase dedup +// ============================================================================= + +test("consolidate dedups knowledge units with identical content across sessions", async () => { + seedSession("dedup-s1", "dedupproj", DENSE_CONVERSATION); + seedSession("dedup-s2", "dedupproj", DENSE_CONVERSATION); + updateDensityScore(db, "dedup-s1", 0.9); + updateDensityScore(db, "dedup-s2", 0.9); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ + units: [{ topic: "JWT token expiry bug", category: "bug/fix", relevance: 9, entities: ["JWT"] }], + })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 0.5, + outputDir: join(tmpDir, "dedup-output"), + }); + + expect(result.sessionsSegmented).toBe(2); + expect(result.unitsStored).toBe(1); + expect(result.unitsSkipped).toBe(1); + expect(result.errors).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Promotion threshold +// ============================================================================= + +test("promote phase only promotes units clearing the retrieval/relevance bar", async () => { + const belowThreshold: KnowledgeUnit = { + id: "unit-below", + topic: "Minor formatting note", + category: "code/pattern", + relevance: 3, + entities: [], + files: [], + plainText: "Use consistent indentation.", + lineRanges: [{ start: 0, end: 1 }], + }; + const aboveThreshold: KnowledgeUnit = { + id: "unit-above", + topic: "Redis caching decision", + category: "architecture/decision", + relevance: 9, + entities: ["Redis"], + files: [], + plainText: "Use Redis with a 5-minute TTL for API responses.", + lineRanges: [{ start: 0, end: 1 }], + }; + + insertKnowledgeUnit(db, belowThreshold, "promote-s1", "promoteproj", "hash-below"); + insertKnowledgeUnit(db, aboveThreshold, "promote-s2", "promoteproj", "hash-above"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ units: [] })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, // no sessions qualify for the segment phase — isolates promote phase + minRetrievals: 3, + minRelevance: 8, + outputDir: join(tmpDir, "promote-output"), + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const canonical = listKnowledgeUnits(db, { tier: "canonical" }); + expect(canonical.map((u) => u.id)).toContain("unit-above"); + expect(canonical.map((u) => u.id)).not.toContain("unit-below"); + + const promoted = canonical.find((u) => u.id === "unit-above")!; + expect(promoted.canonical_doc_path).toContain("architecture-decision"); + expect(promoted.share_id).toBeTruthy(); + + const shareRow = db + .prepare(`SELECT * FROM smriti_shares WHERE unit_id = ?`) + .get("unit-above") as any; + expect(shareRow).toBeTruthy(); + expect(shareRow.session_id).toBe("promote-s2"); + + const writtenFile = join(tmpDir, "promote-output", promoted.canonical_doc_path!); + expect(existsSync(writtenFile)).toBe(true); + + const stillSegmented = listKnowledgeUnits(db, { tier: "segmented" }); + expect(stillSegmented.map((u) => u.id)).toContain("unit-below"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Graceful degradation +// ============================================================================= + +test("promote phase continues past a per-unit failure and records the error", async () => { + // Pre-create a *file* at the exact path the loop will try to mkdir for + // category "bug/fix" (slug "bug-fix"), forcing that unit's write to throw. + const outputDir = join(tmpDir, "degrade-output"); + const knowledgeDir = join(outputDir, "knowledge"); + mkdirSync(knowledgeDir, { recursive: true }); + writeFileSync(join(knowledgeDir, "bug-fix"), "occupied"); + + const willFail: KnowledgeUnit = { + id: "unit-fails", + topic: "Broken unit", + category: "bug/fix", + relevance: 9, + entities: [], + files: [], + plainText: "This unit's category dir collides with a file.", + lineRanges: [{ start: 0, end: 1 }], + }; + const willSucceed: KnowledgeUnit = { + id: "unit-succeeds", + topic: "Working unit", + category: "topic/learning", + relevance: 9, + entities: [], + files: [], + plainText: "This unit writes fine.", + lineRanges: [{ start: 0, end: 1 }], + }; + + insertKnowledgeUnit(db, willFail, "degrade-s1", "degradeproj", "hash-fails"); + insertKnowledgeUnit(db, willSucceed, "degrade-s2", "degradeproj", "hash-succeeds"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ units: [] })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, // exclude leftover segmented units from earlier tests; only relevance-9 units below qualify + minRelevance: 8, + outputDir, + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors.length).toBe(1); + expect(result.errors[0]).toContain("unit-fails"); + + const canonical = listKnowledgeUnits(db, { tier: "canonical" }); + expect(canonical.map((u) => u.id)).toContain("unit-succeeds"); + expect(canonical.map((u) => u.id)).not.toContain("unit-fails"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Retrieval tracking (recall -> incrementRetrievalCount) +// ============================================================================= + +test("recall increments retrieval_count for knowledge units of the recalled session", async () => { + seedSession("track-s1", "trackproj", [ + { role: "user", content: "How do we configure the rate limiter for the public API?" }, + { role: "assistant", content: "Use a token bucket with 100 requests per minute per API key." }, + ]); + + const unit: KnowledgeUnit = { + id: "unit-tracked", + topic: "Rate limiter config", + category: "code/pattern", + relevance: 7, + entities: [], + files: [], + plainText: "Token bucket rate limiting for the public API.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, unit, "track-s1", "trackproj", "hash-tracked"); + + await recall(db, "rate limiter", { project: "trackproj" }); + + const tracked = db + .prepare(`SELECT retrieval_count FROM smriti_knowledge_units WHERE id = ?`) + .get("unit-tracked") as { retrieval_count: number }; + expect(tracked).toBeDefined(); + expect(tracked.retrieval_count).toBeGreaterThanOrEqual(1); +}); + +test("recall does not throw for sessions with no consolidated knowledge units", async () => { + seedSession("untracked-s1", "untrackedproj", [ + { role: "user", content: "What's our deploy process look like?" }, + { role: "assistant", content: "Push to main triggers the CI pipeline and auto-deploys." }, + ]); + + await expect(recall(db, "deploy process", { project: "untrackedproj" })).resolves.toBeDefined(); +});