From 31dd61dacf481d2419c5350f0cd08f3a00d28a41 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 14:45:28 +0000 Subject: [PATCH 01/83] refactor(cli): extract confirm and bulk-command helpers - Add confirmOrCancel() to cli/confirm.ts and use it at all 13 confirm sites - Extract addBulkFilterOptions/addBulkRunOptions/selectorFromOpts/runBulk for bulk delete/retag/move - Use initializeAppWithEmbedding() in connect obsidian - Use static connector-config imports in schedule set/remove - Add splitCsv() for identical comma-split option parsing --- src/cli/commands/registry.ts | 12 +- src/cli/confirm.ts | 15 ++ src/cli/index.ts | 360 ++++++++++++++--------------------- tests/unit/confirm.test.ts | 28 ++- 4 files changed, 185 insertions(+), 230 deletions(-) diff --git a/src/cli/commands/registry.ts b/src/cli/commands/registry.ts index cc7645b5..3686bec0 100644 --- a/src/cli/commands/registry.ts +++ b/src/cli/commands/registry.ts @@ -23,7 +23,7 @@ import { getRegistryCacheDir } from "../../registry/types.js"; import { syncRegistryByName, syncAllRegistries } from "../../registry/sync.js"; import { searchRegistries } from "../../registry/search.js"; import { publishPack, publishPackToBranch, unpublishPack } from "../../registry/publish.js"; -import { confirmAction } from "../confirm.js"; +import { confirmOrCancel } from "../confirm.js"; /** Derive a short name from a git URL (e.g. "github.com/org/repo" → "repo"). */ function deriveNameFromUrl(url: string): string { @@ -215,12 +215,11 @@ export function registerRegistryCommands(program: Command): void { .option("-y, --yes", "Skip confirmation prompt") .action(async (name: string, opts: { yes?: boolean }) => { if ( - !(await confirmAction( + !(await confirmOrCancel( `Remove registry "${name}" and its local cache? This cannot be undone.`, - !!opts.yes, + opts.yes, )) ) { - console.log("Cancelled."); return; } @@ -408,12 +407,11 @@ export function registerRegistryCommands(program: Command): void { await requireGit(); if ( - !(await confirmAction( + !(await confirmOrCancel( `Unpublish "${packName}@${opts.version}" from "${opts.registry}"? This cannot be undone.`, - !!opts.yes, + opts.yes, )) ) { - console.log("Cancelled."); return; } diff --git a/src/cli/confirm.ts b/src/cli/confirm.ts index 36871a1d..97862e17 100644 --- a/src/cli/confirm.ts +++ b/src/cli/confirm.ts @@ -22,3 +22,18 @@ export async function confirmAction( rl.close(); } } + +/** + * Like {@link confirmAction}, but prints "Cancelled." when the user declines. + * Returns true when the caller should proceed. + */ +export async function confirmOrCancel( + message: string, + yes: boolean | undefined, + /** Overridable for testing */ + createInterface?: () => readline.Interface, +): Promise { + if (await confirmAction(message, !!yes, createInterface)) return true; + console.log("Cancelled."); + return false; +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 36ceacde..00556add 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -33,7 +33,7 @@ import { getKnowledgeGaps, } from "../core/analytics.js"; import { startRepl } from "./repl.js"; -import { confirmAction } from "./confirm.js"; +import { confirmOrCancel } from "./confirm.js"; import { createReporter, isVerbose } from "./reporter.js"; import { addTagsToDocument, @@ -43,7 +43,7 @@ import { suggestTags, } from "../core/tags.js"; import { bulkDelete, bulkRetag, bulkMove } from "../core/bulk.js"; -import type { BulkSelector } from "../core/bulk.js"; +import type { BulkResult, BulkSelector } from "../core/bulk.js"; import { createWorkspace, deleteWorkspace, @@ -878,7 +878,7 @@ program try { const limit = parseIntOption(opts.limit, "--limit"); const minScore = opts.minScore !== undefined ? Number.parseFloat(opts.minScore) : undefined; - const tags = opts.tags ? opts.tags.split(",").map((t) => t.trim()) : undefined; + const tags = opts.tags ? splitCsv(opts.tags) : undefined; let result; try { @@ -1275,12 +1275,11 @@ docsCmd try { const doc = getDocument(db, documentId); if ( - !(await confirmAction( + !(await confirmOrCancel( `Delete document "${doc.title}" (${documentId})? This cannot be undone.`, - !!opts.yes, + opts.yes, )) ) { - console.log("Cancelled."); return; } deleteDocument(db, documentId); @@ -1604,6 +1603,11 @@ function initializeAppWithEmbedding(): { return { config, db, provider }; } +/** Split a comma-separated option value and trim each entry. */ +function splitCsv(value: string): string[] { + return value.split(",").map((t) => t.trim()); +} + /** Recursively find files matching given extensions. */ function findFiles(dir: string, extensions: Set): string[] { const results: string[] = []; @@ -2075,8 +2079,7 @@ workspaceCmd .description("Delete a workspace") .option("-y, --yes", "Skip confirmation prompt") .action(async (name: string, opts: { yes?: boolean }) => { - if (!(await confirmAction(`Delete workspace "${name}"? This cannot be undone.`, !!opts.yes))) { - console.log("Cancelled."); + if (!(await confirmOrCancel(`Delete workspace "${name}"? This cannot be undone.`, opts.yes))) { return; } deleteWorkspace(name); @@ -2122,12 +2125,11 @@ packCmd .option("-y, --yes", "Skip confirmation prompt") .action(async (name: string, opts: { yes?: boolean }) => { if ( - !(await confirmAction( + !(await confirmOrCancel( `Remove pack "${name}" and its documents? This cannot be undone.`, - !!opts.yes, + opts.yes, )) ) { - console.log("Cancelled."); return; } const { db } = initializeApp(); @@ -2206,9 +2208,7 @@ packCmd // Source mode: build pack directly from files/URLs (no database needed) // Default to .json.gz for source packs (they can be large) const outputPath = opts.output ?? `${opts.name}.json.gz`; - const extensionList = opts.extensions - ? opts.extensions.split(",").map((e) => e.trim()) - : undefined; + const extensionList = opts.extensions ? splitCsv(opts.extensions) : undefined; const pack = await createPackFromSource({ name: opts.name, @@ -2385,12 +2385,11 @@ disconnectCmd .option("-y, --yes", "Skip confirmation prompt") .action(async (opts: { yes?: boolean }) => { if ( - !(await confirmAction( + !(await confirmOrCancel( "Disconnect OneNote and remove all its data? This cannot be undone.", - !!opts.yes, + opts.yes, )) ) { - console.log("Cancelled."); return; } const config = loadConfig(); @@ -2429,9 +2428,7 @@ connectCmd vaultPath: string, cmdOpts: { sync?: boolean; topicMapping?: string; exclude?: string[] }, ) => { - const { config, db } = initializeApp(); - const provider = createEmbeddingProvider(config); - createVectorTable(db, provider.dimensions); + const { db, provider } = initializeAppWithEmbedding(); try { const { syncObsidianVault } = await import("../connectors/obsidian.js"); @@ -2507,13 +2504,13 @@ connectCmd } slackConfig = { token: opts.token, - channels: opts.channels.split(",").map((c) => c.trim()), + channels: splitCsv(opts.channels), threadMode: opts.threadMode === "separate" ? "separate" : "aggregate", }; if (opts.exclude) { slackConfig = { ...slackConfig, - excludeChannels: opts.exclude.split(",").map((c) => c.trim()), + excludeChannels: splitCsv(opts.exclude), }; } } @@ -2571,10 +2568,8 @@ connectCmd const email = opts.email ?? process.env["CONFLUENCE_EMAIL"] ?? undefined; const token = opts.token ?? process.env["CONFLUENCE_TOKEN"] ?? ""; - const spaces = (opts.spaces ?? "all").split(",").map((s) => s.trim()); - const excludeSpaces = opts.excludeSpaces - ? opts.excludeSpaces.split(",").map((s) => s.trim()) - : undefined; + const spaces = splitCsv(opts.spaces ?? "all"); + const excludeSpaces = opts.excludeSpaces ? splitCsv(opts.excludeSpaces) : undefined; const result = await syncConfluence(db, provider, { baseUrl: url, @@ -2607,12 +2602,11 @@ disconnectCmd .option("-y, --yes", "Skip confirmation prompt") .action(async (vaultPath: string, opts: { yes?: boolean }) => { if ( - !(await confirmAction( + !(await confirmOrCancel( `Disconnect Obsidian vault "${vaultPath}" and remove its documents? This cannot be undone.`, - !!opts.yes, + opts.yes, )) ) { - console.log("Cancelled."); return; } const { db } = initializeApp(); @@ -2678,12 +2672,11 @@ disconnectCmd .option("-y, --yes", "Skip confirmation prompt") .action(async (opts: { yes?: boolean }) => { if ( - !(await confirmAction( + !(await confirmOrCancel( "Disconnect Notion and remove all its documents? This cannot be undone.", - !!opts.yes, + opts.yes, )) ) { - console.log("Cancelled."); return; } const { db } = initializeApp(); @@ -2701,12 +2694,11 @@ disconnectCmd .option("-y, --yes", "Skip confirmation prompt") .action(async (opts: { yes?: boolean }) => { if ( - !(await confirmAction( + !(await confirmOrCancel( "Disconnect Slack and remove all its data? This cannot be undone.", - !!opts.yes, + opts.yes, )) ) { - console.log("Cancelled."); return; } const { db } = initializeApp(); @@ -2724,12 +2716,11 @@ disconnectCmd .option("-y, --yes", "Skip confirmation prompt") .action(async (opts: { yes?: boolean }) => { if ( - !(await confirmAction( + !(await confirmOrCancel( "Disconnect Confluence and remove all synced content? This cannot be undone.", - !!opts.yes, + opts.yes, )) ) { - console.log("Cancelled."); return; } const { disconnectConfluence } = await import("../connectors/confluence.js"); @@ -2781,182 +2772,121 @@ program // bulk const bulkCmd = program.command("bulk").description("Bulk operations on documents"); -bulkCmd - .command("delete") - .description("Delete multiple documents matching filters") - .option("--topic ", "Filter by topic ID") - .option("--library ", "Filter by library name") - .option("--source-type ", "Filter by source type") - .option("--tags ", "Filter by tags (comma-separated)") - .option("--dry-run", "Show what would be affected without making changes") - .option("-y, --yes", "Skip confirmation prompt") - .action( - async (opts: { - topic?: string; - library?: string; - sourceType?: string; - tags?: string; - dryRun?: boolean; - yes?: boolean; - }) => { - const { db } = initializeApp(); - try { - const selector: BulkSelector = {}; - if (opts.topic) selector.topicId = opts.topic; - if (opts.library) selector.library = opts.library; - if (opts.sourceType) selector.sourceType = opts.sourceType; - if (opts.tags) selector.tags = opts.tags.split(",").map((t) => t.trim()); - - const result = bulkDelete(db, selector, true); - console.log(`Found ${result.affected} document(s) matching filters.`); - - if (result.affected === 0) return; - - if (opts.dryRun) { - for (const id of result.documentIds) { - console.log(` - ${id}`); - } - console.log("(dry run — no changes made)"); - return; - } - - if ( - !(await confirmAction( - `Delete ${result.affected} document(s)? This cannot be undone.`, - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - - const actual = bulkDelete(db, selector); - console.log(`✓ Deleted ${actual.affected} document(s).`); - } finally { - closeDatabase(); - } - }, - ); +interface BulkFilterOpts { + topic?: string; + library?: string; + sourceType?: string; + tags?: string; + dryRun?: boolean; + yes?: boolean; +} -bulkCmd - .command("retag") - .description("Add or remove tags from multiple documents") - .option("--topic ", "Filter by topic ID") - .option("--library ", "Filter by library name") - .option("--source-type ", "Filter by source type") - .option("--tags ", "Filter by tags (comma-separated)") - .option("--add-tags ", "Tags to add (comma-separated)") - .option("--remove-tags ", "Tags to remove (comma-separated)") - .option("--dry-run", "Show what would be affected without making changes") - .option("-y, --yes", "Skip confirmation prompt") - .action( - async (opts: { - topic?: string; - library?: string; - sourceType?: string; - tags?: string; - addTags?: string; - removeTags?: string; - dryRun?: boolean; - yes?: boolean; - }) => { - const { db } = initializeApp(); - try { - const selector: BulkSelector = {}; - if (opts.topic) selector.topicId = opts.topic; - if (opts.library) selector.library = opts.library; - if (opts.sourceType) selector.sourceType = opts.sourceType; - if (opts.tags) selector.tags = opts.tags.split(",").map((t) => t.trim()); - - const addTags = opts.addTags ? opts.addTags.split(",").map((t) => t.trim()) : undefined; - const removeTags = opts.removeTags - ? opts.removeTags.split(",").map((t) => t.trim()) - : undefined; +/** Add the document filter options shared by all bulk commands. */ +function addBulkFilterOptions(cmd: Command): Command { + return cmd + .option("--topic ", "Filter by topic ID") + .option("--library ", "Filter by library name") + .option("--source-type ", "Filter by source type") + .option("--tags ", "Filter by tags (comma-separated)"); +} - const result = bulkRetag(db, selector, addTags, removeTags, true); - console.log(`Found ${result.affected} document(s) matching filters.`); +/** Add the --dry-run and --yes options shared by all bulk commands. */ +function addBulkRunOptions(cmd: Command): Command { + return cmd + .option("--dry-run", "Show what would be affected without making changes") + .option("-y, --yes", "Skip confirmation prompt"); +} - if (result.affected === 0) return; +function selectorFromOpts(opts: BulkFilterOpts): BulkSelector { + const selector: BulkSelector = {}; + if (opts.topic) selector.topicId = opts.topic; + if (opts.library) selector.library = opts.library; + if (opts.sourceType) selector.sourceType = opts.sourceType; + if (opts.tags) selector.tags = splitCsv(opts.tags); + return selector; +} - if (opts.dryRun) { - for (const id of result.documentIds) { - console.log(` - ${id}`); - } - console.log("(dry run — no changes made)"); - return; - } +/** + * Shared bulk flow: preview matches, list them on --dry-run, confirm, then apply. + * `run(db, dryRun)` performs the bulk operation (as a preview when dryRun is true). + */ +async function runBulk( + opts: BulkFilterOpts, + run: (db: ReturnType, dryRun?: boolean) => BulkResult, + confirmMessage: (affected: number) => string, + doneMessage: (affected: number) => string, +): Promise { + const { db } = initializeApp(); + try { + const result = run(db, true); + console.log(`Found ${result.affected} document(s) matching filters.`); - if (!(await confirmAction(`Retag ${result.affected} document(s)?`, !!opts.yes))) { - console.log("Cancelled."); - return; - } + if (result.affected === 0) return; - const actual = bulkRetag(db, selector, addTags, removeTags); - console.log(`✓ Retagged ${actual.affected} document(s).`); - } finally { - closeDatabase(); + if (opts.dryRun) { + for (const id of result.documentIds) { + console.log(` - ${id}`); } - }, - ); - -bulkCmd - .command("move") - .description("Move multiple documents to a different topic") - .option("--topic ", "Filter by topic ID") - .option("--library ", "Filter by library name") - .option("--source-type ", "Filter by source type") - .option("--tags ", "Filter by tags (comma-separated)") - .requiredOption("--to ", "Target topic ID to move documents to") - .option("--dry-run", "Show what would be affected without making changes") - .option("-y, --yes", "Skip confirmation prompt") - .action( - async (opts: { - topic?: string; - library?: string; - sourceType?: string; - tags?: string; - to: string; - dryRun?: boolean; - yes?: boolean; - }) => { - const { db } = initializeApp(); - try { - const selector: BulkSelector = {}; - if (opts.topic) selector.topicId = opts.topic; - if (opts.library) selector.library = opts.library; - if (opts.sourceType) selector.sourceType = opts.sourceType; - if (opts.tags) selector.tags = opts.tags.split(",").map((t) => t.trim()); - - const result = bulkMove(db, selector, opts.to, true); - console.log(`Found ${result.affected} document(s) matching filters.`); + console.log("(dry run — no changes made)"); + return; + } - if (result.affected === 0) return; + if (!(await confirmOrCancel(confirmMessage(result.affected), opts.yes))) { + return; + } - if (opts.dryRun) { - for (const id of result.documentIds) { - console.log(` - ${id}`); - } - console.log("(dry run — no changes made)"); - return; - } + const actual = run(db); + console.log(doneMessage(actual.affected)); + } finally { + closeDatabase(); + } +} - if ( - !(await confirmAction( - `Move ${result.affected} document(s) to topic "${opts.to}"?`, - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } +addBulkRunOptions( + addBulkFilterOptions( + bulkCmd.command("delete").description("Delete multiple documents matching filters"), + ), +).action(async (opts: BulkFilterOpts) => { + const selector = selectorFromOpts(opts); + await runBulk( + opts, + (db, dryRun) => bulkDelete(db, selector, dryRun), + (n) => `Delete ${n} document(s)? This cannot be undone.`, + (n) => `✓ Deleted ${n} document(s).`, + ); +}); - const actual = bulkMove(db, selector, opts.to); - console.log(`✓ Moved ${actual.affected} document(s) to topic "${opts.to}".`); - } finally { - closeDatabase(); - } - }, +addBulkRunOptions( + addBulkFilterOptions( + bulkCmd.command("retag").description("Add or remove tags from multiple documents"), + ) + .option("--add-tags ", "Tags to add (comma-separated)") + .option("--remove-tags ", "Tags to remove (comma-separated)"), +).action(async (opts: BulkFilterOpts & { addTags?: string; removeTags?: string }) => { + const selector = selectorFromOpts(opts); + const addTags = opts.addTags ? splitCsv(opts.addTags) : undefined; + const removeTags = opts.removeTags ? splitCsv(opts.removeTags) : undefined; + await runBulk( + opts, + (db, dryRun) => bulkRetag(db, selector, addTags, removeTags, dryRun), + (n) => `Retag ${n} document(s)?`, + (n) => `✓ Retagged ${n} document(s).`, ); +}); + +addBulkRunOptions( + addBulkFilterOptions( + bulkCmd.command("move").description("Move multiple documents to a different topic"), + ).requiredOption("--to ", "Target topic ID to move documents to"), +).action(async (opts: BulkFilterOpts & { to: string }) => { + const selector = selectorFromOpts(opts); + await runBulk( + opts, + (db, dryRun) => bulkMove(db, selector, opts.to, dryRun), + (n) => `Move ${n} document(s) to topic "${opts.to}"?`, + (n) => `✓ Moved ${n} document(s) to topic "${opts.to}".`, + ); +}); // Webhooks const webhookCmd = program.command("webhooks").description("Manage webhooks"); @@ -3081,22 +3011,16 @@ scheduleCmd process.exit(1); } - const { - loadNamedConnectorConfig: loadCfg, - saveNamedConnectorConfig: saveCfg, - hasNamedConnectorConfig: hasCfg, - } = await import("../connectors/index.js"); - - if (!hasCfg(connector)) { + if (!hasNamedConnectorConfig(connector)) { console.error( `No connector config found for "${connector}". Run 'libscope connect ${connector}' first.`, ); process.exit(1); } - const config = loadCfg>(connector); + const config = loadNamedConnectorConfig>(connector); config.schedule = { cronExpression: cronExpr }; - saveCfg(connector, config); + saveNamedConnectorConfig(connector, config); console.log(`✓ Schedule set for ${connector}: ${cronExpr}`); console.log( "The schedule will be active when the API server is running (libscope serve --api)", @@ -3106,25 +3030,19 @@ scheduleCmd scheduleCmd .command("remove ") .description("Remove the sync schedule for a connector") - .action(async (connector: string) => { - const { - loadNamedConnectorConfig: loadCfg, - saveNamedConnectorConfig: saveCfg, - hasNamedConnectorConfig: hasCfg, - } = await import("../connectors/index.js"); - - if (!hasCfg(connector)) { + .action((connector: string) => { + if (!hasNamedConnectorConfig(connector)) { console.error(`No connector config found for "${connector}".`); process.exit(1); } - const config = loadCfg>(connector); + const config = loadNamedConnectorConfig>(connector); if (!config.schedule) { console.error(`No schedule configured for "${connector}". Nothing to remove.`); process.exit(1); } delete config.schedule; - saveCfg(connector, config); + saveNamedConnectorConfig(connector, config); console.log(`✓ Schedule removed for ${connector}`); }); diff --git a/tests/unit/confirm.test.ts b/tests/unit/confirm.test.ts index 1a81b025..61a71ad8 100644 --- a/tests/unit/confirm.test.ts +++ b/tests/unit/confirm.test.ts @@ -1,5 +1,5 @@ -import { describe, it, expect, vi } from "vitest"; -import { confirmAction } from "../../src/cli/confirm.js"; +import { afterEach, describe, it, expect, vi } from "vitest"; +import { confirmAction, confirmOrCancel } from "../../src/cli/confirm.js"; import { EventEmitter } from "node:events"; function createMockInterface(answer: string) { @@ -55,3 +55,27 @@ describe("confirmAction", () => { expect(rl.close).toHaveBeenCalled(); }); }); + +describe("confirmOrCancel", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("returns true without printing when yes flag is set", async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + expect(await confirmOrCancel("Delete?", true)).toBe(true); + expect(log).not.toHaveBeenCalled(); + }); + + it("returns true without printing when user confirms", async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + expect(await confirmOrCancel("Delete?", undefined, createMockInterface("y"))).toBe(true); + expect(log).not.toHaveBeenCalled(); + }); + + it("prints 'Cancelled.' and returns false when user declines", async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + expect(await confirmOrCancel("Delete?", false, createMockInterface("n"))).toBe(false); + expect(log).toHaveBeenCalledWith("Cancelled."); + }); +}); From 8b3b8065c0a52bcdd92fcfc049fbc062b5f85527 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 14:46:13 +0000 Subject: [PATCH 02/83] refactor(mcp,api): dedupe sync/async tool formatting and route matching - mcp: add runMaybeAsync() so each async-capable tool formats its output once - mcp: add internal textResult() helper and use ToolResult instead of inline types - mcp: collapse duplicate cancelled branches in startAsyncTask; drop dead fallbacks - api: replace hand-written match*/is*Path/dispatch* helpers with a pattern route table - api: remove dead placeholder/never-read state in spider handler --- src/api/routes.ts | 396 ++++++----------------------- src/mcp/errors.ts | 5 + src/mcp/server.ts | 627 +++++++++++++++------------------------------- 3 files changed, 277 insertions(+), 751 deletions(-) diff --git a/src/api/routes.ts b/src/api/routes.ts index b43d6d82..ffb061b7 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -55,7 +55,7 @@ import { import type { WebhookEvent } from "../core/webhooks.js"; import { loadScheduleEntries } from "../core/scheduler.js"; import { spiderUrl } from "../core/spider.js"; -import type { SpiderOptions, SpiderStats } from "../core/spider.js"; +import type { SpiderOptions } from "../core/spider.js"; import { loadReposConfig, createRepoLibScope } from "./indexing/repoConfig.js"; import { indexRepo, type IndexJobStats } from "./indexing/repoIndexer.js"; @@ -101,133 +101,6 @@ function extractPathSegments(pathname: string): string[] { return pathname.split("/").filter(Boolean); } -/** Match a path like /api/v1/documents/:id and return the id, or null. */ -function matchDocumentId(segments: string[]): string | null { - // ["api", "v1", "documents", ""] - if ( - segments.length === 4 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "documents" - ) { - const id = segments[3]; - // Exclude sub-paths that are named routes - if (id === "url") return null; - return id ?? null; - } - return null; -} - -/** Match /api/v1/documents/:id/tags */ -function matchDocumentTags(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "documents" && - segments[4] === "tags" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/documents/:id/suggest-tags */ -function matchDocumentSuggestTags(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "documents" && - segments[4] === "suggest-tags" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/documents/:id/links */ -function matchDocumentLinks(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "documents" && - segments[4] === "links" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/links/:id */ -function matchLinkId(segments: string[]): string | null { - if ( - segments.length === 4 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "links" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/searches/:id */ -function matchSearchId(segments: string[]): string | null { - if ( - segments.length === 4 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "searches" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/searches/:id/run */ -function matchSearchRun(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "searches" && - segments[4] === "run" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/webhooks/:id */ -function matchWebhookId(segments: string[]): string | null { - if ( - segments.length === 4 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "webhooks" - ) { - const id = segments[3]; - return id ?? null; - } - return null; -} - -/** Match /api/v1/webhooks/:id/test */ -function matchWebhookTest(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "webhooks" && - segments[4] === "test" - ) { - return segments[3] ?? null; - } - return null; -} - // --------------------------------------------------------------------------- // Parameter parsing helpers // --------------------------------------------------------------------------- @@ -466,7 +339,6 @@ async function handleSpiderUrl( const spiderOptions = buildSpiderOptions(b, fetchOptions); const indexedDocs: Array<{ id: string; title: string; url: string }> = []; const errors: Array<{ url: string; error: string }> = []; - let stats: SpiderStats = { pagesFetched: 0, pagesCrawled: 0, pagesSkipped: 0, errors }; const gen = spiderUrl(urlStr, spiderOptions); let result = await gen.next(); @@ -488,10 +360,7 @@ async function handleSpiderUrl( result = await gen.next(); } // result.value is SpiderStats when done (generator is exhausted) - if (result.done && result.value) { - stats = result.value; - stats.errors = errors; - } + const stats = result.value; sendJson( ctx.res, @@ -925,20 +794,6 @@ function handleDeleteWebhook(ctx: RouteContext, webhookId: string): void { // Repo indexing webhook handler // --------------------------------------------------------------------------- -/** Match /api/v1/index/jobs/:jobId */ -function matchIndexJobId(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "index" && - segments[3] === "jobs" - ) { - return segments[4] ?? null; - } - return null; -} - function handleGetIndexJob(ctx: RouteContext, jobId: string): void { const job = repoIndexJobs.get(jobId); if (!job) { @@ -948,20 +803,6 @@ function handleGetIndexJob(ctx: RouteContext, jobId: string): void { sendJson(ctx.res, 200, job, elapsed(ctx.start)); } -/** Match /api/v1/index/repos/:repoSlug */ -function matchRepoSlugForIndex(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "index" && - segments[3] === "repos" - ) { - return segments[4] ?? null; - } - return null; -} - async function handleIndexRepo(ctx: RouteContext, repoSlug: string): Promise { const log = getLogger(); @@ -1057,164 +898,6 @@ function handleRouteError( sendError(res, 500, "INTERNAL_ERROR", "Internal server error"); } -// --------------------------------------------------------------------------- -// Path-based route matching helpers (for segment-matched routes) -// --------------------------------------------------------------------------- - -function isApiV1Path(segments: string[], resource: string): boolean { - return ( - segments.length === 3 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === resource - ); -} - -function isBulkPath(segments: string[]): boolean { - return ( - segments.length === 4 && segments[0] === "api" && segments[1] === "v1" && segments[2] === "bulk" - ); -} - -// --------------------------------------------------------------------------- -// Segment-based route dispatcher (document/:id, links, searches, webhooks, bulk) -// --------------------------------------------------------------------------- - -/** Dispatch single-document CRUD routes (GET/DELETE/PATCH /documents/:id). */ -async function dispatchDocumentCrudRoutes( - ctx: RouteContext, - docId: string, - method: string, -): Promise { - if (method === "GET") { - handleGetDocument(ctx, docId); - return true; - } - if (method === "DELETE") { - handleDeleteDocument(ctx, docId); - return true; - } - if (method === "PATCH") { - await handleUpdateDocument(ctx, docId); - return true; - } - return false; -} - -/** Dispatch document-related segment routes (tags, suggest, CRUD, links). */ -async function dispatchDocumentRoutes( - ctx: RouteContext, - segments: string[], - method: string, -): Promise { - const tagDocId = matchDocumentTags(segments); - if (tagDocId && method === "POST") { - await handleAddTagsToDocument(ctx, tagDocId); - return true; - } - const suggestDocId = matchDocumentSuggestTags(segments); - if (suggestDocId && method === "GET") { - handleSuggestTags(ctx, suggestDocId); - return true; - } - const docId = matchDocumentId(segments); - if (docId) return dispatchDocumentCrudRoutes(ctx, docId, method); - const linksDocId = matchDocumentLinks(segments); - if (linksDocId && method === "GET") { - handleGetDocumentLinks(ctx, linksDocId); - return true; - } - if (linksDocId && method === "POST") { - await handleCreateDocumentLink(ctx, linksDocId); - return true; - } - const linkId = matchLinkId(segments); - if (linkId && method === "DELETE") { - handleDeleteLink(ctx, linkId); - return true; - } - return false; -} - -/** Dispatch webhook segment routes. */ -async function dispatchWebhookRoutes( - ctx: RouteContext, - segments: string[], - method: string, -): Promise { - if (isApiV1Path(segments, "webhooks") && method === "GET") { - handleListWebhooks(ctx); - return true; - } - if (isApiV1Path(segments, "webhooks") && method === "POST") { - await handleCreateWebhook(ctx); - return true; - } - const webhookTestId = matchWebhookTest(segments); - if (webhookTestId && method === "POST") { - await handleTestWebhook(ctx, webhookTestId); - return true; - } - const webhookId = matchWebhookId(segments); - if (webhookId && method === "DELETE") { - handleDeleteWebhook(ctx, webhookId); - return true; - } - return false; -} - -/** Dispatch search, bulk, and webhook segment routes. */ -async function dispatchMiscSegmentRoutes( - ctx: RouteContext, - segments: string[], - method: string, -): Promise { - const searchRunId = matchSearchRun(segments); - if (searchRunId && method === "POST") { - await handleRunSavedSearch(ctx, searchRunId); - return true; - } - if (isApiV1Path(segments, "searches") && method === "GET") { - handleListSavedSearches(ctx); - return true; - } - if (isApiV1Path(segments, "searches") && method === "POST") { - await handleCreateSavedSearch(ctx); - return true; - } - if (isBulkPath(segments) && method === "POST") { - const operation = segments[3] as string; - await handleBulkOperation(ctx, operation); - return true; - } - const savedSearchId = matchSearchId(segments); - if (savedSearchId && method === "DELETE") { - handleDeleteSavedSearch(ctx, savedSearchId); - return true; - } - const indexJobId = matchIndexJobId(segments); - if (indexJobId && method === "GET") { - handleGetIndexJob(ctx, indexJobId); - return true; - } - const repoSlug = matchRepoSlugForIndex(segments); - if (repoSlug && method === "POST") { - await handleIndexRepo(ctx, repoSlug); - return true; - } - return dispatchWebhookRoutes(ctx, segments, method); -} - -async function dispatchSegmentRoutes( - ctx: RouteContext, - segments: string[], - method: string, -): Promise { - const docHandled = await dispatchDocumentRoutes(ctx, segments, method); - if (docHandled) return true; - return dispatchMiscSegmentRoutes(ctx, segments, method); -} - // --------------------------------------------------------------------------- // Pathname-based route dispatcher (simple /api/v1/... paths) // --------------------------------------------------------------------------- @@ -1249,6 +932,77 @@ async function dispatchPathnameRoutes( return true; } +// --------------------------------------------------------------------------- +// Pattern-based route dispatcher (/documents/:id, links, searches, webhooks, bulk, index) +// --------------------------------------------------------------------------- + +interface PatternRoute { + method: string; + parts: string[]; + handler: (ctx: RouteContext, param: string) => void | Promise; + /** A param value this route must not match (it belongs to a literal route). */ + exclude?: string | undefined; +} + +/** Build a pattern route from "METHOD /path/:param" (at most one ":param" segment). */ +function route(spec: string, handler: PatternRoute["handler"], exclude?: string): PatternRoute { + const [method = "", path = ""] = spec.split(" "); + return { method, parts: extractPathSegments(path), handler, exclude }; +} + +/** + * Match path segments against pattern parts. Returns the ":param" segment value ("" when the + * pattern has no param), or null when the segments do not match. + */ +function matchPattern(parts: string[], segments: string[]): string | null { + if (parts.length !== segments.length) return null; + let param = ""; + for (const [i, part] of parts.entries()) { + const segment = segments[i] ?? ""; + if (part.startsWith(":")) param = segment; + else if (part !== segment) return null; + } + return param; +} + +/** Pattern route table, checked in order after PATHNAME_ROUTES. */ +const PATTERN_ROUTES: PatternRoute[] = [ + route("POST /api/v1/documents/:id/tags", handleAddTagsToDocument), + route("GET /api/v1/documents/:id/suggest-tags", handleSuggestTags), + route("GET /api/v1/documents/:id", handleGetDocument, "url"), + route("DELETE /api/v1/documents/:id", handleDeleteDocument, "url"), + route("PATCH /api/v1/documents/:id", handleUpdateDocument, "url"), + route("GET /api/v1/documents/:id/links", handleGetDocumentLinks), + route("POST /api/v1/documents/:id/links", handleCreateDocumentLink), + route("DELETE /api/v1/links/:id", handleDeleteLink), + route("POST /api/v1/searches/:id/run", handleRunSavedSearch), + route("GET /api/v1/searches", handleListSavedSearches), + route("POST /api/v1/searches", handleCreateSavedSearch), + route("POST /api/v1/bulk/:operation", handleBulkOperation), + route("DELETE /api/v1/searches/:id", handleDeleteSavedSearch), + route("GET /api/v1/index/jobs/:jobId", handleGetIndexJob), + route("POST /api/v1/index/repos/:repoSlug", handleIndexRepo), + route("GET /api/v1/webhooks", handleListWebhooks), + route("POST /api/v1/webhooks", handleCreateWebhook), + route("POST /api/v1/webhooks/:id/test", handleTestWebhook), + route("DELETE /api/v1/webhooks/:id", handleDeleteWebhook), +]; + +async function dispatchPatternRoutes( + ctx: RouteContext, + segments: string[], + method: string, +): Promise { + for (const r of PATTERN_ROUTES) { + if (r.method !== method) continue; + const param = matchPattern(r.parts, segments); + if (param === null || param === r.exclude) continue; + await r.handler(ctx, param); + return true; + } + return false; +} + // --------------------------------------------------------------------------- // Main entry point // --------------------------------------------------------------------------- @@ -1272,8 +1026,8 @@ export async function handleRequest( const pathnameHandled = await dispatchPathnameRoutes(ctx, pathname, method); if (pathnameHandled) return; - // Try segment-matched routes (document/:id, links, searches, webhooks, bulk) - const segmentHandled = await dispatchSegmentRoutes(ctx, segments, method); + // Try segment-pattern routes (document/:id, links, searches, webhooks, bulk, index) + const segmentHandled = await dispatchPatternRoutes(ctx, segments, method); if (segmentHandled) return; // Unknown route — don't leak method/pathname to prevent endpoint enumeration diff --git a/src/mcp/errors.ts b/src/mcp/errors.ts index 38f07b85..49278d23 100644 --- a/src/mcp/errors.ts +++ b/src/mcp/errors.ts @@ -26,6 +26,11 @@ export function errorResponse(err: unknown): { export type ToolResult = { content: Array<{ type: "text"; text: string }>; isError?: boolean }; +/** Wrap plain text as a single-item MCP tool result. */ +export function textResult(text: string): ToolResult { + return { content: [{ type: "text" as const, text }] }; +} + /** Wraps a tool handler so that thrown errors are converted to MCP error responses. */ export function withErrorHandling

( handler: (params: P) => ToolResult | Promise, diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 9071f2f9..f07b8cb1 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -33,10 +33,10 @@ import { spiderUrl } from "../core/spider.js"; import type { SpiderOptions } from "../core/spider.js"; import { initLogger, getLogger } from "../logger.js"; import { ConfigError, ValidationError } from "../errors.js"; -import { errorResponse, withErrorHandling } from "./errors.js"; +import { errorResponse, textResult, withErrorHandling, type ToolResult } from "./errors.js"; export { errorResponse, withErrorHandling, type ToolResult } from "./errors.js"; import { taskRegistry } from "./tasks.js"; -import type { TaskType } from "./tasks.js"; +import type { Task, TaskType } from "./tasks.js"; /** Build SpiderOptions from submit-document params. */ function buildSpiderOptions( @@ -75,7 +75,7 @@ async function handleSpiderSubmit( excludePatterns?: string[] | undefined; }, fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): Promise<{ content: Array<{ type: "text"; text: string }> }> { +): Promise { const { url, library, version, topic } = params; if (!url) { throw new ValidationError("Field 'url' is required when spider is true"); @@ -109,18 +109,16 @@ async function handleSpiderSubmit( } const stats = result.value; - const summary = [ + return [ `Spider complete.`, `Pages indexed: ${indexed.length}`, - `Pages crawled: ${stats?.pagesCrawled ?? indexed.length}`, - `Pages skipped: ${stats?.pagesSkipped ?? 0}`, + `Pages crawled: ${stats.pagesCrawled}`, + `Pages skipped: ${stats.pagesSkipped}`, errors.length > 0 ? `Errors: ${errors.length}` : null, - stats?.abortReason ? `Stopped early: ${stats.abortReason}` : null, + stats.abortReason ? `Stopped early: ${stats.abortReason}` : null, ] .filter(Boolean) .join("\n"); - - return { content: [{ type: "text" as const, text: summary }] }; } /** Handle single-document submission for submit-document. */ @@ -137,7 +135,7 @@ async function handleSingleDocSubmit( sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; }, fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): Promise<{ content: Array<{ type: "text"; text: string }> }> { +): Promise { let { title, content } = params; const { url, library, version, topic } = params; @@ -167,59 +165,61 @@ async function handleSingleDocSubmit( submittedBy: "model", }); - return { - content: [ - { - type: "text" as const, - text: - `Document indexed successfully.\n` + - `Title: ${title}\n` + - `ID: ${result.id}\n` + - `Chunks: ${result.chunkCount}` + - (url ? `\nSource: ${url}` : ""), - }, - ], - }; + return ( + `Document indexed successfully.\n` + + `Title: ${title}\n` + + `ID: ${result.id}\n` + + `Chunks: ${result.chunkCount}` + + (url ? `\nSource: ${url}` : "") + ); } +/** Background work for a tool: receives the task's abort signal and a progress reporter. */ +type TaskWork = ( + signal: AbortSignal, + onProgress: (current: number, total: number) => void, +) => Promise; + /** Fire-and-forget helper: creates a task, runs `work` in background, returns task ID response. */ -function startAsyncTask( - type: TaskType, - work: ( - signal: AbortSignal, - onProgress: (current: number, total: number) => void, - ) => Promise, -): { content: Array<{ type: "text"; text: string }> } { +function startAsyncTask(type: TaskType, work: TaskWork): ToolResult { const { task, signal } = taskRegistry.create(type); taskRegistry.update(task.id, { status: "running", startedAt: new Date() }); const onProgress = (current: number, total: number): void => { taskRegistry.update(task.id, { progress: { current, total } }); }; + const finish = (updates: Partial): void => { + taskRegistry.update( + task.id, + signal.aborted ? { status: "cancelled", completedAt: new Date() } : updates, + ); + }; void work(signal, onProgress).then( - (result) => { - if (signal.aborted) { - taskRegistry.update(task.id, { status: "cancelled", completedAt: new Date() }); - } else { - taskRegistry.update(task.id, { status: "completed", completedAt: new Date(), result }); - } - }, - (err: unknown) => { - if (signal.aborted) { - taskRegistry.update(task.id, { status: "cancelled", completedAt: new Date() }); - } else { - taskRegistry.update(task.id, { - status: "failed", - completedAt: new Date(), - error: err instanceof Error ? err.message : String(err), - }); - } - }, + (result) => finish({ status: "completed", completedAt: new Date(), result }), + (err: unknown) => + finish({ + status: "failed", + completedAt: new Date(), + error: err instanceof Error ? err.message : String(err), + }), ); - return { - content: [ - { type: "text" as const, text: `Task queued. ID: ${task.id}\nUse get-task to check status.` }, - ], - }; + return textResult(`Task queued. ID: ${task.id}\nUse get-task to check status.`); +} + +const noProgress = (): void => { + // Inline (sync) runs have no task to report progress to. +}; + +/** + * Runs `work` as a background task when `isAsync` is set; otherwise runs it inline with a + * never-aborted signal and returns its text as the tool result. + */ +async function runMaybeAsync( + isAsync: boolean | undefined, + type: TaskType, + work: TaskWork, +): Promise { + if (isAsync) return startAsyncTask(type, work); + return textResult(await work(new AbortController().signal, noProgress)); } // Start the server @@ -330,9 +330,7 @@ async function main(): Promise { }); if (results.length === 0) { - return { - content: [{ type: "text" as const, text: "No documents found matching your query." }], - }; + return textResult("No documents found matching your query."); } const text = @@ -360,7 +358,7 @@ async function main(): Promise { }) .join("\n---\n\n"); - return { content: [{ type: "text" as const, text }] }; + return textResult(text); }), ); @@ -401,14 +399,7 @@ async function main(): Promise { ...(minScore !== undefined && { minScore }), ...(includeLinkedDocuments !== undefined && { includeLinkedDocuments }), }); - return { - content: [ - { - type: "text" as const, - text: JSON.stringify(result, null, 2), - }, - ], - }; + return textResult(JSON.stringify(result, null, 2)); }, ), ); @@ -433,7 +424,7 @@ async function main(): Promise { `**Rating:** ${ratings.averageRating.toFixed(1)}/5 (${ratings.totalRatings} ratings)\n\n` + doc.content; - return { content: [{ type: "text" as const, text }] }; + return textResult(text); }), ); @@ -447,14 +438,7 @@ async function main(): Promise { withErrorHandling((params) => { deleteDocument(db, params.documentId); - return { - content: [ - { - type: "text" as const, - text: `Document ${params.documentId} has been deleted successfully.`, - }, - ], - }; + return textResult(`Document ${params.documentId} has been deleted successfully.`); }), ); @@ -491,9 +475,7 @@ async function main(): Promise { }) : undefined, }); - return { - content: [{ type: "text" as const, text: `Document updated: ${doc.title} (${doc.id})` }], - }; + return textResult(`Document updated: ${doc.title} (${doc.id})`); }), ); @@ -521,17 +503,11 @@ async function main(): Promise { ratedBy: "model", }); - return { - content: [ - { - type: "text" as const, - text: - `Rating submitted: ${result.rating}/5 for document ${result.documentId}` + - (result.feedback ? `\nFeedback: ${result.feedback}` : "") + - (result.suggestedCorrection ? `\nCorrection suggested.` : ""), - }, - ], - }; + return textResult( + `Rating submitted: ${result.rating}/5 for document ${result.documentId}` + + (result.feedback ? `\nFeedback: ${result.feedback}` : "") + + (result.suggestedCorrection ? `\nCorrection suggested.` : ""), + ); }), ); @@ -604,22 +580,11 @@ async function main(): Promise { allowSelfSignedCerts: config.indexing.allowSelfSignedCerts, }; - if (params.async) { - return startAsyncTask("index_document", async () => { - if (params.spider) { - const r = await handleSpiderSubmit(db, provider, params, fetchOptions); - return r.content[0]?.text ?? "Done"; - } - const r = await handleSingleDocSubmit(db, provider, params, fetchOptions); - return r.content[0]?.text ?? "Done"; - }); - } - - if (params.spider) { - return handleSpiderSubmit(db, provider, params, fetchOptions); - } - - return handleSingleDocSubmit(db, provider, params, fetchOptions); + return runMaybeAsync(params.async, "index_document", () => + params.spider + ? handleSpiderSubmit(db, provider, params, fetchOptions) + : handleSingleDocSubmit(db, provider, params, fetchOptions), + ); }), ); @@ -634,9 +599,7 @@ async function main(): Promise { const topics = listTopics(db, params.parentId); if (topics.length === 0) { - return { - content: [{ type: "text" as const, text: "No topics found." }], - }; + return textResult("No topics found."); } const text = topics @@ -646,7 +609,7 @@ async function main(): Promise { }) .join("\n"); - return { content: [{ type: "text" as const, text: `## Topics\n\n${text}` }] }; + return textResult(`## Topics\n\n${text}`); }), ); @@ -699,9 +662,7 @@ async function main(): Promise { getLogger().warn({ err }, "Health check: FTS5 index query failed"); } - return { - content: [{ type: "text" as const, text: JSON.stringify(health, null, 2) }], - }; + return textResult(JSON.stringify(health, null, 2)); } catch (err) { return errorResponse(err); } @@ -730,7 +691,7 @@ async function main(): Promise { }); if (docs.length === 0) { - return { content: [{ type: "text" as const, text: "No documents found." }] }; + return textResult("No documents found."); } const text = docs @@ -745,9 +706,7 @@ async function main(): Promise { }) .join("\n"); - return { - content: [{ type: "text" as const, text: `## Documents (${docs.length})\n\n${text}` }], - }; + return textResult(`## Documents (${docs.length})\n\n${text}`); }), ); @@ -783,9 +742,7 @@ async function main(): Promise { .join("\n") : ""; - return { - content: [{ type: "text" as const, text: contextPrompt + sourcesText }], - }; + return textResult(contextPrompt + sourcesText); } if (!llmProvider) { @@ -814,9 +771,7 @@ async function main(): Promise { ? `\n\n_Model: ${result.model} | Tokens: ${result.tokensUsed}_` : ""; - return { - content: [{ type: "text" as const, text: result.answer + sourcesText + metaText }], - }; + return textResult(result.answer + sourcesText + metaText); }), ); @@ -853,47 +808,27 @@ async function main(): Promise { withErrorHandling(async (params) => { const { reindex } = await import("../core/reindex.js"); - if (params.async) { - return startAsyncTask("reindex_library", async (signal, onProgress) => { - const result = await reindex(db, provider, { - documentIds: params.documentIds, - since: params.since, - before: params.before, - batchSize: params.batchSize, - onProgress: (p) => { - if (signal.aborted) throw new Error("Task cancelled"); - onProgress(p.completed, p.total); - }, - }); - return ( - `Reindex complete.\n` + - `Total chunks: ${result.total}\n` + - `Updated: ${result.completed}\n` + - `Failed: ${result.failed}` + - (result.failedChunkIds.length > 0 - ? `\nFailed chunk IDs: ${result.failedChunkIds.join(", ")}` - : "") - ); + return runMaybeAsync(params.async, "reindex_library", async (signal, onProgress) => { + const result = await reindex(db, provider, { + documentIds: params.documentIds, + since: params.since, + before: params.before, + batchSize: params.batchSize, + onProgress: (p) => { + if (signal.aborted) throw new Error("Task cancelled"); + onProgress(p.completed, p.total); + }, }); - } - - const result = await reindex(db, provider, { - documentIds: params.documentIds, - since: params.since, - before: params.before, - batchSize: params.batchSize, + return ( + `Reindex complete.\n` + + `Total chunks: ${result.total}\n` + + `Updated: ${result.completed}\n` + + `Failed: ${result.failed}` + + (result.failedChunkIds.length > 0 + ? `\nFailed chunk IDs: ${result.failedChunkIds.join(", ")}` + : "") + ); }); - - const text = - `Reindex complete.\n` + - `Total chunks: ${result.total}\n` + - `Updated: ${result.completed}\n` + - `Failed: ${result.failed}` + - (result.failedChunkIds.length > 0 - ? `\nFailed chunk IDs: ${result.failedChunkIds.join(", ")}` - : ""); - - return { content: [{ type: "text" as const, text }] }; }), ); @@ -933,35 +868,18 @@ async function main(): Promise { threadMode: params.threadMode ?? ("aggregate" as const), }; - if (params.async) { - return startAsyncTask("sync_connector", async () => { - const result = await doSyncSlack(db, provider, slackConfig); - const slackErrorLines = result.errors - .map((e) => ` #${e.channel}: ${e.error}`) - .join("\n"); - const slackErrors = result.errors.length > 0 ? `\nErrors:\n${slackErrorLines}` : ""; - return ( - `Slack sync complete.\n` + - `Channels: ${result.channels}\n` + - `Messages indexed: ${result.messagesIndexed}\n` + - `Threads indexed: ${result.threadsIndexed}` + - slackErrors - ); - }); - } - - const result = await doSyncSlack(db, provider, slackConfig); - - const slackErrorLines = result.errors.map((e) => ` #${e.channel}: ${e.error}`).join("\n"); - const slackErrors = result.errors.length > 0 ? `\nErrors:\n${slackErrorLines}` : ""; - const text = - `Slack sync complete.\n` + - `Channels: ${result.channels}\n` + - `Messages indexed: ${result.messagesIndexed}\n` + - `Threads indexed: ${result.threadsIndexed}` + - slackErrors; - - return { content: [{ type: "text" as const, text }] }; + return runMaybeAsync(params.async, "sync_connector", async () => { + const result = await doSyncSlack(db, provider, slackConfig); + const slackErrorLines = result.errors.map((e) => ` #${e.channel}: ${e.error}`).join("\n"); + const slackErrors = result.errors.length > 0 ? `\nErrors:\n${slackErrorLines}` : ""; + return ( + `Slack sync complete.\n` + + `Channels: ${result.channels}\n` + + `Messages indexed: ${result.messagesIndexed}\n` + + `Threads indexed: ${result.threadsIndexed}` + + slackErrors + ); + }); }), ); @@ -982,41 +900,18 @@ async function main(): Promise { withErrorHandling(async (params) => { const { installPack } = await import("../core/packs.js"); - if (params.async) { - return startAsyncTask("install_pack", async (signal, onProgress) => { - const result = await installPack(db, provider, params.nameOrPath, { - registryUrl: params.registryUrl, - onProgress: (current, total) => { - if (signal.aborted) throw new Error("Task cancelled"); - onProgress(current, total); - }, - }); - return result.alreadyInstalled - ? `Pack "${result.packName}" is already installed.` - : `Pack "${result.packName}" installed successfully (${result.documentsInstalled} documents).`; + return runMaybeAsync(params.async, "install_pack", async (signal, onProgress) => { + const result = await installPack(db, provider, params.nameOrPath, { + registryUrl: params.registryUrl, + onProgress: (current, total) => { + if (signal.aborted) throw new Error("Task cancelled"); + onProgress(current, total); + }, }); - } - - const result = await installPack(db, provider, params.nameOrPath, { - registryUrl: params.registryUrl, + return result.alreadyInstalled + ? `Pack "${result.packName}" is already installed.` + : `Pack "${result.packName}" installed successfully (${result.documentsInstalled} documents).`; }); - - if (result.alreadyInstalled) { - return { - content: [ - { type: "text" as const, text: `Pack "${result.packName}" is already installed.` }, - ], - }; - } - - return { - content: [ - { - type: "text" as const, - text: `Pack "${result.packName}" installed successfully (${result.documentsInstalled} documents).`, - }, - ], - }; }), ); @@ -1036,20 +931,18 @@ async function main(): Promise { const { listAvailablePacks } = await import("../core/packs.js"); const packs = await listAvailablePacks(params.registryUrl); if (packs.length === 0) { - return { - content: [{ type: "text" as const, text: "No packs available in the registry." }], - }; + return textResult("No packs available in the registry."); } const text = packs .map((p) => `- **${p.name}** v${p.version} — ${p.description} (${p.docCount} docs)`) .join("\n"); - return { content: [{ type: "text" as const, text: `## Available Packs\n\n${text}` }] }; + return textResult(`## Available Packs\n\n${text}`); } const { listInstalledPacks } = await import("../core/packs.js"); const packs = listInstalledPacks(db); if (packs.length === 0) { - return { content: [{ type: "text" as const, text: "No packs installed." }] }; + return textResult("No packs installed."); } const text = packs .map( @@ -1057,7 +950,7 @@ async function main(): Promise { `- **${p.name}** v${p.version} — ${p.description ?? ""} (${p.docCount} docs, installed ${p.installedAt})`, ) .join("\n"); - return { content: [{ type: "text" as const, text: `## Installed Packs\n\n${text}` }] }; + return textResult(`## Installed Packs\n\n${text}`); }), ); @@ -1086,37 +979,20 @@ async function main(): Promise { excludeSections: [] as string[], }; - if (params.async) { - return startAsyncTask("sync_connector", async () => { - const result = await syncOneNote(db, provider, oneNoteConfig); - const oneNoteErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); - const oneNoteErrors = result.errors.length > 0 ? `\nErrors: ${oneNoteErrorLines}` : ""; - return ( - `OneNote sync complete.\n` + - `Notebooks: ${result.notebooks}\n` + - `Sections: ${result.sections}\n` + - `Pages added: ${result.pagesAdded}\n` + - `Pages updated: ${result.pagesUpdated}\n` + - `Pages deleted: ${result.pagesDeleted}` + - oneNoteErrors - ); - }); - } - - const result = await syncOneNote(db, provider, oneNoteConfig); - - const oneNoteErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); - const oneNoteErrors = result.errors.length > 0 ? `\nErrors: ${oneNoteErrorLines}` : ""; - const text = - `OneNote sync complete.\n` + - `Notebooks: ${result.notebooks}\n` + - `Sections: ${result.sections}\n` + - `Pages added: ${result.pagesAdded}\n` + - `Pages updated: ${result.pagesUpdated}\n` + - `Pages deleted: ${result.pagesDeleted}` + - oneNoteErrors; - - return { content: [{ type: "text" as const, text }] }; + return runMaybeAsync(params.async, "sync_connector", async () => { + const result = await syncOneNote(db, provider, oneNoteConfig); + const oneNoteErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); + const oneNoteErrors = result.errors.length > 0 ? `\nErrors: ${oneNoteErrorLines}` : ""; + return ( + `OneNote sync complete.\n` + + `Notebooks: ${result.notebooks}\n` + + `Sections: ${result.sections}\n` + + `Pages added: ${result.pagesAdded}\n` + + `Pages updated: ${result.pagesUpdated}\n` + + `Pages deleted: ${result.pagesDeleted}` + + oneNoteErrors + ); + }); }), ); @@ -1150,31 +1026,17 @@ async function main(): Promise { excludePages: params.excludePages, }; - if (params.async) { - return startAsyncTask("sync_connector", async () => { - const result = await syncNotion(db, provider, notionConfig); - const notionErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); - const notionErrors = result.errors.length > 0 ? `\nErrors: ${notionErrorLines}` : ""; - return ( - `Notion sync complete.\n` + - `Pages indexed: ${result.pagesIndexed}\n` + - `Databases indexed: ${result.databasesIndexed}` + - notionErrors - ); - }); - } - - const result = await syncNotion(db, provider, notionConfig); - - const notionErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); - const notionErrors = result.errors.length > 0 ? `\nErrors: ${notionErrorLines}` : ""; - const text = - `Notion sync complete.\n` + - `Pages indexed: ${result.pagesIndexed}\n` + - `Databases indexed: ${result.databasesIndexed}` + - notionErrors; - - return { content: [{ type: "text" as const, text }] }; + return runMaybeAsync(params.async, "sync_connector", async () => { + const result = await syncNotion(db, provider, notionConfig); + const notionErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); + const notionErrors = result.errors.length > 0 ? `\nErrors: ${notionErrorLines}` : ""; + return ( + `Notion sync complete.\n` + + `Pages indexed: ${result.pagesIndexed}\n` + + `Databases indexed: ${result.databasesIndexed}` + + notionErrors + ); + }); }), ); @@ -1200,33 +1062,18 @@ async function main(): Promise { excludePatterns: [] as string[], }; - if (params.async) { - return startAsyncTask("sync_connector", async () => { - const result = await syncObsidianVault(db, provider, obsidianConfig); - const obsidianErrorLines = result.errors.map((e) => `${e.file}: ${e.error}`).join(", "); - const obsidianErrors = result.errors.length > 0 ? `\nErrors: ${obsidianErrorLines}` : ""; - return ( - `Obsidian vault sync complete.\n` + - `Added: ${result.added}\n` + - `Updated: ${result.updated}\n` + - `Deleted: ${result.deleted}` + - obsidianErrors - ); - }); - } - - const result = await syncObsidianVault(db, provider, obsidianConfig); - - const obsidianErrorLines = result.errors.map((e) => `${e.file}: ${e.error}`).join(", "); - const obsidianErrors = result.errors.length > 0 ? `\nErrors: ${obsidianErrorLines}` : ""; - const text = - `Obsidian vault sync complete.\n` + - `Added: ${result.added}\n` + - `Updated: ${result.updated}\n` + - `Deleted: ${result.deleted}` + - obsidianErrors; - - return { content: [{ type: "text" as const, text }] }; + return runMaybeAsync(params.async, "sync_connector", async () => { + const result = await syncObsidianVault(db, provider, obsidianConfig); + const obsidianErrorLines = result.errors.map((e) => `${e.file}: ${e.error}`).join(", "); + const obsidianErrors = result.errors.length > 0 ? `\nErrors: ${obsidianErrorLines}` : ""; + return ( + `Obsidian vault sync complete.\n` + + `Added: ${result.added}\n` + + `Updated: ${result.updated}\n` + + `Deleted: ${result.deleted}` + + obsidianErrors + ); + }); }), ); @@ -1261,34 +1108,19 @@ async function main(): Promise { excludeSpaces: params.excludeSpaces, }; - if (params.async) { - return startAsyncTask("sync_connector", async () => { - const result = await syncConfluence(db, provider, confluenceConfig); - const confluenceErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join(", "); - const confluenceErrors = - result.errors.length > 0 ? `\nErrors: ${confluenceErrorLines}` : ""; - return ( - `Confluence sync complete.\n` + - `Spaces: ${result.spaces}\n` + - `Pages indexed: ${result.pagesIndexed}\n` + - `Pages updated: ${result.pagesUpdated}` + - confluenceErrors - ); - }); - } - - const result = await syncConfluence(db, provider, confluenceConfig); - - const confluenceErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join(", "); - const confluenceErrors = result.errors.length > 0 ? `\nErrors: ${confluenceErrorLines}` : ""; - const text = - `Confluence sync complete.\n` + - `Spaces: ${result.spaces}\n` + - `Pages indexed: ${result.pagesIndexed}\n` + - `Pages updated: ${result.pagesUpdated}` + - confluenceErrors; - - return { content: [{ type: "text" as const, text }] }; + return runMaybeAsync(params.async, "sync_connector", async () => { + const result = await syncConfluence(db, provider, confluenceConfig); + const confluenceErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join(", "); + const confluenceErrors = + result.errors.length > 0 ? `\nErrors: ${confluenceErrorLines}` : ""; + return ( + `Confluence sync complete.\n` + + `Spaces: ${result.spaces}\n` + + `Pages indexed: ${result.pagesIndexed}\n` + + `Pages updated: ${result.pagesUpdated}` + + confluenceErrors + ); + }); }), ); @@ -1319,7 +1151,7 @@ async function main(): Promise { "Knowledge gaps:", ...gaps.map((g) => ` ${g.count}x ${g.query} (last: ${g.lastSearched})`), ]; - return { content: [{ type: "text" as const, text: lines.join("\n") }] }; + return textResult(lines.join("\n")); }), ); @@ -1344,14 +1176,9 @@ async function main(): Promise { params.label, ); const linkLabel = link.label ? ` — ${link.label}` : ""; - return { - content: [ - { - type: "text" as const, - text: `✓ Link created: ${link.sourceId} → ${link.targetId} (${link.linkType})${linkLabel}`, - }, - ], - }; + return textResult( + `✓ Link created: ${link.sourceId} → ${link.targetId} (${link.linkType})${linkLabel}`, + ); }), ); @@ -1365,7 +1192,7 @@ async function main(): Promise { withErrorHandling((params) => { const { outgoing, incoming } = getDocumentLinks(db, params.documentId); if (outgoing.length === 0 && incoming.length === 0) { - return { content: [{ type: "text" as const, text: "No links found for this document." }] }; + return textResult("No links found for this document."); } const lines: string[] = []; @@ -1383,7 +1210,7 @@ async function main(): Promise { lines.push(` ← [${l.linkType}] ${l.sourceTitle} (${l.sourceId})${inLabel}`); } } - return { content: [{ type: "text" as const, text: lines.join("\n") }] }; + return textResult(lines.join("\n")); }), ); @@ -1396,7 +1223,7 @@ async function main(): Promise { }, withErrorHandling((params) => { deleteLink(db, params.linkId); - return { content: [{ type: "text" as const, text: `✓ Link ${params.linkId} deleted.` }] }; + return textResult(`✓ Link ${params.linkId} deleted.`); }), ); @@ -1431,14 +1258,7 @@ async function main(): Promise { query, Object.keys(filters).length > 0 ? filters : undefined, ); - return { - content: [ - { - type: "text" as const, - text: JSON.stringify(saved, null, 2), - }, - ], - }; + return textResult(JSON.stringify(saved, null, 2)); }), ); @@ -1452,14 +1272,7 @@ async function main(): Promise { }, withErrorHandling((params) => { const suggestions = suggestTags(db, params.documentId, params.maxSuggestions); - return { - content: [ - { - type: "text" as const, - text: JSON.stringify({ documentId: params.documentId, suggestions }, null, 2), - }, - ], - }; + return textResult(JSON.stringify({ documentId: params.documentId, suggestions }, null, 2)); }), ); @@ -1470,14 +1283,9 @@ async function main(): Promise { {}, withErrorHandling(() => { const searches = listSavedSearches(db); - return { - content: [ - { - type: "text" as const, - text: searches.length === 0 ? "No saved searches." : JSON.stringify(searches, null, 2), - }, - ], - }; + return textResult( + searches.length === 0 ? "No saved searches." : JSON.stringify(searches, null, 2), + ); }), ); @@ -1490,14 +1298,7 @@ async function main(): Promise { }, withErrorHandling(async (params) => { const { search, results } = await runSavedSearch(db, provider, params.nameOrId); - return { - content: [ - { - type: "text" as const, - text: JSON.stringify({ search, resultCount: results.length, results }, null, 2), - }, - ], - }; + return textResult(JSON.stringify({ search, resultCount: results.length, results }, null, 2)); }), ); @@ -1510,9 +1311,7 @@ async function main(): Promise { }, withErrorHandling((params) => { deleteSavedSearch(db, params.nameOrId); - return { - content: [{ type: "text" as const, text: `✓ Saved search "${params.nameOrId}" deleted.` }], - }; + return textResult(`✓ Saved search "${params.nameOrId}" deleted.`); }), ); @@ -1539,9 +1338,7 @@ async function main(): Promise { params.events as WebhookEvent[], params.secret, ); - return { - content: [{ type: "text" as const, text: JSON.stringify(redactWebhook(webhook), null, 2) }], - }; + return textResult(JSON.stringify(redactWebhook(webhook), null, 2)); }), ); @@ -1552,11 +1349,7 @@ async function main(): Promise { {}, withErrorHandling(() => { const webhooks = listWebhooks(db); - return { - content: [ - { type: "text" as const, text: JSON.stringify(webhooks.map(redactWebhook), null, 2) }, - ], - }; + return textResult(JSON.stringify(webhooks.map(redactWebhook), null, 2)); }), ); @@ -1569,9 +1362,7 @@ async function main(): Promise { }, withErrorHandling((params) => { deleteWebhook(db, params.id); - return { - content: [{ type: "text" as const, text: `✓ Webhook "${params.id}" deleted.` }], - }; + return textResult(`✓ Webhook "${params.id}" deleted.`); }), ); @@ -1585,16 +1376,11 @@ async function main(): Promise { withErrorHandling((params) => { const task = taskRegistry.get(params.taskId); if (!task) { - return { - content: [ - { - type: "text" as const, - text: `Task ${params.taskId} not found or has expired (tasks are kept for 1 hour after completion).`, - }, - ], - }; + return textResult( + `Task ${params.taskId} not found or has expired (tasks are kept for 1 hour after completion).`, + ); } - return { content: [{ type: "text" as const, text: JSON.stringify(task, null, 2) }] }; + return textResult(JSON.stringify(task, null, 2)); }), ); @@ -1608,35 +1394,16 @@ async function main(): Promise { withErrorHandling((params) => { const outcome = taskRegistry.cancel(params.taskId); if (outcome === "not_found") { - return { - content: [ - { - type: "text" as const, - text: `Task ${params.taskId} not found or has expired.`, - }, - ], - }; + return textResult(`Task ${params.taskId} not found or has expired.`); } if (outcome === "already_terminal") { const task = taskRegistry.get(params.taskId); const status = task?.status ?? "unknown"; - return { - content: [ - { - type: "text" as const, - text: `Task ${params.taskId} cannot be cancelled (current status: ${status}).`, - }, - ], - }; + return textResult(`Task ${params.taskId} cannot be cancelled (current status: ${status}).`); } - return { - content: [ - { - type: "text" as const, - text: `Cancellation requested for task ${params.taskId}. Running operations will stop at the next checkpoint.`, - }, - ], - }; + return textResult( + `Cancellation requested for task ${params.taskId}. Running operations will stop at the next checkpoint.`, + ); }), ); From 937a288fe4c14285472371564077cd7d6a8773ff Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 14:46:39 +0000 Subject: [PATCH 03/83] refactor(connectors): hoist repeated work and share fetch helpers - confluence: compute html.toLowerCase() once per helper call - obsidian: build the vault file map once per sync instead of per file - obsidian/onenote: call createTopic directly (ON CONFLICT returns existing) - slack/spider/repo/onenote: use node:timers/promises setTimeout for delays - url-fetcher: fetchAndConvert reuses fetchRaw; share getInsecureAgent with http-utils - graph/docs: has-only Maps become Sets - rag: extract shared retrieval helper - config: truthy() and LLM_PROVIDERS helpers in getEnvOverrides - lite/normalize: file and buffer inputs share one path - parsers: MarkdownParser extends PlainTextParser --- src/config.ts | 45 ++++++++++------------ src/connectors/confluence.ts | 20 ++++++---- src/connectors/docs.ts | 12 +++--- src/connectors/http-utils.ts | 10 +---- src/connectors/obsidian.ts | 44 ++++++++++----------- src/connectors/onenote.ts | 19 ++++----- src/connectors/slack.ts | 5 +-- src/core/graph.ts | 6 +-- src/core/parsers/markdown.ts | 12 ++---- src/core/parsers/text.ts | 2 +- src/core/rag.ts | 42 +++++++++----------- src/core/repo.ts | 5 +-- src/core/spider.ts | 5 +-- src/core/url-fetcher.ts | 71 ++++++++++------------------------ src/lite/normalize.ts | 74 +++++++++++++++--------------------- 15 files changed, 150 insertions(+), 222 deletions(-) diff --git a/src/config.ts b/src/config.ts index e5249874..02926c78 100644 --- a/src/config.ts +++ b/src/config.ts @@ -74,6 +74,18 @@ function loadJsonFile(path: string): Partial { } } +const LLM_PROVIDERS = ["openai", "ollama", "anthropic", "passthrough"] as const; +type LlmProviderName = (typeof LLM_PROVIDERS)[number]; + +function isLlmProvider(value: string | undefined): value is LlmProviderName { + return value !== undefined && (LLM_PROVIDERS as readonly string[]).includes(value); +} + +/** Env-var flag parsing: only "true" and "1" enable a flag. */ +function truthy(value: string | undefined): boolean { + return value === "true" || value === "1"; +} + function getEnvOverrides(): Partial { const overrides: Partial = {}; const provider = process.env["LIBSCOPE_EMBEDDING_PROVIDER"]; @@ -95,39 +107,22 @@ function getEnvOverrides(): Partial { const llmProvider = process.env["LIBSCOPE_LLM_PROVIDER"]; const llmModel = process.env["LIBSCOPE_LLM_MODEL"]; - const allowPrivate = process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"]; - const allowSelfSigned = process.env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"]; + const allowPrivate = truthy(process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"]); + const allowSelfSigned = truthy(process.env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"]); - if ( - allowPrivate === "true" || - allowPrivate === "1" || - allowSelfSigned === "true" || - allowSelfSigned === "1" - ) { + if (allowPrivate || allowSelfSigned) { overrides.indexing = { ...DEFAULT_CONFIG.indexing, - ...(allowPrivate === "true" || allowPrivate === "1" ? { allowPrivateUrls: true } : {}), - ...(allowSelfSigned === "true" || allowSelfSigned === "1" - ? { allowSelfSignedCerts: true } - : {}), + ...(allowPrivate ? { allowPrivateUrls: true } : {}), + ...(allowSelfSigned ? { allowSelfSignedCerts: true } : {}), }; } - if ( - llmProvider === "openai" || - llmProvider === "ollama" || - llmProvider === "anthropic" || - llmProvider === "passthrough" || - llmModel - ) { + const validLlmProvider = isLlmProvider(llmProvider); + if (validLlmProvider || llmModel) { const anthropicKey = process.env["LIBSCOPE_ANTHROPIC_API_KEY"]; overrides.llm = { - ...(llmProvider === "openai" || - llmProvider === "ollama" || - llmProvider === "anthropic" || - llmProvider === "passthrough" - ? { provider: llmProvider } - : {}), + ...(validLlmProvider ? { provider: llmProvider } : {}), ...(llmModel ? { model: llmModel } : {}), ...(anthropicKey ? { anthropicApiKey: anthropicKey } : {}), }; diff --git a/src/connectors/confluence.ts b/src/connectors/confluence.ts index 14e58997..4a4f508e 100644 --- a/src/connectors/confluence.ts +++ b/src/connectors/confluence.ts @@ -137,11 +137,12 @@ function replaceStructuredMacros( ): string { const OPEN = " string): string { const openPrefix = `<${tagName}`; const closeTag = ``; + const lower = html.toLowerCase(); + const lowerOpen = openPrefix.toLowerCase(); + const lowerClose = closeTag.toLowerCase(); let result = ""; let pos = 0; while (pos < html.length) { - const start = html.toLowerCase().indexOf(openPrefix.toLowerCase(), pos); + const start = lower.indexOf(lowerOpen, pos); if (start === -1) { result += html.slice(pos); break; @@ -188,7 +192,7 @@ function replaceTagPairs(html: string, tagName: string, cb: (inner: string) => s result += html.slice(pos); break; } - const closeStart = html.toLowerCase().indexOf(closeTag.toLowerCase(), tagEnd + 1); + const closeStart = lower.indexOf(lowerClose, tagEnd + 1); if (closeStart === -1) { result += html.slice(pos); break; @@ -204,10 +208,11 @@ function replaceTagPairs(html: string, tagName: string, cb: (inner: string) => s function extractTagContent(html: string, tagName: string): string { const open = `<${tagName}>`; const close = ``; - const start = html.toLowerCase().indexOf(open.toLowerCase()); + const lower = html.toLowerCase(); + const start = lower.indexOf(open.toLowerCase()); if (start === -1) return ""; const contentStart = start + open.length; - const end = html.toLowerCase().indexOf(close.toLowerCase(), contentStart); + const end = lower.indexOf(close.toLowerCase(), contentStart); if (end === -1) return ""; return html.slice(contentStart, end); } @@ -218,11 +223,12 @@ function extractTagContent(html: string, tagName: string): string { */ function removeSelfClosingMacros(html: string, nameTest: RegExp): string { const OPEN = "; + /** Normalised URLs of existing documents, for update detection. */ + existingUrls: Set; result: DocSiteSyncResult; } @@ -520,7 +520,7 @@ async function processPage(url: string, html: string, ctx: PageContext): Promise } const normalised = normalizeUrl(url); - const isKnown = ctx.existingUrlMap.has(normalised); + const isKnown = ctx.existingUrls.has(normalised); const indexed = await indexDocument(ctx.db, ctx.provider, { title, @@ -678,10 +678,10 @@ export async function syncDocSite( // --- Build existing-URL index for update tracking --- const existingDocs = listDocuments(db, { sourceType: SOURCE_TYPE, library: config.library }); - const existingUrlMap = new Map( + const existingUrls = new Set( existingDocs .filter((d): d is typeof d & { url: string } => d.url !== null) - .map((d) => [normalizeUrl(d.url), d.id]), + .map((d) => normalizeUrl(d.url)), ); const ctx: PageContext = { @@ -689,7 +689,7 @@ export async function syncDocSite( db, provider, config, - existingUrlMap, + existingUrls, result, }; diff --git a/src/connectors/http-utils.ts b/src/connectors/http-utils.ts index e8d10718..b2cad595 100644 --- a/src/connectors/http-utils.ts +++ b/src/connectors/http-utils.ts @@ -1,14 +1,8 @@ -import { Agent } from "undici"; +import type { Agent } from "undici"; import { getLogger } from "../logger.js"; import { FetchError } from "../errors.js"; import { loadConfig } from "../config.js"; - -/** Lazy singleton undici Agent that skips TLS certificate verification. */ -let _insecureAgent: Agent | undefined; -function getInsecureAgent(): Agent { - _insecureAgent ??= new Agent({ connect: { rejectUnauthorized: false } }); - return _insecureAgent; -} +import { getInsecureAgent } from "../core/url-fetcher.js"; export interface RetryConfig { maxRetries?: number; diff --git a/src/connectors/obsidian.ts b/src/connectors/obsidian.ts index 3c59a6ad..308fd6ea 100644 --- a/src/connectors/obsidian.ts +++ b/src/connectors/obsidian.ts @@ -5,7 +5,7 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; -import { createTopic, listTopics } from "../core/topics.js"; +import { createTopic } from "../core/topics.js"; import { addTagsToDocument, createTag } from "../core/tags.js"; import { createLink, resolveDocumentByTitle } from "../core/links.js"; import { getLogger } from "../logger.js"; @@ -199,19 +199,24 @@ function collectTags(body: string, frontmatter: Record): string return [...tagSet]; } -export function parseObsidianMarkdown( - content: string, - vaultFiles: string[], -): { +interface ParsedObsidianMarkdown { frontmatter: Record; body: string; tags: string[]; wikilinks: string[]; -} { +} + +export function parseObsidianMarkdown( + content: string, + vaultFiles: string[], +): ParsedObsidianMarkdown { + return parseWithFileMap(content, buildVaultFileMap(vaultFiles)); +} + +function parseWithFileMap(content: string, fileMap: Map): ParsedObsidianMarkdown { const safeContent = content.length > MAX_PARSE_SIZE ? content.slice(0, MAX_PARSE_SIZE) : content; const { frontmatter, body: rawBody } = extractFrontmatter(safeContent); - const fileMap = buildVaultFileMap(vaultFiles); const wikilinks = collectWikilinks(rawBody); const body = transformObsidianBody(rawBody, fileMap); const tags = collectTags(body, frontmatter); @@ -222,15 +227,9 @@ export function parseObsidianMarkdown( function resolveEmbeds( body: string, vaultPath: string, - vaultFiles: string[], + fileMap: Map, _visited: Set = new Set(), ): string { - const fileMap = new Map(); - for (const f of vaultFiles) { - const name = basename(f, ".md"); - fileMap.set(name.toLowerCase(), f); - } - return body.replace( /!\[\[([^\]|]+)(?:\|([^\]]*))?\]\]/g, (_match, link: string, display?: string) => { @@ -261,13 +260,9 @@ function folderToTopic(relPath: string): string | undefined { return dir; } +/** createTopic returns the existing topic when the name is already taken (ON CONFLICT). */ function getOrCreateTopic(db: Database.Database, topicPath: string): string { - const existing = listTopics(db); - const found = existing.find((t) => t.name === topicPath); - if (found) return found.id; - - const topic = createTopic(db, { name: topicPath }); - return topic.id; + return createTopic(db, { name: topicPath }).id; } /** Determine the topic ID for a file based on the topic mapping strategy. */ @@ -345,7 +340,7 @@ async function processVaultFile( provider: EmbeddingProvider, config: ObsidianConfig, relPath: string, - vaultFiles: string[], + fileMap: Map, tracked: VaultFileEntry | undefined, log: ReturnType, ): Promise<{ entry: VaultFileEntry; isUpdate: boolean } | "unchanged"> { @@ -356,8 +351,8 @@ async function processVaultFile( if (tracked?.mtime === mtime) return "unchanged"; const rawContent = readFileSync(fullPath, "utf-8"); - const contentWithEmbeds = resolveEmbeds(rawContent, config.vaultPath, vaultFiles); - const parsed = parseObsidianMarkdown(contentWithEmbeds, vaultFiles); + const contentWithEmbeds = resolveEmbeds(rawContent, config.vaultPath, fileMap); + const parsed = parseWithFileMap(contentWithEmbeds, fileMap); const title = typeof parsed.frontmatter.title === "string" @@ -432,6 +427,7 @@ async function syncVaultFiles( result: SyncResult, ): Promise { const log = getLogger(); + const fileMap = buildVaultFileMap(vaultFiles); for (const relPath of vaultFiles) { try { const outcome = await processVaultFile( @@ -439,7 +435,7 @@ async function syncVaultFiles( provider, config, relPath, - vaultFiles, + fileMap, trackedFiles[relPath], log, ); diff --git a/src/connectors/onenote.ts b/src/connectors/onenote.ts index fb3b9474..feb3eb41 100644 --- a/src/connectors/onenote.ts +++ b/src/connectors/onenote.ts @@ -1,11 +1,12 @@ import type Database from "better-sqlite3"; +import { setTimeout as sleep } from "node:timers/promises"; import { NodeHtmlMarkdown } from "node-html-markdown"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { getLogger } from "../logger.js"; import { LibScopeError } from "../errors.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; -import { createTopic, listTopics } from "../core/topics.js"; +import { createTopic } from "../core/topics.js"; import { loadConnectorConfig, saveConnectorConfig } from "./index.js"; import { startSync, completeSync, failSync } from "./sync-tracker.js"; @@ -76,7 +77,7 @@ async function rateLimitedFetch(url: string, options: RequestInit): Promise= MAX_REQUESTS_PER_MINUTE) { const waitMs = 60_000 - (now - (requestTimestamps[0] ?? now)); log.debug({ waitMs }, "Rate limit reached, waiting"); - await new Promise((resolve) => setTimeout(resolve, waitMs)); + await sleep(waitMs); } unlock!(); @@ -97,7 +98,7 @@ async function rateLimitedFetch(url: string, options: RequestInit): Promise setTimeout(resolve, delayMs)); + await sleep(delayMs); continue; } @@ -289,7 +290,7 @@ export async function authenticateDeviceCode( const deadline = Date.now() + dcData.expires_in * 1000; while (Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, interval)); + await sleep(interval); const tokenRes = await fetch(tokenUrl, { method: "POST", @@ -321,7 +322,7 @@ export async function authenticateDeviceCode( continue; } if (errData.error === "slow_down") { - await new Promise((resolve) => setTimeout(resolve, 5000)); + await sleep(5000); continue; } throw new LibScopeError(`Authentication failed: ${errData.error}`, "ONENOTE_AUTH_ERROR"); @@ -380,13 +381,9 @@ export async function refreshAccessToken( // Sync // --------------------------------------------------------------------------- +/** createTopic returns the existing topic when the name is already taken (ON CONFLICT). */ function ensureOrCreateTopic(db: Database.Database, name: string, parentId?: string): string { - const existing = listTopics(db, parentId).find((t) => t.name === name); - if (existing) { - return existing.id; - } - const topic = createTopic(db, { name, parentId }); - return topic.id; + return createTopic(db, { name, parentId }).id; } function buildSourceUrl(notebook: string, section: string, pageTitle: string): string { diff --git a/src/connectors/slack.ts b/src/connectors/slack.ts index 26dc1cda..02b04071 100644 --- a/src/connectors/slack.ts +++ b/src/connectors/slack.ts @@ -1,4 +1,5 @@ import type Database from "better-sqlite3"; +import { setTimeout as delay } from "node:timers/promises"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { indexDocument } from "../core/indexing.js"; import { getLogger } from "../logger.js"; @@ -69,10 +70,6 @@ export function _clearUserCache(): void { userCache.clear(); } -async function delay(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - async function slackApi( method: string, token: string, diff --git a/src/core/graph.ts b/src/core/graph.ts index 37613031..85c215b5 100644 --- a/src/core/graph.ts +++ b/src/core/graph.ts @@ -187,17 +187,17 @@ function buildTagNodesAndEdges( docIds: Set, ): { nodes: GraphNode[]; edges: GraphEdge[] } { const tags = db.prepare("SELECT id, name FROM tags ORDER BY name").all() as TagRow[]; - const tagIdMap = new Map(); + const tagIds = new Set(); const nodes: GraphNode[] = []; for (const tag of tags) { - tagIdMap.set(tag.id, tag.name); + tagIds.add(tag.id); nodes.push({ id: `tag:${tag.id}`, label: tag.name, type: "tag", metadata: {} }); } const docTags = db.prepare("SELECT document_id, tag_id FROM document_tags").all() as DocTagRow[]; const edges: GraphEdge[] = []; for (const dt of docTags) { - if (docIds.has(dt.document_id) && tagIdMap.has(dt.tag_id)) { + if (docIds.has(dt.document_id) && tagIds.has(dt.tag_id)) { edges.push({ source: dt.document_id, target: `tag:${dt.tag_id}`, diff --git a/src/core/parsers/markdown.ts b/src/core/parsers/markdown.ts index 82cf9f08..a3b28585 100644 --- a/src/core/parsers/markdown.ts +++ b/src/core/parsers/markdown.ts @@ -1,10 +1,6 @@ -import type { DocumentParser } from "./index.js"; +import { PlainTextParser } from "./text.js"; -/** Pass-through parser for Markdown files. */ -export class MarkdownParser implements DocumentParser { - readonly extensions = [".md", ".markdown", ".mdx"]; - - parse(content: Buffer): Promise { - return Promise.resolve(content.toString("utf-8")); - } +/** Pass-through parser for Markdown files (same UTF-8 decoding as plain text). */ +export class MarkdownParser extends PlainTextParser { + override readonly extensions = [".md", ".markdown", ".mdx"]; } diff --git a/src/core/parsers/text.ts b/src/core/parsers/text.ts index 1721c4c2..a549e068 100644 --- a/src/core/parsers/text.ts +++ b/src/core/parsers/text.ts @@ -2,7 +2,7 @@ import type { DocumentParser } from "./index.js"; /** Pass-through parser for plain text files. */ export class PlainTextParser implements DocumentParser { - readonly extensions = [".txt"]; + readonly extensions: string[] = [".txt"]; parse(content: Buffer): Promise { return Promise.resolve(content.toString("utf-8")); diff --git a/src/core/rag.ts b/src/core/rag.ts index 935f8d8c..248ddc78 100644 --- a/src/core/rag.ts +++ b/src/core/rag.ts @@ -100,6 +100,21 @@ export function createLlmProvider(config: LibScopeConfig): LlmProvider { ); } +/** Retrieve the top-K search results for a RAG question (default K = 5). */ +async function retrieveResults( + db: Database.Database, + embeddingProvider: EmbeddingProvider, + options: RagOptions, +): Promise { + const { results } = await searchDocuments(db, embeddingProvider, { + query: options.question, + topic: options.topic, + library: options.library, + limit: options.topK ?? 5, + }); + return results; +} + export interface PassthroughResult { contextPrompt: string; sources: RagSource[]; @@ -114,14 +129,7 @@ export async function getContextForQuestion( embeddingProvider: EmbeddingProvider, options: RagOptions, ): Promise { - const topK = options.topK ?? 5; - - const { results } = await searchDocuments(db, embeddingProvider, { - query: options.question, - topic: options.topic, - library: options.library, - limit: topK, - }); + const results = await retrieveResults(db, embeddingProvider, options); return { contextPrompt: buildContextPrompt(options.question, results), @@ -328,14 +336,7 @@ export async function* askQuestionStream( llmProvider: LlmProvider, options: RagOptions, ): AsyncGenerator { - const topK = options.topK ?? 5; - - const { results } = await searchDocuments(db, embeddingProvider, { - query: options.question, - topic: options.topic, - library: options.library, - limit: topK, - }); + const results = await retrieveResults(db, embeddingProvider, options); const contextPrompt = buildContextPrompt(options.question, results); const systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT; @@ -364,14 +365,7 @@ export async function askQuestion( llmProvider: LlmProvider, options: RagOptions, ): Promise { - const topK = options.topK ?? 5; - - const { results } = await searchDocuments(db, embeddingProvider, { - query: options.question, - topic: options.topic, - library: options.library, - limit: topK, - }); + const results = await retrieveResults(db, embeddingProvider, options); const contextPrompt = buildContextPrompt(options.question, results); const systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT; diff --git a/src/core/repo.ts b/src/core/repo.ts index 1854cfeb..793de555 100644 --- a/src/core/repo.ts +++ b/src/core/repo.ts @@ -1,5 +1,6 @@ import type Database from "better-sqlite3"; import { createHash } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { FetchError, ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; @@ -209,10 +210,6 @@ async function fetchWithRetry({ url, token, accept }: FetchWithRetryOptions): Pr ); } -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - // ── Tree Fetching ──────────────────────────────────────────────────────────── interface GitHubTreeItem { diff --git a/src/core/spider.ts b/src/core/spider.ts index 56c5311f..be23fbec 100644 --- a/src/core/spider.ts +++ b/src/core/spider.ts @@ -9,6 +9,7 @@ * - Private/internal IPs are blocked by the underlying url-fetcher */ +import { setTimeout as sleep } from "node:timers/promises"; import { getLogger } from "../logger.js"; import { FetchError } from "../errors.js"; import { fetchRaw, type FetchOptions } from "./url-fetcher.js"; @@ -540,7 +541,3 @@ function extractTextTitle(text: string, url: string): string { return url; } } - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/src/core/url-fetcher.ts b/src/core/url-fetcher.ts index 2bbe7dd3..887ed14d 100644 --- a/src/core/url-fetcher.ts +++ b/src/core/url-fetcher.ts @@ -7,7 +7,7 @@ import { getLogger } from "../logger.js"; /** Lazy singleton undici Agent that skips TLS certificate verification. */ let _insecureAgent: Agent | undefined; -function getInsecureAgent(): Agent { +export function getInsecureAgent(): Agent { _insecureAgent ??= new Agent({ connect: { rejectUnauthorized: false } }); return _insecureAgent; } @@ -160,21 +160,6 @@ async function readBodyWithLimit(response: Response, limit: number): Promise { - // Pass a per-request undici Agent when self-signed certs are allowed. - // This is scoped to this specific request chain and does not affect other - // concurrent requests (unlike mutating process.env["NODE_TLS_REJECT_UNAUTHORIZED"]). - const dispatcher = allowSelfSignedCerts ? getInsecureAgent() : undefined; - return _fetchWithRedirects(url, timeout, maxRedirects, allowPrivateUrls, dispatcher); -} - /** Re-resolve DNS after a fetch to detect DNS rebinding attacks. */ async function checkDnsRebinding(hostname: string, allowPrivateUrls: boolean): Promise { const recheck = await Promise.allSettled([dns.resolve4(hostname), dns.resolve6(hostname)]); @@ -201,7 +186,8 @@ async function checkDnsRebinding(hostname: string, allowPrivateUrls: boolean): P } } -async function _fetchWithRedirects( +/** Follow redirects manually so we can enforce a configurable limit. DNS-pinned to prevent rebinding. */ +async function fetchWithRedirects( url: string, timeout: number, maxRedirects: number, @@ -266,12 +252,16 @@ export async function fetchRaw(url: string, options?: FetchOptions): Promise { + const ext = extname(filename).toLowerCase(); + const title = explicitTitle ?? basename(filename, ext); + + if (CODE_EXTENSIONS.has(ext)) { + const chunker = await getTreeSitterChunker(); + const lang = extToLang(ext); + if (chunker?.supports(lang)) { + const codeChunks = await chunker.chunk(buf.toString("utf-8"), lang); + return { + title, + content: codeChunks[0]?.content ?? "", + chunks: codeChunks.map((c) => c.content), + }; + } + } + + const parser = getParserForFile(filename); + const content = parser ? await parser.parse(buf) : buf.toString("utf-8"); + return { title, content }; +} + export async function normalizeRawInput(input: RawInput): Promise { switch (input.type) { case "text": return { title: input.title, content: input.content }; - case "file": { - const ext = extname(input.path).toLowerCase(); - const buf = readFileSync(input.path); - const title = input.title ?? basename(input.path, ext); + case "file": + return normalizeBuffer(readFileSync(input.path), input.path, input.title); - if (CODE_EXTENSIONS.has(ext)) { - const chunker = await getTreeSitterChunker(); - const lang = extToLang(ext); - if (chunker?.supports(lang)) { - const codeChunks = await chunker.chunk(buf.toString("utf-8"), lang); - return { - title, - content: codeChunks[0]?.content ?? "", - chunks: codeChunks.map((c) => c.content), - }; - } - } - - const parser = getParserForFile(input.path); - const content = parser ? await parser.parse(buf) : buf.toString("utf-8"); - return { title, content }; - } - - case "buffer": { - const ext = extname(input.filename).toLowerCase(); - const title = input.title ?? basename(input.filename, ext); - - if (CODE_EXTENSIONS.has(ext)) { - const chunker = await getTreeSitterChunker(); - const lang = extToLang(ext); - if (chunker?.supports(lang)) { - const codeChunks = await chunker.chunk(input.buffer.toString("utf-8"), lang); - return { - title, - content: codeChunks[0]?.content ?? "", - chunks: codeChunks.map((c) => c.content), - }; - } - } - - const parser = getParserForFile(input.filename); - const content = parser ? await parser.parse(input.buffer) : input.buffer.toString("utf-8"); - return { title, content }; - } + case "buffer": + return normalizeBuffer(input.buffer, input.filename, input.title); case "url": { const fetched = await fetchAndConvert(input.url); From 87b28e230fe9216344b19fbd64c38f70ce829eec Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 14:47:31 +0000 Subject: [PATCH 04/83] refactor(core): share search row mapping and chunk-embedding cleanup - search: shared doc column list, rating join helpers, base zod row schema with .extend, toResult/vectorRowToResult mappers, runFtsQuery for the FTS5 AND/OR-fallback queries; hoist zod schemas to module scope; drop unused vectorSearch offset param - documents/topics/tags: DOC_COLUMNS + rowToDocument, TOPIC_COLUMNS + rowToTopic - add deleteChunkEmbeddings(db, docIds) and use it in documents, indexing, watcher, packs and ttl with each site's existing error handling - registry/publish: reuse readPackFile from core/packs - registry/resolve: remove unreachable manifest alt-path fallback - packs: drop redundant batchInstalled counter; tags: dedupe STOPWORDS --- src/core/documents.ts | 123 +++++----- src/core/indexing.ts | 5 +- src/core/packs.ts | 12 +- src/core/search.ts | 493 ++++++++++++++++------------------------ src/core/tags.ts | 19 +- src/core/topics.ts | 131 +++-------- src/core/ttl.ts | 8 +- src/core/watcher.ts | 7 +- src/registry/publish.ts | 13 +- src/registry/resolve.ts | 25 +- 10 files changed, 298 insertions(+), 538 deletions(-) diff --git a/src/core/documents.ts b/src/core/documents.ts index d3ff7e5d..804dc741 100644 --- a/src/core/documents.ts +++ b/src/core/documents.ts @@ -21,36 +21,28 @@ export interface Document { updatedAt: string; } -/** Get a document by ID. */ -export function getDocument(db: Database.Database, documentId: string): Document { - const row = db - .prepare( - ` - SELECT id, source_type, library, version, topic_id, title, content, url, content_hash, submitted_by, created_at, updated_at - FROM documents WHERE id = ? - `, - ) - .get(documentId) as - | { - id: string; - source_type: string; - library: string | null; - version: string | null; - topic_id: string | null; - title: string; - content: string; - url: string | null; - content_hash: string | null; - submitted_by: string; - created_at: string; - updated_at: string; - } - | undefined; +/** Column list matching {@link DocumentRow}, for `SELECT ... FROM documents`. */ +export const DOC_COLUMNS = + "id, source_type, library, version, topic_id, title, content, url, content_hash, submitted_by, created_at, updated_at"; - if (!row) { - throw new DocumentNotFoundError(documentId); - } +/** Raw `documents` row as returned by a `SELECT ${DOC_COLUMNS}` query. */ +export interface DocumentRow { + id: string; + source_type: string; + library: string | null; + version: string | null; + topic_id: string | null; + title: string; + content: string; + url: string | null; + content_hash: string | null; + submitted_by: string; + created_at: string; + updated_at: string; +} +/** Map a raw `documents` row to a {@link Document}. */ +export function rowToDocument(row: DocumentRow): Document { return { id: row.id, sourceType: row.source_type, @@ -67,17 +59,37 @@ export function getDocument(db: Database.Database, documentId: string): Document }; } +/** + * Delete vector embeddings for every chunk of the given documents. + * Throws on any SQLite error (including a missing `chunk_embeddings` table); + * callers decide how to handle it. + */ +export function deleteChunkEmbeddings(db: Database.Database, docIds: string[]): void { + if (docIds.length === 0) return; + const placeholders = docIds.map(() => "?").join(", "); + db.prepare( + `DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id IN (${placeholders}))`, + ).run(...docIds); +} + +/** Get a document by ID. */ +export function getDocument(db: Database.Database, documentId: string): Document { + const row = db.prepare(`SELECT ${DOC_COLUMNS} FROM documents WHERE id = ?`).get(documentId) as + | DocumentRow + | undefined; + + if (!row) { + throw new DocumentNotFoundError(documentId); + } + + return rowToDocument(row); +} + /** Delete a document and all its chunks/ratings (cascade). */ export function deleteDocument(db: Database.Database, documentId: string): void { // Clean up chunk_embeddings (no foreign key cascade for virtual tables) try { - db.prepare( - ` - DELETE FROM chunk_embeddings WHERE chunk_id IN ( - SELECT id FROM chunks WHERE document_id = ? - ) - `, - ).run(documentId); + deleteChunkEmbeddings(db, [documentId]); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); if (msg.includes("no such table")) { @@ -108,10 +120,7 @@ export function listDocuments( limit?: number | undefined; }, ): Document[] { - let sql = ` - SELECT id, source_type, library, version, topic_id, title, content, url, content_hash, submitted_by, created_at, updated_at - FROM documents WHERE 1=1 - `; + let sql = `SELECT ${DOC_COLUMNS} FROM documents WHERE 1=1`; const params: unknown[] = []; if (options?.library) { @@ -138,35 +147,9 @@ export function listDocuments( sql += " ORDER BY updated_at DESC LIMIT ?"; params.push(options?.limit ?? 50); - const rows = db.prepare(sql).all(...params) as Array<{ - id: string; - source_type: string; - library: string | null; - version: string | null; - topic_id: string | null; - title: string; - content: string; - url: string | null; - content_hash: string | null; - submitted_by: string; - created_at: string; - updated_at: string; - }>; - - return rows.map((row) => ({ - id: row.id, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - title: row.title, - content: row.content, - url: row.url, - contentHash: row.content_hash, - submittedBy: row.submitted_by, - createdAt: row.created_at, - updatedAt: row.updated_at, - })); + const rows = db.prepare(sql).all(...params) as DocumentRow[]; + + return rows.map((row) => rowToDocument(row)); } export interface UpdateDocumentInput { @@ -231,9 +214,7 @@ export async function updateDocument( saveVersion(db, documentId); try { - db.prepare( - "DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)", - ).run(documentId); + deleteChunkEmbeddings(db, [documentId]); } catch (err: unknown) { // chunk_embeddings table may not exist log.debug({ err, documentId }, "Skipped chunk_embeddings cleanup during update"); diff --git a/src/core/indexing.ts b/src/core/indexing.ts index 03641e4d..0d660b16 100644 --- a/src/core/indexing.ts +++ b/src/core/indexing.ts @@ -8,6 +8,7 @@ import { ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; import { checkDuplicate } from "./dedup.js"; import type { DedupOptions } from "./dedup.js"; +import { deleteChunkEmbeddings } from "./documents.js"; import { extractAndStoreDocumentLinks } from "./links.js"; import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; @@ -334,9 +335,7 @@ function handleUrlDedup( log.info({ docId: existing.id, url }, "Document updated, re-indexing"); try { - db.prepare( - "DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)", - ).run(existing.id); + deleteChunkEmbeddings(db, [existing.id]); } catch (err: unknown) { log.debug({ err, docId: existing.id }, "Skipped chunk_embeddings cleanup during re-index"); } diff --git a/src/core/packs.ts b/src/core/packs.ts index 04832dcd..1ca96063 100644 --- a/src/core/packs.ts +++ b/src/core/packs.ts @@ -15,6 +15,7 @@ import type { EmbeddingProvider } from "../providers/embedding.js"; import { ValidationError, FetchError } from "../errors.js"; import { getLogger } from "../logger.js"; import { chunkContent, chunkContentStreaming, STREAMING_THRESHOLD } from "./indexing.js"; +import { deleteChunkEmbeddings } from "./documents.js"; import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; import { suggestTagsFromText } from "./tags.js"; import { fetchAndConvert } from "./url-fetcher.js"; @@ -124,7 +125,7 @@ function writePackFile(filePath: string, pack: KnowledgePack): void { } /** Read a pack file, auto-detecting gzip by magic bytes or extension. */ -function readPackFile(filePath: string): string { +export function readPackFile(filePath: string): string { const raw = readFileSync(filePath); if (raw.length >= 2 && raw[0] === GZIP_MAGIC[0] && raw[1] === GZIP_MAGIC[1]) { return gunzipSync(raw).toString("utf-8"); @@ -369,9 +370,7 @@ function insertBatchIntoDb( insertEmbedding: Database.Statement, ): number { const log = getLogger(); - let batchInstalled = 0; const doInsert = db.transaction(() => { - batchInstalled = 0; for (const info of batch.docInfos) { insertDoc.run( info.docId, @@ -398,11 +397,10 @@ function insertBatchIntoDb( } } } - batchInstalled++; } }); doInsert(); - return batchInstalled; + return batch.docInfos.length; } /** Install a pack from a local JSON file path or registry name. */ @@ -589,9 +587,7 @@ export function removePack(db: Database.Database, packName: string): void { const deleteTransaction = db.transaction(() => { for (const { id } of docIds) { try { - db.prepare( - "DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)", - ).run(id); + deleteChunkEmbeddings(db, [id]); } catch (err) { log.debug( { err, documentId: id }, diff --git a/src/core/search.ts b/src/core/search.ts index c0570fd1..3de6762d 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -215,6 +215,13 @@ function reciprocalRankFusion(listA: SearchResult[], listB: SearchResult[]): Sea return computeRrfScores(map); } +const ContextCurrentRowSchema = z.object({ chunk_index: z.number() }).optional(); +const ContextChunkRowSchema = z.object({ + id: z.string(), + content: z.string(), + chunk_index: z.number(), +}); + /** Fetch neighboring chunks for a given chunk within its document. */ function fetchContextChunks( db: Database.Database, @@ -222,9 +229,8 @@ function fetchContextChunks( documentId: string, contextSize: number, ): { before: ContextChunk[]; after: ContextChunk[] } { - const CurrentRowSchema = z.object({ chunk_index: z.number() }).optional(); const currentRow = validateRow( - CurrentRowSchema, + ContextCurrentRowSchema, db .prepare(`SELECT chunk_index FROM chunks WHERE id = ? AND document_id = ?`) .get(chunkId, documentId), @@ -235,9 +241,8 @@ function fetchContextChunks( const idx = currentRow.chunk_index; - const ChunkRowSchema = z.object({ id: z.string(), content: z.string(), chunk_index: z.number() }); const beforeRows = validateRows( - ChunkRowSchema, + ContextChunkRowSchema, db .prepare( `SELECT id, content, chunk_index FROM chunks @@ -249,7 +254,7 @@ function fetchContextChunks( ); const afterRows = validateRows( - ChunkRowSchema, + ContextChunkRowSchema, db .prepare( `SELECT id, content, chunk_index FROM chunks @@ -521,7 +526,7 @@ export async function searchDocuments( const candidateLimit = Math.min((offset + limit) * overfetchFactor, maxCandidateLimit); try { - const vectorResults = vectorSearch(db, options, vecBuffer, candidateLimit, 0); + const vectorResults = vectorSearch(db, options, vecBuffer, candidateLimit); let ftsResults: SearchResult[] | null = null; let ftsTotalCount = 0; @@ -562,6 +567,102 @@ export async function searchDocuments( } } +// --------------------------------------------------------------------------- +// Shared SQL fragments, row schemas and row -> SearchResult mapping +// --------------------------------------------------------------------------- + +/** Document columns selected alongside every chunk search hit (alias `d`). */ +const DOC_RESULT_COLUMNS = `d.title, + d.source_type, + d.library, + d.version, + d.topic_id, + d.url`; + +/** Optional `avg_r.avg_rating` select column, present only when the rating join is used. */ +function ratingColumn(needsRatingJoin: boolean): string { + return needsRatingJoin ? ",\n avg_r.avg_rating" : ""; +} + +/** Optional LEFT JOIN of per-document average ratings (alias `avg_r`, joined to `d`). */ +function ratingJoin(needsRatingJoin: boolean): string { + return needsRatingJoin + ? ` + LEFT JOIN ( + SELECT document_id, AVG(rating) AS avg_rating + FROM ratings + GROUP BY document_id + ) avg_r ON avg_r.document_id = d.id` + : ""; +} + +const BaseResultRowSchema = z.object({ + chunk_id: z.string(), + document_id: z.string(), + chunk_content: z.string(), + title: z.string(), + source_type: z.string(), + library: z.string().nullable(), + version: z.string().nullable(), + topic_id: z.string().nullable(), + url: z.string().nullable(), +}); + +const RatedResultRowSchema = BaseResultRowSchema.extend({ + avg_rating: z.number().nullable().optional(), +}); + +const VectorRowSchema = RatedResultRowSchema.extend({ distance: z.number() }); +const KeywordRowSchema = RatedResultRowSchema; +const Fts5RowSchema = RatedResultRowSchema.extend({ fts_rank: z.number() }); +const RelatedRowSchema = BaseResultRowSchema.extend({ distance: z.number() }); + +const LinkedChunkSchema = z.object({ + id: z.string(), + document_id: z.string(), + content: z.string(), + chunk_index: z.number(), + title: z.string(), + source_type: z.string(), + library: z.string().nullable(), + version: z.string().nullable(), + topic_id: z.string().nullable(), + url: z.string().nullable(), +}); + +type ResultRow = z.infer & { + avg_rating?: number | null | undefined; +}; + +/** Map a validated search row to a SearchResult with the given score and explanation. */ +function toResult(row: ResultRow, score: number, scoreExplanation: ScoreExplanation): SearchResult { + return { + documentId: row.document_id, + chunkId: row.chunk_id, + title: row.title, + content: row.chunk_content, + sourceType: row.source_type, + library: row.library, + version: row.version, + topicId: row.topic_id, + url: row.url, + score, + avgRating: row.avg_rating ?? null, + scoreExplanation, + }; +} + +/** Map a vector ANN row (with `distance`) to a SearchResult scored by similarity. */ +function vectorRowToResult(row: ResultRow & { distance: number }): SearchResult { + const similarity = 1 - row.distance; + return toResult(row, similarity, { + method: "vector" as SearchMethod, + rawScore: row.distance, + boostFactors: [], + details: `Vector similarity: distance=${row.distance.toFixed(4)}, similarity=${similarity.toFixed(4)}`, + }); +} + /** Pure vector search — returns candidates for fusion/pagination. * `limit` should already account for offset (i.e. caller passes offset+limit). */ function vectorSearch( @@ -569,11 +670,7 @@ function vectorSearch( options: SearchOptions, vecBuffer: Buffer, limit: number, - _offset: number, ): SearchResponse { - // The caller already over-fetches; use the limit directly. - const annCandidateLimit = limit; - const needsRatingJoin = options.minRating !== undefined; let sql = ` @@ -582,12 +679,7 @@ function vectorSearch( candidates.distance, c.document_id, c.content AS chunk_content, - d.title, - d.source_type, - d.library, - d.version, - d.topic_id, - d.url${needsRatingJoin ? ",\n avg_r.avg_rating" : ""} + ${DOC_RESULT_COLUMNS}${ratingColumn(needsRatingJoin)} FROM ( SELECT chunk_id, distance FROM chunk_embeddings @@ -596,37 +688,16 @@ function vectorSearch( LIMIT ? ) candidates JOIN chunks c ON c.id = candidates.chunk_id - JOIN documents d ON d.id = c.document_id${ - needsRatingJoin - ? ` - LEFT JOIN ( - SELECT document_id, AVG(rating) AS avg_rating - FROM ratings - GROUP BY document_id - ) avg_r ON avg_r.document_id = d.id` - : "" - } + JOIN documents d ON d.id = c.document_id${ratingJoin(needsRatingJoin)} WHERE 1=1 `; - const params: unknown[] = [vecBuffer, annCandidateLimit]; + // The caller already over-fetches; use the limit directly as the ANN candidate limit. + const params: unknown[] = [vecBuffer, limit]; sql = appendFilters(sql, params, options, "d"); sql += ` ORDER BY candidates.distance`; - const VectorRowSchema = z.object({ - chunk_id: z.string(), - distance: z.number(), - document_id: z.string(), - chunk_content: z.string(), - title: z.string(), - source_type: z.string(), - library: z.string().nullable(), - version: z.string().nullable(), - topic_id: z.string().nullable(), - url: z.string().nullable(), - avg_rating: z.number().nullable().optional(), - }); const rows = validateRows(VectorRowSchema, db.prepare(sql).all(...params), "vectorSearch.rows"); // totalCount: if we got fewer rows than the ANN candidate limit, we know @@ -637,28 +708,7 @@ function vectorSearch( return { totalCount, - results: rows.map((row) => { - const similarity = 1 - row.distance; - return { - documentId: row.document_id, - chunkId: row.chunk_id, - title: row.title, - content: row.chunk_content, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - url: row.url, - score: similarity, - avgRating: row.avg_rating ?? null, - scoreExplanation: { - method: "vector" as SearchMethod, - rawScore: row.distance, - boostFactors: [], - details: `Vector similarity: distance=${row.distance.toFixed(4)}, similarity=${similarity.toFixed(4)}`, - }, - }; - }), + results: rows.map((row) => vectorRowToResult(row)), }; } @@ -692,23 +742,9 @@ function keywordSearch( c.id AS chunk_id, c.document_id, c.content AS chunk_content, - d.title, - d.source_type, - d.library, - d.version, - d.topic_id, - d.url${needsRatingJoin ? ",\n avg_r.avg_rating" : ""} + ${DOC_RESULT_COLUMNS}${ratingColumn(needsRatingJoin)} FROM chunks c - JOIN documents d ON d.id = c.document_id${ - needsRatingJoin - ? ` - LEFT JOIN ( - SELECT document_id, AVG(rating) AS avg_rating - FROM ratings - GROUP BY document_id - ) avg_r ON avg_r.document_id = d.id` - : "" - } + JOIN documents d ON d.id = c.document_id${ratingJoin(needsRatingJoin)} WHERE (${likeConditions}) `; @@ -721,18 +757,6 @@ function keywordSearch( sql += " LIMIT ? OFFSET ?"; params.push(limit, offset); - const KeywordRowSchema = z.object({ - chunk_id: z.string(), - document_id: z.string(), - chunk_content: z.string(), - title: z.string(), - source_type: z.string(), - library: z.string().nullable(), - version: z.string().nullable(), - topic_id: z.string().nullable(), - url: z.string().nullable(), - avg_rating: z.number().nullable().optional(), - }); const rows = validateRows(KeywordRowSchema, db.prepare(sql).all(...params), "keywordSearch.rows"); const totalCount = lazyCount( @@ -749,25 +773,12 @@ function keywordSearch( totalCount, results: rows.map((row, index) => { const rankScore = Math.max(0, 1 - index * 0.1); - return { - documentId: row.document_id, - chunkId: row.chunk_id, - title: row.title, - content: row.chunk_content, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - url: row.url, - score: rankScore, - avgRating: row.avg_rating ?? null, - scoreExplanation: { - method: "keyword" as SearchMethod, - rawScore: rankScore, - boostFactors: [], - details: `Keyword LIKE match: rank=${index + 1}, score=${rankScore.toFixed(4)}`, - }, - }; + return toResult(row, rankScore, { + method: "keyword" as SearchMethod, + rawScore: rankScore, + boostFactors: [], + details: `Keyword LIKE match: rank=${index + 1}, score=${rankScore.toFixed(4)}`, + }); }), }; } @@ -792,12 +803,13 @@ function sanitizeFtsWord(word: string): string { return word; } +const RatingRowSchema = z.object({ document_id: z.string(), avg_rating: z.number().nullable() }); + /** Fetch avg ratings for a small set of documents and attach to results. */ function attachRatings(db: Database.Database, results: SearchResult[]): SearchResult[] { if (results.length === 0) return results; const ids = [...new Set(results.map((r) => r.documentId))]; const placeholders = ids.map(() => "?").join(", "); - const RatingRowSchema = z.object({ document_id: z.string(), avg_rating: z.number().nullable() }); const rows = validateRows( RatingRowSchema, db @@ -814,6 +826,14 @@ function attachRatings(db: Database.Database, results: SearchResult[]): SearchRe return results.map((r) => ({ ...r, avgRating: ratingMap.get(r.documentId) ?? null })); } +const SourceChunkSchema = z.object({ + id: z.string(), + document_id: z.string(), + content: z.string(), + chunk_index: z.number(), +}); +const EmbeddingRowSchema = z.object({ embedding: z.instanceof(Buffer) }); + /** * Find chunks related to a given chunk by vector similarity. * Looks up the source chunk's embedding, then searches for similar chunks @@ -828,12 +848,6 @@ export function getRelatedChunks( const minScore = options.minScore ?? 0; // Look up the source chunk - const SourceChunkSchema = z.object({ - id: z.string(), - document_id: z.string(), - content: z.string(), - chunk_index: z.number(), - }); const sourceChunkRow = validateRow( SourceChunkSchema.optional(), db @@ -855,7 +869,6 @@ export function getRelatedChunks( const excludeDocumentId = options.excludeDocumentId ?? sourceChunkRow.document_id; // Fetch the embedding for the source chunk - const EmbeddingRowSchema = z.object({ embedding: z.instanceof(Buffer) }); const embeddingRow = validateRow( EmbeddingRowSchema.optional(), db.prepare(`SELECT embedding FROM chunk_embeddings WHERE chunk_id = ?`).get(chunkId), @@ -876,12 +889,7 @@ export function getRelatedChunks( candidates.distance, c.document_id, c.content AS chunk_content, - d.title, - d.source_type, - d.library, - d.version, - d.topic_id, - d.url + ${DOC_RESULT_COLUMNS} FROM ( SELECT chunk_id, distance FROM chunk_embeddings @@ -910,47 +918,13 @@ export function getRelatedChunks( sql += ` ORDER BY candidates.distance LIMIT ?`; params.push(limit * 2); // over-fetch to allow minScore filtering - const RelatedRowSchema = z.object({ - chunk_id: z.string(), - distance: z.number(), - document_id: z.string(), - chunk_content: z.string(), - title: z.string(), - source_type: z.string(), - library: z.string().nullable(), - version: z.string().nullable(), - topic_id: z.string().nullable(), - url: z.string().nullable(), - }); - const rows = validateRows( RelatedRowSchema, db.prepare(sql).all(...params), "getRelatedChunks.rows", ); - let results: SearchResult[] = rows.map((row) => { - const similarity = 1 - row.distance; - return { - documentId: row.document_id, - chunkId: row.chunk_id, - title: row.title, - content: row.chunk_content, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - url: row.url, - score: similarity, - avgRating: null, - scoreExplanation: { - method: "vector" as SearchMethod, - rawScore: row.distance, - boostFactors: [], - details: `Vector similarity: distance=${row.distance.toFixed(4)}, similarity=${similarity.toFixed(4)}`, - }, - }; - }); + let results: SearchResult[] = rows.map((row) => vectorRowToResult(row)); // Apply minScore filter if (minScore > 0) { @@ -970,19 +944,6 @@ export function getRelatedChunks( linked_doc_id: string; }[]; - const LinkedChunkSchema = z.object({ - id: z.string(), - document_id: z.string(), - content: z.string(), - chunk_index: z.number(), - title: z.string(), - source_type: z.string(), - library: z.string().nullable(), - version: z.string().nullable(), - topic_id: z.string().nullable(), - url: z.string().nullable(), - }); - const presentDocIds = new Set(results.map((r) => r.documentId)); for (const { linked_doc_id } of linkedDocs) { if (!presentDocIds.has(linked_doc_id)) { @@ -1002,25 +963,18 @@ export function getRelatedChunks( "getRelatedChunks.linkedChunk", ); if (linkedChunk) { - results.push({ - documentId: linkedChunk.document_id, - chunkId: linkedChunk.id, - title: linkedChunk.title, - content: linkedChunk.content, - sourceType: linkedChunk.source_type, - library: linkedChunk.library, - version: linkedChunk.version, - topicId: linkedChunk.topic_id, - url: linkedChunk.url, - score: 0.6, - avgRating: null, - scoreExplanation: { - method: "vector" as SearchMethod, - rawScore: 0.6, - boostFactors: ["linked_document"], - details: "Explicitly linked document", - }, - }); + results.push( + toResult( + { ...linkedChunk, chunk_id: linkedChunk.id, chunk_content: linkedChunk.content }, + 0.6, + { + method: "vector" as SearchMethod, + rawScore: 0.6, + boostFactors: ["linked_document"], + details: "Explicitly linked document", + }, + ), + ); } } } @@ -1033,150 +987,93 @@ export function getRelatedChunks( return { chunks: results, sourceChunk }; } -/** FTS5-based full-text search with BM25 ranking. Uses AND logic by default. */ -function fts5Search( +/** Quote a sanitized word as an FTS5 string literal. */ +function quoteFtsWord(word: string): string { + return `"${word.replaceAll('"', '""')}"`; +} + +/** + * Run a single FTS5 MATCH query with the standard filters and pagination. + * Returns the validated rows plus the unpaginated SQL/params for lazy counting. + */ +function runFtsQuery( db: Database.Database, options: SearchOptions, + matchExpr: string, limit: number, offset: number, -): SearchResponse { - // Sanitize and escape FTS5 query: strip dangerous syntax, wrap each word in quotes. - // AND-by-default: require all terms to match for better precision. - const words = options.query - .split(/\s+/) - .map((w) => sanitizeFtsWord(w)) - .filter((w) => w.length > 0); - if (words.length === 0) return { results: [], totalCount: 0 }; - + label: string, +): { rows: Array>; baseSql: string; baseParams: unknown[] } { const needsRatingJoin = options.minRating !== undefined; - - const ftsQuery = words.map((w) => `"${w.replaceAll('"', '""')}"`).join(" AND "); - const params: unknown[] = [ftsQuery]; + const params: unknown[] = [matchExpr]; let sql = ` SELECT f.chunk_id, f.document_id, f.content AS chunk_content, - d.title, - d.source_type, - d.library, - d.version, - d.topic_id, - d.url, - rank AS fts_rank${needsRatingJoin ? ",\n avg_r.avg_rating" : ""} + ${DOC_RESULT_COLUMNS}, + rank AS fts_rank${ratingColumn(needsRatingJoin)} FROM chunks_fts f - JOIN documents d ON d.id = f.document_id${ - needsRatingJoin - ? ` - LEFT JOIN ( - SELECT document_id, AVG(rating) AS avg_rating - FROM ratings - GROUP BY document_id - ) avg_r ON avg_r.document_id = d.id` - : "" - } + JOIN documents d ON d.id = f.document_id${ratingJoin(needsRatingJoin)} WHERE chunks_fts MATCH ? `; sql = appendFilters(sql, params, options, "d"); - // Lazy count – may be updated if OR fallback is used - let baseSql = sql; - let baseParams = [...params]; + const baseSql = sql; + const baseParams = [...params]; sql += " ORDER BY rank LIMIT ? OFFSET ?"; params.push(limit, offset); - const Fts5RowSchema = z.object({ - chunk_id: z.string(), - document_id: z.string(), - chunk_content: z.string(), - title: z.string(), - source_type: z.string(), - library: z.string().nullable(), - version: z.string().nullable(), - topic_id: z.string().nullable(), - url: z.string().nullable(), - fts_rank: z.number(), - avg_rating: z.number().nullable().optional(), - }); - let rows = validateRows(Fts5RowSchema, db.prepare(sql).all(...params), "fts5Search.rows"); - - // If AND returned nothing, retry with OR for recall - if (rows.length === 0 && words.length > 1) { - const orQuery = words.map((w) => `"${w.replace(/"/g, '""')}"`).join(" OR "); - const orParams: unknown[] = [orQuery]; - let orSql = ` - SELECT - f.chunk_id, - f.document_id, - f.content AS chunk_content, - d.title, - d.source_type, - d.library, - d.version, - d.topic_id, - d.url, - rank AS fts_rank${needsRatingJoin ? ",\n avg_r.avg_rating" : ""} - FROM chunks_fts f - JOIN documents d ON d.id = f.document_id${ - needsRatingJoin - ? ` - LEFT JOIN ( - SELECT document_id, AVG(rating) AS avg_rating - FROM ratings - GROUP BY document_id - ) avg_r ON avg_r.document_id = d.id` - : "" - } - WHERE chunks_fts MATCH ? - `; - orSql = appendFilters(orSql, orParams, options, "d"); + const rows = validateRows(Fts5RowSchema, db.prepare(sql).all(...params), label); + return { rows, baseSql, baseParams }; +} - // Update count base to use OR query - baseSql = orSql; - baseParams = [...orParams]; +/** FTS5-based full-text search with BM25 ranking. Uses AND logic by default. */ +function fts5Search( + db: Database.Database, + options: SearchOptions, + limit: number, + offset: number, +): SearchResponse { + // Sanitize and escape FTS5 query: strip dangerous syntax, wrap each word in quotes. + // AND-by-default: require all terms to match for better precision. + const words = options.query + .split(/\s+/) + .map((w) => sanitizeFtsWord(w)) + .filter((w) => w.length > 0); + if (words.length === 0) return { results: [], totalCount: 0 }; - orSql += " ORDER BY rank LIMIT ? OFFSET ?"; - orParams.push(limit, offset); + const quoted = words.map((w) => quoteFtsWord(w)); + let fts = runFtsQuery(db, options, quoted.join(" AND "), limit, offset, "fts5Search.rows"); - rows = validateRows(Fts5RowSchema, db.prepare(orSql).all(...orParams), "fts5Search.orRows"); + // If AND returned nothing, retry with OR for recall (the lazy count then uses the OR query) + if (fts.rows.length === 0 && words.length > 1) { + fts = runFtsQuery(db, options, quoted.join(" OR "), limit, offset, "fts5Search.orRows"); } const totalCount = lazyCount( db, - baseSql, - baseParams, + fts.baseSql, + fts.baseParams, offset, - rows.length, + fts.rows.length, limit, "FTS5 search count", ); return { totalCount, - results: rows.map((row) => { + results: fts.rows.map((row) => { const bm25Score = -row.fts_rank; - return { - documentId: row.document_id, - chunkId: row.chunk_id, - title: row.title, - content: row.chunk_content, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - url: row.url, - score: bm25Score, - avgRating: row.avg_rating ?? null, - scoreExplanation: { - method: "fts5" as SearchMethod, - rawScore: row.fts_rank, - boostFactors: [], - details: `FTS5 BM25 ranking: raw_rank=${row.fts_rank.toFixed(4)}, score=${bm25Score.toFixed(4)}`, - }, - }; + return toResult(row, bm25Score, { + method: "fts5" as SearchMethod, + rawScore: row.fts_rank, + boostFactors: [], + details: `FTS5 BM25 ranking: raw_rank=${row.fts_rank.toFixed(4)}, score=${bm25Score.toFixed(4)}`, + }); }), }; } diff --git a/src/core/tags.ts b/src/core/tags.ts index 2f85b699..5026d019 100644 --- a/src/core/tags.ts +++ b/src/core/tags.ts @@ -4,7 +4,7 @@ import { z } from "zod"; import { DocumentNotFoundError, ValidationError } from "../errors.js"; import { validateRow, validateRows } from "../db/validate.js"; import { createChildLogger } from "../logger.js"; -import type { Document } from "./documents.js"; +import { rowToDocument, type Document } from "./documents.js"; const TagRowSchema = z.object({ id: z.string(), @@ -131,7 +131,6 @@ const STOPWORDS = new Set([ "your", "his", "her", - "our", "out", "then", "there", @@ -140,7 +139,6 @@ const STOPWORDS = new Set([ "them", "they", "you", - "your", "only", "own", "same", @@ -354,20 +352,7 @@ export function getDocumentsByTag( log.info({ tagNames: normalized, resultCount: rows.length }, "Documents retrieved by tags"); - return rows.map((row) => ({ - id: row.id, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - title: row.title, - content: row.content, - url: row.url, - contentHash: row.content_hash, - submittedBy: row.submitted_by, - createdAt: row.created_at, - updatedAt: row.updated_at, - })); + return rows.map((row) => rowToDocument(row)); } /** Tokenize text into lowercase words, filtering stopwords and short words. */ diff --git a/src/core/topics.ts b/src/core/topics.ts index 59eb2afc..9408ed5a 100644 --- a/src/core/topics.ts +++ b/src/core/topics.ts @@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto"; import { ValidationError, TopicNotFoundError } from "../errors.js"; import { createChildLogger } from "../logger.js"; import { validateRow } from "../utils/db-validation.js"; -import type { Document } from "./documents.js"; +import { DOC_COLUMNS, rowToDocument, type Document, type DocumentRow } from "./documents.js"; export interface Topic { id: string; @@ -14,6 +14,28 @@ export interface Topic { updatedAt: string; } +const TOPIC_COLUMNS = "id, name, description, parent_id, created_at, updated_at"; + +type TopicRow = { + id: string; + name: string; + description: string | null; + parent_id: string | null; + created_at: string; + updated_at: string; +}; + +function rowToTopic(row: TopicRow): Topic { + return { + id: row.id, + name: row.name, + description: row.description, + parentId: row.parent_id, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; +} + export interface CreateTopicInput { name: string; description?: string | undefined; @@ -69,41 +91,20 @@ export function createTopic(db: Database.Database, input: CreateTopicInput): Top // Topic already existed — fetch and return it log.info({ name: input.name }, "Topic already exists, returning existing"); - const row = db - .prepare( - "SELECT id, name, description, parent_id, created_at, updated_at FROM topics WHERE name = ?", - ) - .get(input.name); + const row = db.prepare(`SELECT ${TOPIC_COLUMNS} FROM topics WHERE name = ?`).get(input.name); - const validated = validateRow<{ - id: string; - name: string; - description: string | null; - parent_id: string | null; - created_at: string; - updated_at: string; - }>( + const validated = validateRow( row, ["id", "name", "description", "parent_id", "created_at", "updated_at"], "existing topic lookup", ); - return { - id: validated.id, - name: validated.name, - description: validated.description, - parentId: validated.parent_id, - createdAt: validated.created_at, - updatedAt: validated.updated_at, - }; + return rowToTopic(validated); } /** List topics, optionally filtered by parent. */ export function listTopics(db: Database.Database, parentId?: string): Topic[] { - let sql = ` - SELECT id, name, description, parent_id, created_at, updated_at - FROM topics - `; + let sql = `SELECT ${TOPIC_COLUMNS} FROM topics`; const params: unknown[] = []; if (parentId === undefined) { @@ -115,55 +116,20 @@ export function listTopics(db: Database.Database, parentId?: string): Topic[] { sql += " ORDER BY name"; - const rows = db.prepare(sql).all(...params) as Array<{ - id: string; - name: string; - description: string | null; - parent_id: string | null; - created_at: string; - updated_at: string; - }>; + const rows = db.prepare(sql).all(...params) as TopicRow[]; - return rows.map((row) => ({ - id: row.id, - name: row.name, - description: row.description, - parentId: row.parent_id, - createdAt: row.created_at, - updatedAt: row.updated_at, - })); + return rows.map((row) => rowToTopic(row)); } /** Get a topic by ID. */ export function getTopic(db: Database.Database, topicId: string): Topic { - const row = db - .prepare( - ` - SELECT id, name, description, parent_id, created_at, updated_at - FROM topics WHERE id = ? - `, - ) - .get(topicId) as - | { - id: string; - name: string; - description: string | null; - parent_id: string | null; - created_at: string; - updated_at: string; - } + const row = db.prepare(`SELECT ${TOPIC_COLUMNS} FROM topics WHERE id = ?`).get(topicId) as + | TopicRow | undefined; if (!row) throw new TopicNotFoundError(topicId); - return { - id: row.id, - name: row.name, - description: row.description, - parentId: row.parent_id, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; + return rowToTopic(row); } export interface GetDocumentsByTopicOptions { @@ -223,37 +189,10 @@ export function getDocumentsByTopic( const offset = options?.offset ?? 0; const rows = db .prepare( - `SELECT id, source_type, library, version, topic_id, title, content, url, content_hash, submitted_by, created_at, updated_at - FROM documents WHERE topic_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?`, + `SELECT ${DOC_COLUMNS} FROM documents WHERE topic_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?`, ) - .all(topicId, limit, offset) as Array<{ - id: string; - source_type: string; - library: string | null; - version: string | null; - topic_id: string | null; - title: string; - content: string; - url: string | null; - content_hash: string | null; - submitted_by: string; - created_at: string; - updated_at: string; - }>; - return rows.map((row) => ({ - id: row.id, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - title: row.title, - content: row.content, - url: row.url, - contentHash: row.content_hash, - submittedBy: row.submitted_by, - createdAt: row.created_at, - updatedAt: row.updated_at, - })); + .all(topicId, limit, offset) as DocumentRow[]; + return rows.map((row) => rowToDocument(row)); } /** Get topics with document counts. */ diff --git a/src/core/ttl.ts b/src/core/ttl.ts index 3b24c5c4..b8ce0e2b 100644 --- a/src/core/ttl.ts +++ b/src/core/ttl.ts @@ -1,5 +1,6 @@ import type Database from "better-sqlite3"; import { getLogger } from "../logger.js"; +import { deleteChunkEmbeddings } from "./documents.js"; export interface PruneResult { pruned: number; @@ -28,12 +29,7 @@ export function pruneExpiredDocuments(db: Database.Database): PruneResult { const pruneTransaction = db.transaction(() => { // Remove embeddings first (foreign key dependency) try { - db.prepare( - `DELETE FROM chunk_embeddings - WHERE chunk_id IN ( - SELECT id FROM chunks WHERE document_id IN (${placeholders}) - )`, - ).run(...ids); + deleteChunkEmbeddings(db, ids); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); if (msg.includes("no such table")) { diff --git a/src/core/watcher.ts b/src/core/watcher.ts index 5ac7ad9b..b2d50686 100644 --- a/src/core/watcher.ts +++ b/src/core/watcher.ts @@ -4,6 +4,7 @@ import { createHash } from "node:crypto"; import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { indexDocument } from "./indexing.js"; +import { deleteChunkEmbeddings } from "./documents.js"; import { createChildLogger } from "../logger.js"; export const DEFAULT_WATCH_EXTENSIONS = [".md", ".mdx", ".txt", ".rst"]; @@ -133,11 +134,7 @@ export class FileWatcher { if (!existing) return; try { - this.db - .prepare( - "DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)", - ) - .run(existing.id); + deleteChunkEmbeddings(this.db, [existing.id]); } catch { // chunk_embeddings table may not exist } diff --git a/src/registry/publish.ts b/src/registry/publish.ts index cb0ba342..42f91f85 100644 --- a/src/registry/publish.ts +++ b/src/registry/publish.ts @@ -4,7 +4,6 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync, copyFileSync, rmSync } from "node:fs"; import { join } from "node:path"; -import { gunzipSync } from "node:zlib"; import { getLogger } from "../logger.js"; import { ValidationError } from "../errors.js"; import type { @@ -25,7 +24,7 @@ import { import { getRegistry } from "./config.js"; import { commitAndPush, fetchRegistry, git, clearIndexCache } from "./git.js"; import { computeChecksum, writeChecksumFile } from "./checksum.js"; -import type { KnowledgePack } from "../core/packs.js"; +import { readPackFile, type KnowledgePack } from "../core/packs.js"; /** Remove an entire pack directory and its entry from the index. */ function removeEntirePack(packDir: string, cacheDir: string, packName: string): void { @@ -100,20 +99,12 @@ function bumpPatchVersion(version: string): string { return `${parts[0]}.${parts[1]}.${Number.isNaN(patch) ? 1 : patch + 1}`; } -/** Gzip magic number: first two bytes of a gzip stream. */ -const GZIP_MAGIC = Buffer.from([0x1f, 0x8b]); - /** * Read a pack JSON file (plain or gzip-compressed). * Auto-detects gzip by checking for magic bytes. */ function readPackJson(filePath: string): KnowledgePack { - const raw = readFileSync(filePath); - const text = - raw.length >= 2 && raw[0] === GZIP_MAGIC[0] && raw[1] === GZIP_MAGIC[1] - ? gunzipSync(raw).toString("utf-8") - : raw.toString("utf-8"); - return JSON.parse(text) as KnowledgePack; + return JSON.parse(readPackFile(filePath)) as KnowledgePack; } /** Validate that the registry exists and has a local cache; return the cache directory. */ diff --git a/src/registry/resolve.ts b/src/registry/resolve.ts index 6eac6efb..c757ed14 100644 --- a/src/registry/resolve.ts +++ b/src/registry/resolve.ts @@ -3,7 +3,6 @@ */ import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; import { getLogger } from "../logger.js"; import { ValidationError } from "../errors.js"; import type { @@ -13,12 +12,7 @@ import type { RegistryConflict, ConflictResolution, } from "./types.js"; -import { - getRegistryCacheDir, - getPackManifestPath, - getPackDataPath, - PACK_MANIFEST_FILE, -} from "./types.js"; +import { getRegistryCacheDir, getPackManifestPath, getPackDataPath } from "./types.js"; import { loadRegistries } from "./config.js"; import { readIndex } from "./git.js"; import { verifyChecksum } from "./checksum.js"; @@ -86,22 +80,7 @@ export function findPackInRegistries(packName: string): { */ export function readPackManifest(registryName: string, packName: string): PackManifest | null { const manifestPath = getPackManifestPath(registryName, packName); - if (!existsSync(manifestPath)) { - // Fall back: try reading from the packs directory directly - const cacheDir = getRegistryCacheDir(registryName); - const altPath = join(cacheDir, "packs", packName, PACK_MANIFEST_FILE); - if (!existsSync(altPath)) return null; - try { - return JSON.parse(readFileSync(altPath, "utf-8")) as PackManifest; - } catch (err) { - const log = getLogger(); - log.warn( - { registryName, packName, err: err instanceof Error ? err.message : String(err) }, - "Failed to parse pack manifest (alt path)", - ); - return null; - } - } + if (!existsSync(manifestPath)) return null; try { return JSON.parse(readFileSync(manifestPath, "utf-8")) as PackManifest; } catch (err) { From e939eed7cd2b0ccf7c80fd72d270a1ce52127102 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 15:10:05 +0000 Subject: [PATCH 05/83] fix(mcp): include document, chunk, and link IDs in tool output search-docs results now show a Document ID / Chunk ID line, link-documents prints the created link's ID, and get-document-links shows each link's ID, so get-related, rate-document, and delete-link are usable from tool output. The link-documents enum now comes from LINK_TYPES in core/links.ts and accepts "references". --- docs/reference/mcp-tools.md | 8 +- src/core/links.ts | 11 ++- src/mcp/format.ts | 67 +++++++++++++++++ src/mcp/server.ts | 76 ++----------------- tests/unit/mcp-format.test.ts | 135 ++++++++++++++++++++++++++++++++++ 5 files changed, 223 insertions(+), 74 deletions(-) create mode 100644 src/mcp/format.ts create mode 100644 tests/unit/mcp-format.test.ts diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 55db89b8..92d6d15b 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -16,6 +16,8 @@ Semantic search across your knowledge base. | `limit` | number | | Max results (default: 10) | | `offset` | number | | Pagination offset | +Each result shows a `**Document ID:** … | **Chunk ID:** …` line. Pass these IDs to `get-document`, `rate-document`, `get-related`, and `link-documents`. + **Search results** include a `scoreExplanation` object on each result: ```typescript @@ -236,12 +238,14 @@ Create a typed cross-reference relationship between two documents. | ---------- | ------ | -------- | -------------------------------------------------------------------- | | `sourceId` | string | ✅ | The source document ID | | `targetId` | string | ✅ | The target document ID | -| `linkType` | string | ✅ | Relationship type: `see_also`, `prerequisite`, `supersedes`, `related` | +| `linkType` | string | ✅ | Relationship type: `see_also`, `prerequisite`, `supersedes`, `related`, `references` | | `label` | string | | Optional human-readable description of the relationship | +The output includes a `Link ID:` line. Pass this ID to `delete-link`. + ## get-document-links -Get all cross-reference links for a document, both outgoing and incoming. +Get all cross-reference links for a document, both outgoing and incoming. Each line ends with `[link ID: …]`, which you can pass to `delete-link`. | Parameter | Type | Required | Description | | ------------ | ------ | -------- | --------------- | diff --git a/src/core/links.ts b/src/core/links.ts index e725d71c..6814edfd 100644 --- a/src/core/links.ts +++ b/src/core/links.ts @@ -4,15 +4,18 @@ import { ValidationError, DocumentNotFoundError } from "../errors.js"; import { createChildLogger } from "../logger.js"; import { extractMarkdownLinks, extractWikilinks } from "./link-extractor.js"; -export type LinkType = "see_also" | "prerequisite" | "supersedes" | "related" | "references"; - -const VALID_LINK_TYPES: ReadonlySet = new Set([ +/** Every link type accepted by {@link createLink}. */ +export const LINK_TYPES = [ "see_also", "prerequisite", "supersedes", "related", "references", -]); +] as const; + +export type LinkType = (typeof LINK_TYPES)[number]; + +const VALID_LINK_TYPES: ReadonlySet = new Set(LINK_TYPES); export interface DocumentLink { id: string; diff --git a/src/mcp/format.ts b/src/mcp/format.ts new file mode 100644 index 00000000..80b1483a --- /dev/null +++ b/src/mcp/format.ts @@ -0,0 +1,67 @@ +import type { SearchResult } from "../core/search.js"; +import type { DocumentLink, DocumentLinks } from "../core/links.js"; + +/** Join context chunks into a labelled block, or return "" when there are none. */ +function formatContext(label: string, chunks: SearchResult["contextBefore"]): string { + if (!chunks || chunks.length === 0) return ""; + return `\n**Context (${label}):**\n${chunks.map((c) => c.content).join("\n\n")}\n`; +} + +/** Format one search-docs result, including the IDs that follow-up tools need. */ +function formatSearchResult(r: SearchResult, index: number): string { + const libraryVersion = r.version ? ` v${r.version}` : ""; + return ( + `## Result ${index + 1}: ${r.title} (score: ${r.score.toFixed(2)})\n` + + `**Document ID:** ${r.documentId} | **Chunk ID:** ${r.chunkId}\n` + + (r.library ? `**Library:** ${r.library}${libraryVersion}\n` : "") + + (r.url ? `**Source:** ${r.url}\n` : "") + + (r.avgRating ? `**Rating:** ${r.avgRating.toFixed(1)}/5\n` : "") + + formatContext("before", r.contextBefore) + + `\n${r.content}\n` + + formatContext("after", r.contextAfter) + ); +} + +/** Format the search-docs tool output. */ +export function formatSearchResults(results: SearchResult[], totalCount: number): string { + if (results.length === 0) return "No documents found matching your query."; + return ( + `**Total results: ${totalCount}**\n\n` + + results.map((r, i) => formatSearchResult(r, i)).join("\n---\n\n") + ); +} + +/** Format the link-documents tool output, including the link ID needed by delete-link. */ +export function formatLinkCreated(link: DocumentLink): string { + const linkLabel = link.label ? ` — ${link.label}` : ""; + return ( + `✓ Link created: ${link.sourceId} → ${link.targetId} (${link.linkType})${linkLabel}\n` + + `Link ID: ${link.id}` + ); +} + +/** Format the get-document-links tool output, including each link's ID. */ +export function formatDocumentLinks({ outgoing, incoming }: DocumentLinks): string { + if (outgoing.length === 0 && incoming.length === 0) return "No links found for this document."; + + const lines: string[] = []; + if (outgoing.length > 0) { + lines.push("**Outgoing links:**"); + for (const l of outgoing) { + const outLabel = l.label ? ` — ${l.label}` : ""; + lines.push( + ` → [${l.linkType}] ${l.targetTitle} (${l.targetId})${outLabel} [link ID: ${l.id}]`, + ); + } + } + if (incoming.length > 0) { + lines.push("**Incoming links:**"); + for (const l of incoming) { + const inLabel = l.label ? ` — ${l.label}` : ""; + lines.push( + ` ← [${l.linkType}] ${l.sourceTitle} (${l.sourceId})${inLabel} [link ID: ${l.id}]`, + ); + } + } + return lines.join("\n"); +} diff --git a/src/mcp/server.ts b/src/mcp/server.ts index f07b8cb1..4d034e55 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -17,8 +17,7 @@ import { getDocument, listDocuments, deleteDocument, updateDocument } from "../c import { rateDocument, getDocumentRatings } from "../core/ratings.js"; import { indexDocument } from "../core/indexing.js"; import { listTopics } from "../core/topics.js"; -import { createLink, getDocumentLinks, deleteLink } from "../core/links.js"; -import type { LinkType } from "../core/links.js"; +import { createLink, getDocumentLinks, deleteLink, LINK_TYPES } from "../core/links.js"; import { createSavedSearch, listSavedSearches, @@ -35,6 +34,7 @@ import { initLogger, getLogger } from "../logger.js"; import { ConfigError, ValidationError } from "../errors.js"; import { errorResponse, textResult, withErrorHandling, type ToolResult } from "./errors.js"; export { errorResponse, withErrorHandling, type ToolResult } from "./errors.js"; +import { formatDocumentLinks, formatLinkCreated, formatSearchResults } from "./format.js"; import { taskRegistry } from "./tasks.js"; import type { Task, TaskType } from "./tasks.js"; @@ -329,36 +329,7 @@ async function main(): Promise { contextChunks: params.contextChunks, }); - if (results.length === 0) { - return textResult("No documents found matching your query."); - } - - const text = - `**Total results: ${totalCount}**\n\n` + - results - .map((r, i) => { - const libraryVersion = r.version ? ` v${r.version}` : ""; - let entry = - `## Result ${i + 1}: ${r.title} (score: ${r.score.toFixed(2)})\n` + - (r.library ? `**Library:** ${r.library}${libraryVersion}\n` : "") + - (r.url ? `**Source:** ${r.url}\n` : "") + - (r.avgRating ? `**Rating:** ${r.avgRating.toFixed(1)}/5\n` : ""); - - if (r.contextBefore && r.contextBefore.length > 0) { - entry += `\n**Context (before):**\n${r.contextBefore.map((c) => c.content).join("\n\n")}\n`; - } - - entry += `\n${r.content}\n`; - - if (r.contextAfter && r.contextAfter.length > 0) { - entry += `\n**Context (after):**\n${r.contextAfter.map((c) => c.content).join("\n\n")}\n`; - } - - return entry; - }) - .join("\n---\n\n"); - - return textResult(text); + return textResult(formatSearchResults(results, totalCount)); }), ); @@ -1158,27 +1129,16 @@ async function main(): Promise { // Tool: link-documents server.tool( "link-documents", - "Create a relationship between two documents (see_also, prerequisite, supersedes, related)", + `Create a relationship between two documents (${LINK_TYPES.join(", ")})`, { sourceId: z.string().describe("The source document ID"), targetId: z.string().describe("The target document ID"), - linkType: z - .enum(["see_also", "prerequisite", "supersedes", "related"]) - .describe("Type of relationship"), + linkType: z.enum(LINK_TYPES).describe("Type of relationship"), label: z.string().optional().describe("Optional human-readable description of the link"), }, withErrorHandling((params) => { - const link = createLink( - db, - params.sourceId, - params.targetId, - params.linkType as LinkType, - params.label, - ); - const linkLabel = link.label ? ` — ${link.label}` : ""; - return textResult( - `✓ Link created: ${link.sourceId} → ${link.targetId} (${link.linkType})${linkLabel}`, - ); + const link = createLink(db, params.sourceId, params.targetId, params.linkType, params.label); + return textResult(formatLinkCreated(link)); }), ); @@ -1190,27 +1150,7 @@ async function main(): Promise { documentId: z.string().describe("The document ID"), }, withErrorHandling((params) => { - const { outgoing, incoming } = getDocumentLinks(db, params.documentId); - if (outgoing.length === 0 && incoming.length === 0) { - return textResult("No links found for this document."); - } - - const lines: string[] = []; - if (outgoing.length > 0) { - lines.push("**Outgoing links:**"); - for (const l of outgoing) { - const outLabel = l.label ? ` — ${l.label}` : ""; - lines.push(` → [${l.linkType}] ${l.targetTitle} (${l.targetId})${outLabel}`); - } - } - if (incoming.length > 0) { - lines.push("**Incoming links:**"); - for (const l of incoming) { - const inLabel = l.label ? ` — ${l.label}` : ""; - lines.push(` ← [${l.linkType}] ${l.sourceTitle} (${l.sourceId})${inLabel}`); - } - } - return textResult(lines.join("\n")); + return textResult(formatDocumentLinks(getDocumentLinks(db, params.documentId))); }), ); diff --git a/tests/unit/mcp-format.test.ts b/tests/unit/mcp-format.test.ts new file mode 100644 index 00000000..92f2426e --- /dev/null +++ b/tests/unit/mcp-format.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import type Database from "better-sqlite3"; +import { z } from "zod"; +import { createTestDb } from "../fixtures/test-db.js"; +import { insertDoc } from "../fixtures/helpers.js"; +import { createLink, deleteLink, getDocumentLinks, LINK_TYPES } from "../../src/core/links.js"; +import type { SearchResult } from "../../src/core/search.js"; +import { + formatDocumentLinks, + formatLinkCreated, + formatSearchResults, +} from "../../src/mcp/format.js"; + +function makeResult(overrides: Partial = {}): SearchResult { + return { + documentId: "doc-1", + chunkId: "chunk-1", + title: "Doc One", + content: "chunk body", + sourceType: "manual", + library: null, + version: null, + topicId: null, + url: null, + score: 0.5, + avgRating: null, + scoreExplanation: { method: "vector", rawScore: 0.5, boostFactors: [], details: "" }, + ...overrides, + }; +} + +describe("MCP output formatting", () => { + describe("formatSearchResults", () => { + it("returns the no-results message for an empty list", () => { + expect(formatSearchResults([], 0)).toBe("No documents found matching your query."); + }); + + it("includes the document ID and chunk ID of every result", () => { + const text = formatSearchResults( + [makeResult(), makeResult({ documentId: "doc-2", chunkId: "chunk-2", title: "Doc Two" })], + 2, + ); + expect(text).toContain("**Document ID:** doc-1 | **Chunk ID:** chunk-1"); + expect(text).toContain("**Document ID:** doc-2 | **Chunk ID:** chunk-2"); + }); + + it("keeps the existing header, metadata, context, and separator layout", () => { + const text = formatSearchResults( + [ + makeResult({ + library: "react", + version: "18", + url: "https://example.com", + avgRating: 4, + contextBefore: [{ chunkId: "c0", content: "before text", chunkIndex: 0 }], + contextAfter: [{ chunkId: "c2", content: "after text", chunkIndex: 2 }], + }), + makeResult({ title: "Second" }), + ], + 7, + ); + expect(text.startsWith("**Total results: 7**\n\n## Result 1: Doc One (score: 0.50)\n")).toBe( + true, + ); + expect(text).toContain("**Library:** react v18\n"); + expect(text).toContain("**Source:** https://example.com\n"); + expect(text).toContain("**Rating:** 4.0/5\n"); + expect(text).toContain("**Context (before):**\nbefore text\n"); + expect(text).toContain("\nchunk body\n"); + expect(text).toContain("**Context (after):**\nafter text\n"); + expect(text).toContain("\n---\n\n## Result 2: Second"); + }); + }); + + describe("link formatting", () => { + let db: Database.Database; + + beforeEach(() => { + db = createTestDb(); + insertDoc(db, "doc-a", "Document A"); + insertDoc(db, "doc-b", "Document B"); + }); + + afterEach(() => { + db.close(); + }); + + it("formatLinkCreated includes the link ID, which delete-link accepts", () => { + const link = createLink(db, "doc-a", "doc-b", "see_also", "read next"); + const text = formatLinkCreated(link); + expect(text).toContain("doc-a → doc-b (see_also) — read next"); + const id = /Link ID: (\S+)/.exec(text)?.[1]; + expect(id).toBe(link.id); + deleteLink(db, id!); + expect(getDocumentLinks(db, "doc-a").outgoing).toHaveLength(0); + }); + + it("formatDocumentLinks includes the link ID on outgoing and incoming lines", () => { + const link = createLink(db, "doc-a", "doc-b", "references"); + const outgoing = formatDocumentLinks(getDocumentLinks(db, "doc-a")); + const incoming = formatDocumentLinks(getDocumentLinks(db, "doc-b")); + expect(outgoing).toContain( + `**Outgoing links:**\n → [references] Document B (doc-b) [link ID: ${link.id}]`, + ); + expect(incoming).toContain( + `**Incoming links:**\n ← [references] Document A (doc-a) [link ID: ${link.id}]`, + ); + }); + + it("formatDocumentLinks returns the no-links message when there are none", () => { + expect(formatDocumentLinks(getDocumentLinks(db, "doc-a"))).toBe( + "No links found for this document.", + ); + }); + }); + + describe("LINK_TYPES", () => { + it("includes 'references' and every value is accepted by createLink", () => { + const db = createTestDb(); + insertDoc(db, "doc-a", "Document A"); + insertDoc(db, "doc-b", "Document B"); + expect(LINK_TYPES).toContain("references"); + for (const type of LINK_TYPES) { + expect(createLink(db, "doc-a", "doc-b", type).linkType).toBe(type); + } + db.close(); + }); + + it("works as the link-documents zod enum", () => { + const schema = z.enum(LINK_TYPES); + expect(schema.parse("references")).toBe("references"); + expect(() => schema.parse("bogus")).toThrow(); + }); + }); +}); From 3a8a0eab29081542bbc97977c98e6fb6950e24c0 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 15:11:23 +0000 Subject: [PATCH 06/83] fix(logger): write logs to stderr so MCP stdio stream stays clean pino defaults to stdout, so the MCP stdio server interleaved JSON log lines (DB open, migrations) with protocol messages. initLogger now takes a destination option (default stderr); MCP main requests stderr explicitly. The CLI configures logging before loading config so validation warnings honour the CLI log level. --- src/cli/index.ts | 9 ++++--- src/logger.ts | 25 +++++++++++++++--- src/mcp/server.ts | 3 ++- tests/unit/logger.test.ts | 55 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 82 insertions(+), 10 deletions(-) diff --git a/src/cli/index.ts b/src/cli/index.ts index 00556add..627503ba 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1524,6 +1524,7 @@ configCmd .command("set ") .description("Set a configuration value (e.g., embedding.provider local)") .action((key: string, value: string) => { + setupLogging(program.opts()); if (key === "embedding.provider") { if (value !== "local" && value !== "ollama" && value !== "openai") { console.error("Invalid provider. Must be: local, ollama, or openai"); @@ -1549,6 +1550,7 @@ configCmd .command("show") .description("Show current configuration") .action(() => { + setupLogging(program.opts()); const config = loadConfig(); console.log(JSON.stringify(config, null, 2)); }); @@ -1576,9 +1578,10 @@ function initializeApp(): { config: ReturnType; db: ReturnType; } { - const config = loadConfig(); + // Configure logging first so warnings emitted while loading config honour the CLI log level. const opts = program.opts(); setupLogging(opts); + const config = loadConfig(); if (opts.workspace) { process.env["LIBSCOPE_WORKSPACE"] = opts.workspace; @@ -2313,8 +2316,8 @@ connectCmd .option("--sync", "Incremental re-sync only") .option("--notebook ", "Sync a specific notebook") .action(async (opts: { token?: string; sync?: boolean; notebook?: string }) => { - const config = loadConfig(); setupLogging(program.opts()); + const config = loadConfig(); const workspace = program.opts().workspace as string | undefined; if (workspace) { @@ -2392,8 +2395,8 @@ disconnectCmd ) { return; } - const config = loadConfig(); setupLogging(program.opts()); + const config = loadConfig(); const workspace2 = program.opts().workspace as string | undefined; if (workspace2) { diff --git a/src/logger.ts b/src/logger.ts index 5a16a9cc..c78daa23 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -3,11 +3,28 @@ import pino from "pino"; export type LogLevel = "debug" | "info" | "warn" | "error" | "silent"; -let currentLogger: pino.Logger = pino({ level: "info" }); +/** Where log lines are written. */ +export type LogDestination = "stdout" | "stderr"; -/** Initialize the logger with a specific level. */ -export function initLogger(level: LogLevel): void { - currentLogger = pino({ level }); +export interface LoggerOptions { + /** + * Output stream for log lines (default: "stderr"). + * Logs go to stderr by default so they never mix with program output on stdout + * (the MCP stdio transport, `--json` CLI output, piped results). + */ + destination?: LogDestination; +} + +function createLogger(level: LogLevel, destination: LogDestination): pino.Logger { + const fd = destination === "stdout" ? 1 : 2; + return pino({ level }, pino.destination({ dest: fd, sync: true })); +} + +let currentLogger: pino.Logger = createLogger("info", "stderr"); + +/** Initialize the logger with a specific level and output stream (stderr by default). */ +export function initLogger(level: LogLevel, options?: LoggerOptions): void { + currentLogger = createLogger(level, options?.destination ?? "stderr"); } /** Get the current logger instance. */ diff --git a/src/mcp/server.ts b/src/mcp/server.ts index f07b8cb1..f3868197 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -232,7 +232,8 @@ async function main(): Promise { process.exit(1); } - initLogger(config.logging.level); + // stdout carries the MCP protocol stream — logs must go to stderr. + initLogger(config.logging.level, { destination: "stderr" }); let db; try { diff --git a/tests/unit/logger.test.ts b/tests/unit/logger.test.ts index 2a76554f..d6de4e77 100644 --- a/tests/unit/logger.test.ts +++ b/tests/unit/logger.test.ts @@ -1,5 +1,32 @@ -import { describe, it, expect, beforeEach } from "vitest"; -import { initLogger, getLogger, createChildLogger, withCorrelationId } from "../../src/logger.js"; +import { describe, it, expect, beforeEach, vi } from "vitest"; + +// Record which file descriptor each logger writes to, while still building real pino loggers. +const destinationCalls = vi.hoisted(() => [] as unknown[]); + +interface PinoModuleLike { + default: ((...args: unknown[]) => unknown) & { destination: (opts: unknown) => unknown }; +} + +vi.mock("pino", async (importOriginal) => { + const actual = await importOriginal(); + const realPino = actual.default; + const destination = (opts: unknown): unknown => { + destinationCalls.push(opts); + return realPino.destination(opts); + }; + const wrapped = Object.assign((...args: unknown[]) => realPino(...args), realPino, { + destination, + }); + return { ...actual, default: wrapped }; +}); + +const { initLogger, getLogger, createChildLogger, withCorrelationId } = + await import("../../src/logger.js"); + +function lastDestFd(): unknown { + const last = destinationCalls.at(-1) as { dest?: unknown } | undefined; + return last?.dest; +} describe("logger", () => { beforeEach(() => { @@ -30,3 +57,27 @@ describe("logger", () => { expect(logger.level).toBe("silent"); }); }); + +describe("logger destination", () => { + it("module-level logger (used before initLogger, e.g. config warnings) writes to stderr", () => { + const first = destinationCalls[0] as { dest?: unknown } | undefined; + expect(first?.dest).toBe(2); + }); + + it("writes to stderr (fd 2) by default so stdout stays clean", () => { + initLogger("info"); + expect(lastDestFd()).toBe(2); + }); + + it("writes to stderr when explicitly requested (MCP stdio server)", () => { + initLogger("info", { destination: "stderr" }); + expect(lastDestFd()).toBe(2); + expect(getLogger().level).toBe("info"); + }); + + it("can still target stdout when requested", () => { + initLogger("warn", { destination: "stdout" }); + expect(lastDestFd()).toBe(1); + expect(getLogger().level).toBe("warn"); + }); +}); From e1647e129cb0be813d82d2a302d3bc8b43f2e154 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 15:12:12 +0000 Subject: [PATCH 07/83] fix(webhooks): fire webhooks from core mutations and share one delivery path fireWebhooks had no callers, so webhooks never fired. Add a small core/events module; indexDocument, updateDocument, deleteDocument, rateDocument and searchDocuments emit document.created/updated/deleted, document.rated and search.executed. Webhooks subscribe to it. Emission never throws into the caller and never waits on delivery. Add deliverWebhook(webhook, event, data): decrypts the stored secret, signs, re-runs the SSRF check and POSTs with redirect: "error". Used by fireWebhooks, the CLI 'webhooks test' command and the REST test route, which previously signed with the encrypted secret (and the CLI skipped the SSRF check and followed redirects). --- README.md | 14 +- docs/guide/webhooks.md | 91 ++++++----- src/api/routes.ts | 19 +-- src/cli/index.ts | 16 +- src/core/documents.ts | 8 + src/core/events.ts | 58 +++++++ src/core/indexing.ts | 7 + src/core/ratings.ts | 6 + src/core/search.ts | 6 + src/core/webhooks.ts | 128 +++++++++------ tests/unit/webhook-events.test.ts | 253 ++++++++++++++++++++++++++++++ 11 files changed, 479 insertions(+), 127 deletions(-) create mode 100644 src/core/events.ts create mode 100644 tests/unit/webhook-events.test.ts diff --git a/README.md b/README.md index 92d7c76b..ab1ec232 100644 --- a/README.md +++ b/README.md @@ -386,23 +386,25 @@ export LIBSCOPE_ALLOW_SELF_SIGNED_CERTS=true LibScope can push events to any HTTP endpoint. Useful for triggering CI pipelines, Slack notifications, or custom workflows whenever documents are created or updated. -```bash -libscope serve --api # webhooks require the REST API -``` +Events fire from every surface (CLI, MCP server, REST API, SDK) when that process changes the knowledge base. Delivery runs in the background and never fails or delays the operation that caused it. ```bash -# Create a webhook +# Create a webhook (CLI) +libscope webhooks create https://hooks.example.com/libscope --events document.created,document.updated + +# Or via the REST API curl -X POST http://localhost:3378/api/v1/webhooks \ -H "Content-Type: application/json" \ -d '{"url": "https://hooks.example.com/libscope", "events": ["document.created", "document.updated"], "secret": "my-hmac-secret"}' # Send a test ping +libscope webhooks test curl -X POST http://localhost:3378/api/v1/webhooks//test ``` -Webhook payloads are signed with HMAC-SHA256 when a secret is set. The signature is in the `X-LibScope-Signature` header. +Webhook payloads are signed with HMAC-SHA256 when a secret is set (storing a secret requires `LIBSCOPE_SECRET_KEY`). The hex signature is in the `X-LibScope-Signature` header. -Supported events: `document.created`, `document.updated`, `document.deleted`. +Supported events: `document.created`, `document.updated`, `document.deleted`, `document.rated`, `search.executed`. ## Other Tools diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 8125b184..aa38f584 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -1,9 +1,19 @@ # Webhooks -LibScope can send HTTP POST notifications to external URLs when documents are created, updated, or deleted. This lets you integrate LibScope with CI pipelines, Slack bots, or any other HTTP-capable service. +LibScope can send HTTP POST notifications to external URLs when documents are created, updated, deleted, or rated, and when a search runs. This lets you integrate LibScope with CI pipelines, Slack bots, or any other HTTP-capable service. + +Events fire from every surface — CLI, MCP server, REST API, and the SDK — in the process that made the change. ## Creating a Webhook +### Via CLI + +```bash +libscope webhooks create https://hooks.example.com/libscope \ + --events document.created,document.updated,document.deleted \ + --secret my-hmac-secret +``` + ### Via REST API ```bash @@ -16,21 +26,17 @@ curl -X POST http://localhost:3378/api/v1/webhooks \ }' ``` -### Via CLI - -Webhook management is available through the REST API only. Start the API server first: - -```bash -libscope serve --api --port 3378 -``` +Storing a secret requires the `LIBSCOPE_SECRET_KEY` environment variable. LibScope encrypts the secret at rest with AES-256-GCM. Webhook URLs must use http or https and must not resolve to a private or internal IP address. ## Supported Events -| Event | Fired when | -| ------------------- | ---------------------------------------------- | -| `document.created` | A new document is indexed | -| `document.updated` | A document's content or metadata is updated | -| `document.deleted` | A document is deleted | +| Event | Fired when | +| ------------------ | ------------------------------------------- | +| `document.created` | A new document is indexed | +| `document.updated` | A document's content or metadata is updated | +| `document.deleted` | A document is deleted | +| `document.rated` | A document or chunk is rated | +| `search.executed` | A search runs | ## Payload Format @@ -41,27 +47,31 @@ Every webhook delivery sends a `POST` request with `Content-Type: application/js "event": "document.created", "timestamp": "2026-03-18T12:00:00.000Z", "data": { - "id": "doc_abc123", + "documentId": "3f1c…", "title": "Auth Guide", "library": "my-lib", - "version": "2.0.0", - "topic": "security", - "sourceType": "manual", - "url": null, - "createdAt": "2026-03-18T12:00:00.000Z", - "updatedAt": "2026-03-18T12:00:00.000Z" + "version": "2.0.0" } } ``` -For `document.deleted`, the `data` object contains only `id` and `title`. +The `data` object depends on the event: + +| Event | `data` fields | +| --------------------------------------- | -------------------------------------------------- | +| `document.created`, `document.updated` | `documentId`, `title`, `library`?, `version`? | +| `document.deleted` | `documentId` | +| `document.rated` | `documentId`, `rating`, `feedback`? | +| `search.executed` | `query`, `resultCount`, `topicId`? | + +Fields marked `?` are omitted when they have no value. ## Verifying Signatures -When you create a webhook with a `secret`, LibScope signs every delivery with HMAC-SHA256. The signature is sent in the `X-LibScope-Signature` header: +When you create a webhook with a `secret`, LibScope signs every delivery with HMAC-SHA256 over the raw request body. The hex digest is sent in the `X-LibScope-Signature` header: ``` -X-LibScope-Signature: sha256=abc123... +X-LibScope-Signature: 9f86d081884c7d65... ``` To verify in Node.js: @@ -69,19 +79,17 @@ To verify in Node.js: ```typescript import { createHmac, timingSafeEqual } from "crypto"; -function verifySignature(secret: string, body: string, header: string): boolean { - const expected = "sha256=" + createHmac("sha256", secret).update(body).digest("hex"); +function verifySignature(secret: string, rawBody: string, header: string): boolean { + const expected = Buffer.from(createHmac("sha256", secret).update(rawBody).digest("hex")); const received = Buffer.from(header); - const expectedBuf = Buffer.from(expected); - if (received.length !== expectedBuf.length) return false; - return timingSafeEqual(received, expectedBuf); + if (received.length !== expected.length) return false; + return timingSafeEqual(received, expected); } -// Express example -app.post("/webhook", (req, res) => { - const rawBody = JSON.stringify(req.body); // or use express.raw() +// Express example — verify against the raw body, not re-serialized JSON +app.post("/webhook", express.raw({ type: "application/json" }), (req, res) => { const sig = req.headers["x-libscope-signature"] as string; - if (!verifySignature("my-hmac-secret", rawBody, sig)) { + if (!verifySignature("my-hmac-secret", req.body.toString("utf8"), sig)) { return res.status(401).send("Invalid signature"); } // handle event @@ -96,26 +104,31 @@ Use `timingSafeEqual` to prevent timing attacks — never use `===` for comparin Send a test ping to verify your endpoint is reachable: ```bash +libscope webhooks test +# or curl -X POST http://localhost:3378/api/v1/webhooks//test ``` -This sends a `POST` to your webhook URL with `event: "ping"` and no `data` payload. +The ping is a `document.created` event with `data: { "test": true, "message": "Webhook test ping" }`. It is signed and SSRF-checked exactly like a real delivery, and the command prints the HTTP status your endpoint returned. ## Managing Webhooks ```bash # List all webhooks +libscope webhooks list curl http://localhost:3378/api/v1/webhooks # Delete a webhook +libscope webhooks delete curl -X DELETE http://localhost:3378/api/v1/webhooks/ ``` ## Delivery Behavior -- Deliveries are attempted synchronously as part of the request that triggered the event -- Failed deliveries (non-2xx response or network error) are logged but not retried automatically -- Webhook secrets are stored hashed and cannot be retrieved after creation — store them securely +- Deliveries run in the background. They never delay or fail the operation that triggered the event. +- Before each delivery, LibScope resolves the URL again and refuses private or internal IPs. Redirects are refused. Each request times out after 5 seconds. +- Failed deliveries (non-2xx response, network error, or SSRF rejection) are logged and counted, but not retried. After 10 consecutive failures the webhook is deactivated. A successful delivery resets the count. +- A short-lived CLI command can exit before a slow endpoint responds. The delivery is still sent, but its result may not be recorded. ## Example: Notify Slack on New Documents @@ -146,10 +159,8 @@ app.post("/relay", async (req, res) => { app.listen(4000); ``` -Then register the relay as a LibScope webhook: +Deploy the relay at a public address, then register it as a LibScope webhook (localhost and private addresses are rejected): ```bash -curl -X POST http://localhost:3378/api/v1/webhooks \ - -H "Content-Type: application/json" \ - -d '{"url": "http://localhost:4000/relay", "events": ["document.created"]}' +libscope webhooks create https://relay.example.com/relay --events document.created ``` diff --git a/src/api/routes.ts b/src/api/routes.ts index ffb061b7..0adecb7e 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -47,10 +47,9 @@ import { listWebhooks, deleteWebhook, getWebhook, - buildPayload, - signPayload, + deliverWebhook, + TEST_PING_DATA, redactWebhook, - validateWebhookUrlSsrf, } from "../core/webhooks.js"; import type { WebhookEvent } from "../core/webhooks.js"; import { loadScheduleEntries } from "../core/scheduler.js"; @@ -768,19 +767,7 @@ async function handleCreateWebhook(ctx: RouteContext): Promise { async function handleTestWebhook(ctx: RouteContext, webhookId: string): Promise { const webhook = getWebhook(ctx.db, webhookId); - await validateWebhookUrlSsrf(webhook.url); - const body = buildPayload("document.created", { test: true, message: "Webhook test ping" }); - const headers: Record = { "Content-Type": "application/json" }; - if (webhook.secret) { - headers["X-LibScope-Signature"] = signPayload(body, webhook.secret); - } - const resp = await fetch(webhook.url, { - method: "POST", - headers, - body, - redirect: "error", - signal: AbortSignal.timeout(5000), - }); + const resp = await deliverWebhook(webhook, "document.created", TEST_PING_DATA); sendJson(ctx.res, 200, { status: resp.status, statusText: resp.statusText }, elapsed(ctx.start)); } diff --git a/src/cli/index.ts b/src/cli/index.ts index 00556add..51abf5cf 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -92,8 +92,8 @@ import { listWebhooks, deleteWebhook, getWebhook, - buildPayload, - signPayload, + deliverWebhook, + TEST_PING_DATA, } from "../core/webhooks.js"; import type { WebhookEvent } from "../core/webhooks.js"; import { registerRegistryCommands } from "./commands/registry.js"; @@ -2958,18 +2958,8 @@ webhookCmd const { db } = initializeApp(); try { const webhook = getWebhook(db, id); - const body = buildPayload("document.created", { test: true, message: "Webhook test ping" }); - const headers: Record = { "Content-Type": "application/json" }; - if (webhook.secret) { - headers["X-LibScope-Signature"] = signPayload(body, webhook.secret); - } console.log(`Sending test ping to ${webhook.url}...`); - const resp = await fetch(webhook.url, { - method: "POST", - headers, - body, - signal: AbortSignal.timeout(5000), - }); + const resp = await deliverWebhook(webhook, "document.created", TEST_PING_DATA); console.log(`✓ Response: ${resp.status} ${resp.statusText}`); } finally { closeDatabase(); diff --git a/src/core/documents.ts b/src/core/documents.ts index 804dc741..2569a4eb 100644 --- a/src/core/documents.ts +++ b/src/core/documents.ts @@ -5,6 +5,7 @@ import { DocumentNotFoundError, ValidationError } from "../errors.js"; import { chunkContent, chunkContentStreaming, STREAMING_THRESHOLD } from "./indexing.js"; import { getLogger } from "../logger.js"; import { saveVersion } from "./versioning.js"; +import { emitEvent } from "./events.js"; export interface Document { id: string; @@ -106,6 +107,7 @@ export function deleteDocument(db: Database.Database, documentId: string): void if (result.changes === 0) { throw new DocumentNotFoundError(documentId); } + emitEvent(db, "document.deleted", { documentId }); } /** List documents with optional filters. */ @@ -270,5 +272,11 @@ export async function updateDocument( transaction(); } + emitEvent(db, "document.updated", { + documentId, + title: newTitle, + library: newLibrary, + version: newVersion, + }); return getDocument(db, documentId); } diff --git a/src/core/events.ts b/src/core/events.ts new file mode 100644 index 00000000..b1c3f6dc --- /dev/null +++ b/src/core/events.ts @@ -0,0 +1,58 @@ +import type Database from "better-sqlite3"; +import { getLogger } from "../logger.js"; + +/** Events emitted by core mutation functions. Webhooks subscribe to these. */ +export const LIBSCOPE_EVENTS = [ + "document.created", + "document.updated", + "document.deleted", + "document.rated", + "search.executed", +] as const; + +export type LibScopeEvent = (typeof LIBSCOPE_EVENTS)[number]; + +/** + * A listener receives every emitted event. It runs synchronously inside the emitting call, + * so it must start any slow work (network I/O) without awaiting it. + */ +export type EventListener = ( + db: Database.Database, + event: LibScopeEvent, + data: Record, +) => void | Promise; + +const listeners = new Set(); + +/** Subscribe to all events. Returns a function that removes the listener. */ +export function onEvent(listener: EventListener): () => void { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +function logListenerError(err: unknown, event: LibScopeEvent): void { + getLogger().warn({ err, event }, "Event listener failed"); +} + +/** + * Emit an event to all listeners. Never throws into the caller and never waits for + * asynchronous listener work; listener errors and rejections are logged. + */ +export function emitEvent( + db: Database.Database, + event: LibScopeEvent, + data: Record, +): void { + for (const listener of listeners) { + try { + const pending = listener(db, event, data); + if (pending instanceof Promise) { + pending.catch((err: unknown) => logListenerError(err, event)); + } + } catch (err) { + logListenerError(err, event); + } + } +} diff --git a/src/core/indexing.ts b/src/core/indexing.ts index 0d660b16..1e717d63 100644 --- a/src/core/indexing.ts +++ b/src/core/indexing.ts @@ -10,6 +10,7 @@ import { checkDuplicate } from "./dedup.js"; import type { DedupOptions } from "./dedup.js"; import { deleteChunkEmbeddings } from "./documents.js"; import { extractAndStoreDocumentLinks } from "./links.js"; +import { emitEvent } from "./events.js"; import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; export interface IndexDocumentInput { @@ -498,6 +499,12 @@ export async function indexDocument( log.warn({ err, docId }, "Failed to extract document links"); } + emitEvent(db, "document.created", { + documentId: docId, + title: input.title, + library: input.library, + version: input.version, + }); return { id: docId, chunkCount: chunks.length }; } diff --git a/src/core/ratings.ts b/src/core/ratings.ts index b7a9e52e..1de0c324 100644 --- a/src/core/ratings.ts +++ b/src/core/ratings.ts @@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto"; import { z } from "zod"; import { ValidationError, DocumentNotFoundError } from "../errors.js"; import { validateRow, validateRows } from "../db/validate.js"; +import { emitEvent } from "./events.js"; const RatingSummaryRowSchema = z.object({ avg_rating: z.number().nullable(), @@ -90,6 +91,11 @@ export function rateDocument(db: Database.Database, input: RateDocumentInput): R ratedBy, ); + emitEvent(db, "document.rated", { + documentId: input.documentId, + rating: input.rating, + feedback: input.feedback, + }); return { id, documentId: input.documentId, diff --git a/src/core/search.ts b/src/core/search.ts index 3de6762d..52b96a7d 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -5,6 +5,7 @@ import { withCorrelationId, createChildLogger } from "../logger.js"; import { validateCountRow } from "../utils/db-validation.js"; import { validateRow, validateRows } from "../db/validate.js"; import { logSearch, recordSearchQuery } from "./analytics.js"; +import { emitEvent } from "./events.js"; import { performance } from "node:perf_hooks"; /** Build SQL clause and params for AND-logic tag filtering on a document alias. */ @@ -501,6 +502,11 @@ function finalizeResponse( if (options.contextChunks) { response.results = attachContext(db, response.results, options.contextChunks); } + emitEvent(db, "search.executed", { + query: options.query, + resultCount: response.totalCount, + topicId: options.topic, + }); return response; } diff --git a/src/core/webhooks.ts b/src/core/webhooks.ts index c4ae6705..2e951989 100644 --- a/src/core/webhooks.ts +++ b/src/core/webhooks.ts @@ -12,6 +12,7 @@ import type Database from "better-sqlite3"; import { ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; import { isPrivateIP } from "./url-fetcher.js"; +import { LIBSCOPE_EVENTS, onEvent, type LibScopeEvent } from "./events.js"; const lookupAsync = promisify(dnsLookup); @@ -43,15 +44,9 @@ function decryptSecret(stored: string): string { return decipher.update(Buffer.from(encHex!, "hex")).toString("utf8") + decipher.final("utf8"); } -export const WEBHOOK_EVENTS = [ - "document.created", - "document.updated", - "document.deleted", - "document.rated", - "search.executed", -] as const; +export const WEBHOOK_EVENTS = LIBSCOPE_EVENTS; -export type WebhookEvent = (typeof WEBHOOK_EVENTS)[number]; +export type WebhookEvent = LibScopeEvent; export interface Webhook { id: string; @@ -68,8 +63,10 @@ export interface Webhook { * Payload sent to webhook subscribers. * * The `data` field shape varies by event type: - * - "document.created" / "document.updated" / "document.deleted": + * - "document.created" / "document.updated": * { documentId: string; title: string; library?: string; version?: string } + * - "document.deleted": + * { documentId: string } * - "document.rated": * { documentId: string; rating: number; feedback?: string } * - "search.executed": @@ -318,6 +315,60 @@ export function buildPayload(event: WebhookEvent, data: Record) return JSON.stringify(payload); } +const DELIVERY_TIMEOUT_MS = 5000; + +/** Payload data sent by the CLI and REST "test webhook" commands. */ +export const TEST_PING_DATA: Readonly> = Object.freeze({ + test: true, + message: "Webhook test ping", +}); + +/** + * Deliver one event to one webhook: decrypt the stored secret, sign the payload, + * re-check the URL against private IPs, and POST it (redirects are refused). + * Throws on SSRF rejection or network failure; returns the HTTP response otherwise. + */ +export async function deliverWebhook( + webhook: Webhook, + event: WebhookEvent, + data: Record, +): Promise { + const body = buildPayload(event, data); + const headers: Record = { "Content-Type": "application/json" }; + if (webhook.secret) { + headers["X-LibScope-Signature"] = signPayload(body, decryptSecret(webhook.secret)); + } + await validateWebhookUrlSsrf(webhook.url); + return fetch(webhook.url, { + method: "POST", + headers, + body, + redirect: "error", + signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS), + }); +} + +/** Record a delivery outcome; skipped when the caller already closed the database. */ +function recordDelivery( + db: Database.Database, + log: ReturnType, + webhookId: string, + ok: boolean, +): void { + if (!db.open) return; + try { + if (ok) { + db.prepare( + "UPDATE webhooks SET last_triggered_at = datetime('now'), failure_count = 0 WHERE id = ?", + ).run(webhookId); + } else { + recordFailure(db, log, webhookId); + } + } catch (dbErr: unknown) { + log.error({ err: dbErr, webhookId }, "DB error recording webhook delivery"); + } +} + /** * Fire webhooks for a given event. Sends HTTP POST to all active webhooks * subscribed to this event. Fire-and-forget — errors are caught internally. @@ -334,56 +385,29 @@ export function fireWebhooks( ) .all() as WebhookRow[]; - const body = buildPayload(event, data); - for (const row of rows) { const webhook = rowToWebhook(row); if (!webhook.events.includes(event)) { continue; } - const headers: Record = { - "Content-Type": "application/json", - }; - if (webhook.secret) { - headers["X-LibScope-Signature"] = signPayload(body, decryptSecret(webhook.secret)); - } - - // SSRF check before firing - validateWebhookUrlSsrf(webhook.url) - .then(() => - fetch(webhook.url, { - method: "POST", - headers, - body, - redirect: "error", - signal: AbortSignal.timeout(5000), - }), - ) - .then((resp) => { - try { - if (!resp.ok) { - log.warn( - { webhookId: webhook.id, url: webhook.url, status: resp.status }, - "Webhook delivery received non-2xx response", - ); - recordFailure(db, log, webhook.id); - return; - } - db.prepare( - "UPDATE webhooks SET last_triggered_at = datetime('now'), failure_count = 0 WHERE id = ?", - ).run(webhook.id); - } catch (dbErr: unknown) { - log.error({ err: dbErr, webhookId: webhook.id }, "DB error recording webhook success"); + deliverWebhook(webhook, event, data).then( + (resp) => { + if (!resp.ok) { + log.warn( + { webhookId: webhook.id, url: webhook.url, status: resp.status }, + "Webhook delivery received non-2xx response", + ); } - }) - .catch((err: unknown) => { + recordDelivery(db, log, webhook.id, resp.ok); + }, + (err: unknown) => { log.warn({ err, webhookId: webhook.id, url: webhook.url }, "Webhook delivery failed"); - try { - recordFailure(db, log, webhook.id); - } catch (dbErr: unknown) { - log.error({ err: dbErr, webhookId: webhook.id }, "DB error recording webhook failure"); - } - }); + recordDelivery(db, log, webhook.id, false); + }, + ); } } + +// Deliver core events (document/search mutations) to registered webhooks. +onEvent(fireWebhooks); diff --git a/tests/unit/webhook-events.test.ts b/tests/unit/webhook-events.test.ts new file mode 100644 index 00000000..68a81886 --- /dev/null +++ b/tests/unit/webhook-events.test.ts @@ -0,0 +1,253 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { IncomingMessage, ServerResponse } from "node:http"; +import { Socket } from "node:net"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { initLogger } from "../../src/logger.js"; +import { ValidationError } from "../../src/errors.js"; + +const resolve4 = vi.fn(); +vi.mock("node:dns", async (importOriginal) => { + const actual: typeof import("node:dns") = await importOriginal(); + return { + ...actual, + promises: { + ...actual.promises, + resolve4: (host: string): Promise => resolve4(host) as Promise, + resolve6: vi.fn().mockResolvedValue([]), + }, + }; +}); + +const { emitEvent, onEvent } = await import("../../src/core/events.js"); +const { createWebhook, deliverWebhook, signPayload } = await import("../../src/core/webhooks.js"); +const { indexDocument } = await import("../../src/core/indexing.js"); +const { updateDocument, deleteDocument } = await import("../../src/core/documents.js"); +const { rateDocument } = await import("../../src/core/ratings.js"); +const { searchDocuments } = await import("../../src/core/search.js"); +const { handleRequest } = await import("../../src/api/routes.js"); + +interface SentPayload { + event: string; + data: Record; +} + +function sentPayloads(mockFetch: ReturnType): SentPayload[] { + return mockFetch.mock.calls.map((c) => { + const init = c[1] as RequestInit; + return JSON.parse(init.body as string) as SentPayload; + }); +} + +describe("core events", () => { + beforeEach(() => { + initLogger("silent"); + }); + + it("delivers events to listeners until unsubscribed", () => { + const db = createTestDbWithVec(); + const listener = vi.fn(); + const off = onEvent(listener); + emitEvent(db, "document.deleted", { documentId: "d1" }); + off(); + emitEvent(db, "document.deleted", { documentId: "d2" }); + expect(listener).toHaveBeenCalledOnce(); + expect(listener).toHaveBeenCalledWith(db, "document.deleted", { documentId: "d1" }); + db.close(); + }); + + it("never throws into the caller when a listener throws or rejects", async () => { + const db = createTestDbWithVec(); + const offThrow = onEvent(() => { + throw new Error("sync boom"); + }); + const rejected = vi.fn().mockRejectedValue(new Error("async boom")); + const offReject = onEvent(rejected); + const after = vi.fn(); + const offAfter = onEvent(after); + expect(() => emitEvent(db, "search.executed", { query: "q" })).not.toThrow(); + await Promise.resolve(); + expect(rejected).toHaveBeenCalledOnce(); + expect(after).toHaveBeenCalledOnce(); + offThrow(); + offReject(); + offAfter(); + db.close(); + }); +}); + +describe("webhook delivery wiring", () => { + let db: Database.Database; + let provider: MockEmbeddingProvider; + let mockFetch: ReturnType; + + beforeEach(() => { + initLogger("silent"); + resolve4.mockResolvedValue(["93.184.216.34"]); + db = createTestDbWithVec(); + provider = new MockEmbeddingProvider(); + mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" }); + vi.stubGlobal("fetch", mockFetch); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + delete process.env["LIBSCOPE_SECRET_KEY"]; + db.close(); + }); + + it("fires document.created, updated, rated, deleted and search.executed", async () => { + await createWebhook(db, "https://example.com/hook", [ + "document.created", + "document.updated", + "document.deleted", + "document.rated", + "search.executed", + ]); + + const doc = await indexDocument(db, provider, { + title: "Webhook Doc", + content: "Some content about webhooks firing.", + sourceType: "manual", + library: "lib", + }); + await updateDocument(db, provider, doc.id, { title: "Webhook Doc v2" }); + rateDocument(db, { documentId: doc.id, rating: 4, feedback: "good" }); + await searchDocuments(db, provider, { query: "webhooks" }); + deleteDocument(db, doc.id); + + await vi.waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(5)); + const byEvent = new Map(sentPayloads(mockFetch).map((p) => [p.event, p.data])); + expect(byEvent.get("document.created")).toEqual({ + documentId: doc.id, + title: "Webhook Doc", + library: "lib", + }); + expect(byEvent.get("document.updated")).toMatchObject({ + documentId: doc.id, + title: "Webhook Doc v2", + }); + expect(byEvent.get("document.rated")).toEqual({ + documentId: doc.id, + rating: 4, + feedback: "good", + }); + expect(byEvent.get("search.executed")).toMatchObject({ query: "webhooks" }); + expect(byEvent.get("document.deleted")).toEqual({ documentId: doc.id }); + }); + + it("only delivers subscribed events and does not block or fail the mutation", async () => { + await createWebhook(db, "https://example.com/hook", ["document.deleted"]); + mockFetch.mockRejectedValue(new Error("network down")); + + const doc = await indexDocument(db, provider, { + title: "Quiet", + content: "No created hook for this one.", + sourceType: "manual", + }); + expect(mockFetch).not.toHaveBeenCalled(); + + expect(() => deleteDocument(db, doc.id)).not.toThrow(); + await vi.waitFor(() => { + const row = db.prepare("SELECT failure_count FROM webhooks").get() as { + failure_count: number; + }; + expect(row.failure_count).toBe(1); + }); + expect(sentPayloads(mockFetch)[0]!.event).toBe("document.deleted"); + }); + + it("does not throw into the mutation when the webhooks table is missing", async () => { + db.exec("DROP TABLE webhooks"); + await expect( + indexDocument(db, provider, { title: "T", content: "Body text", sourceType: "manual" }), + ).resolves.toMatchObject({ chunkCount: 1 }); + }); +}); + +describe("deliverWebhook", () => { + let db: Database.Database; + let mockFetch: ReturnType; + + beforeEach(() => { + initLogger("silent"); + resolve4.mockResolvedValue(["93.184.216.34"]); + db = createTestDbWithVec(); + mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 202, statusText: "Accepted" }); + vi.stubGlobal("fetch", mockFetch); + process.env["LIBSCOPE_SECRET_KEY"] = "test-master-key"; + }); + + afterEach(() => { + vi.unstubAllGlobals(); + delete process.env["LIBSCOPE_SECRET_KEY"]; + db.close(); + }); + + it("signs with the decrypted secret and refuses redirects", async () => { + const hook = await createWebhook( + db, + "https://example.com/hook", + ["document.created"], + "s3cr3t", + ); + expect(hook.secret).not.toBe("s3cr3t"); + + const resp = await deliverWebhook(hook, "document.created", { test: true }); + expect(resp.status).toBe(202); + + const init = mockFetch.mock.calls[0]![1] as RequestInit & { headers: Record }; + expect(init.redirect).toBe("error"); + expect(init.headers["X-LibScope-Signature"]).toBe(signPayload(init.body as string, "s3cr3t")); + }); + + it("rejects URLs that now resolve to a private IP without sending", async () => { + const hook = await createWebhook(db, "https://example.com/hook", ["document.created"]); + resolve4.mockResolvedValue(["10.0.0.5"]); + await expect(deliverWebhook(hook, "document.created", {})).rejects.toThrow(ValidationError); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it("REST test route signs with the decrypted secret and checks SSRF", async () => { + const hook = await createWebhook( + db, + "https://example.com/hook", + ["document.created"], + "s3cr3t", + ); + + const send = async (): Promise<{ status: number; body: string }> => { + const req = new IncomingMessage(new Socket()); + req.method = "POST"; + req.url = `/api/v1/webhooks/${hook.id}/test`; + req.headers = { host: "localhost:3378" }; + process.nextTick(() => { + req.push(null); + }); + const res = new ServerResponse(new IncomingMessage(new Socket())); + let body = ""; + res.end = ((chunk?: unknown) => { + if (typeof chunk === "string") body = chunk; + return res; + }) as typeof res.end; + await handleRequest(req, res, db, new MockEmbeddingProvider()); + return { status: res.statusCode, body }; + }; + + const ok = await send(); + expect(ok.status).toBe(200); + expect(JSON.parse(ok.body)).toMatchObject({ data: { status: 202 } }); + const init = mockFetch.mock.calls[0]![1] as RequestInit & { headers: Record }; + expect(init.headers["X-LibScope-Signature"]).toBe(signPayload(init.body as string, "s3cr3t")); + expect(JSON.parse(init.body as string)).toMatchObject({ + event: "document.created", + data: { test: true }, + }); + + resolve4.mockResolvedValue(["127.0.0.1"]); + const blocked = await send(); + expect(blocked.status).toBe(400); + expect(mockFetch).toHaveBeenCalledOnce(); + }); +}); From a90a56907d9cf3cb9314f5001ea3e5e7be5bf459 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 15:12:50 +0000 Subject: [PATCH 08/83] fix(db): only backfill chunks_fts when the FTS index is empty The FTS backfill ran after every migration, so each schema upgrade re-inserted the whole corpus into chunks_fts and doubled keyword hits. Backfill now runs only when chunks_fts has no rows. runMigrations takes an optional target version so tests can build older schemas. --- src/db/schema.ts | 26 +++++++++--- tests/unit/schema-migrations.test.ts | 61 ++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 tests/unit/schema-migrations.test.ts diff --git a/src/db/schema.ts b/src/db/schema.ts index 5b0fb613..b117c4e8 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -276,8 +276,22 @@ const FTS_BACKFILL_SQL = ` SELECT content, id, document_id FROM chunks; `; -/** Run pending migrations on the database. */ -export function runMigrations(db: Database.Database): void { +/** + * Populate chunks_fts from chunks, but only when the FTS index is empty (for example, + * right after it was created). A non-empty index is kept in sync by triggers, so + * backfilling it again would duplicate every row. + */ +function backfillFtsIfEmpty(db: Database.Database): void { + const hasFtsRows = db.prepare("SELECT 1 FROM chunks_fts LIMIT 1").get() !== undefined; + if (hasFtsRows) return; + db.exec(FTS_BACKFILL_SQL); +} + +/** + * Run pending migrations on the database. + * @param targetVersion - Stop at this schema version (defaults to the latest). Used by tests. + */ +export function runMigrations(db: Database.Database, targetVersion: number = SCHEMA_VERSION): void { const log = getLogger(); try { @@ -294,15 +308,15 @@ export function runMigrations(db: Database.Database): void { currentVersion = row?.version ?? 0; } - if (currentVersion >= SCHEMA_VERSION) { + if (currentVersion >= targetVersion) { log.debug({ currentVersion }, "Database schema is up to date"); return; } - log.info({ from: currentVersion, to: SCHEMA_VERSION }, "Running database migrations"); + log.info({ from: currentVersion, to: targetVersion }, "Running database migrations"); const migrate = db.transaction(() => { - for (let v = currentVersion + 1; v <= SCHEMA_VERSION; v++) { + for (let v = currentVersion + 1; v <= targetVersion; v++) { const sql = MIGRATIONS[v]; if (!sql) { throw new DatabaseError(`Missing migration for version ${v}`); @@ -315,7 +329,7 @@ export function runMigrations(db: Database.Database): void { migrate(); try { - db.exec(FTS_BACKFILL_SQL); + backfillFtsIfEmpty(db); } catch (err) { log.warn({ err }, "FTS backfill failed — new chunks will still be indexed via triggers"); } diff --git a/tests/unit/schema-migrations.test.ts b/tests/unit/schema-migrations.test.ts new file mode 100644 index 00000000..bbeffbe5 --- /dev/null +++ b/tests/unit/schema-migrations.test.ts @@ -0,0 +1,61 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import DatabaseConstructor from "better-sqlite3"; +import type Database from "better-sqlite3"; +import { runMigrations } from "../../src/db/schema.js"; +import { initLogger } from "../../src/logger.js"; + +function newDb(): Database.Database { + const db = new DatabaseConstructor(":memory:"); + db.pragma("foreign_keys = ON"); + return db; +} + +function seedChunks(db: Database.Database, docCount: number, chunksPerDoc: number): void { + const insertDoc = db.prepare( + "INSERT INTO documents (id, source_type, title, content) VALUES (?, 'manual', ?, ?)", + ); + const insertChunk = db.prepare( + "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", + ); + for (let d = 0; d < docCount; d++) { + insertDoc.run(`doc-${d}`, `Document ${d}`, `content of document ${d}`); + for (let c = 0; c < chunksPerDoc; c++) { + insertChunk.run(`chunk-${d}-${c}`, `doc-${d}`, `alpha bravo doc${d} part${c}`, c); + } + } +} + +function count(db: Database.Database, table: string): number { + return (db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n; +} + +describe("schema migrations", () => { + beforeEach(() => { + initLogger("silent"); + }); + + describe("FTS backfill", () => { + it("does not duplicate FTS rows when upgrading a populated database", () => { + const db = newDb(); + runMigrations(db, 16); + seedChunks(db, 3, 4); + expect(count(db, "chunks_fts")).toBe(12); + + runMigrations(db); + + expect(count(db, "chunks_fts")).toBe(count(db, "chunks")); + }); + + it("backfills an empty FTS index from existing chunks", () => { + const db = newDb(); + runMigrations(db, 16); + seedChunks(db, 2, 3); + db.exec("DELETE FROM chunks_fts"); + expect(count(db, "chunks_fts")).toBe(0); + + runMigrations(db); + + expect(count(db, "chunks_fts")).toBe(6); + }); + }); +}); From fc7129bad8c9d9127791c35fab81e2c280f3eb5a Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 15:13:22 +0000 Subject: [PATCH 09/83] fix(config): env overrides no longer reset file settings getEnvOverrides spread DEFAULT_CONFIG sections into the override layer, so setting any LIBSCOPE_* variable reset sibling keys from config files to defaults (e.g. file provider=openai + LIBSCOPE_OPENAI_API_KEY gave provider=local). The env layer now contains only variables that are set. --- src/config.ts | 94 +++++++++++++------------ tests/unit/config-layers.test.ts | 114 +++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 42 deletions(-) create mode 100644 tests/unit/config-layers.test.ts diff --git a/src/config.ts b/src/config.ts index 02926c78..7d669e46 100644 --- a/src/config.ts +++ b/src/config.ts @@ -64,71 +64,81 @@ function getProjectConfigPath(): string { return join(process.cwd(), ".libscope.json"); } -function loadJsonFile(path: string): Partial { +/** A partial config layer (file or env): every key optional, only set keys present. */ +export interface ConfigLayer { + embedding?: Partial; + llm?: LibScopeConfig["llm"]; + database?: Partial; + indexing?: Partial; + logging?: Partial; +} + +function loadJsonFile(path: string): ConfigLayer { try { if (!existsSync(path)) return {}; const content = readFileSync(path, "utf-8"); - return JSON.parse(content) as Partial; + return JSON.parse(content) as ConfigLayer; } catch (err) { throw new ConfigError(`Failed to read config file: ${path}`, err); } } +const EMBEDDING_PROVIDERS = ["local", "ollama", "openai"] as const; const LLM_PROVIDERS = ["openai", "ollama", "anthropic", "passthrough"] as const; +type EmbeddingProviderName = (typeof EMBEDDING_PROVIDERS)[number]; type LlmProviderName = (typeof LLM_PROVIDERS)[number]; +function isEmbeddingProvider(value: string | undefined): value is EmbeddingProviderName { + return value !== undefined && (EMBEDDING_PROVIDERS as readonly string[]).includes(value); +} + function isLlmProvider(value: string | undefined): value is LlmProviderName { return value !== undefined && (LLM_PROVIDERS as readonly string[]).includes(value); } /** Env-var flag parsing: only "true" and "1" enable a flag. */ -function truthy(value: string | undefined): boolean { - return value === "true" || value === "1"; +function truthy(value: string | undefined): true | undefined { + return value === "true" || value === "1" ? true : undefined; } -function getEnvOverrides(): Partial { - const overrides: Partial = {}; - const provider = process.env["LIBSCOPE_EMBEDDING_PROVIDER"]; - const openaiKey = process.env["LIBSCOPE_OPENAI_API_KEY"]; - const ollamaUrl = process.env["LIBSCOPE_OLLAMA_URL"]; +type Compact = { [K in keyof T]?: Exclude }; - if (provider === "local" || provider === "ollama" || provider === "openai") { - overrides.embedding = { ...DEFAULT_CONFIG.embedding, provider }; - } - if (openaiKey) { - overrides.embedding = { - ...(overrides.embedding ?? DEFAULT_CONFIG.embedding), - openaiApiKey: openaiKey, - }; - } - if (ollamaUrl) { - overrides.embedding = { ...(overrides.embedding ?? DEFAULT_CONFIG.embedding), ollamaUrl }; - } - - const llmProvider = process.env["LIBSCOPE_LLM_PROVIDER"]; - const llmModel = process.env["LIBSCOPE_LLM_MODEL"]; - const allowPrivate = truthy(process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"]); - const allowSelfSigned = truthy(process.env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"]); - - if (allowPrivate || allowSelfSigned) { - overrides.indexing = { - ...DEFAULT_CONFIG.indexing, - ...(allowPrivate ? { allowPrivateUrls: true } : {}), - ...(allowSelfSigned ? { allowSelfSignedCerts: true } : {}), - }; +/** Drop undefined and empty-string values so a layer only carries keys that were actually set. */ +function compact>(obj: T): Compact { + const out: Record = {}; + for (const [key, value] of Object.entries(obj)) { + if (value !== undefined && value !== "") out[key] = value; } + return out as Compact; +} +/** Collect config values from LIBSCOPE_* env vars. Only variables that are set appear in the result. */ +function getEnvOverrides(): ConfigLayer { + const env = process.env; + const provider = env["LIBSCOPE_EMBEDDING_PROVIDER"]; + const llmProvider = env["LIBSCOPE_LLM_PROVIDER"]; + const llmModel = env["LIBSCOPE_LLM_MODEL"]; const validLlmProvider = isLlmProvider(llmProvider); - if (validLlmProvider || llmModel) { - const anthropicKey = process.env["LIBSCOPE_ANTHROPIC_API_KEY"]; - overrides.llm = { - ...(validLlmProvider ? { provider: llmProvider } : {}), - ...(llmModel ? { model: llmModel } : {}), - ...(anthropicKey ? { anthropicApiKey: anthropicKey } : {}), - }; - } - return overrides; + return { + embedding: compact({ + provider: isEmbeddingProvider(provider) ? provider : undefined, + openaiApiKey: env["LIBSCOPE_OPENAI_API_KEY"], + ollamaUrl: env["LIBSCOPE_OLLAMA_URL"], + }), + llm: + validLlmProvider || llmModel + ? compact({ + provider: validLlmProvider ? llmProvider : undefined, + model: llmModel, + anthropicApiKey: env["LIBSCOPE_ANTHROPIC_API_KEY"], + }) + : {}, + indexing: compact({ + allowPrivateUrls: truthy(env["LIBSCOPE_ALLOW_PRIVATE_URLS"]), + allowSelfSignedCerts: truthy(env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"]), + }), + }; } let _configCache: LibScopeConfig | null = null; diff --git a/tests/unit/config-layers.test.ts b/tests/unit/config-layers.test.ts new file mode 100644 index 00000000..cdc3da08 --- /dev/null +++ b/tests/unit/config-layers.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { loadConfig, invalidateConfigCache } from "../../src/config.js"; +import { initLogger } from "../../src/logger.js"; + +/** Env vars that influence config loading; cleared for every test. */ +const CONFIG_ENV_VARS = [ + "HOME", + "LIBSCOPE_EMBEDDING_PROVIDER", + "LIBSCOPE_OPENAI_API_KEY", + "LIBSCOPE_OLLAMA_URL", + "LIBSCOPE_OLLAMA_MODEL", + "LIBSCOPE_LLM_PROVIDER", + "LIBSCOPE_LLM_MODEL", + "LIBSCOPE_ANTHROPIC_API_KEY", + "LIBSCOPE_ALLOW_PRIVATE_URLS", + "LIBSCOPE_ALLOW_SELF_SIGNED_CERTS", + "LIBSCOPE_WORKSPACE", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", +]; + +describe("config layering (temp HOME and cwd)", () => { + let tempHome: string; + let tempCwd: string; + const savedEnv: Record = {}; + + function writeUserConfig(data: unknown): void { + mkdirSync(join(tempHome, ".libscope"), { recursive: true }); + writeFileSync(join(tempHome, ".libscope", "config.json"), JSON.stringify(data), "utf-8"); + } + + function writeProjectConfig(data: unknown): void { + writeFileSync(join(tempCwd, ".libscope.json"), JSON.stringify(data), "utf-8"); + } + + function load(env: Record = {}): ReturnType { + Object.assign(process.env, env); + invalidateConfigCache(); + return loadConfig(); + } + + beforeEach(() => { + initLogger("silent"); + for (const key of CONFIG_ENV_VARS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + tempHome = mkdtempSync(join(tmpdir(), "libscope-config-home-")); + tempCwd = mkdtempSync(join(tmpdir(), "libscope-config-cwd-")); + process.env["HOME"] = tempHome; + vi.spyOn(process, "cwd").mockReturnValue(tempCwd); + }); + + afterEach(() => { + vi.restoreAllMocks(); + for (const [key, val] of Object.entries(savedEnv)) { + if (val === undefined) delete process.env[key]; + else process.env[key] = val; + } + invalidateConfigCache(); + rmSync(tempHome, { recursive: true, force: true }); + rmSync(tempCwd, { recursive: true, force: true }); + }); + + describe("env overrides only replace keys that are set", () => { + it("keeps provider=openai from the config file when LIBSCOPE_OPENAI_API_KEY is set", () => { + writeUserConfig({ embedding: { provider: "openai" } }); + const config = load({ LIBSCOPE_OPENAI_API_KEY: "sk-env" }); + expect(config.embedding.provider).toBe("openai"); + expect(config.embedding.openaiApiKey).toBe("sk-env"); + }); + + it("keeps file ollamaModel/ollamaUrl when LIBSCOPE_EMBEDDING_PROVIDER is set", () => { + writeUserConfig({ + embedding: { ollamaUrl: "http://gpu-box:11434", ollamaModel: "mxbai-embed-large" }, + }); + const config = load({ LIBSCOPE_EMBEDDING_PROVIDER: "ollama" }); + expect(config.embedding.provider).toBe("ollama"); + expect(config.embedding.ollamaUrl).toBe("http://gpu-box:11434"); + expect(config.embedding.ollamaModel).toBe("mxbai-embed-large"); + }); + + it("keeps project-level indexing settings when one indexing env flag is set", () => { + writeProjectConfig({ indexing: { allowPrivateUrls: true, maxDocumentSize: 1234 } }); + const config = load({ LIBSCOPE_ALLOW_SELF_SIGNED_CERTS: "1" }); + expect(config.indexing.allowSelfSignedCerts).toBe(true); + expect(config.indexing.allowPrivateUrls).toBe(true); + expect(config.indexing.maxDocumentSize).toBe(1234); + }); + + it("keeps file llm.model when only LIBSCOPE_LLM_PROVIDER is set", () => { + writeUserConfig({ llm: { provider: "ollama", model: "llama3" } }); + const config = load({ LIBSCOPE_LLM_PROVIDER: "openai" }); + expect(config.llm?.provider).toBe("openai"); + expect(config.llm?.model).toBe("llama3"); + }); + + it("ignores invalid LIBSCOPE_EMBEDDING_PROVIDER without resetting the file value", () => { + writeUserConfig({ embedding: { provider: "ollama" } }); + const config = load({ LIBSCOPE_EMBEDDING_PROVIDER: "bogus" }); + expect(config.embedding.provider).toBe("ollama"); + }); + + it("applies defaults when no files and no env vars exist", () => { + const config = load(); + expect(config.embedding.provider).toBe("local"); + expect(config.embedding.ollamaUrl).toBe("http://localhost:11434"); + expect(config.indexing.allowPrivateUrls).toBe(false); + }); + }); +}); From d6b8342e743e4bb68abd7b00f540646671d52fed Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 15:13:45 +0000 Subject: [PATCH 10/83] fix(connectors): track all existing doc-site pages, not just the first 50 syncDocSite built its existing-URL set from listDocuments, which applies a default LIMIT 50 and loads full content. Sites with more than 50 pages had unchanged and changed pages miscounted as new. Query the url column directly with no limit. --- src/connectors/docs.ts | 24 +++++++++++++++++------- tests/unit/docs-connector.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/connectors/docs.ts b/src/connectors/docs.ts index c6400f97..fc8865e7 100644 --- a/src/connectors/docs.ts +++ b/src/connectors/docs.ts @@ -12,7 +12,7 @@ import { getLogger } from "../logger.js"; import { fetchRaw } from "../core/url-fetcher.js"; import type { FetchOptions } from "../core/url-fetcher.js"; import { indexDocument } from "../core/indexing.js"; -import { listDocuments, deleteDocument } from "../core/documents.js"; +import { deleteDocument } from "../core/documents.js"; import { startSync, completeSync, failSync } from "./sync-tracker.js"; import type { EmbeddingProvider } from "../providers/embedding.js"; @@ -576,6 +576,21 @@ function validateDocSiteConfig(config: DocSiteConfig): URL { return baseUrl; } +/** + * Normalised URLs of every docs-connector document (optionally scoped to one library). + * Reads only the url column and applies no row limit, so large sites are tracked fully. + */ +function loadExistingDocUrls(db: Database.Database, library: string | undefined): Set { + let sql = "SELECT id, url FROM documents WHERE source_type = ? AND url IS NOT NULL"; + const params: string[] = [SOURCE_TYPE]; + if (library) { + sql += " AND library = ?"; + params.push(library); + } + const rows = db.prepare(sql).all(...params) as Array<{ id: string; url: string }>; + return new Set(rows.map((r) => normalizeUrl(r.url))); +} + /** Discover URLs via sitemap.xml and root page links, populating the BFS queue. */ async function discoverUrls( config: DocSiteConfig, @@ -677,12 +692,7 @@ export async function syncDocSite( await discoverUrls(config, baseUrl, rootHtml, pathPrefix, fetchOptions, visited, queue); // --- Build existing-URL index for update tracking --- - const existingDocs = listDocuments(db, { sourceType: SOURCE_TYPE, library: config.library }); - const existingUrls = new Set( - existingDocs - .filter((d): d is typeof d & { url: string } => d.url !== null) - .map((d) => normalizeUrl(d.url)), - ); + const existingUrls = loadExistingDocUrls(db, config.library); const ctx: PageContext = { siteType: result.detectedType, diff --git a/tests/unit/docs-connector.test.ts b/tests/unit/docs-connector.test.ts index 9a974179..679b16ef 100644 --- a/tests/unit/docs-connector.test.ts +++ b/tests/unit/docs-connector.test.ts @@ -796,6 +796,30 @@ describe("syncDocSite — mocked fetch", () => { expect(result2.pagesUpdated).toBe(0); }); + it("recognises existing pages beyond the first 50 documents", async () => { + mockFetch + .mockResolvedValueOnce(htmlResponse(SPHINX_ROOT_SIMPLE)) // root + .mockResolvedValueOnce(notFoundResponse()); // sitemap.xml + await syncDocSite(db, provider, { url: "https://docs.example.com/docs/" }); + + // Make the root doc the oldest of 61 docs-connector documents. + db.prepare("UPDATE documents SET updated_at = '2000-01-01 00:00:00'").run(); + const insert = db.prepare( + "INSERT INTO documents (id, source_type, title, content, url, updated_at) VALUES (?, 'library', 'T', 'B', ?, '2030-01-01 00:00:00')", + ); + for (let i = 0; i < 60; i++) { + insert.run(`filler-${i}`, `https://docs.example.com/docs/filler-${i}`); + } + + mockFetch + .mockResolvedValueOnce(htmlResponse(SPHINX_ROOT_SIMPLE)) // root (unchanged) + .mockResolvedValueOnce(notFoundResponse()); // sitemap.xml + const result = await syncDocSite(db, provider, { url: "https://docs.example.com/docs/" }); + + expect(result.pagesSkipped).toBe(1); + expect(result.pagesIndexed).toBe(0); + }); + it("records sync history in the connector_syncs table", async () => { mockFetch .mockResolvedValueOnce(htmlResponse(SPHINX_ROOT_SIMPLE)) // root From 79914d2901f290c4ba018ab35d103c9a79d0ecf6 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Wed, 7 Oct 2026 15:14:31 +0000 Subject: [PATCH 11/83] fix(cli): rename subcommand --version flags shadowed by global --version The root program's .version() registers a global --version option, so `add/import/import-batch/docs update/pack install/pack create/registry publish --version X` printed the CLI version and exited, and `registry unpublish` (required --version) could never run. - Document metadata flag: --version -> --lib-version (add, import, import-batch, docs update) - Pack/registry flag: --version -> --pack-version (pack install, pack create, registry publish, registry unpublish) - registry unpublish accepts @; --pack-version is now optional and takes precedence over the suffix - Only call program.parse() when the module is the process entry point, and export the program so tests can parse arguments without running it --- README.md | 2 +- docs/guide/knowledge-packs.md | 2 +- docs/guide/pack-registries.md | 10 +- docs/reference/cli.md | 48 ++++---- docs/reference/registry.md | 18 +-- src/cli/commands/registry.ts | 32 +++-- src/cli/index.ts | 75 ++++++++---- src/registry/publish.ts | 2 +- tests/unit/cli-version-options.test.ts | 159 +++++++++++++++++++++++++ 9 files changed, 272 insertions(+), 76 deletions(-) create mode 100644 tests/unit/cli-version-options.test.ts diff --git a/README.md b/README.md index 92d7c76b..d0e1ec23 100644 --- a/README.md +++ b/README.md @@ -451,7 +451,7 @@ libscope pack install react-docs@1.2.0 # specific version libscope registry create ./my-registry # Publish a pack file to your registry -libscope registry publish ./my-pack.json -r my-registry --version 1.0.0 +libscope registry publish ./my-pack.json -r my-registry --pack-version 1.0.0 # Submit a pack to someone else's registry (creates a feature branch) libscope registry publish ./my-pack.json -r community --submit diff --git a/docs/guide/knowledge-packs.md b/docs/guide/knowledge-packs.md index e9a7dbd5..edb97644 100644 --- a/docs/guide/knowledge-packs.md +++ b/docs/guide/knowledge-packs.md @@ -28,7 +28,7 @@ You can export documents from your knowledge base as a pack: libscope pack create \ --name "react-docs" \ --topic react \ - --version 1.0.0 \ + --pack-version 1.0.0 \ --description "React documentation" \ --author "team" ``` diff --git a/docs/guide/pack-registries.md b/docs/guide/pack-registries.md index 0323c032..13b58ee4 100644 --- a/docs/guide/pack-registries.md +++ b/docs/guide/pack-registries.md @@ -49,7 +49,7 @@ The existing `pack install` command now resolves packs from your configured regi libscope pack install react-docs # Install a specific version -libscope pack install react-docs --version 1.2.0 +libscope pack install react-docs --pack-version 1.2.0 # or libscope pack install react-docs@1.2.0 @@ -89,7 +89,7 @@ This creates a git repo with the correct folder structure (`index.json`, `packs/ ```bash # Publish a pack file to a registry you own -libscope registry publish ./my-pack.json -r my-registry --version 1.0.0 +libscope registry publish ./my-pack.json -r my-registry --pack-version 1.0.0 # Auto-bump patch version (from latest in registry) libscope registry publish ./my-pack.json -r my-registry @@ -98,7 +98,9 @@ libscope registry publish ./my-pack.json -r my-registry libscope registry publish ./my-pack.json -r community --submit # Unpublish a specific version -libscope registry unpublish my-pack -r my-registry --version 1.0.0 +libscope registry unpublish my-pack -r my-registry --pack-version 1.0.0 +# or +libscope registry unpublish my-pack@1.0.0 -r my-registry ``` Publishing assembles the pack into the registry's folder structure, generates a SHA-256 checksum, updates `index.json` and `pack.json`, and commits + pushes. The `--submit` flag pushes to a `feature/add-` branch instead — you then create a pull request manually. @@ -109,7 +111,7 @@ Every published pack version includes a `checksum.sha256` file. On install, libs ## Versioning -Pack versions follow [semver](https://semver.org/) (e.g. `1.0.0`, `1.2.3`). When you publish without `--version`, the patch version is auto-bumped from the latest. Old versions are preserved in the registry. `pack install` defaults to the latest version unless you specify one. +Pack versions follow [semver](https://semver.org/) (e.g. `1.0.0`, `1.2.3`). When you publish without `--pack-version`, the patch version is auto-bumped from the latest. Old versions are preserved in the registry. `pack install` defaults to the latest version unless you specify one. ## MCP Usage diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 2919a2b1..94e47ccb 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -22,12 +22,12 @@ libscope add ./path/to/doc.md --topic deployment --title "Deploy Guide" libscope add https://reactjs.org/docs/hooks-intro.html --library react ``` -| Option | Description | -| ------------------ | --------------------------- | -| `--library ` | Assign to a library | -| `--topic ` | Assign to a topic | -| `--title ` | Override the document title | -| `--version <ver>` | Library version tag | +| Option | Description | +| --------------------- | --------------------------- | +| `--library <name>` | Assign to a library | +| `--topic <name>` | Assign to a topic | +| `--title <title>` | Override the document title | +| `--lib-version <ver>` | Library version tag | ### `libscope import` @@ -41,6 +41,7 @@ libscope import ./wiki/ --topic internal --extensions .md,.mdx,.txt | --------------------- | ------------------------------------------------ | | `--library <name>` | Assign all docs to a library | | `--topic <name>` | Assign all docs to a topic | +| `--lib-version <ver>` | Library version tag | | `--extensions <exts>` | Comma-separated file extensions (default: `.md`) | ### `libscope import-batch` @@ -51,11 +52,12 @@ Parallel batch import with progress tracking. libscope import-batch ./docs/ --concurrency 10 --filter "**/*.md" --library my-lib ``` -| Option | Description | -| ------------------- | ---------------------------- | -| `--concurrency <n>` | Number of parallel workers | -| `--filter <glob>` | Glob pattern to match files | -| `--library <name>` | Assign all docs to a library | +| Option | Description | +| --------------------- | ---------------------------- | +| `--concurrency <n>` | Number of parallel workers | +| `--filter <glob>` | Glob pattern to match files | +| `--library <name>` | Assign all docs to a library | +| `--lib-version <ver>` | Library version tag | ### `libscope search` @@ -142,7 +144,7 @@ Update an existing document's title, content, or metadata. Changing content trig ```bash libscope docs update <documentId> --title "New Title" libscope docs update <documentId> --content "Updated content here" -libscope docs update <documentId> --library vue --version 3.0.0 +libscope docs update <documentId> --library vue --lib-version 3.0.0 ``` | Option | Description | @@ -150,7 +152,7 @@ libscope docs update <documentId> --library vue --version 3.0.0 | `--title <title>` | New document title | | `--content <content>` | New content (triggers re-chunking) | | `--library <name>` | New library name | -| `--version <ver>` | New library version | +| `--lib-version <ver>` | New library version | | `--url <url>` | New source URL | | `--topic <topicId>` | New topic ID | @@ -252,14 +254,14 @@ When no local file path is given, `pack install` searches configured registries: ```bash libscope pack install react-docs # latest from any registry libscope pack install react-docs@1.2.0 # specific version -libscope pack install react-docs --version 1.2.0 +libscope pack install react-docs --pack-version 1.2.0 libscope pack install react-docs --registry official ``` -| Option | Description | -| -------------------- | -------------------------------------------- | -| `--version <semver>` | Install a specific version (default: latest) | -| `--registry <name>` | Install from a specific registry | +| Option | Description | +| ------------------------- | -------------------------------------------- | +| `--pack-version <semver>` | Install a specific version (default: latest) | +| `--registry <name>` | Install from a specific registry | ## Pack Registries @@ -272,7 +274,7 @@ libscope pack install react-docs --registry official | `libscope registry search <query> [-r <name>]` | Search registry pack indexes | | `libscope registry create <path>` | Initialize a new registry repo | | `libscope registry publish <file> -r <name>` | Publish a pack file to a registry | -| `libscope registry unpublish <pack> -r <name> --version <v>` | Remove a pack version from a registry | +| `libscope registry unpublish <pack> -r <name> --pack-version <v>` | Remove a pack version from a registry (or `<pack>@<v>`) | ### `libscope registry add` @@ -300,7 +302,7 @@ libscope registry search "kubernetes" -r official ```bash # Direct publish (you have write access) -libscope registry publish ./my-pack.json -r my-registry --version 1.0.0 +libscope registry publish ./my-pack.json -r my-registry --pack-version 1.0.0 # Auto-bump patch version libscope registry publish ./my-pack.json -r my-registry @@ -312,20 +314,20 @@ libscope registry publish ./my-pack.json -r community --submit | Option | Description | | ------------------------ | -------------------------------------------------------- | | `-r, --registry <name>` | Target registry (required) | -| `--version <semver>` | Version to publish as (default: auto-bump patch) | +| `--pack-version <semver>` | Version to publish as (default: auto-bump patch) | | `-m, --message <msg>` | Git commit message | | `--submit` | Push to a feature branch instead of main | ### `libscope registry unpublish` ```bash -libscope registry unpublish my-pack -r my-registry --version 1.0.0 +libscope registry unpublish my-pack -r my-registry --pack-version 1.0.0 ``` | Option | Description | | ------------------------ | --------------------------------- | | `-r, --registry <name>` | Target registry (required) | -| `--version <semver>` | Version to remove (required) | +| `--pack-version <semver>` | Version to remove (required unless given as `<pack>@<version>`) | | `-m, --message <msg>` | Git commit message | | `-y, --yes` | Skip confirmation prompt | diff --git a/docs/reference/registry.md b/docs/reference/registry.md index d5b6a568..7310b10e 100644 --- a/docs/reference/registry.md +++ b/docs/reference/registry.md @@ -103,13 +103,13 @@ libscope registry publish <packFile> -r <name> [options] | ----------------------- | -------------------------------------------------------- | | `<packFile>` | Path to the pack `.json` file to publish | | `-r, --registry <name>` | Target registry (required) | -| `--version <semver>` | Version to publish as (default: auto-bump patch) | +| `--pack-version <semver>` | Version to publish as (default: auto-bump patch) | | `-m, --message <msg>` | Git commit message | | `--submit` | Push to a feature branch instead of main (for PR workflow) | **Direct publish** (you have write access): ```bash -libscope registry publish ./react-docs.json -r my-registry --version 1.0.0 +libscope registry publish ./react-docs.json -r my-registry --pack-version 1.0.0 ``` **Submit for inclusion** (you don't have write access): @@ -124,14 +124,14 @@ The `--submit` flag creates a `feature/add-<pack-name>` branch and pushes it. Yo Remove a specific pack version from a registry. ```bash -libscope registry unpublish <packName> -r <name> --version <semver> [options] +libscope registry unpublish <packName>[@<semver>] -r <name> [--pack-version <semver>] [options] ``` | Option | Description | | ----------------------- | --------------------------------- | | `<packName>` | Name of the pack to unpublish | | `-r, --registry <name>` | Target registry (required) | -| `--version <semver>` | Version to remove (required) | +| `--pack-version <semver>` | Version to remove (required unless given as `<packName>@<semver>`) | | `-m, --message <msg>` | Git commit message | | `-y, --yes` | Skip confirmation prompt | @@ -142,12 +142,12 @@ If the last version of a pack is unpublished, the entire pack is removed from th The existing `pack install` command is extended to resolve packs from registries. ```bash -libscope pack install <name> [--version <semver>] [--registry <name>] +libscope pack install <name> [--pack-version <semver>] [--registry <name>] ``` | Option | Description | | -------------------- | ---------------------------------------------- | -| `--version <semver>` | Install a specific version (default: latest) | +| `--pack-version <semver>` | Install a specific version (default: latest) | | `--registry <name>` | Install from a specific registry | ```bash @@ -156,7 +156,7 @@ libscope pack install react-docs # Install specific version libscope pack install react-docs@1.2.0 -libscope pack install react-docs --version 1.2.0 +libscope pack install react-docs --pack-version 1.2.0 # Install from a specific registry libscope pack install react-docs --registry official @@ -342,9 +342,9 @@ Pack versions follow [semver](https://semver.org/): - Versions must be valid semver strings (e.g. `1.0.0`, `2.3.1`) - `pack install <name>` installs the latest version -- `pack install <name>@1.0.0` or `--version 1.0.0` installs a specific version +- `pack install <name>@1.0.0` or `--pack-version 1.0.0` installs a specific version - Old versions are preserved in the registry — publishing a new version does not remove previous ones -- When publishing without `--version`, the patch version is auto-bumped from the latest +- When publishing without `--pack-version`, the patch version is auto-bumped from the latest - The `latestVersion` in `index.json` always points to the most recently published version ## Conflict Resolution diff --git a/src/cli/commands/registry.ts b/src/cli/commands/registry.ts index 3686bec0..6cd2ab83 100644 --- a/src/cli/commands/registry.ts +++ b/src/cli/commands/registry.ts @@ -23,6 +23,7 @@ import { getRegistryCacheDir } from "../../registry/types.js"; import { syncRegistryByName, syncAllRegistries } from "../../registry/sync.js"; import { searchRegistries } from "../../registry/search.js"; import { publishPack, publishPackToBranch, unpublishPack } from "../../registry/publish.js"; +import { parsePackSpecifier } from "../../registry/resolve.js"; import { confirmOrCancel } from "../confirm.js"; /** Derive a short name from a git URL (e.g. "github.com/org/repo" → "repo"). */ @@ -349,13 +350,13 @@ export function registerRegistryCommands(program: Command): void { .command("publish <packFile>") .description("Publish a pack file to a registry") .requiredOption("-r, --registry <name>", "Target registry name") - .option("--version <semver>", "Version to publish as (default: auto-bump patch)") + .option("--pack-version <semver>", "Version to publish as (default: auto-bump patch)") .option("-m, --message <msg>", "Git commit message") .option("--submit", "Push to a feature branch instead of main (for PR workflow)") .action( async ( packFile: string, - opts: { registry: string; version?: string; message?: string; submit?: boolean }, + opts: { registry: string; packVersion?: string; message?: string; submit?: boolean }, ) => { await requireGit(); @@ -366,7 +367,7 @@ export function registerRegistryCommands(program: Command): void { const result = await publishPackToBranch({ registryName: opts.registry, packFilePath: resolved, - version: opts.version, + version: opts.packVersion, commitMessage: opts.message, }); console.log( @@ -377,7 +378,7 @@ export function registerRegistryCommands(program: Command): void { const result = await publishPack({ registryName: opts.registry, packFilePath: resolved, - version: opts.version, + version: opts.packVersion, commitMessage: opts.message, }); console.log( @@ -394,21 +395,30 @@ export function registerRegistryCommands(program: Command): void { // --- registry unpublish --- registryCmd .command("unpublish <packName>") - .description("Remove a pack version from a registry") + .description("Remove a pack version from a registry (accepts name@version)") .requiredOption("-r, --registry <name>", "Target registry name") - .requiredOption("--version <semver>", "Version to unpublish") + .option("--pack-version <semver>", "Version to unpublish (or use name@version)") .option("-m, --message <msg>", "Git commit message") .option("-y, --yes", "Skip confirmation prompt") .action( async ( - packName: string, - opts: { registry: string; version: string; message?: string; yes?: boolean }, + packSpec: string, + opts: { registry: string; packVersion?: string; message?: string; yes?: boolean }, ) => { + const { name: packName, version: specVersion } = parsePackSpecifier(packSpec); + const version = opts.packVersion ?? specVersion; + if (!version) { + console.error( + "Error: specify a version with <name>@<version> or --pack-version <semver>.", + ); + process.exit(1); + } + await requireGit(); if ( !(await confirmOrCancel( - `Unpublish "${packName}@${opts.version}" from "${opts.registry}"? This cannot be undone.`, + `Unpublish "${packName}@${version}" from "${opts.registry}"? This cannot be undone.`, opts.yes, )) ) { @@ -419,10 +429,10 @@ export function registerRegistryCommands(program: Command): void { await unpublishPack({ registryName: opts.registry, packName, - version: opts.version, + version, commitMessage: opts.message, }); - console.log(`Pack "${packName}@${opts.version}" unpublished from "${opts.registry}".`); + console.log(`Pack "${packName}@${version}" unpublished from "${opts.registry}".`); } catch (err) { console.error(`Error: ${formatError(err)}`); process.exit(1); diff --git a/src/cli/index.ts b/src/cli/index.ts index 00556add..d71a8f1c 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -15,8 +15,9 @@ import { createLink, getDocumentLinks, deleteLink, getPrerequisiteChain } from " import type { LinkType } from "../core/links.js"; import { getVersionHistory, rollbackToVersion } from "../core/versioning.js"; import { initLogger, type LogLevel } from "../logger.js"; -import { readFileSync, readdirSync, statSync } from "node:fs"; +import { readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; import { join, extname, basename } from "node:path"; +import { fileURLToPath } from "node:url"; import { fetchAndConvert } from "../core/url-fetcher.js"; import { spiderUrl } from "../core/spider.js"; import type { SpiderOptions } from "../core/spider.js"; @@ -340,7 +341,7 @@ async function handlePackConflict( /** Try to resolve a pack from git registries; returns true if installed. */ async function tryGitRegistryInstall( nameOrPath: string, - opts: { fromRegistry?: string; version?: string; yes?: boolean }, + opts: { fromRegistry?: string; packVersion?: string; yes?: boolean }, db: ReturnType<typeof getDatabase>, provider: EmbeddingProvider, installOpts: { @@ -354,7 +355,7 @@ async function tryGitRegistryInstall( if (isLocalFile || loadRegistries().length === 0) return false; const { name: packName, version: specVersion } = parsePackSpecifier(nameOrPath); - const version = opts.version ?? specVersion; + const version = opts.packVersion ?? specVersion; const { resolved, conflict, warnings } = resolvePackFromRegistries(packName, { version, @@ -398,7 +399,7 @@ async function executePackInstall( opts: { registry?: string; fromRegistry?: string; - version?: string; + packVersion?: string; yes?: boolean; batchSize?: string; resumeFrom?: string; @@ -512,7 +513,7 @@ async function runSpiderAdd( opts: { topic?: string; library?: string; - version?: string; + libVersion?: string; dedup?: string; maxPages?: number; maxDepth?: number; @@ -538,7 +539,7 @@ async function runSpiderAdd( content: page.content, sourceType, library: opts.library, - version: opts.version, + version: opts.libVersion, topicId: opts.topic, url: page.url, dedup: opts.dedup as "skip" | "warn" | "force" | undefined, @@ -568,7 +569,7 @@ program ) .option("--topic <topicId>", "Assign to a topic") .option("--library <name>", "Mark as library documentation") - .option("--version <version>", "Library version") + .option("--lib-version <version>", "Library version") .option("--title <title>", "Override document title") .option("--format <ext>", "Force file format (e.g. .pdf, .csv, .yaml)") .option("--dedup <mode>", "Dedup mode: skip, warn, or force") @@ -588,7 +589,7 @@ program opts: { topic?: string; library?: string; - version?: string; + libVersion?: string; title?: string; format?: string; dedup?: string; @@ -618,7 +619,7 @@ program content: fetched.content, sourceType: resolveSourceType(opts.library, opts.topic), library: opts.library, - version: opts.version, + version: opts.libVersion, topicId: opts.topic, url: fileOrUrl, dedup: opts.dedup as "skip" | "warn" | "force" | undefined, @@ -631,7 +632,7 @@ program title: opts.title, topic: opts.topic, library: opts.library, - version: opts.version, + version: opts.libVersion, format: opts.format, dedup: opts.dedup as "skip" | "warn" | "force" | undefined, }); @@ -651,7 +652,7 @@ program .description("Bulk import files from a directory (supports all document formats)") .option("--topic <topicId>", "Assign all to a topic") .option("--library <name>", "Mark all as library documentation") - .option("--version <version>", "Library version") + .option("--lib-version <version>", "Library version") .option( "--extensions <exts>", "Comma-separated file extensions to include", @@ -664,7 +665,7 @@ program opts: { topic?: string; library?: string; - version?: string; + libVersion?: string; extensions: string; dedup?: string; }, @@ -689,7 +690,7 @@ program const result = await indexFile(db, provider, file, { topic: opts.topic, library: opts.library, - version: opts.version, + version: opts.libVersion, dedup: opts.dedup as "skip" | "warn" | "force" | undefined, }); @@ -722,7 +723,7 @@ program .option("--dry-run", "Preview files without importing") .option("--topic <topicId>", "Assign all to a topic") .option("--library <name>", "Mark all as library documentation") - .option("--version <version>", "Library version") + .option("--lib-version <version>", "Library version") .action( async ( directory: string, @@ -732,7 +733,7 @@ program dryRun?: boolean; topic?: string; library?: string; - version?: string; + libVersion?: string; }, ) => { const { db, provider } = initializeAppWithEmbedding(); @@ -762,7 +763,7 @@ program const result = await batchImport(db, provider, files, { concurrency: parseIntOption(opts.concurrency, "--concurrency"), library: opts.library, - version: opts.version, + version: opts.libVersion, topicId: opts.topic, onProgress: (progress) => { const done = progress.completed + progress.failed; @@ -1295,7 +1296,7 @@ docsCmd .option("--title <title>", "New title") .option("--content <content>", "New content (will re-chunk and re-index)") .option("--library <name>", "New library name") - .option("--version <ver>", "New version") + .option("--lib-version <ver>", "New library version") .option("--url <url>", "New URL") .option("--topic <topicId>", "New topic ID") .action( @@ -1305,7 +1306,7 @@ docsCmd title?: string; content?: string; library?: string; - version?: string; + libVersion?: string; url?: string; topic?: string; }, @@ -1314,7 +1315,7 @@ docsCmd try { const metadata: Record<string, string | null | undefined> = {}; if (opts.library !== undefined) metadata.library = opts.library; - if (opts.version !== undefined) metadata.version = opts.version; + if (opts.libVersion !== undefined) metadata.version = opts.libVersion; if (opts.url !== undefined) metadata.url = opts.url; if (opts.topic !== undefined) metadata.topicId = opts.topic; @@ -2097,7 +2098,10 @@ packCmd ) .option("--registry <url>", "Custom registry URL (for URL-based registries)") .option("--from-registry <name>", "Install from a specific git registry by name") - .option("--version <semver>", "Install a specific version (for git registries)") + .option( + "--pack-version <semver>", + "Install a specific version (for git registries; or use name@version)", + ) .option("-y, --yes", "Non-interactive mode (fail on conflicts instead of prompting)") .option("--batch-size <n>", "Number of documents to embed per batch (default: 10)") .option("--resume-from <n>", "Skip the first N documents (resume a partial install)") @@ -2108,7 +2112,7 @@ packCmd opts: { registry?: string; fromRegistry?: string; - version?: string; + packVersion?: string; yes?: boolean; batchSize?: string; resumeFrom?: string; @@ -2181,7 +2185,7 @@ packCmd .requiredOption("--name <name>", "Pack name") .option("--from <sources...>", "Source folder(s), file(s), or URL(s) to build pack from") .option("--topic <topic>", "Filter documents by topic ID (database mode only)") - .option("--version <version>", "Pack version (default: 1.0.0)") + .option("--pack-version <version>", "Pack version (default: 1.0.0)") .option("--description <desc>", "Pack description") .option("--author <author>", "Pack author") .option("--output <path>", "Output file path") @@ -2196,7 +2200,7 @@ packCmd name: string; from?: string[]; topic?: string; - version?: string; + packVersion?: string; description?: string; author?: string; output?: string; @@ -2213,7 +2217,7 @@ packCmd const pack = await createPackFromSource({ name: opts.name, from: opts.from, - version: opts.version, + version: opts.packVersion, description: opts.description, author: opts.author, outputPath, @@ -2238,7 +2242,7 @@ packCmd try { const pack = createPack(db, { name: opts.name, - version: opts.version, + version: opts.packVersion, description: opts.description, author: opts.author, topic: opts.topic, @@ -3049,4 +3053,23 @@ scheduleCmd // Registry commands registerRegistryCommands(program); -program.parse(); +/** True when this module is the process entry point (the `libscope` bin), not an import. */ +function isCliEntryPoint(): boolean { + const entry = process.argv[1]; + if (!entry) return false; + const self = realpathSync(fileURLToPath(import.meta.url)); + // Node also accepts an entry path without the ".js" extension. + return [entry, `${entry}.js`].some((candidate) => { + try { + return realpathSync(candidate) === self; + } catch { + return false; + } + }); +} + +if (isCliEntryPoint()) { + program.parse(); +} + +export { program }; diff --git a/src/registry/publish.ts b/src/registry/publish.ts index 42f91f85..88e14322 100644 --- a/src/registry/publish.ts +++ b/src/registry/publish.ts @@ -183,7 +183,7 @@ function resolveVersionAndManifest( if (manifest.versions.some((v) => v.version === version)) { throw new ValidationError( `Version ${version} of "${pack.name}" already exists in "${registryName}". ` + - "Use --version to specify a different version.", + "Use --pack-version to specify a different version.", ); } diff --git a/tests/unit/cli-version-options.test.ts b/tests/unit/cli-version-options.test.ts new file mode 100644 index 00000000..d6f8c5b0 --- /dev/null +++ b/tests/unit/cli-version-options.test.ts @@ -0,0 +1,159 @@ +import { describe, it, expect, vi, beforeAll, afterEach } from "vitest"; +import { Command, CommanderError } from "commander"; +import { readFileSync } from "node:fs"; + +vi.mock("../../src/registry/git.js", async (importOriginal) => { + const original = await importOriginal<typeof import("../../src/registry/git.js")>(); + return { ...original, checkGitAvailable: vi.fn().mockResolvedValue(true) }; +}); + +vi.mock("../../src/registry/publish.js", () => ({ + publishPack: vi.fn(), + publishPackToBranch: vi.fn(), + unpublishPack: vi.fn().mockResolvedValue(undefined), +})); + +import { program } from "../../src/cli/index.js"; +import { registerRegistryCommands } from "../../src/cli/commands/registry.js"; +import { unpublishPack } from "../../src/registry/publish.js"; + +const pkgVersion = ( + JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf-8")) as { + version: string; + } +).version; + +/** Make every command throw instead of exiting, and swallow help/version output. */ +function makeTestable(cmd: Command, output: string[]): void { + cmd.exitOverride(); + cmd.configureOutput({ + writeOut: (str) => output.push(str), + writeErr: (str) => output.push(str), + }); + for (const sub of cmd.commands) makeTestable(sub, output); +} + +function findCommand(root: Command, path: string[]): Command { + let cmd = root; + for (const name of path) { + const next = cmd.commands.find((c) => c.name() === name); + if (!next) throw new Error(`Command not found: ${path.join(" ")}`); + cmd = next; + } + return cmd; +} + +/** Parse argv against the real CLI program with the target command's action stubbed out. */ +async function parseWithStubbedAction( + path: string[], + args: string[], +): Promise<Record<string, unknown>> { + const cmd = findCommand(program, path); + const action = vi.fn(); + cmd.action(action); + await program.parseAsync([...path, ...args], { from: "user" }); + expect(action).toHaveBeenCalledTimes(1); + return cmd.opts(); +} + +describe("CLI subcommand version flags", () => { + const output: string[] = []; + + beforeAll(() => { + makeTestable(program, output); + }); + + it("keeps the root --version flag printing the CLI version", async () => { + output.length = 0; + await expect(program.parseAsync(["--version"], { from: "user" })).rejects.toBeInstanceOf( + CommanderError, + ); + expect(output.join("")).toContain(pkgVersion); + }); + + it.each([ + [["add"], ["doc.md"]], + [["import"], ["./docs"]], + [["import-batch"], ["./docs"]], + [["docs", "update"], ["doc-1"]], + ])("%j accepts --lib-version", async (path, positional) => { + const opts = await parseWithStubbedAction(path, [...positional, "--lib-version", "2.1.0"]); + expect(opts.libVersion).toBe("2.1.0"); + }); + + it.each([ + [["pack", "install"], ["react-docs"]], + [ + ["pack", "create"], + ["--name", "react-docs"], + ], + [ + ["registry", "publish"], + ["pack.json", "-r", "main"], + ], + [ + ["registry", "unpublish"], + ["react-docs", "-r", "main"], + ], + ])("%j accepts --pack-version", async (path, rest) => { + const opts = await parseWithStubbedAction(path, [...rest, "--pack-version", "3.0.0"]); + expect(opts.packVersion).toBe("3.0.0"); + }); +}); + +describe("registry unpublish version resolution", () => { + function buildProgram(): Command { + const root = new Command(); + root.exitOverride(); + registerRegistryCommands(root); + return root; + } + + afterEach(() => { + vi.restoreAllMocks(); + vi.mocked(unpublishPack).mockClear(); + }); + + function stubExit(): void { + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process, "exit").mockImplementation((code) => { + throw new Error(`process.exit(${String(code)})`); + }); + } + + it("accepts name@version", async () => { + stubExit(); + await buildProgram().parseAsync( + ["registry", "unpublish", "react-docs@1.2.0", "-r", "main", "-y"], + { + from: "user", + }, + ); + expect(unpublishPack).toHaveBeenCalledWith( + expect.objectContaining({ registryName: "main", packName: "react-docs", version: "1.2.0" }), + ); + }); + + it("prefers --pack-version over the name@version suffix", async () => { + stubExit(); + await buildProgram().parseAsync( + ["registry", "unpublish", "react-docs@1.2.0", "-r", "main", "-y", "--pack-version", "1.3.0"], + { from: "user" }, + ); + expect(unpublishPack).toHaveBeenCalledWith( + expect.objectContaining({ packName: "react-docs", version: "1.3.0" }), + ); + }); + + it("exits with an error when no version is given", async () => { + stubExit(); + const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + await expect( + buildProgram().parseAsync(["registry", "unpublish", "react-docs", "-r", "main", "-y"], { + from: "user", + }), + ).rejects.toThrow("process.exit(1)"); + expect(errSpy).toHaveBeenCalledWith(expect.stringContaining("--pack-version")); + expect(unpublishPack).not.toHaveBeenCalled(); + }); +}); From ab199cd36010551a17dc7596a3e3aa9a04c6f072 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:14:53 +0000 Subject: [PATCH 12/83] perf(db): key chunks_fts by chunk rowid (schema v18) The FTS delete and update triggers matched rows on the UNINDEXED chunk_id column, so deleting one chunk scanned the whole FTS table. Migration 18 recreates chunks_fts with rowid = chunks.rowid, makes the triggers delete by rowid, and repopulates the index once from chunks, which also removes duplicate rows left by earlier backfills. Column names are unchanged, so search queries are unchanged. Deleting a 40-chunk document at 20k chunks: 94 ms -> 0.15 ms. --- CLAUDE.md | 2 +- src/db/schema.ts | 50 ++++++++++-- tests/unit/schema-migrations.test.ts | 109 +++++++++++++++++++++++++++ tests/unit/schema.test.ts | 2 +- 4 files changed, 155 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b89685c0..76ceecea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -120,7 +120,7 @@ Always use the appropriate error subclass. MCP tool handlers must be wrapped wit ## Database - **Engine:** better-sqlite3 + sqlite-vec for vector search -- **Schema version:** 17 (migrations in `src/db/schema.ts`) +- **Schema version:** 18 (migrations in `src/db/schema.ts`) - **Adding migrations:** Increment `SCHEMA_VERSION`, add entry to `MIGRATIONS` object with the new version number as key - **Key tables:** documents, chunks, chunks_fts (FTS5), chunk_embeddings (vector), topics, ratings, schema_version diff --git a/src/db/schema.ts b/src/db/schema.ts index b117c4e8..47704117 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -2,7 +2,13 @@ import type Database from "better-sqlite3"; import { DatabaseError } from "../errors.js"; import { getLogger } from "../logger.js"; -const SCHEMA_VERSION = 17; +const SCHEMA_VERSION = 18; + +/** Copy every chunk into chunks_fts, keyed by the chunk's rowid. */ +const FTS_BACKFILL_SQL = ` + INSERT INTO chunks_fts(rowid, content, chunk_id, document_id) + SELECT rowid, content, id, document_id FROM chunks; +`; const MIGRATIONS: Record<number, string> = { 1: ` @@ -269,12 +275,44 @@ const MIGRATIONS: Record<number, string> = { INSERT INTO schema_version (version) VALUES (17); `, -}; + // Rebuild chunks_fts keyed by chunks.rowid. The old triggers matched rows on the + // UNINDEXED chunk_id column, so every chunk delete scanned the whole FTS table. + // Repopulating from chunks also drops duplicate rows left by earlier backfills. + 18: ` + DROP TRIGGER IF EXISTS chunks_ai; + DROP TRIGGER IF EXISTS chunks_ad; + DROP TRIGGER IF EXISTS chunks_au; + DROP TABLE IF EXISTS chunks_fts; + + CREATE VIRTUAL TABLE chunks_fts USING fts5( + content, + chunk_id UNINDEXED, + document_id UNINDEXED, + tokenize='porter unicode61' + ); -const FTS_BACKFILL_SQL = ` - INSERT INTO chunks_fts(content, chunk_id, document_id) - SELECT content, id, document_id FROM chunks; -`; + -- The DELETE before each INSERT clears a stale row that may hold the same rowid. + CREATE TRIGGER chunks_ai AFTER INSERT ON chunks BEGIN + DELETE FROM chunks_fts WHERE rowid = new.rowid; + INSERT INTO chunks_fts(rowid, content, chunk_id, document_id) + VALUES (new.rowid, new.content, new.id, new.document_id); + END; + + CREATE TRIGGER chunks_ad AFTER DELETE ON chunks BEGIN + DELETE FROM chunks_fts WHERE rowid = old.rowid; + END; + + CREATE TRIGGER chunks_au AFTER UPDATE ON chunks BEGIN + DELETE FROM chunks_fts WHERE rowid = old.rowid; + INSERT INTO chunks_fts(rowid, content, chunk_id, document_id) + VALUES (new.rowid, new.content, new.id, new.document_id); + END; + + ${FTS_BACKFILL_SQL} + + INSERT INTO schema_version (version) VALUES (18); + `, +}; /** * Populate chunks_fts from chunks, but only when the FTS index is empty (for example, diff --git a/tests/unit/schema-migrations.test.ts b/tests/unit/schema-migrations.test.ts index bbeffbe5..45645e73 100644 --- a/tests/unit/schema-migrations.test.ts +++ b/tests/unit/schema-migrations.test.ts @@ -1,8 +1,13 @@ import { describe, it, expect, beforeEach } from "vitest"; import DatabaseConstructor from "better-sqlite3"; import type Database from "better-sqlite3"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { runMigrations } from "../../src/db/schema.js"; +import { searchDocuments } from "../../src/core/search.js"; import { initLogger } from "../../src/logger.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; function newDb(): Database.Database { const db = new DatabaseConstructor(":memory:"); @@ -25,10 +30,43 @@ function seedChunks(db: Database.Database, docCount: number, chunksPerDoc: numbe } } +function seedChunksForDoc(db: Database.Database, docId: string, chunkCount: number): void { + db.prepare( + "INSERT INTO documents (id, source_type, title, content) VALUES (?, 'manual', ?, '')", + ).run(docId, docId); + const insertChunk = db.prepare( + "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", + ); + for (let c = 0; c < chunkCount; c++) { + insertChunk.run(`${docId}-chunk-${c}`, docId, `charlie ${docId} part${c}`, c); + } +} + function count(db: Database.Database, table: string): number { return (db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n; } +/** FTS rows whose rowid does not point at the chunk they describe. */ +function misalignedFtsRows(db: Database.Database): number { + return ( + db + .prepare( + `SELECT COUNT(*) AS n FROM chunks_fts f + LEFT JOIN chunks c ON c.rowid = f.rowid + WHERE c.id IS NULL OR c.id != f.chunk_id`, + ) + .get() as { n: number } + ).n; +} + +async function keywordHits(db: Database.Database, query: string): Promise<string[]> { + const { results } = await searchDocuments(db, new MockEmbeddingProvider(), { + query, + limit: 50, + }); + return results.map((r) => `${r.chunkId}:${r.score.toFixed(6)}`); +} + describe("schema migrations", () => { beforeEach(() => { initLogger("silent"); @@ -58,4 +96,75 @@ describe("schema migrations", () => { expect(count(db, "chunks_fts")).toBe(6); }); }); + + describe("migration 18: chunks_fts keyed by chunk rowid", () => { + it("keeps keyword search results unchanged", async () => { + const db = newDb(); + runMigrations(db, 17); + seedChunks(db, 4, 5); + const before = await keywordHits(db, "bravo doc2"); + expect(before.length).toBeGreaterThan(0); + + runMigrations(db); + + expect(await keywordHits(db, "bravo doc2")).toEqual(before); + expect(count(db, "chunks_fts")).toBe(count(db, "chunks")); + expect(misalignedFtsRows(db)).toBe(0); + }); + + it("removes duplicate FTS rows left by earlier backfills", async () => { + const db = newDb(); + runMigrations(db, 17); + seedChunks(db, 3, 4); + const clean = await keywordHits(db, "alpha"); + // Simulate the old backfill bug: every chunk indexed twice. + db.exec( + "INSERT INTO chunks_fts(content, chunk_id, document_id) SELECT content, id, document_id FROM chunks", + ); + expect(count(db, "chunks_fts")).toBe(24); + + runMigrations(db); + + expect(count(db, "chunks_fts")).toBe(12); + expect(await keywordHits(db, "alpha")).toEqual(clean); + }); + + it("keeps chunks_fts in sync on insert, update and delete", () => { + const db = newDb(); + runMigrations(db); + seedChunks(db, 3, 4); + expect(count(db, "chunks_fts")).toBe(12); + + db.prepare("UPDATE chunks SET content = 'zulu replaced' WHERE id = 'chunk-0-0'").run(); + const updated = db + .prepare("SELECT chunk_id FROM chunks_fts WHERE chunks_fts MATCH 'zulu'") + .all() as Array<{ chunk_id: string }>; + expect(updated).toEqual([{ chunk_id: "chunk-0-0" }]); + + db.prepare("DELETE FROM documents WHERE id = 'doc-1'").run(); + expect(count(db, "chunks_fts")).toBe(8); + expect(misalignedFtsRows(db)).toBe(0); + }); + + it("stays aligned with chunks after VACUUM", () => { + const dir = mkdtempSync(join(tmpdir(), "libscope-fts-")); + try { + const db = new DatabaseConstructor(join(dir, "test.db")); + db.pragma("foreign_keys = ON"); + runMigrations(db); + seedChunks(db, 6, 5); + db.prepare("DELETE FROM documents WHERE id IN ('doc-1', 'doc-3')").run(); + db.exec("VACUUM"); + expect(misalignedFtsRows(db)).toBe(0); + + db.prepare("DELETE FROM documents WHERE id = 'doc-4'").run(); + seedChunksForDoc(db, "doc-new", 3); + expect(count(db, "chunks_fts")).toBe(count(db, "chunks")); + expect(misalignedFtsRows(db)).toBe(0); + db.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + }); }); diff --git a/tests/unit/schema.test.ts b/tests/unit/schema.test.ts index e1f71c18..00288016 100644 --- a/tests/unit/schema.test.ts +++ b/tests/unit/schema.test.ts @@ -38,7 +38,7 @@ describe("database schema", () => { const version = db.prepare("SELECT MAX(version) as v FROM schema_version").get() as { v: number; }; - expect(version.v).toBe(17); + expect(version.v).toBe(18); }); it("should create expected indexes", () => { From 654411586cc7c7d03ddd04b3f3bb0e44bfbd74ca Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:15:47 +0000 Subject: [PATCH 13/83] perf(db): index documents.title for the dedup lookup indexDocument checks for an existing document with WHERE title = ? AND LENGTH(content) = ?, which scanned the whole documents table on every index call. Migration 18 adds idx_documents_title. --- src/db/schema.ts | 3 +++ tests/unit/schema-migrations.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/db/schema.ts b/src/db/schema.ts index 47704117..ca240cf2 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -310,6 +310,9 @@ const MIGRATIONS: Record<number, string> = { ${FTS_BACKFILL_SQL} + -- Serves the title + length dedup lookup that runs on every index call. + CREATE INDEX IF NOT EXISTS idx_documents_title ON documents(title); + INSERT INTO schema_version (version) VALUES (18); `, }; diff --git a/tests/unit/schema-migrations.test.ts b/tests/unit/schema-migrations.test.ts index 45645e73..60aa0ac6 100644 --- a/tests/unit/schema-migrations.test.ts +++ b/tests/unit/schema-migrations.test.ts @@ -167,4 +167,27 @@ describe("schema migrations", () => { } }); }); + + describe("migration 18: documents title index", () => { + it("indexes documents.title and uses it for the dedup lookup", () => { + const db = newDb(); + runMigrations(db, 17); + seedChunks(db, 2, 1); + + runMigrations(db); + + const index = db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_documents_title'", + ) + .get(); + expect(index).toBeDefined(); + const plan = db + .prepare( + "EXPLAIN QUERY PLAN SELECT id FROM documents WHERE title = ? AND LENGTH(content) = ?", + ) + .all("Document 1", 22) as Array<{ detail: string }>; + expect(plan.map((p) => p.detail).join(" ")).toContain("idx_documents_title"); + }); + }); }); From 6c3c4bfc01cc4d65d156945aed968b3b1a9c7325 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:16:07 +0000 Subject: [PATCH 14/83] fix(mcp): make task cancellation real and report final status accurately - Add signal?: AbortSignal to ReindexOptions, InstallOptions, and SpiderOptions; each checks throwIfAborted() between batches/pages. - MCP submit-document (single and spider), reindex-documents, and install-pack pass the task signal instead of throwing from onProgress. - installPack no longer hangs (with an unhandled rejection) when a batch callback throws inside its scheduler; it now rejects and rolls back the partially installed pack. - TaskRegistry.run records completed when work resolves (even after a late cancel), cancelled only when it rejects because of the abort, else failed. - Rename the reindex task type from reindex_library to reindex_documents. --- docs/reference/mcp-tools.md | 24 ++++++ src/core/packs.ts | 144 +++++++++++++++++++++------------ src/core/reindex.ts | 3 + src/core/spider.ts | 4 + src/mcp/server.ts | 53 ++++-------- src/mcp/tasks.ts | 48 ++++++++++- tests/unit/packs.test.ts | 88 ++++++++++++++++++++ tests/unit/reindex.test.ts | 33 ++++++++ tests/unit/spider.test.ts | 37 +++++++++ tests/unit/tasks.test.ts | 155 +++++++++++++++++++++++++++++++++--- 10 files changed, 490 insertions(+), 99 deletions(-) diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 92d6d15b..c3fd22b2 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -296,3 +296,27 @@ Delete a saved search by name or ID. | Parameter | Type | Required | Description | | ---------- | ------ | -------- | -------------------------------------------- | | `nameOrId` | string | ✅ | The name or ID of the saved search to delete | + +## get-task + +Get the status, progress, and result of a background task. Tools called with `async: true` return a task ID. + +| Parameter | Type | Required | Description | +| --------- | ------ | -------- | ---------------------------------------- | +| `taskId` | string | ✅ | Task ID returned by an async tool call | + +The task `type` is one of `index_document`, `reindex_documents`, `sync_connector`, or `install_pack`. + +## cancel-task + +Request cancellation of a pending or running background task. + +| Parameter | Type | Required | Description | +| --------- | ------ | -------- | ------------------ | +| `taskId` | string | ✅ | Task ID to cancel | + +`submit-document` (including spider mode), `reindex-documents`, and `install-pack` stop at the next page or batch. A cancelled `install-pack` removes the documents it already inserted. The final status is: + +- `cancelled` — the task stopped because of the cancellation. +- `completed` — the task finished before it could stop. The result is kept. +- `failed` — the task stopped because of a different error. diff --git a/src/core/packs.ts b/src/core/packs.ts index 1ca96063..aea8d5af 100644 --- a/src/core/packs.ts +++ b/src/core/packs.ts @@ -72,6 +72,11 @@ export interface InstallOptions { concurrency?: number | undefined; /** Called after each batch of documents is processed. */ onProgress?: ((current: number, total: number, docTitle: string) => void) | undefined; + /** + * Abort between batches. The install is then rolled back (pack row and inserted documents + * removed) and the returned promise rejects with the signal's reason. + */ + signal?: AbortSignal | undefined; } export interface CreatePackOptions { @@ -403,6 +408,67 @@ function insertBatchIntoDb( return batch.docInfos.length; } +/** + * Run `start(i)` for each batch index with at most `concurrency` in flight, calling + * `afterEach()` each time one settles. Stops scheduling and rejects when `start` rejects, + * `afterEach` throws, or `signal` is aborted (with the signal's reason). + */ +function runBatchesConcurrently( + batchCount: number, + concurrency: number, + start: (i: number) => Promise<void>, + afterEach: () => void, + signal: AbortSignal | undefined, +): Promise<void> { + return new Promise<void>((resolve, reject) => { + let activeCount = 0; + let scheduleIdx = 0; + let settled = false; + + const fail = (err: unknown): void => { + if (settled) return; + settled = true; + reject(err instanceof Error ? err : new Error(String(err))); + }; + + const onBatchSettled = (): void => { + activeCount--; + if (settled) return; + try { + signal?.throwIfAborted(); + afterEach(); + if (scheduleIdx < batchCount) { + scheduleNext(); + } else if (activeCount === 0) { + settled = true; + resolve(); + } + } catch (err) { + fail(err); + } + }; + + const onBatchFailed = (err: unknown): void => { + activeCount--; + fail(err); + }; + + function scheduleNext(): void { + while (activeCount < concurrency && scheduleIdx < batchCount) { + const i = scheduleIdx++; + activeCount++; + start(i).then(onBatchSettled, onBatchFailed); + } + } + + if (batchCount === 0) { + resolve(); + return; + } + scheduleNext(); + }); +} + /** Install a pack from a local JSON file path or registry name. */ export async function installPack( db: Database.Database, @@ -431,6 +497,7 @@ export async function installPack( const { batchSize, concurrency, resumeFrom } = validateInstallOptions(options, total); const onProgress = options?.onProgress; const docs = resumeFrom > 0 ? pack.documents.slice(resumeFrom) : pack.documents; + options?.signal?.throwIfAborted(); log.info( { pack: pack.name, docCount: total, batchSize, concurrency, resumeFrom }, @@ -509,58 +576,35 @@ export async function installPack( } } - // Semaphore-based concurrent embedding - await new Promise<void>((resolve) => { - if (batches.length === 0) { - resolve(); - return; - } - - let activeCount = 0; - let scheduleIdx = 0; - - function scheduleNext(): void { - while (activeCount < concurrency && scheduleIdx < batches.length) { - const i = scheduleIdx++; - const resolved = resolveBatch(batches[i]!.batchDocs); - activeCount++; - - let embedPromise: Promise<number[][]>; - if (resolved.allChunks.length > 0) { - try { - embedPromise = provider.embedBatch(resolved.allChunks); - } catch (err) { - embedPromise = Promise.reject(err instanceof Error ? err : new Error(String(err))); - } - } else { - embedPromise = Promise.resolve([] as number[][]); - } - - embedPromise - .then((embeddings) => { - embedResults[i] = { resolved, embeddings, success: true }; - }) - .catch((err) => { - log.warn( - { err, pack: pack.name, batchIndex: i }, - "Failed to embed batch, skipping these documents", - ); - embedResults[i] = { resolved, embeddings: [], success: false }; - }) - .finally(() => { - activeCount--; - flushInserts(); - if (scheduleIdx < batches.length) { - scheduleNext(); - } else if (activeCount === 0) { - resolve(); - } - }); - } + /** Chunk and embed batch `i`; embedding failures are recorded, not thrown. */ + async function embedBatchAt(i: number): Promise<void> { + const resolved = resolveBatch(batches[i]!.batchDocs); + try { + const embeddings = + resolved.allChunks.length > 0 ? await provider.embedBatch(resolved.allChunks) : []; + embedResults[i] = { resolved, embeddings, success: true }; + } catch (err) { + log.warn( + { err, pack: pack.name, batchIndex: i }, + "Failed to embed batch, skipping these documents", + ); + embedResults[i] = { resolved, embeddings: [], success: false }; } + } - scheduleNext(); - }); + try { + await runBatchesConcurrently( + batches.length, + concurrency, + embedBatchAt, + flushInserts, + options?.signal, + ); + } catch (err) { + log.warn({ err, pack: pack.name }, "Pack install stopped, rolling back"); + removePack(db, pack.name); + throw err; + } db.prepare("UPDATE packs SET doc_count = ? WHERE name = ?").run(installed, pack.name); diff --git a/src/core/reindex.ts b/src/core/reindex.ts index 08a9ee79..199227af 100644 --- a/src/core/reindex.ts +++ b/src/core/reindex.ts @@ -15,6 +15,8 @@ export interface ReindexOptions { batchSize?: number | undefined; /** Called after each batch completes. */ onProgress?: ((progress: ReindexProgress) => void) | undefined; + /** Abort between batches; the returned promise rejects with the signal's reason. */ + signal?: AbortSignal | undefined; } export interface ReindexProgress { @@ -76,6 +78,7 @@ export async function reindex( const failedChunkIds: string[] = []; for (let i = 0; i < total; i += batchSize) { + options.signal?.throwIfAborted(); const batch = chunks.slice(i, i + batchSize); const texts = batch.map((c) => c.content); const ids = batch.map((c) => c.id); diff --git a/src/core/spider.ts b/src/core/spider.ts index be23fbec..a7081773 100644 --- a/src/core/spider.ts +++ b/src/core/spider.ts @@ -45,6 +45,8 @@ export interface SpiderOptions { FetchOptions, "allowPrivateUrls" | "allowSelfSignedCerts" | "timeout" | "maxBodySize" >; + /** Abort the crawl between pages; the generator then throws the signal's reason. */ + signal?: AbortSignal; } export interface SpiderResult { @@ -486,6 +488,7 @@ export async function* spiderUrl( const deadline = Date.now() + HARD_TOTAL_TIMEOUT_MS; while (queue.length > 0 && stats.pagesFetched < config.maxPages) { + options.signal?.throwIfAborted(); if (checkDeadline(deadline, stats, log)) break; const { url, depth } = queue.shift()!; @@ -495,6 +498,7 @@ export async function* spiderUrl( if (depth > 0 && (await shouldSkipNonSeedUrl(url, config, robotsCache, stats, log))) continue; const raw = await fetchSpiderPage(url, config, stats, log); + options.signal?.throwIfAborted(); if (!raw) continue; const result = convertPage(raw, url, depth); diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 4d034e55..5e62b752 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -36,7 +36,7 @@ import { errorResponse, textResult, withErrorHandling, type ToolResult } from ". export { errorResponse, withErrorHandling, type ToolResult } from "./errors.js"; import { formatDocumentLinks, formatLinkCreated, formatSearchResults } from "./format.js"; import { taskRegistry } from "./tasks.js"; -import type { Task, TaskType } from "./tasks.js"; +import type { TaskType, TaskWork } from "./tasks.js"; /** Build SpiderOptions from submit-document params. */ function buildSpiderOptions( @@ -75,6 +75,7 @@ async function handleSpiderSubmit( excludePatterns?: string[] | undefined; }, fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, + signal?: AbortSignal, ): Promise<string> { const { url, library, version, topic } = params; if (!url) { @@ -82,6 +83,7 @@ async function handleSpiderSubmit( } const spiderOptions = buildSpiderOptions(params, fetchOptions); + if (signal) spiderOptions.signal = signal; const indexed: Array<{ id: string; title: string }> = []; const errors: Array<{ url: string; error: string }> = []; const sourceType = params.sourceType ?? (library ? "library" : "manual"); @@ -135,6 +137,7 @@ async function handleSingleDocSubmit( sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; }, fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, + signal?: AbortSignal, ): Promise<string> { let { title, content } = params; const { url, library, version, topic } = params; @@ -154,6 +157,7 @@ async function handleSingleDocSubmit( const sourceType = params.sourceType ?? (library ? "library" : "manual"); + signal?.throwIfAborted(); const result = await indexDocument(db, provider, { title, content, @@ -174,34 +178,9 @@ async function handleSingleDocSubmit( ); } -/** Background work for a tool: receives the task's abort signal and a progress reporter. */ -type TaskWork = ( - signal: AbortSignal, - onProgress: (current: number, total: number) => void, -) => Promise<string>; - /** Fire-and-forget helper: creates a task, runs `work` in background, returns task ID response. */ function startAsyncTask(type: TaskType, work: TaskWork): ToolResult { - const { task, signal } = taskRegistry.create(type); - taskRegistry.update(task.id, { status: "running", startedAt: new Date() }); - const onProgress = (current: number, total: number): void => { - taskRegistry.update(task.id, { progress: { current, total } }); - }; - const finish = (updates: Partial<Task>): void => { - taskRegistry.update( - task.id, - signal.aborted ? { status: "cancelled", completedAt: new Date() } : updates, - ); - }; - void work(signal, onProgress).then( - (result) => finish({ status: "completed", completedAt: new Date(), result }), - (err: unknown) => - finish({ - status: "failed", - completedAt: new Date(), - error: err instanceof Error ? err.message : String(err), - }), - ); + const { task } = taskRegistry.run(type, work); return textResult(`Task queued. ID: ${task.id}\nUse get-task to check status.`); } @@ -551,10 +530,10 @@ async function main(): Promise<void> { allowSelfSignedCerts: config.indexing.allowSelfSignedCerts, }; - return runMaybeAsync(params.async, "index_document", () => + return runMaybeAsync(params.async, "index_document", (signal) => params.spider - ? handleSpiderSubmit(db, provider, params, fetchOptions) - : handleSingleDocSubmit(db, provider, params, fetchOptions), + ? handleSpiderSubmit(db, provider, params, fetchOptions, signal) + : handleSingleDocSubmit(db, provider, params, fetchOptions, signal), ); }), ); @@ -779,16 +758,14 @@ async function main(): Promise<void> { withErrorHandling(async (params) => { const { reindex } = await import("../core/reindex.js"); - return runMaybeAsync(params.async, "reindex_library", async (signal, onProgress) => { + return runMaybeAsync(params.async, "reindex_documents", async (signal, onProgress) => { const result = await reindex(db, provider, { documentIds: params.documentIds, since: params.since, before: params.before, batchSize: params.batchSize, - onProgress: (p) => { - if (signal.aborted) throw new Error("Task cancelled"); - onProgress(p.completed, p.total); - }, + signal, + onProgress: (p) => onProgress(p.completed, p.total), }); return ( `Reindex complete.\n` + @@ -874,10 +851,8 @@ async function main(): Promise<void> { return runMaybeAsync(params.async, "install_pack", async (signal, onProgress) => { const result = await installPack(db, provider, params.nameOrPath, { registryUrl: params.registryUrl, - onProgress: (current, total) => { - if (signal.aborted) throw new Error("Task cancelled"); - onProgress(current, total); - }, + signal, + onProgress: (current, total) => onProgress(current, total), }); return result.alreadyInstalled ? `Pack "${result.packName}" is already installed.` diff --git a/src/mcp/tasks.ts b/src/mcp/tasks.ts index 34795c78..4c4e365c 100644 --- a/src/mcp/tasks.ts +++ b/src/mcp/tasks.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; export type TaskStatus = "pending" | "running" | "completed" | "failed" | "cancelled"; -export type TaskType = "index_document" | "reindex_library" | "sync_connector" | "install_pack"; +export type TaskType = "index_document" | "reindex_documents" | "sync_connector" | "install_pack"; export interface TaskProgress { current: number; @@ -21,6 +21,18 @@ export interface Task { completedAt?: Date | undefined; } +/** Background work for a task: receives the task's abort signal and a progress reporter. */ +export type TaskWork = ( + signal: AbortSignal, + onProgress: (current: number, total: number) => void, +) => Promise<string>; + +/** True when `err` is the rejection produced by aborting `signal`. */ +export function isAbortRejection(err: unknown, signal: AbortSignal): boolean { + if (!signal.aborted) return false; + return err === signal.reason || (err instanceof Error && err.name === "AbortError"); +} + /** TTL for completed/failed/cancelled tasks before they are pruned (1 hour). */ const TASK_TTL_MS = 60 * 60 * 1000; @@ -43,6 +55,40 @@ export class TaskRegistry { return { task, signal: controller.signal }; } + /** + * Create a task and run `work` in the background. + * Final status: "completed" if `work` resolves (even when cancellation was requested too late + * to stop it), "cancelled" if it rejects because the task signal was aborted, else "failed". + * `done` settles after the final status is recorded; it never rejects. + */ + run(type: TaskType, work: TaskWork): { task: Task; done: Promise<void> } { + const { task, signal } = this.create(type); + this.update(task.id, { status: "running", startedAt: new Date() }); + const onProgress = (current: number, total: number): void => { + this.update(task.id, { progress: { current, total } }); + }; + const done = new Promise<string>((resolve) => { + resolve(work(signal, onProgress)); + }).then( + (result) => { + this.update(task.id, { status: "completed", completedAt: new Date(), result }); + }, + (err: unknown) => { + this.update( + task.id, + isAbortRejection(err, signal) + ? { status: "cancelled", completedAt: new Date() } + : { + status: "failed", + completedAt: new Date(), + error: err instanceof Error ? err.message : String(err), + }, + ); + }, + ); + return { task, done }; + } + /** Retrieve a task by ID. Returns undefined if not found or expired. */ get(id: string): Task | undefined { this.prune(); diff --git a/tests/unit/packs.test.ts b/tests/unit/packs.test.ts index 9ca8dd4a..f07fd9d4 100644 --- a/tests/unit/packs.test.ts +++ b/tests/unit/packs.test.ts @@ -125,6 +125,94 @@ describe("knowledge packs", () => { }); }); + describe("installPack — cancellation", () => { + function writeManyDocPack(count: number): string { + const documents = Array.from({ length: count }, (_, i) => ({ + title: `Doc ${i}`, + content: `# Doc ${i}\n\nBody of document ${i}.`, + source: "", + })); + const packPath = join(tempDir, "many.json"); + writeFileSync(packPath, JSON.stringify(makeSamplePack({ documents })), "utf-8"); + return packPath; + } + + function countRows(table: "documents" | "packs" | "chunks"): number { + return (db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n; + } + + it.each([1, 4])( + "rejects with the abort reason and rolls back when aborted mid-install (concurrency=%i)", + async (concurrency) => { + const packPath = writeManyDocPack(6); + const controller = new AbortController(); + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + try { + const promise = installPack(db, provider, packPath, { + batchSize: 1, + concurrency, + signal: controller.signal, + onProgress: () => controller.abort(), + }); + await expect(promise).rejects.toHaveProperty("name", "AbortError"); + await new Promise((r) => setTimeout(r, 20)); + } finally { + process.off("unhandledRejection", onUnhandled); + } + expect(unhandled).toEqual([]); + expect(countRows("packs")).toBe(0); + expect(countRows("documents")).toBe(0); + expect(countRows("chunks")).toBe(0); + }, + ); + + it("rejects instead of hanging when onProgress throws", async () => { + const packPath = writeManyDocPack(4); + const promise = installPack(db, provider, packPath, { + batchSize: 1, + onProgress: () => { + throw new Error("progress sink failed"); + }, + }); + await expect(promise).rejects.toThrow("progress sink failed"); + expect(countRows("packs")).toBe(0); + expect(countRows("documents")).toBe(0); + }); + + it("does not create the pack when the signal is already aborted", async () => { + const packPath = writeManyDocPack(2); + const controller = new AbortController(); + controller.abort(); + const embedSpy = vi.spyOn(provider, "embedBatch"); + await expect(installPack(db, provider, packPath, { signal: controller.signal })).rejects.toBe( + controller.signal.reason, + ); + expect(embedSpy).not.toHaveBeenCalled(); + expect(countRows("packs")).toBe(0); + }); + + it("can be installed again after a cancelled install", async () => { + const packPath = writeManyDocPack(3); + const controller = new AbortController(); + await expect( + installPack(db, provider, packPath, { + batchSize: 1, + concurrency: 1, + signal: controller.signal, + onProgress: () => controller.abort(), + }), + ).rejects.toHaveProperty("name", "AbortError"); + + const result = await installPack(db, provider, packPath); + expect(result.alreadyInstalled).toBe(false); + expect(result.documentsInstalled).toBe(3); + }); + }); + describe("removePack", () => { it("should remove a pack and its associated documents", async () => { const pack = makeSamplePack(); diff --git a/tests/unit/reindex.test.ts b/tests/unit/reindex.test.ts index f6b5646e..8fb33f04 100644 --- a/tests/unit/reindex.test.ts +++ b/tests/unit/reindex.test.ts @@ -228,3 +228,36 @@ describe("reindex", () => { expect(result.completed).toBe(0); }); }); + +describe("reindex cancellation", () => { + const chunks = [ + { id: "c1", content: "chunk 1" }, + { id: "c2", content: "chunk 2" }, + { id: "c3", content: "chunk 3" }, + ]; + + it("rejects with the abort reason and stops between batches", async () => { + const { db } = createMockDb(chunks); + const { provider, embedBatchFn } = createMockProvider(); + const controller = new AbortController(); + + const promise = reindex(db, provider, { + batchSize: 1, + signal: controller.signal, + onProgress: () => controller.abort(), + }); + + await expect(promise).rejects.toHaveProperty("name", "AbortError"); + expect(embedBatchFn).toHaveBeenCalledTimes(1); + }); + + it("does not embed anything when the signal is already aborted", async () => { + const { db } = createMockDb(chunks); + const { provider, embedBatchFn } = createMockProvider(); + const controller = new AbortController(); + controller.abort(); + + await expect(reindex(db, provider, { signal: controller.signal })).rejects.toThrow(); + expect(embedBatchFn).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/spider.test.ts b/tests/unit/spider.test.ts index 24d93488..8cf59070 100644 --- a/tests/unit/spider.test.ts +++ b/tests/unit/spider.test.ts @@ -494,4 +494,41 @@ describe("spiderUrl", () => { const { stats } = await collectPages(gen); expect((stats as { abortReason?: string }).abortReason).toBe("maxPages"); }); + + it("throws the abort reason between pages when the signal is aborted", async () => { + let counter = 0; + mockFetchRaw.mockImplementation((url: string) => { + if (url.endsWith("/robots.txt")) return Promise.reject(new Error("404")); + counter++; + const links = [`https://example.com/page${counter + 100}`]; + return Promise.resolve(pageResponse(htmlPage(`Page ${counter}`, links), url)); + }); + const controller = new AbortController(); + + const gen = spiderUrl("https://example.com/", { + maxPages: 10, + maxDepth: 5, + requestDelay: 0, + signal: controller.signal, + }); + const first = await gen.next(); + expect(first.done).toBe(false); + controller.abort(); + + await expect(gen.next()).rejects.toBe(controller.signal.reason); + // Only the seed page was fetched (robots.txt calls excluded). + expect(counter).toBe(1); + }); + + it("throws the abort reason when a page fetch finishes after abort", async () => { + const controller = new AbortController(); + mockFetchRaw.mockImplementation((url: string) => { + if (url.endsWith("/robots.txt")) return Promise.reject(new Error("404")); + controller.abort(); + return Promise.resolve(pageResponse(htmlPage("Seed"), url)); + }); + + const gen = spiderUrl("https://example.com/", { requestDelay: 0, signal: controller.signal }); + await expect(gen.next()).rejects.toHaveProperty("name", "AbortError"); + }); }); diff --git a/tests/unit/tasks.test.ts b/tests/unit/tasks.test.ts index 5aa32118..c126db56 100644 --- a/tests/unit/tasks.test.ts +++ b/tests/unit/tasks.test.ts @@ -1,6 +1,11 @@ -import { describe, it, expect, beforeEach } from "vitest"; -import { TaskRegistry } from "../../src/mcp/tasks.js"; +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { TaskRegistry, isAbortRejection } from "../../src/mcp/tasks.js"; import type { TaskType } from "../../src/mcp/tasks.js"; +import { reindex } from "../../src/core/reindex.js"; +import { initLogger } from "../../src/logger.js"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { insertChunk, insertDoc } from "../fixtures/helpers.js"; function makeRegistry(): TaskRegistry { return new TaskRegistry(); @@ -23,7 +28,7 @@ describe("TaskRegistry", () => { }); it("returns an AbortSignal that is not yet aborted", () => { - const { signal } = registry.create("reindex_library"); + const { signal } = registry.create("reindex_documents"); expect(signal.aborted).toBe(false); }); @@ -36,7 +41,7 @@ describe("TaskRegistry", () => { it("supports all task types", () => { const types: TaskType[] = [ "index_document", - "reindex_library", + "reindex_documents", "sync_connector", "install_pack", ]; @@ -60,7 +65,7 @@ describe("TaskRegistry", () => { }); it("prunes tasks whose completedAt is older than 1 hour", () => { - const { task } = registry.create("reindex_library"); + const { task } = registry.create("reindex_documents"); registry.update(task.id, { status: "completed", completedAt: new Date(Date.now() - 61 * 60 * 1000), // 61 minutes ago @@ -88,7 +93,7 @@ describe("TaskRegistry", () => { }); it("updates progress fields", () => { - const { task } = registry.create("reindex_library"); + const { task } = registry.create("reindex_documents"); registry.update(task.id, { progress: { current: 10, total: 100 } }); const updated = registry.get(task.id); expect(updated?.progress?.current).toBe(10); @@ -127,7 +132,7 @@ describe("TaskRegistry", () => { }); it("returns already_terminal for a failed task", () => { - const { task } = registry.create("reindex_library"); + const { task } = registry.create("reindex_documents"); registry.update(task.id, { status: "failed", completedAt: new Date() }); expect(registry.cancel(task.id)).toBe("already_terminal"); }); @@ -139,7 +144,7 @@ describe("TaskRegistry", () => { }); it("aborts the signal when cancelling a running task", () => { - const { task, signal } = registry.create("reindex_library"); + const { task, signal } = registry.create("reindex_documents"); registry.update(task.id, { status: "running", startedAt: new Date() }); const outcome = registry.cancel(task.id); expect(outcome).toBe("cancelled"); @@ -149,6 +154,106 @@ describe("TaskRegistry", () => { }); }); + describe("run", () => { + it("marks the task completed with the work's result", async () => { + const { task, done } = registry.run("index_document", (_signal, onProgress) => { + onProgress(1, 2); + return Promise.resolve("all done"); + }); + expect(registry.get(task.id)?.status).toBe("running"); + await done; + const finished = registry.get(task.id); + expect(finished?.status).toBe("completed"); + expect(finished?.result).toBe("all done"); + expect(finished?.progress).toEqual({ current: 1, total: 2 }); + expect(finished?.completedAt).toBeInstanceOf(Date); + }); + + it("marks the task failed when the work rejects without cancellation", async () => { + const { task, done } = registry.run("install_pack", () => Promise.reject(new Error("boom"))); + await done; + expect(registry.get(task.id)?.status).toBe("failed"); + expect(registry.get(task.id)?.error).toBe("boom"); + }); + + it("marks the task failed when the work throws synchronously", async () => { + const { task, done } = registry.run("install_pack", () => { + throw new Error("sync boom"); + }); + await done; + expect(registry.get(task.id)?.status).toBe("failed"); + expect(registry.get(task.id)?.error).toBe("sync boom"); + }); + + it("marks the task completed when the work finishes even though cancel was requested", async () => { + let release!: () => void; + const gate = new Promise<void>((r) => { + release = r; + }); + const { task, done } = registry.run("index_document", async () => { + await gate; + return "finished anyway"; + }); + expect(registry.cancel(task.id)).toBe("cancelled"); + release(); + await done; + expect(registry.get(task.id)?.status).toBe("completed"); + expect(registry.get(task.id)?.result).toBe("finished anyway"); + }); + + it("marks the task cancelled when the work rejects because of the abort", async () => { + let release!: () => void; + const gate = new Promise<void>((r) => { + release = r; + }); + const { task, done } = registry.run("reindex_documents", async (signal) => { + await gate; + signal.throwIfAborted(); + return "unreachable"; + }); + registry.cancel(task.id); + release(); + await done; + expect(registry.get(task.id)?.status).toBe("cancelled"); + expect(registry.get(task.id)?.error).toBeUndefined(); + }); + + it("marks the task failed when an unrelated error happens after cancel", async () => { + let release!: () => void; + const gate = new Promise<void>((r) => { + release = r; + }); + const { task, done } = registry.run("install_pack", async () => { + await gate; + throw new Error("disk full"); + }); + registry.cancel(task.id); + release(); + await done; + expect(registry.get(task.id)?.status).toBe("failed"); + expect(registry.get(task.id)?.error).toBe("disk full"); + }); + }); + + describe("isAbortRejection", () => { + it("is false when the signal is not aborted", () => { + const controller = new AbortController(); + const err = new Error("x"); + err.name = "AbortError"; + expect(isAbortRejection(err, controller.signal)).toBe(false); + }); + + it("matches the signal's reason and AbortError-named errors once aborted", () => { + const controller = new AbortController(); + controller.abort(); + expect(isAbortRejection(controller.signal.reason, controller.signal)).toBe(true); + const named = new Error("stopped"); + named.name = "AbortError"; + expect(isAbortRejection(named, controller.signal)).toBe(true); + expect(isAbortRejection(new Error("other"), controller.signal)).toBe(false); + }); + }); + describe("TTL pruning", () => { it("does not prune tasks without a completedAt", () => { const { task } = registry.create("index_document"); @@ -172,7 +277,7 @@ describe("TaskRegistry", () => { describe("async task lifecycle simulation", () => { it("transitions through pending -> running -> completed", () => { - const { task, signal } = registry.create("reindex_library"); + const { task, signal } = registry.create("reindex_documents"); expect(task.status).toBe("pending"); registry.update(task.id, { status: "running", startedAt: new Date() }); @@ -240,3 +345,35 @@ describe("taskRegistry singleton", () => { expect(taskRegistry).toBeInstanceOf(TaskRegistry); }); }); + +describe("TaskRegistry.run with core operations", () => { + it("reports cancelled when cancel-task aborts a running reindex", async () => { + initLogger("silent"); + const db = createTestDbWithVec(); + const provider = new MockEmbeddingProvider(); + insertDoc(db, "d1", "Doc"); + for (let i = 0; i < 3; i++) insertChunk(db, `c${i}`, "d1", `chunk ${i}`, i); + const embedSpy = vi.spyOn(provider, "embedBatch"); + + const registry = new TaskRegistry(); + let taskId = ""; + const { task, done } = registry.run("reindex_documents", async (signal, onProgress) => { + await reindex(db, provider, { + batchSize: 1, + signal, + onProgress: (p) => { + onProgress(p.completed, p.total); + registry.cancel(taskId); + }, + }); + return "Reindex complete."; + }); + taskId = task.id; + await done; + + expect(registry.get(task.id)?.status).toBe("cancelled"); + expect(registry.get(task.id)?.progress).toEqual({ current: 1, total: 3 }); + expect(embedSpy).toHaveBeenCalledTimes(1); + db.close(); + }); +}); From 5da0482095104303a9c7cd2783331c5755eac76e Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:16:30 +0000 Subject: [PATCH 15/83] fix(config): one API-key rule for embedding and LLM providers Keys now resolve in loadConfig as LIBSCOPE_<P>_API_KEY > <P>_API_KEY > config file, for both embedding and LLM (OPENAI_API_KEY was accepted by validation but never read by providers). LIBSCOPE_ANTHROPIC_API_KEY no longer requires LIBSCOPE_LLM_PROVIDER, LIBSCOPE_OLLAMA_MODEL is read, validation checks the resolved config (incl. anthropic), the config-file key warning names the file and the real env vars, and config show masks keys. Docs: env var tables, .env.example (nothing loads .env), REST API auth header. --- .env.example | 17 +++-- README.md | 14 +++- docs/guide/configuration.md | 17 +++-- docs/guide/troubleshooting.md | 16 ++--- docs/reference/configuration.md | 55 ++++++++++----- src/cli/index.ts | 4 +- src/config.ts | 113 +++++++++++++++++++++---------- tests/unit/config-layers.test.ts | 104 +++++++++++++++++++++++++++- tests/unit/config.test.ts | 15 +++- 9 files changed, 274 insertions(+), 81 deletions(-) diff --git a/.env.example b/.env.example index 7ac109c9..17615e42 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,21 @@ -# LibScope Environment Variables -# Copy this file to .env and fill in your values. +# LibScope environment variables +# LibScope does not load .env files. Export these in your shell, or put them in +# the "env" block of your MCP client config. This file is a reference only. # Embedding provider: "local" | "ollama" | "openai" (default: "local") LIBSCOPE_EMBEDDING_PROVIDER=local -# Ollama settings (required when provider is "ollama") +# Ollama settings (used when the provider is "ollama") LIBSCOPE_OLLAMA_URL=http://localhost:11434 LIBSCOPE_OLLAMA_MODEL=nomic-embed-text -# OpenAI settings (required when provider is "openai") +# OpenAI key (used for embeddings and for the LLM). +# Precedence: LIBSCOPE_OPENAI_API_KEY > OPENAI_API_KEY > config file. LIBSCOPE_OPENAI_API_KEY= + +# LLM for `ask` / RAG: "openai" | "ollama" | "anthropic" | "passthrough" +# LIBSCOPE_LLM_PROVIDER= +# LIBSCOPE_LLM_MODEL= + +# Anthropic key. Precedence: LIBSCOPE_ANTHROPIC_API_KEY > ANTHROPIC_API_KEY > config file. +# LIBSCOPE_ANTHROPIC_API_KEY= diff --git a/README.md b/README.md index 92d7c76b..2488e525 100644 --- a/README.md +++ b/README.md @@ -316,13 +316,20 @@ export LIBSCOPE_LLM_MODEL=gpt-4o-mini # optional | Variable | Description | Default | | ---------------------------------- | ---------------------------------------- | ------------------------ | | `LIBSCOPE_EMBEDDING_PROVIDER` | `local`, `ollama`, or `openai` | `local` | -| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key | — | +| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key (embeddings and LLM) | — | +| `OPENAI_API_KEY` | Used when `LIBSCOPE_OPENAI_API_KEY` is not set | — | | `LIBSCOPE_OLLAMA_URL` | Ollama server URL | `http://localhost:11434` | +| `LIBSCOPE_OLLAMA_MODEL` | Ollama embedding model | `nomic-embed-text` | | `LIBSCOPE_LLM_PROVIDER` | LLM for RAG (`openai` / `ollama` / `anthropic`) | — | | `LIBSCOPE_LLM_MODEL` | LLM model override | — | | `LIBSCOPE_ANTHROPIC_API_KEY` | Anthropic API key (for Claude models) | — | +| `ANTHROPIC_API_KEY` | Used when `LIBSCOPE_ANTHROPIC_API_KEY` is not set | — | | `LIBSCOPE_ALLOW_PRIVATE_URLS` | Allow fetching from private/internal IPs | `false` | | `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | Accept self-signed TLS certificates | `false` | +| `LIBSCOPE_WORKSPACE` | Active workspace for this shell | `default` | +| `LIBSCOPE_API_KEY` | REST API key (`Authorization: Bearer <key>`) | — | +| `LIBSCOPE_SECRET_KEY` | Encrypts stored webhook secrets | — | +| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (stderr) in the CLI | — | | `ONENOTE_CLIENT_ID` | Microsoft app registration client ID | — | | `ONENOTE_TENANT_ID` | Microsoft tenant ID | `common` | | `NOTION_TOKEN` | Notion integration token | — | @@ -330,6 +337,8 @@ export LIBSCOPE_LLM_MODEL=gpt-4o-mini # optional | `CONFLUENCE_EMAIL` | Confluence user email | — | | `CONFLUENCE_TOKEN` | Confluence API token | — | +API keys use one rule for embeddings and the LLM: `LIBSCOPE_<PROVIDER>_API_KEY`, then `<PROVIDER>_API_KEY`, then the key in a config file. LibScope does not load `.env` files. + </details> <details> @@ -345,8 +354,7 @@ export LIBSCOPE_LLM_MODEL=gpt-4o-mini # optional }, "llm": { "provider": "openai", - "model": "gpt-4o-mini", - "anthropicApiKey": "sk-ant-..." + "model": "gpt-4o-mini" }, "database": { "path": "~/.libscope/libscope.db" diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 7a47ed87..d308941f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -78,14 +78,14 @@ export LIBSCOPE_LLM_MODEL=gpt-4o-mini Supported providers: `openai`, `ollama`, `anthropic`, `passthrough`. -The `anthropic` provider uses Anthropic's Claude models. Set the API key via config or environment variable: +The `anthropic` provider uses Anthropic's Claude models. Set the API key with an environment variable (`LIBSCOPE_ANTHROPIC_API_KEY` or `ANTHROPIC_API_KEY`): ```bash export LIBSCOPE_LLM_PROVIDER=anthropic export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-... ``` -Or in your config file, set `llm.provider` to `"anthropic"` and `llm.anthropicApiKey` to your key. You can optionally set `llm.model` to choose a specific Claude model. +The key is read even when `LIBSCOPE_LLM_PROVIDER` is not set, so you can set `llm.provider` to `"anthropic"` in a config file and keep the key in the environment. A `llm.anthropicApiKey` value in a config file also works, but environment variables take precedence. You can optionally set `llm.model` to choose a specific Claude model. The `passthrough` provider is for advanced integrations where you supply your own LLM responses externally. When set, the `ask` command emits an event stream that your application handles rather than calling an LLM directly. @@ -94,13 +94,20 @@ The `passthrough` provider is for advanced integrations where you supply your ow | Variable | Description | Default | | ---------------------------------- | -------------------------------------------------- | ------------------------ | | `LIBSCOPE_EMBEDDING_PROVIDER` | Embedding provider (`local` / `ollama` / `openai`) | `local` | -| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key | — | +| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key (embeddings and LLM) | — | +| `OPENAI_API_KEY` | Used when `LIBSCOPE_OPENAI_API_KEY` is not set | — | | `LIBSCOPE_OLLAMA_URL` | Ollama server URL | `http://localhost:11434` | +| `LIBSCOPE_OLLAMA_MODEL` | Ollama embedding model | `nomic-embed-text` | | `LIBSCOPE_LLM_PROVIDER` | LLM provider for RAG (`openai` / `ollama` / `anthropic`) | — | | `LIBSCOPE_LLM_MODEL` | LLM model override | — | | `LIBSCOPE_ANTHROPIC_API_KEY` | Anthropic API key (for Claude models) | — | +| `ANTHROPIC_API_KEY` | Used when `LIBSCOPE_ANTHROPIC_API_KEY` is not set | — | | `LIBSCOPE_ALLOW_PRIVATE_URLS` | Allow fetching from private/internal IPs | `false` | | `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | Accept self-signed TLS certificates | `false` | +| `LIBSCOPE_WORKSPACE` | Active workspace for this shell | `default` | +| `LIBSCOPE_API_KEY` | REST API key (`Authorization: Bearer <key>`) | — | +| `LIBSCOPE_SECRET_KEY` | Encrypts stored webhook secrets | — | +| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (stderr) in the CLI | — | | `ONENOTE_CLIENT_ID` | Microsoft app registration client ID | — | | `ONENOTE_TENANT_ID` | Microsoft tenant ID | `common` | | `NOTION_TOKEN` | Notion integration token | — | @@ -108,7 +115,9 @@ The `passthrough` provider is for advanced integrations where you supply your ow | `CONFLUENCE_EMAIL` | Confluence user email | — | | `CONFLUENCE_TOKEN` | Confluence API token | — | -Environment variables always take precedence over config files. +Environment variables always take precedence over config files. LibScope does not load `.env` files; export the variables in your shell or MCP client config. + +API keys use one rule for embeddings and the LLM: `LIBSCOPE_<PROVIDER>_API_KEY`, then `<PROVIDER>_API_KEY` (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`), then the key in a config file. `libscope config show` masks keys. ## Workspaces diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index d382bd25..45cf4720 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -67,7 +67,7 @@ libscope pack install ... # re-install packs ### `401 Unauthorized` from API -The REST API requires an `X-API-Key` header. Find your key: `libscope config show` +When the server was started with `LIBSCOPE_API_KEY` set, every request must send `Authorization: Bearer <key>` with that value. `libscope config show` does not show this key. ### MCP tools not appearing in Claude / Cursor @@ -138,16 +138,10 @@ OneNote uses device code auth which caches a refresh token. If the cache is miss ### `401 Unauthorized` from API -The REST API requires an `X-API-Key` header for write operations. Retrieve your key: +Authentication is on only when the server process has `LIBSCOPE_API_KEY` set. Then every request must send that value as a bearer token: ```bash -libscope config show -``` - -Then include it in requests: - -```bash -curl -H "X-API-Key: <your-key>" http://localhost:3378/api/v1/documents +curl -H "Authorization: Bearer $LIBSCOPE_API_KEY" http://localhost:3378/api/v1/documents ``` ### `429 Too Many Requests` @@ -221,14 +215,14 @@ The default `import` command is sequential. `import-batch` processes files in pa The `ask` command requires an LLM provider. Configure one: ```bash -# OpenAI +# OpenAI (LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY) libscope config set llm.provider openai export LIBSCOPE_OPENAI_API_KEY=sk-... # Ollama (must be running locally) libscope config set llm.provider ollama -# Anthropic +# Anthropic (LIBSCOPE_ANTHROPIC_API_KEY or ANTHROPIC_API_KEY) libscope config set llm.provider anthropic export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-... ``` diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 30e06aa7..077ce716 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -21,6 +21,7 @@ Complete reference for all configuration options. | `embedding.ollamaUrl` | string | `"http://localhost:11434"` | Ollama server URL | | `embedding.ollamaModel` | string | `"nomic-embed-text"` | Ollama embedding model | | `embedding.openaiModel` | string | `"text-embedding-3-small"` | OpenAI embedding model | +| `embedding.openaiApiKey` | string | — | OpenAI API key (env vars take precedence, see [API keys](#api-keys)) | ### LLM (for RAG) @@ -29,7 +30,8 @@ Complete reference for all configuration options. | `llm.provider` | string | — | `openai`, `ollama`, `anthropic`, or `passthrough` | | `llm.model` | string | — | Model name override | | `llm.ollamaUrl` | string | — | Ollama server URL (overrides embedding URL) | -| `llm.anthropicApiKey` | string | — | Anthropic API key (or use `LIBSCOPE_ANTHROPIC_API_KEY` env var) | +| `llm.openaiApiKey` | string | — | OpenAI API key for the LLM (env vars take precedence, see [API keys](#api-keys)) | +| `llm.anthropicApiKey` | string | — | Anthropic API key (env vars take precedence, see [API keys](#api-keys)) | ### Database @@ -52,22 +54,41 @@ Complete reference for all configuration options. ## Environment Variables -| Variable | Maps to | Default | -| ---------------------------------- | ------------------------------- | ------------------------ | -| `LIBSCOPE_EMBEDDING_PROVIDER` | `embedding.provider` | `local` | -| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key | — | -| `LIBSCOPE_OLLAMA_URL` | `embedding.ollamaUrl` | `http://localhost:11434` | -| `LIBSCOPE_LLM_PROVIDER` | `llm.provider` | — | -| `LIBSCOPE_LLM_MODEL` | `llm.model` | — | -| `LIBSCOPE_ANTHROPIC_API_KEY` | `llm.anthropicApiKey` | — | -| `LIBSCOPE_ALLOW_PRIVATE_URLS` | `indexing.allowPrivateUrls` | `false` | -| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | `indexing.allowSelfSignedCerts` | `false` | -| `ONENOTE_CLIENT_ID` | OneNote app client ID | — | -| `ONENOTE_TENANT_ID` | OneNote tenant ID | `common` | -| `NOTION_TOKEN` | Notion integration token | — | -| `CONFLUENCE_URL` | Confluence base URL | — | -| `CONFLUENCE_EMAIL` | Confluence user email | — | -| `CONFLUENCE_TOKEN` | Confluence API token | — | +| Variable | Maps to | Default | +| ---------------------------------- | ------------------------------------------ | ------------------------ | +| `LIBSCOPE_EMBEDDING_PROVIDER` | `embedding.provider` | `local` | +| `LIBSCOPE_OPENAI_API_KEY` | `embedding.openaiApiKey`, `llm.openaiApiKey` | — | +| `OPENAI_API_KEY` | Same as above, used when `LIBSCOPE_OPENAI_API_KEY` is not set | — | +| `LIBSCOPE_OLLAMA_URL` | `embedding.ollamaUrl` | `http://localhost:11434` | +| `LIBSCOPE_OLLAMA_MODEL` | `embedding.ollamaModel` | `nomic-embed-text` | +| `LIBSCOPE_LLM_PROVIDER` | `llm.provider` | — | +| `LIBSCOPE_LLM_MODEL` | `llm.model` | — | +| `LIBSCOPE_ANTHROPIC_API_KEY` | `llm.anthropicApiKey` | — | +| `ANTHROPIC_API_KEY` | Same as above, used when `LIBSCOPE_ANTHROPIC_API_KEY` is not set | — | +| `LIBSCOPE_ALLOW_PRIVATE_URLS` | `indexing.allowPrivateUrls` | `false` | +| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | `indexing.allowSelfSignedCerts` | `false` | +| `LIBSCOPE_WORKSPACE` | Active workspace (overrides `libscope workspace use`) | `default` | +| `LIBSCOPE_API_KEY` | REST API key. When set, requests must send `Authorization: Bearer <key>` | — | +| `LIBSCOPE_SECRET_KEY` | Key used to encrypt stored webhook secrets | — | +| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (to stderr) in the CLI | — | +| `ONENOTE_CLIENT_ID` | OneNote app client ID | — | +| `ONENOTE_TENANT_ID` | OneNote tenant ID | `common` | +| `NOTION_TOKEN` | Notion integration token | — | +| `CONFLUENCE_URL` | Confluence base URL | — | +| `CONFLUENCE_EMAIL` | Confluence user email | — | +| `CONFLUENCE_TOKEN` | Confluence API token | — | + +### API keys + +OpenAI and Anthropic keys use the same rule for embeddings and for the LLM: + +1. `LIBSCOPE_OPENAI_API_KEY` / `LIBSCOPE_ANTHROPIC_API_KEY` +2. `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` +3. The key in a config file (`embedding.openaiApiKey`, `llm.openaiApiKey`, `llm.anthropicApiKey`) + +`libscope config show` masks keys (for example `sk-…abcd`). + +LibScope does not load `.env` files. Export variables in your shell, or set them in the `env` block of your MCP client config. ## Setting Values diff --git a/src/cli/index.ts b/src/cli/index.ts index 627503ba..b6e60cf3 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import { Command } from "commander"; -import { loadConfig, saveUserConfig } from "../config.js"; +import { loadConfig, saveUserConfig, maskConfigSecrets } from "../config.js"; import { getDatabase, runMigrations, createVectorTable, closeDatabase } from "../db/index.js"; import { createEmbeddingProvider, type EmbeddingProvider } from "../providers/index.js"; import { indexDocument, indexFile } from "../core/indexing.js"; @@ -1552,7 +1552,7 @@ configCmd .action(() => { setupLogging(program.opts<ProgramOpts>()); const config = loadConfig(); - console.log(JSON.stringify(config, null, 2)); + console.log(JSON.stringify(maskConfigSecrets(config), null, 2)); }); interface ProgramOpts { diff --git a/src/config.ts b/src/config.ts index 7d669e46..40c0b4a4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -112,28 +112,37 @@ function compact<T extends Record<string, unknown>>(obj: T): Compact<T> { return out as Compact<T>; } -/** Collect config values from LIBSCOPE_* env vars. Only variables that are set appear in the result. */ +/** Return the first value that is set and non-empty. */ +function firstSet(...values: Array<string | undefined>): string | undefined { + return values.find((v) => v !== undefined && v !== ""); +} + +/** + * Collect config values from env vars. Only variables that are set appear in the result. + * + * API keys follow one rule for embedding and LLM providers: + * LIBSCOPE_<P>_API_KEY > <P>_API_KEY > config file. + */ function getEnvOverrides(): ConfigLayer { const env = process.env; const provider = env["LIBSCOPE_EMBEDDING_PROVIDER"]; const llmProvider = env["LIBSCOPE_LLM_PROVIDER"]; - const llmModel = env["LIBSCOPE_LLM_MODEL"]; - const validLlmProvider = isLlmProvider(llmProvider); + const openaiApiKey = firstSet(env["LIBSCOPE_OPENAI_API_KEY"], env["OPENAI_API_KEY"]); + const anthropicApiKey = firstSet(env["LIBSCOPE_ANTHROPIC_API_KEY"], env["ANTHROPIC_API_KEY"]); return { embedding: compact({ provider: isEmbeddingProvider(provider) ? provider : undefined, - openaiApiKey: env["LIBSCOPE_OPENAI_API_KEY"], + openaiApiKey, ollamaUrl: env["LIBSCOPE_OLLAMA_URL"], + ollamaModel: env["LIBSCOPE_OLLAMA_MODEL"], + }), + llm: compact({ + provider: isLlmProvider(llmProvider) ? llmProvider : undefined, + model: env["LIBSCOPE_LLM_MODEL"], + openaiApiKey, + anthropicApiKey, }), - llm: - validLlmProvider || llmModel - ? compact({ - provider: validLlmProvider ? llmProvider : undefined, - model: llmModel, - anthropicApiKey: env["LIBSCOPE_ANTHROPIC_API_KEY"], - }) - : {}, indexing: compact({ allowPrivateUrls: truthy(env["LIBSCOPE_ALLOW_PRIVATE_URLS"]), allowSelfSignedCerts: truthy(env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"]), @@ -141,6 +150,34 @@ function getEnvOverrides(): ConfigLayer { }; } +/** Config keys that hold secrets: masked by `config show`/`config get`, never written by `config set`. */ +export const SECRET_CONFIG_KEYS = [ + "embedding.openaiApiKey", + "llm.openaiApiKey", + "llm.anthropicApiKey", +] as const; + +function hasApiKey(layer: ConfigLayer): boolean { + return Boolean( + layer.embedding?.openaiApiKey ?? layer.llm?.openaiApiKey ?? layer.llm?.anthropicApiKey, + ); +} + +/** Mask a secret for display: `sk-…abcd` for long values, `****` otherwise. */ +export function maskSecret(value: string): string { + return value.length >= 12 ? `${value.slice(0, 3)}…${value.slice(-4)}` : "****"; +} + +/** Return a copy of the config with every API key masked (for display). */ +export function maskConfigSecrets(config: LibScopeConfig): LibScopeConfig { + const masked: LibScopeConfig = structuredClone(config); + const { embedding, llm } = masked; + if (embedding.openaiApiKey) embedding.openaiApiKey = maskSecret(embedding.openaiApiKey); + if (llm?.openaiApiKey) llm.openaiApiKey = maskSecret(llm.openaiApiKey); + if (llm?.anthropicApiKey) llm.anthropicApiKey = maskSecret(llm.anthropicApiKey); + return masked; +} + let _configCache: LibScopeConfig | null = null; let _configCacheAt = 0; const CONFIG_CACHE_TTL_MS = 30_000; @@ -161,16 +198,17 @@ export function loadConfig(): LibScopeConfig { const projectConfig = loadJsonFile(getProjectConfigPath()); const envOverrides = getEnvOverrides(); - if ( - userConfig.embedding?.openaiApiKey || - userConfig.llm?.openaiApiKey || - userConfig.llm?.anthropicApiKey - ) { - getLogger().warn( - "API keys found in config file (~/.libscope/config.json). " + - "This is deprecated — please use environment variables (OPENAI_API_KEY, ANTHROPIC_API_KEY) instead. " + - "Keys in the config file will no longer be written back after the next save.", - ); + for (const [path, layer] of [ + [getUserConfigPath(), userConfig], + [getProjectConfigPath(), projectConfig], + ] as const) { + if (hasApiKey(layer)) { + getLogger().warn( + `API keys found in config file ${path}. Prefer environment variables ` + + "(LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY, LIBSCOPE_ANTHROPIC_API_KEY or ANTHROPIC_API_KEY); " + + "they take precedence over keys in config files.", + ); + } } const config: LibScopeConfig = { @@ -210,27 +248,28 @@ export function loadConfig(): LibScopeConfig { return config; } +const OPENAI_KEY_HINT = "Set LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY"; + /** Check embedding and LLM provider configuration for missing keys/URLs. */ function validateProviderConfig(config: LibScopeConfig, warnings: string[]): void { - if (config.embedding.provider === "openai") { - const hasKey = config.embedding.openaiApiKey ?? process.env["OPENAI_API_KEY"]; - if (!hasKey) { - warnings.push( - 'embedding.provider is "openai" but no API key found. Set embedding.openaiApiKey or OPENAI_API_KEY env var.', - ); - } + if (config.embedding.provider === "openai" && !config.embedding.openaiApiKey) { + warnings.push( + `embedding.provider is "openai" but no API key found. ${OPENAI_KEY_HINT} (or embedding.openaiApiKey in a config file).`, + ); } if (config.embedding.provider === "ollama" && !config.embedding.ollamaUrl) { warnings.push('embedding.provider is "ollama" but embedding.ollamaUrl is not set.'); } - if (config.llm?.provider === "openai") { - const hasKey = - config.llm.openaiApiKey ?? config.embedding.openaiApiKey ?? process.env["OPENAI_API_KEY"]; - if (!hasKey) { - warnings.push( - 'llm.provider is "openai" but no API key found. Set llm.openaiApiKey or OPENAI_API_KEY env var.', - ); - } + const llm = config.llm; + if (llm?.provider === "openai" && !(llm.openaiApiKey ?? config.embedding.openaiApiKey)) { + warnings.push( + `llm.provider is "openai" but no API key found. ${OPENAI_KEY_HINT} (or llm.openaiApiKey in a config file).`, + ); + } + if (llm?.provider === "anthropic" && !llm.anthropicApiKey) { + warnings.push( + 'llm.provider is "anthropic" but no API key found. Set LIBSCOPE_ANTHROPIC_API_KEY or ANTHROPIC_API_KEY (or llm.anthropicApiKey in a config file).', + ); } } diff --git a/tests/unit/config-layers.test.ts b/tests/unit/config-layers.test.ts index cdc3da08..20dc4ccc 100644 --- a/tests/unit/config-layers.test.ts +++ b/tests/unit/config-layers.test.ts @@ -2,7 +2,13 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import { loadConfig, invalidateConfigCache } from "../../src/config.js"; +import { + loadConfig, + invalidateConfigCache, + maskSecret, + maskConfigSecrets, +} from "../../src/config.js"; +import * as loggerModule from "../../src/logger.js"; import { initLogger } from "../../src/logger.js"; /** Env vars that influence config loading; cleared for every test. */ @@ -111,4 +117,100 @@ describe("config layering (temp HOME and cwd)", () => { expect(config.indexing.allowPrivateUrls).toBe(false); }); }); + + describe("API keys: LIBSCOPE_<P>_API_KEY > <P>_API_KEY > config file", () => { + it("OPENAI_API_KEY alone is used for embedding and llm", () => { + const config = load({ OPENAI_API_KEY: "sk-plain" }); + expect(config.embedding.openaiApiKey).toBe("sk-plain"); + expect(config.llm?.openaiApiKey).toBe("sk-plain"); + }); + + it("LIBSCOPE_OPENAI_API_KEY wins over OPENAI_API_KEY", () => { + const config = load({ OPENAI_API_KEY: "sk-plain", LIBSCOPE_OPENAI_API_KEY: "sk-scoped" }); + expect(config.embedding.openaiApiKey).toBe("sk-scoped"); + expect(config.llm?.openaiApiKey).toBe("sk-scoped"); + }); + + it("an empty LIBSCOPE_OPENAI_API_KEY falls through to OPENAI_API_KEY", () => { + const config = load({ LIBSCOPE_OPENAI_API_KEY: "", OPENAI_API_KEY: "sk-plain" }); + expect(config.embedding.openaiApiKey).toBe("sk-plain"); + }); + + it("env keys win over keys in the config file", () => { + writeUserConfig({ embedding: { openaiApiKey: "sk-file" } }); + const config = load({ OPENAI_API_KEY: "sk-env" }); + expect(config.embedding.openaiApiKey).toBe("sk-env"); + }); + + it("config file key is used when no env key is set", () => { + writeUserConfig({ embedding: { provider: "openai", openaiApiKey: "sk-file" } }); + const config = load(); + expect(config.embedding.openaiApiKey).toBe("sk-file"); + }); + + it("LIBSCOPE_ANTHROPIC_API_KEY works without LIBSCOPE_LLM_PROVIDER", () => { + writeUserConfig({ llm: { provider: "anthropic" } }); + const config = load({ LIBSCOPE_ANTHROPIC_API_KEY: "sk-ant-scoped" }); + expect(config.llm?.provider).toBe("anthropic"); + expect(config.llm?.anthropicApiKey).toBe("sk-ant-scoped"); + }); + + it("ANTHROPIC_API_KEY is used when LIBSCOPE_ANTHROPIC_API_KEY is not set", () => { + const config = load({ ANTHROPIC_API_KEY: "sk-ant-plain" }); + expect(config.llm?.anthropicApiKey).toBe("sk-ant-plain"); + }); + + it("validation accepts exactly the keys providers will use", () => { + const warn = vi.fn(); + vi.spyOn(loggerModule, "getLogger").mockReturnValue({ + warn, + } as unknown as ReturnType<typeof loggerModule.getLogger>); + writeUserConfig({ embedding: { provider: "openai" }, llm: { provider: "anthropic" } }); + const config = load({ OPENAI_API_KEY: "sk-plain", ANTHROPIC_API_KEY: "sk-ant" }); + expect(config.embedding.openaiApiKey).toBe("sk-plain"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("warns about keys in a config file, naming the file and the env vars", () => { + const warn = vi.fn(); + vi.spyOn(loggerModule, "getLogger").mockReturnValue({ + warn, + } as unknown as ReturnType<typeof loggerModule.getLogger>); + writeProjectConfig({ llm: { anthropicApiKey: "sk-ant-file" } }); + load(); + const messages = warn.mock.calls.map((c) => String(c[0])); + const keyWarning = messages.find((m) => m.includes("API keys found")); + expect(keyWarning).toContain(join(tempCwd, ".libscope.json")); + expect(keyWarning).toContain("ANTHROPIC_API_KEY"); + expect(keyWarning).not.toContain("deprecated"); + }); + + it("LIBSCOPE_OLLAMA_MODEL sets embedding.ollamaModel", () => { + const config = load({ LIBSCOPE_OLLAMA_MODEL: "mxbai-embed-large" }); + expect(config.embedding.ollamaModel).toBe("mxbai-embed-large"); + }); + }); + + describe("secret masking", () => { + it("masks long keys to prefix and last four characters", () => { + expect(maskSecret("sk-abcdefghijklmnop1234")).toBe("sk-…1234"); + }); + + it("fully masks short keys", () => { + expect(maskSecret("short")).toBe("****"); + }); + + it("masks every API key in a config without mutating the original", () => { + const config = load({ + LIBSCOPE_OPENAI_API_KEY: "sk-openai-secret-value-9999", + ANTHROPIC_API_KEY: "sk-ant-secret-value-8888", + }); + const masked = maskConfigSecrets(config); + const text = JSON.stringify(masked); + expect(text).not.toContain("secret-value"); + expect(masked.embedding.openaiApiKey).toBe("sk-…9999"); + expect(masked.llm?.anthropicApiKey).toBe("sk-…8888"); + expect(config.embedding.openaiApiKey).toBe("sk-openai-secret-value-9999"); + }); + }); }); diff --git a/tests/unit/config.test.ts b/tests/unit/config.test.ts index fa34f2e6..788dd36c 100644 --- a/tests/unit/config.test.ts +++ b/tests/unit/config.test.ts @@ -144,11 +144,22 @@ describe("validateConfig", () => { expect(warnings).toHaveLength(0); }); - it("should not warn when embedding provider is openai with OPENAI_API_KEY env", () => { + it("should validate the resolved config only (env keys are resolved by loadConfig)", () => { process.env["OPENAI_API_KEY"] = "sk-env-test"; const config = makeConfig({ embedding: { provider: "openai" } }); const warnings = validateConfig(config); - expect(warnings).toHaveLength(0); + expect(warnings[0]).toContain("LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY"); + }); + + it("should warn when llm provider is anthropic without API key", () => { + const config = makeConfig({ llm: { provider: "anthropic" } }); + const warnings = validateConfig(config); + expect(warnings.some((w) => w.includes("LIBSCOPE_ANTHROPIC_API_KEY"))).toBe(true); + }); + + it("should not warn when llm provider is anthropic with a key", () => { + const config = makeConfig({ llm: { provider: "anthropic", anthropicApiKey: "sk-ant" } }); + expect(validateConfig(config)).toHaveLength(0); }); it("should warn when llm provider is openai without API key", () => { From e122e304b3759cad234160dcc32a7e4ec991eff3 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:17:42 +0000 Subject: [PATCH 16/83] fix(topics): delete vectors when deleteTopic removes documents deleteTopic({ deleteDocuments: true }) deleted the documents (and, by cascade, their chunks) but left their vectors in chunk_embeddings, a vec0 virtual table with no foreign key. Delete the vectors first with deleteChunkEmbeddings, in batches of 500 documents. --- src/core/topics.ts | 27 +++++++++++++++++++++++++-- tests/unit/topics.test.ts | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/core/topics.ts b/src/core/topics.ts index 9408ed5a..16e08365 100644 --- a/src/core/topics.ts +++ b/src/core/topics.ts @@ -3,7 +3,13 @@ import { randomUUID } from "node:crypto"; import { ValidationError, TopicNotFoundError } from "../errors.js"; import { createChildLogger } from "../logger.js"; import { validateRow } from "../utils/db-validation.js"; -import { DOC_COLUMNS, rowToDocument, type Document, type DocumentRow } from "./documents.js"; +import { + DOC_COLUMNS, + deleteChunkEmbeddings, + rowToDocument, + type Document, + type DocumentRow, +} from "./documents.js"; export interface Topic { id: string; @@ -145,7 +151,10 @@ export interface TopicStats { documentCount: number; } -/** Delete a topic and optionally its document associations. */ +/** Documents per DELETE statement when removing a topic's vectors (bounds SQL variables). */ +const EMBEDDING_DELETE_BATCH = 500; + +/** Delete a topic and optionally its documents (with their chunks and vectors). */ export function deleteTopic( db: Database.Database, topicId: string, @@ -155,6 +164,20 @@ export function deleteTopic( getTopic(db, topicId); const run = db.transaction(() => { if (options?.deleteDocuments) { + // chunk_embeddings is a virtual table with no foreign key, so remove vectors explicitly. + const docIds = ( + db.prepare("SELECT id FROM documents WHERE topic_id = ?").all(topicId) as Array<{ + id: string; + }> + ).map((row) => row.id); + try { + for (let i = 0; i < docIds.length; i += EMBEDDING_DELETE_BATCH) { + deleteChunkEmbeddings(db, docIds.slice(i, i + EMBEDDING_DELETE_BATCH)); + } + } catch (err: unknown) { + const tableMissing = err instanceof Error && err.message.includes("no such table"); + log[tableMissing ? "debug" : "warn"]({ err, topicId }, "Skipped chunk_embeddings cleanup"); + } db.prepare("DELETE FROM documents WHERE topic_id = ?").run(topicId); } db.prepare("DELETE FROM topics WHERE id = ?").run(topicId); diff --git a/tests/unit/topics.test.ts b/tests/unit/topics.test.ts index 03035533..f5590a4f 100644 --- a/tests/unit/topics.test.ts +++ b/tests/unit/topics.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach } from "vitest"; -import { createTestDb } from "../fixtures/test-db.js"; +import { createTestDb, createTestDbWithVec } from "../fixtures/test-db.js"; import { createTopic, listTopics, @@ -141,6 +141,43 @@ describe("topics", () => { expect(doc).toBeUndefined(); }); + it("should delete vectors of deleted documents when deleteDocuments is true", () => { + const vdb = createTestDbWithVec(); + createTopic(vdb, { name: "Vectors" }); + const insertDoc = vdb.prepare( + "INSERT INTO documents (id, source_type, title, content, topic_id) VALUES (?, 'topic', ?, 'c', ?)", + ); + insertDoc.run("doc-in", "In topic", "vectors"); + insertDoc.run("doc-out", "Other", null); + const insertChunk = vdb.prepare( + "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, 'text', 0)", + ); + insertChunk.run("chunk-in", "doc-in"); + insertChunk.run("chunk-out", "doc-out"); + const insertVec = vdb.prepare( + "INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, x'00')", + ); + insertVec.run("chunk-in"); + insertVec.run("chunk-out"); + + deleteTopic(vdb, "vectors", { deleteDocuments: true }); + + const remaining = vdb.prepare("SELECT chunk_id FROM chunk_embeddings").all(); + expect(remaining).toEqual([{ chunk_id: "chunk-out" }]); + vdb.close(); + }); + + it("should delete documents when the vector table is missing", () => { + createTopic(db, { name: "NoVec" }); + db.prepare( + "INSERT INTO documents (id, source_type, title, content, topic_id) VALUES ('d', 'topic', 'D', 'c', 'novec')", + ).run(); + + deleteTopic(db, "novec", { deleteDocuments: true }); + + expect(db.prepare("SELECT id FROM documents WHERE id = 'd'").get()).toBeUndefined(); + }); + it("should set child topic parent_id to null on deletion", () => { const parent = createTopic(db, { name: "Parent" }); createTopic(db, { name: "Child", parentId: parent.id }); From f870f67132520e2c3a2aa5c9018f2baf8006f48e Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:21:17 +0000 Subject: [PATCH 17/83] fix(config): config set/get/unset for every key, preserve config.json config set accepted only 3 keys while docs and error messages told users to set others, and saveUserConfig rewrote config.json from 5 known sections: it dropped the registries array, wrote every default, and did not restrict permissions. - config set <key> <value> accepts every key of LibScopeConfig (typed key table; booleans/integers coerced, enums validated). API keys are refused with the env var to use. - New config get, config unset, config path; show/get mask API keys. - saveUserConfig is read-modify-write: only passed keys are written, unknown keys (registries, hand-written values) are kept. - One raw reader/writer of ~/.libscope/config.json (readRawUserConfig / writeRawUserConfig, 0600 file, 0700 dir), used by registry/config.ts. --- README.md | 7 +- docs/guide/configuration.md | 13 +- docs/reference/configuration.md | 17 ++- src/cli/index.ts | 88 +++++++---- src/config.ts | 250 +++++++++++++++++++++++------- src/registry/config.ts | 40 +---- tests/unit/config-save.test.ts | 262 ++++++++++++++++++++++++++++---- 7 files changed, 524 insertions(+), 153 deletions(-) diff --git a/README.md b/README.md index 2488e525..a77c38da 100644 --- a/README.md +++ b/README.md @@ -573,8 +573,11 @@ There's also a web dashboard at `http://localhost:3377` when you run `libscope s | `libscope stats` | Usage overview | | `libscope pack install <name>` | Install a knowledge pack | | `libscope pack create` | Create a knowledge pack | -| `libscope config set <key> <val>` | Set config | -| `libscope config show` | Show config | +| `libscope config set <key> <val>` | Set any config key (not API keys) | +| `libscope config get <key>` | Print one effective value | +| `libscope config unset <key>` | Remove a key from the user config file | +| `libscope config path` | Print the user config file path | +| `libscope config show` | Show config (API keys masked) | **Global flags:** `--verbose`, `--log-level <level>`, `--workspace <name>` diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index d308941f..3a08716b 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -9,13 +9,22 @@ LibScope uses a 3-tier config system. Higher tiers override lower ones: You can set options via the CLI or by editing the config file directly. ```bash -# Set a value +# Set a value (any key from the configuration reference, except API keys) libscope config set embedding.provider ollama -# View current config +# Read or remove one value +libscope config get embedding.provider +libscope config unset embedding.provider + +# Show where the user config file is +libscope config path + +# View current config (API keys masked) libscope config show ``` +`config set` validates the key and value, and changes only that key in `~/.libscope/config.json`. Other content of the file, such as `registries`, is kept. + Example `~/.libscope/config.json`: ```json diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 077ce716..f36ca501 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -49,6 +49,7 @@ Complete reference for all configuration options. | Key | Type | Default | Description | | ------------------------------- | ------- | ------- | ------------------------------------------------- | +| `indexing.maxDocumentSize` | number | `104857600` | Maximum document size in bytes (100 MB) | | `indexing.allowPrivateUrls` | boolean | `false` | Allow fetching from private/internal IP addresses | | `indexing.allowSelfSignedCerts` | boolean | `false` | Accept self-signed or untrusted TLS certificates | @@ -100,7 +101,7 @@ libscope config set embedding.ollamaModel nomic-embed-text libscope config set embedding.openaiModel text-embedding-3-small # LLM (for RAG) -libscope config set llm.provider openai # openai | ollama | passthrough +libscope config set llm.provider openai # openai | ollama | anthropic | passthrough libscope config set llm.model gpt-4o-mini # Database @@ -113,10 +114,22 @@ libscope config set logging.level debug # debug | info | warn | error libscope config set indexing.allowPrivateUrls true libscope config set indexing.allowSelfSignedCerts true -# View current config +# Size limit (bytes) +libscope config set indexing.maxDocumentSize 52428800 + +# Read, remove, and locate +libscope config get embedding.provider # effective value (API keys masked) +libscope config unset llm.model # remove from the user config file +libscope config path # print the user config file path + +# View current config (API keys masked) libscope config show ``` +`config set` accepts every key listed in [All Config Keys](#all-config-keys) except API keys. It checks the value type (booleans accept `true`/`false`/`1`/`0`, numbers must be positive integers, and enum keys must use a listed value). It writes only the key you set to `~/.libscope/config.json` and keeps all other content of that file, including `registries`. The file is written with mode `0600`. + +API keys (`embedding.openaiApiKey`, `llm.openaiApiKey`, `llm.anthropicApiKey`) are not written by `config set`. Use the environment variables in [API keys](#api-keys). + ## Corporate / Internal Networks If you're indexing docs from internal servers (Confluence, wikis, etc.): diff --git a/src/cli/index.ts b/src/cli/index.ts index b6e60cf3..12f4c4a3 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,7 +1,15 @@ #!/usr/bin/env node import { Command } from "commander"; -import { loadConfig, saveUserConfig, maskConfigSecrets } from "../config.js"; +import { + loadConfig, + maskConfigSecrets, + getUserConfigPath, + CONFIG_KEY_NAMES, + setUserConfigValue, + unsetUserConfigValue, + getConfigValue, +} from "../config.js"; import { getDatabase, runMigrations, createVectorTable, closeDatabase } from "../db/index.js"; import { createEmbeddingProvider, type EmbeddingProvider } from "../providers/index.js"; import { indexDocument, indexFile } from "../core/indexing.js"; @@ -1518,41 +1526,67 @@ program }); // config -const configCmd = program.command("config").description("Manage configuration"); +const configCmd = program + .command("config") + .description(`Manage configuration (user file: ${getUserConfigPath()})`); + +/** Run a config subcommand: set up logging, print errors without a stack trace. */ +function runConfigAction(fn: () => void): void { + setupLogging(program.opts<ProgramOpts>()); + try { + fn(); + } catch (err) { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); + } +} configCmd .command("set <key> <value>") - .description("Set a configuration value (e.g., embedding.provider local)") + .description( + `Set a value in the user config file. Keys: ${CONFIG_KEY_NAMES.join(", ")}. API keys must be set with environment variables.`, + ) .action((key: string, value: string) => { - setupLogging(program.opts<ProgramOpts>()); - if (key === "embedding.provider") { - if (value !== "local" && value !== "ollama" && value !== "openai") { - console.error("Invalid provider. Must be: local, ollama, or openai"); - process.exit(1); - } - saveUserConfig({ embedding: { provider: value } }); - console.log(`✓ Embedding provider set to: ${value}`); - } else if (key === "indexing.allowPrivateUrls") { - const bool = value === "true"; - saveUserConfig({ indexing: { ...loadConfig().indexing, allowPrivateUrls: bool } }); - console.log(`✓ indexing.allowPrivateUrls set to: ${bool}`); - } else if (key === "indexing.allowSelfSignedCerts") { - const bool = value === "true"; - saveUserConfig({ indexing: { ...loadConfig().indexing, allowSelfSignedCerts: bool } }); - console.log(`✓ indexing.allowSelfSignedCerts set to: ${bool}`); - } else { - console.error(`Unknown config key: ${key}`); - process.exit(1); - } + runConfigAction(() => { + const stored = setUserConfigValue(key, value); + console.log(`✓ ${key} set to: ${String(stored)}`); + }); + }); + +configCmd + .command("get <key>") + .description("Print the effective value of a config key (API keys are masked)") + .action((key: string) => { + runConfigAction(() => { + const value = getConfigValue(loadConfig(), key); + console.log(value === undefined ? "" : String(value)); + }); + }); + +configCmd + .command("unset <key>") + .description("Remove a key from the user config file (the default or other layers apply)") + .action((key: string) => { + runConfigAction(() => { + const removed = unsetUserConfigValue(key); + console.log(removed ? `✓ ${key} removed` : `${key} is not set in ${getUserConfigPath()}`); + }); + }); + +configCmd + .command("path") + .description("Print the path of the user config file") + .action(() => { + console.log(getUserConfigPath()); }); configCmd .command("show") - .description("Show current configuration") + .description("Show the effective configuration (API keys are masked)") .action(() => { - setupLogging(program.opts<ProgramOpts>()); - const config = loadConfig(); - console.log(JSON.stringify(maskConfigSecrets(config), null, 2)); + runConfigAction(() => { + console.log(JSON.stringify(maskConfigSecrets(loadConfig()), null, 2)); + }); }); interface ProgramOpts { diff --git a/src/config.ts b/src/config.ts index 40c0b4a4..7315a515 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,15 @@ -import { readFileSync, writeFileSync, existsSync, mkdirSync, accessSync, constants } from "node:fs"; +import { + readFileSync, + writeFileSync, + existsSync, + mkdirSync, + accessSync, + chmodSync, + constants, +} from "node:fs"; import { join, dirname } from "node:path"; import { homedir } from "node:os"; -import { ConfigError } from "./errors.js"; +import { ConfigError, ValidationError } from "./errors.js"; import { getLogger } from "./logger.js"; export interface LibScopeConfig { @@ -56,7 +64,8 @@ function getConfigDir(): string { return join(homedir(), ".libscope"); } -function getUserConfigPath(): string { +/** Path to the user config file (~/.libscope/config.json). */ +export function getUserConfigPath(): string { return join(getConfigDir(), "config.json"); } @@ -73,16 +82,51 @@ export interface ConfigLayer { logging?: Partial<LibScopeConfig["logging"]>; } -function loadJsonFile(path: string): ConfigLayer { +type RawConfig = Record<string, unknown>; + +function isPlainObject(value: unknown): value is RawConfig { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function readJsonObject(path: string): RawConfig { try { if (!existsSync(path)) return {}; - const content = readFileSync(path, "utf-8"); - return JSON.parse(content) as ConfigLayer; + const parsed: unknown = JSON.parse(readFileSync(path, "utf-8")); + if (!isPlainObject(parsed)) throw new Error("top-level value must be a JSON object"); + return parsed; } catch (err) { throw new ConfigError(`Failed to read config file: ${path}`, err); } } +function loadJsonFile(path: string): ConfigLayer { + return readJsonObject(path) as ConfigLayer; +} + +/** + * Read ~/.libscope/config.json as raw JSON, including keys this module does not model + * (e.g. `registries`). This and writeRawUserConfig are the only reader/writer of that file. + */ +export function readRawUserConfig(): RawConfig { + return readJsonObject(getUserConfigPath()); +} + +/** Write ~/.libscope/config.json (directory 0700, file 0600). */ +export function writeRawUserConfig(config: RawConfig): void { + const dir = getConfigDir(); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + } + const configPath = getUserConfigPath(); + writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n", { + encoding: "utf-8", + mode: 0o600, + }); + // writeFileSync's mode only applies when the file is created; tighten existing files too. + chmodSync(configPath, 0o600); + invalidateConfigCache(); +} + const EMBEDDING_PROVIDERS = ["local", "ollama", "openai"] as const; const LLM_PROVIDERS = ["openai", "ollama", "anthropic", "passthrough"] as const; type EmbeddingProviderName = (typeof EMBEDDING_PROVIDERS)[number]; @@ -150,12 +194,136 @@ function getEnvOverrides(): ConfigLayer { }; } -/** Config keys that hold secrets: masked by `config show`/`config get`, never written by `config set`. */ -export const SECRET_CONFIG_KEYS = [ - "embedding.openaiApiKey", - "llm.openaiApiKey", - "llm.anthropicApiKey", -] as const; +/** Every settable key, as `section.field`, derived from the LibScopeConfig shape. */ +export type ConfigKey = { + [S in keyof LibScopeConfig]-?: `${S}.${keyof NonNullable<LibScopeConfig[S]> & string}`; +}[keyof LibScopeConfig]; + +type ConfigValue = string | number | boolean; + +interface ConfigKeySpec { + /** "string", "boolean", "positiveInt", or the list of allowed values. */ + type: "string" | "boolean" | "positiveInt" | readonly string[]; + /** For secrets: the env var to use instead of storing the value in a config file. */ + secretEnv?: string; +} + +const CONFIG_KEYS: Record<ConfigKey, ConfigKeySpec> = { + "embedding.provider": { type: EMBEDDING_PROVIDERS }, + "embedding.ollamaUrl": { type: "string" }, + "embedding.ollamaModel": { type: "string" }, + "embedding.openaiApiKey": { type: "string", secretEnv: "LIBSCOPE_OPENAI_API_KEY" }, + "embedding.openaiModel": { type: "string" }, + "llm.provider": { type: LLM_PROVIDERS }, + "llm.model": { type: "string" }, + "llm.ollamaUrl": { type: "string" }, + "llm.openaiApiKey": { type: "string", secretEnv: "LIBSCOPE_OPENAI_API_KEY" }, + "llm.anthropicApiKey": { type: "string", secretEnv: "LIBSCOPE_ANTHROPIC_API_KEY" }, + "database.path": { type: "string" }, + "indexing.maxDocumentSize": { type: "positiveInt" }, + "indexing.allowPrivateUrls": { type: "boolean" }, + "indexing.allowSelfSignedCerts": { type: "boolean" }, + "logging.level": { type: ["debug", "info", "warn", "error", "silent"] }, +}; + +/** All settable config keys. */ +export const CONFIG_KEY_NAMES = Object.keys(CONFIG_KEYS) as ConfigKey[]; + +function isConfigKey(key: string): key is ConfigKey { + return Object.hasOwn(CONFIG_KEYS, key); +} + +/** Validate a dotted key name; throws ValidationError listing the valid keys. */ +export function parseConfigKey(key: string): ConfigKey { + if (!isConfigKey(key)) { + throw new ValidationError( + `Unknown config key: ${key}. Valid keys: ${CONFIG_KEY_NAMES.join(", ")}`, + ); + } + return key; +} + +/** Convert a CLI string to the key's type (booleans, positive integers, enums). */ +export function coerceConfigValue(key: ConfigKey, raw: string): ConfigValue { + const { type } = CONFIG_KEYS[key]; + if (type === "boolean") { + const lower = raw.trim().toLowerCase(); + if (lower === "true" || lower === "1") return true; + if (lower === "false" || lower === "0") return false; + throw new ValidationError(`${key} must be true or false (got "${raw}")`); + } + if (type === "positiveInt") { + const n = Number(raw); + if (!Number.isInteger(n) || n <= 0) { + throw new ValidationError(`${key} must be a positive integer (got "${raw}")`); + } + return n; + } + if (type === "string") { + if (raw.trim() === "") throw new ValidationError(`${key} must not be empty`); + return raw; + } + if (!type.includes(raw)) { + throw new ValidationError(`${key} must be one of: ${type.join(", ")} (got "${raw}")`); + } + return raw; +} + +function splitKey(key: ConfigKey): [section: string, field: string] { + const [section = "", field = ""] = key.split("."); + return [section, field]; +} + +function rawSection(raw: RawConfig, section: string): RawConfig { + const existing = raw[section]; + if (isPlainObject(existing)) return existing; + const created: RawConfig = {}; + raw[section] = created; + return created; +} + +/** + * Set one key in ~/.libscope/config.json. Validates and coerces the value and keeps every + * other key in the file (including `registries`). API keys are refused: use env vars. + */ +export function setUserConfigValue(key: string, rawValue: string): ConfigValue { + const configKey = parseConfigKey(key); + const { secretEnv } = CONFIG_KEYS[configKey]; + if (secretEnv) { + throw new ValidationError( + `${configKey} is a secret and is not written to config files. Set the ${secretEnv} environment variable instead.`, + ); + } + const value = coerceConfigValue(configKey, rawValue); + const raw = readRawUserConfig(); + const [section, field] = splitKey(configKey); + rawSection(raw, section)[field] = value; + writeRawUserConfig(raw); + return value; +} + +/** Remove one key from ~/.libscope/config.json. Returns false if the file did not set it. */ +export function unsetUserConfigValue(key: string): boolean { + const configKey = parseConfigKey(key); + const raw = readRawUserConfig(); + const [section, field] = splitKey(configKey); + const sectionObj = raw[section]; + if (!isPlainObject(sectionObj) || !Object.hasOwn(sectionObj, field)) return false; + delete sectionObj[field]; + if (Object.keys(sectionObj).length === 0) delete raw[section]; + writeRawUserConfig(raw); + return true; +} + +/** Read the effective value of a key from a loaded config. Secrets are masked. */ +export function getConfigValue(config: LibScopeConfig, key: string): ConfigValue | undefined { + const configKey = parseConfigKey(key); + const [section, field] = splitKey(configKey); + const sections = config as unknown as Record<string, RawConfig | undefined>; + const value = sections[section]?.[field] as ConfigValue | undefined; + if (typeof value === "string" && CONFIG_KEYS[configKey].secretEnv) return maskSecret(value); + return value; +} function hasApiKey(layer: ConfigLayer): boolean { return Boolean( @@ -309,48 +477,20 @@ export function validateConfig(config: LibScopeConfig): string[] { return warnings; } -/** Save a config value to the user config file. */ -export function saveUserConfig(config: Partial<LibScopeConfig>): void { - const configDir = getConfigDir(); - if (!existsSync(configDir)) { - mkdirSync(configDir, { recursive: true }); - } - const existing = loadJsonFile(getUserConfigPath()); - const merged: LibScopeConfig = { - embedding: { - ...DEFAULT_CONFIG.embedding, - ...existing.embedding, - ...config.embedding, - }, - llm: { - ...existing.llm, - ...config.llm, - }, - database: { - ...DEFAULT_CONFIG.database, - ...existing.database, - ...config.database, - }, - indexing: { - ...DEFAULT_CONFIG.indexing, - ...existing.indexing, - ...config.indexing, - }, - logging: { - ...DEFAULT_CONFIG.logging, - ...existing.logging, - ...config.logging, - }, - }; - - // Security: never persist API keys to disk — use environment variables instead. - // Keys are read from env vars (OPENAI_API_KEY, ANTHROPIC_API_KEY) at runtime. - delete merged.embedding.openaiApiKey; - if (merged.llm) { - delete merged.llm.openaiApiKey; - delete merged.llm.anthropicApiKey; +/** + * Merge values into ~/.libscope/config.json. Only the keys passed in are written; every + * other key in the file (including `registries` and values added by hand) is kept. + * Security: API keys passed in are never written — use environment variables instead. + */ +export function saveUserConfig(config: ConfigLayer): void { + const raw = readRawUserConfig(); + for (const [section, values] of Object.entries(config)) { + if (!isPlainObject(values)) continue; + for (const [field, value] of Object.entries(values)) { + const key = `${section}.${field}`; + if (value === undefined || !isConfigKey(key) || CONFIG_KEYS[key].secretEnv) continue; + rawSection(raw, section)[field] = value; + } } - - writeFileSync(getUserConfigPath(), JSON.stringify(merged, null, 2), "utf-8"); - invalidateConfigCache(); + writeRawUserConfig(raw); } diff --git a/src/registry/config.ts b/src/registry/config.ts index 21dcad83..4b656346 100644 --- a/src/registry/config.ts +++ b/src/registry/config.ts @@ -3,18 +3,11 @@ * Reads/writes the "registries" array in ~/.libscope/config.json. */ -import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; -import { join } from "node:path"; -import { homedir } from "node:os"; -import { ConfigError, ValidationError } from "../errors.js"; +import { readRawUserConfig, writeRawUserConfig } from "../config.js"; +import { ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; import type { RegistryEntry } from "./types.js"; -/** Path to the user config file. */ -function getUserConfigPath(): string { - return join(homedir(), ".libscope", "config.json"); -} - /** Sanitize a URL for safe display in logs — masks any embedded credentials. */ export function sanitizeUrl(url: string): string { // Replace password in https://user:pass@host or https://token@host patterns @@ -66,32 +59,9 @@ export function validateGitUrl(url: string): string { return normalized; } -/** Read the raw config JSON from disk. */ -function readRawConfig(): Record<string, unknown> { - const configPath = getUserConfigPath(); - if (!existsSync(configPath)) return {}; - try { - const raw = readFileSync(configPath, "utf-8"); - return JSON.parse(raw) as Record<string, unknown>; - } catch (err) { - throw new ConfigError("Failed to read config file", err); - } -} - -/** Write the raw config JSON to disk, preserving all other keys. */ -function writeRawConfig(config: Record<string, unknown>): void { - const dir = join(homedir(), ".libscope"); - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true, mode: 0o700 }); - } - const configPath = getUserConfigPath(); - writeFileSync(configPath, JSON.stringify(config, null, 2), "utf-8"); - chmodSync(configPath, 0o600); -} - /** Load all registry entries from config. */ export function loadRegistries(): RegistryEntry[] { - const config = readRawConfig(); + const config = readRawUserConfig(); const registries = config["registries"]; if (!Array.isArray(registries)) return []; return registries as RegistryEntry[]; @@ -99,9 +69,9 @@ export function loadRegistries(): RegistryEntry[] { /** Save registry entries to config (merges with existing config keys). */ export function saveRegistries(registries: RegistryEntry[]): void { - const config = readRawConfig(); + const config = readRawUserConfig(); config["registries"] = registries; - writeRawConfig(config); + writeRawUserConfig(config); } /** Find a registry by name. Returns undefined if not found. */ diff --git a/tests/unit/config-save.test.ts b/tests/unit/config-save.test.ts index f29867ee..76a5a603 100644 --- a/tests/unit/config-save.test.ts +++ b/tests/unit/config-save.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdirSync, readFileSync, rmSync } from "node:fs"; +import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync, chmodSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; @@ -19,33 +19,59 @@ vi.mock("node:os", async (importOriginal) => { }); // Dynamic import after mock is set up -const { saveUserConfig, invalidateConfigCache } = await import("../../src/config.js"); +const { + saveUserConfig, + invalidateConfigCache, + setUserConfigValue, + unsetUserConfigValue, + getConfigValue, + getUserConfigPath, + loadConfig, + CONFIG_KEY_NAMES, +} = await import("../../src/config.js"); +const { saveRegistries, loadRegistries } = await import("../../src/registry/config.js"); + +function configPath(): string { + return join(tempHome, ".libscope", "config.json"); +} + +function readSavedRaw(): Record<string, unknown> { + return JSON.parse(readFileSync(configPath(), "utf-8")) as Record<string, unknown>; +} function readSavedConfig(): LibScopeConfig { - const written = readFileSync(join(tempHome, ".libscope", "config.json"), "utf-8"); - return JSON.parse(written) as LibScopeConfig; + return readSavedRaw() as unknown as LibScopeConfig; } -describe("saveUserConfig credential stripping", () => { - beforeEach(() => { - initLogger("silent"); - tempHome = join(tmpdir(), `libscope-config-save-test-${randomUUID()}`); - mkdirSync(tempHome, { recursive: true }); - invalidateConfigCache(); - }); +function writeExisting(data: unknown): void { + mkdirSync(join(tempHome, ".libscope"), { recursive: true }); + writeFileSync(configPath(), JSON.stringify(data), "utf-8"); +} - afterEach(() => { - try { - rmSync(tempHome, { recursive: true, force: true }); - } catch { - // ignore cleanup errors - } - }); +const REGISTRY = { + name: "team", + url: "https://example.com/registry.git", + syncInterval: 3600, + priority: 1, + lastSyncedAt: null, +}; + +beforeEach(() => { + initLogger("silent"); + tempHome = join(tmpdir(), `libscope-config-save-test-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + invalidateConfigCache(); +}); + +afterEach(() => { + rmSync(tempHome, { recursive: true, force: true }); +}); +describe("saveUserConfig credential stripping", () => { it("should not persist embedding.openaiApiKey to disk", () => { saveUserConfig({ embedding: { provider: "openai", openaiApiKey: "sk-test-key" } }); - const written = readFileSync(join(tempHome, ".libscope", "config.json"), "utf-8"); + const written = readFileSync(configPath(), "utf-8"); const parsed = readSavedConfig(); expect(parsed.embedding.provider).toBe("openai"); @@ -56,20 +82,16 @@ describe("saveUserConfig credential stripping", () => { it("should not persist llm.openaiApiKey to disk", () => { saveUserConfig({ llm: { provider: "openai", openaiApiKey: "sk-llm-key" } }); - const written = readFileSync(join(tempHome, ".libscope", "config.json"), "utf-8"); - const parsed = readSavedConfig(); - - expect(parsed.llm?.openaiApiKey).toBeUndefined(); + const written = readFileSync(configPath(), "utf-8"); + expect(readSavedConfig().llm?.openaiApiKey).toBeUndefined(); expect(written).not.toContain("sk-llm-key"); }); it("should not persist llm.anthropicApiKey to disk", () => { saveUserConfig({ llm: { provider: "anthropic", anthropicApiKey: "sk-ant-key" } }); - const written = readFileSync(join(tempHome, ".libscope", "config.json"), "utf-8"); - const parsed = readSavedConfig(); - - expect(parsed.llm?.anthropicApiKey).toBeUndefined(); + const written = readFileSync(configPath(), "utf-8"); + expect(readSavedConfig().llm?.anthropicApiKey).toBeUndefined(); expect(written).not.toContain("sk-ant-key"); }); @@ -79,8 +101,7 @@ describe("saveUserConfig credential stripping", () => { llm: { provider: "openai", openaiApiKey: "sk-llm", anthropicApiKey: "sk-ant" }, }); - const written = readFileSync(join(tempHome, ".libscope", "config.json"), "utf-8"); - + const written = readFileSync(configPath(), "utf-8"); expect(written).not.toContain("sk-embed"); expect(written).not.toContain("sk-llm"); expect(written).not.toContain("sk-ant"); @@ -98,10 +119,191 @@ describe("saveUserConfig credential stripping", () => { }); const parsed = readSavedConfig(); - expect(parsed.embedding.provider).toBe("openai"); expect(parsed.embedding.openaiModel).toBe("text-embedding-3-large"); expect(parsed.logging.level).toBe("debug"); expect(parsed.embedding.openaiApiKey).toBeUndefined(); }); }); + +describe("saveUserConfig read-modify-write", () => { + it("keeps registries and unknown top-level keys", () => { + writeExisting({ registries: [REGISTRY], customTool: { a: 1 } }); + saveUserConfig({ embedding: { provider: "ollama" } }); + + const raw = readSavedRaw(); + expect(raw["registries"]).toEqual([REGISTRY]); + expect(raw["customTool"]).toEqual({ a: 1 }); + }); + + it("writes only the values that were set, not defaults", () => { + saveUserConfig({ logging: { level: "warn" } }); + expect(readSavedRaw()).toEqual({ logging: { level: "warn" } }); + }); + + it("keeps sibling keys in the same section", () => { + writeExisting({ embedding: { ollamaUrl: "http://gpu:11434" } }); + saveUserConfig({ embedding: { provider: "ollama" } }); + expect(readSavedRaw()["embedding"]).toEqual({ + ollamaUrl: "http://gpu:11434", + provider: "ollama", + }); + }); + + it("does not delete an API key the user put in the file by hand", () => { + writeExisting({ embedding: { openaiApiKey: "sk-hand-written" } }); + saveUserConfig({ embedding: { provider: "openai" } }); + expect(readSavedConfig().embedding.openaiApiKey).toBe("sk-hand-written"); + }); + + it("writes the file with mode 0600, also when it already existed", () => { + writeExisting({}); + chmodSync(configPath(), 0o644); + saveUserConfig({ logging: { level: "info" } }); + expect(statSync(configPath()).mode & 0o777).toBe(0o600); + }); + + it("invalidates the config cache so the next load sees the change", () => { + loadConfig(); + saveUserConfig({ logging: { level: "error" } }); + expect(loadConfig().logging.level).toBe("error"); + }); +}); + +describe("setUserConfigValue", () => { + it("covers every key in the LibScopeConfig shape", () => { + expect([...CONFIG_KEY_NAMES].sort()).toEqual( + [ + "database.path", + "embedding.ollamaModel", + "embedding.ollamaUrl", + "embedding.openaiApiKey", + "embedding.openaiModel", + "embedding.provider", + "indexing.allowPrivateUrls", + "indexing.allowSelfSignedCerts", + "indexing.maxDocumentSize", + "llm.anthropicApiKey", + "llm.model", + "llm.ollamaUrl", + "llm.openaiApiKey", + "llm.provider", + "logging.level", + ].sort(), + ); + }); + + it.each([ + ["embedding.ollamaModel", "mxbai-embed-large", "mxbai-embed-large"], + ["embedding.openaiModel", "text-embedding-3-large", "text-embedding-3-large"], + ["llm.provider", "anthropic", "anthropic"], + ["llm.model", "gpt-4o", "gpt-4o"], + ["llm.ollamaUrl", "http://localhost:11434", "http://localhost:11434"], + ["database.path", "/data/libscope.db", "/data/libscope.db"], + ["logging.level", "debug", "debug"], + ["indexing.maxDocumentSize", "2048", 2048], + ["indexing.allowPrivateUrls", "true", true], + ["indexing.allowSelfSignedCerts", "0", false], + ] as const)("sets %s", (key, input, expected) => { + expect(setUserConfigValue(key, input)).toBe(expected); + const [section = "", field = ""] = key.split("."); + const sectionObj = readSavedRaw()[section] as Record<string, unknown>; + expect(sectionObj[field]).toBe(expected); + }); + + it("rejects unknown keys and lists valid keys", () => { + expect(() => setUserConfigValue("embedding.nope", "x")).toThrow( + /Unknown config key.*llm\.model/, + ); + }); + + it("rejects values outside an enum", () => { + expect(() => setUserConfigValue("embedding.provider", "cohere")).toThrow( + /one of: local, ollama, openai/, + ); + expect(() => setUserConfigValue("logging.level", "loud")).toThrow(/one of/); + }); + + it("rejects non-boolean and non-numeric values", () => { + expect(() => setUserConfigValue("indexing.allowPrivateUrls", "yes")).toThrow(/true or false/); + expect(() => setUserConfigValue("indexing.maxDocumentSize", "-5")).toThrow(/positive integer/); + expect(() => setUserConfigValue("indexing.maxDocumentSize", "1.5")).toThrow(/positive integer/); + }); + + it("refuses API keys and names the env var to use", () => { + expect(() => setUserConfigValue("embedding.openaiApiKey", "sk-x")).toThrow( + /LIBSCOPE_OPENAI_API_KEY/, + ); + expect(() => setUserConfigValue("llm.anthropicApiKey", "sk-x")).toThrow( + /LIBSCOPE_ANTHROPIC_API_KEY/, + ); + }); + + it("keeps registries written by the registry module", () => { + saveRegistries([REGISTRY]); + setUserConfigValue("embedding.provider", "ollama"); + expect(loadRegistries()).toEqual([REGISTRY]); + expect(loadConfig().embedding.provider).toBe("ollama"); + }); + + it("registry writes keep values written by config set", () => { + setUserConfigValue("llm.model", "gpt-4o"); + saveRegistries([REGISTRY]); + expect(readSavedRaw()["llm"]).toEqual({ model: "gpt-4o" }); + expect(statSync(configPath()).mode & 0o777).toBe(0o600); + }); +}); + +describe("unsetUserConfigValue", () => { + it("removes the key and drops an empty section", () => { + writeExisting({ logging: { level: "debug" }, registries: [REGISTRY] }); + expect(unsetUserConfigValue("logging.level")).toBe(true); + expect(readSavedRaw()).toEqual({ registries: [REGISTRY] }); + }); + + it("keeps other keys in the section", () => { + writeExisting({ embedding: { provider: "ollama", ollamaModel: "m" } }); + unsetUserConfigValue("embedding.ollamaModel"); + expect(readSavedRaw()["embedding"]).toEqual({ provider: "ollama" }); + }); + + it("returns false when the key is not set", () => { + expect(unsetUserConfigValue("llm.model")).toBe(false); + }); + + it("can remove a hand-written API key", () => { + writeExisting({ embedding: { openaiApiKey: "sk-old" } }); + expect(unsetUserConfigValue("embedding.openaiApiKey")).toBe(true); + expect(readFileSync(configPath(), "utf-8")).not.toContain("sk-old"); + }); + + it("rejects unknown keys", () => { + expect(() => unsetUserConfigValue("bogus")).toThrow(/Unknown config key/); + }); +}); + +describe("getConfigValue", () => { + it("returns effective values including defaults", () => { + setUserConfigValue("indexing.maxDocumentSize", "4096"); + const config = loadConfig(); + expect(getConfigValue(config, "indexing.maxDocumentSize")).toBe(4096); + expect(getConfigValue(config, "logging.level")).toBe("info"); + }); + + it("masks API keys", () => { + const config = loadConfig(); + const withKey: LibScopeConfig = { + ...config, + embedding: { ...config.embedding, openaiApiKey: "sk-proj-abcdefghijkl9876" }, + }; + expect(getConfigValue(withKey, "embedding.openaiApiKey")).toBe("sk-…9876"); + }); + + it("returns undefined for unset optional keys", () => { + expect(getConfigValue(loadConfig(), "llm.model")).toBeUndefined(); + }); + + it("reports the user config path under HOME", () => { + expect(getUserConfigPath()).toBe(configPath()); + }); +}); From 94b0c069e4bf2a515fb5a4e6ab262d01c6dc948b Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:21:33 +0000 Subject: [PATCH 18/83] fix(core): embed the same text on every chunk write path Fresh indexDocument embedded a title/library/version prefix plus the chunk and wrote chunk_embedding_metadata, but updateDocument, reindex, and pack install embedded the raw chunk and skipped the metadata row, so a chunk's vector depended on how its document was written. Add buildEmbeddingText, embedChunks, splitIntoChunks, and createChunkWriter in core/indexing.ts and use them on all four paths. updateDocument now also re-extracts document links when content changes, and re-embeds existing chunks in place when title, library, or version change. --- docs/reference/mcp-tools.md | 2 +- src/core/documents.ts | 74 +++++--- src/core/indexing.ts | 156 +++++++++++---- src/core/packs.ts | 69 +++---- src/core/reindex.ts | 16 +- tests/unit/embedding-text-consistency.test.ts | 177 ++++++++++++++++++ tests/unit/update-document.test.ts | 15 +- 7 files changed, 391 insertions(+), 118 deletions(-) create mode 100644 tests/unit/embedding-text-consistency.test.ts diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index c3fd22b2..8be2f726 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -73,7 +73,7 @@ Index a new document. You can provide content directly, or a URL to fetch automa ## update-document -Update an existing document's title, content, or metadata. Changing content triggers re-chunking and re-embedding. +Update an existing document's title, content, or metadata. Changing content triggers re-chunking and re-embedding. Changing the title, library, or version re-embeds the existing chunks, because these fields are part of the embedded text. | Parameter | Type | Required | Description | | ------------ | ------ | -------- | --------------------------------------- | diff --git a/src/core/documents.ts b/src/core/documents.ts index 804dc741..f2098ebf 100644 --- a/src/core/documents.ts +++ b/src/core/documents.ts @@ -1,8 +1,15 @@ import type Database from "better-sqlite3"; -import { createHash, randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { DocumentNotFoundError, ValidationError } from "../errors.js"; -import { chunkContent, chunkContentStreaming, STREAMING_THRESHOLD } from "./indexing.js"; +import { + createChunkWriter, + embedChunks, + embeddingMetaChanged, + splitIntoChunks, + storeDocumentLinks, + type EmbeddingMeta, +} from "./indexing.js"; import { getLogger } from "../logger.js"; import { saveVersion } from "./versioning.js"; @@ -203,12 +210,14 @@ export async function updateDocument( // SQLite's datetime('now') uses the OS clock and cannot be mocked in unit tests. const updatedAt = new Date().toISOString().replace("T", " ").slice(0, 19); + const newMeta = { title: newTitle, library: newLibrary, version: newVersion }; + if (contentChanged) { log.info({ docId: documentId }, "Content changed, re-chunking and re-indexing embeddings"); - const useStreaming = newContent.length > STREAMING_THRESHOLD; - const chunks = useStreaming ? chunkContentStreaming(newContent) : chunkContent(newContent); - const embeddings = await provider.embedBatch(chunks); + const chunks = splitIntoChunks(newContent); + const embeddings = await embedChunks(provider, chunks, newMeta); + const writer = createChunkWriter(db, provider); const transaction = db.transaction(() => { saveVersion(db, documentId); @@ -236,35 +245,26 @@ export async function updateDocument( documentId, ); - const insertChunk = db.prepare( - "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", - ); - const insertEmbedding = db.prepare( - "INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)", - ); - - for (let i = 0; i < chunks.length; i++) { - const chunkId = randomUUID(); - insertChunk.run(chunkId, documentId, chunks[i] ?? "", i); - - try { - const vecBuffer = Buffer.from(new Float32Array(embeddings[i] ?? []).buffer); - insertEmbedding.run(chunkId, vecBuffer); - } catch (err: unknown) { - // chunk_embeddings table may not exist - log.debug({ err, chunkId }, "Skipped embedding insertion during update"); - } - } + writer.insertChunks(documentId, chunks, embeddings); }); transaction(); + storeDocumentLinks(db, documentId, newContent); } else { + // Title/library/version are part of the embedding text, so re-embed existing chunks + // (keeping their IDs) when they change. + const reembed = embeddingMetaChanged(existing, newMeta) + ? await embedExistingChunks(db, provider, documentId, newMeta) + : null; + const transaction = db.transaction(() => { saveVersion(db, documentId); db.prepare( `UPDATE documents SET title = ?, library = ?, version = ?, url = ?, topic_id = ?, updated_at = ? WHERE id = ?`, ).run(newTitle, newLibrary, newVersion, newUrl, newTopicId, updatedAt, documentId); + + reembed?.write(); }); transaction(); @@ -272,3 +272,29 @@ export async function updateDocument( return getDocument(db, documentId); } + +/** + * Embed a document's existing chunks with new metadata. Returns a `write()` step that + * replaces the stored vectors; call it inside the caller's transaction. + */ +async function embedExistingChunks( + db: Database.Database, + provider: EmbeddingProvider, + documentId: string, + meta: EmbeddingMeta, +): Promise<{ write: () => void }> { + const rows = db + .prepare("SELECT id, content FROM chunks WHERE document_id = ? ORDER BY chunk_index") + .all(documentId) as Array<{ id: string; content: string }>; + const embeddings = await embedChunks( + provider, + rows.map((r) => r.content), + meta, + ); + const writer = createChunkWriter(db, provider); + return { + write: (): void => { + rows.forEach((row, i) => writer.replaceEmbedding(row.id, embeddings[i] ?? [])); + }, + }; +} diff --git a/src/core/indexing.ts b/src/core/indexing.ts index 0d660b16..acb264cf 100644 --- a/src/core/indexing.ts +++ b/src/core/indexing.ts @@ -374,15 +374,110 @@ function handleTitleLengthDedup( ); } +/** Document fields that are folded into every chunk's embedding text. */ +export interface EmbeddingMeta { + title?: string | null | undefined; + library?: string | null | undefined; + version?: string | null | undefined; +} + /** Build the metadata prefix for embedding enrichment. */ -function buildMetaPrefix(input: IndexDocumentInput): string { +function buildMetaPrefix(meta: EmbeddingMeta): string { const parts: string[] = []; - if (input.title) parts.push(input.title); - if (input.library) parts.push(`Library: ${input.library}`); - if (input.version) parts.push(`Version: ${input.version}`); + if (meta.title) parts.push(meta.title); + if (meta.library) parts.push(`Library: ${meta.library}`); + if (meta.version) parts.push(`Version: ${meta.version}`); return parts.length > 0 ? parts.join(" | ") + "\n\n" : ""; } +/** + * The text that is embedded for a chunk: a title/library/version prefix plus the chunk. + * Every write path (index, update, reindex, pack install) must embed this text so that + * a chunk's vector does not depend on how its document was written. + */ +export function buildEmbeddingText(chunk: string, meta: EmbeddingMeta): string { + return buildMetaPrefix(meta) + chunk; +} + +/** True when two documents' metadata would produce different embedding text. */ +export function embeddingMetaChanged(a: EmbeddingMeta, b: EmbeddingMeta): boolean { + return buildMetaPrefix(a) !== buildMetaPrefix(b); +} + +/** Split document content into chunks, using windowed chunking for very large documents. */ +export function splitIntoChunks(content: string): string[] { + return content.length > STREAMING_THRESHOLD + ? chunkContentStreaming(content) + : chunkContent(content); +} + +/** Embed a document's chunks using {@link buildEmbeddingText}. */ +export function embedChunks( + provider: EmbeddingProvider, + chunks: string[], + meta: EmbeddingMeta, +): Promise<number[][]> { + return provider.embedBatch(chunks.map((c) => buildEmbeddingText(c, meta))); +} + +function isNoSuchTableError(err: unknown): boolean { + const message = err instanceof Error ? err.message : String(err); + return message.includes("no such table"); +} + +/** Writes chunk rows and vectors the same way for every write path. */ +export interface ChunkWriter { + /** Insert chunk rows for a document with their embeddings (matched by array index). */ + insertChunks(documentId: string, chunks: string[], embeddings: ReadonlyArray<number[]>): void; + /** Replace the stored embedding (and embedding metadata row) of an existing chunk. */ + replaceEmbedding(chunkId: string, embedding: number[]): void; +} + +/** + * Create a {@link ChunkWriter}. Writes are synchronous; callers wrap them in a transaction. + * When the vector table does not exist (sqlite-vec not loaded), vectors are skipped. + */ +export function createChunkWriter( + db: Database.Database, + provider: Pick<EmbeddingProvider, "name">, +): ChunkWriter { + const log = getLogger(); + const insertChunk = db.prepare( + "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", + ); + const insertMeta = tryPrepareMetaInsert(db); + let insertVec: Database.Statement | undefined; + let deleteVec: Database.Statement | undefined; + + const writeVector = (chunkId: string, embedding: number[], replace: boolean): void => { + try { + if (replace) { + deleteVec ??= db.prepare("DELETE FROM chunk_embeddings WHERE chunk_id = ?"); + deleteVec.run(chunkId); + } + insertVec ??= db.prepare("INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)"); + insertVec.run(chunkId, Buffer.from(new Float32Array(embedding).buffer)); + insertMeta?.run(chunkId, provider.name, "unknown"); + } catch (err) { + if (!isNoSuchTableError(err)) throw err; + log.debug({ chunkId }, "Skipped vector insertion (sqlite-vec not loaded)"); + } + }; + + return { + insertChunks(documentId, chunks, embeddings): void { + for (let i = 0; i < chunks.length; i++) { + const chunkId = randomUUID(); + insertChunk.run(chunkId, documentId, chunks[i] ?? "", i); + writeVector(chunkId, embeddings[i] ?? [], false); + } + }, + replaceEmbedding(chunkId, embedding): void { + writeVector(chunkId, embedding, true); + }, + }; +} + /** Try to prepare an insert statement for chunk_embedding_metadata if the table exists. */ function tryPrepareMetaInsert( db: Database.Database, @@ -431,33 +526,20 @@ export async function indexDocument( if (titleResult) return titleResult; const docId = randomUUID(); - let chunks: string[]; - if (input.preChunked && input.preChunked.length > 0) { - chunks = input.preChunked; - } else { - const useStreaming = input.content.length > STREAMING_THRESHOLD; - chunks = useStreaming ? chunkContentStreaming(input.content) : chunkContent(input.content); - } + const chunks = + input.preChunked && input.preChunked.length > 0 + ? input.preChunked + : splitIntoChunks(input.content); log.info({ docId, title: input.title, chunkCount: chunks.length }, "Indexing document"); - const metaPrefix = buildMetaPrefix(input); - const textsForEmbedding = chunks.map((c) => metaPrefix + c); - const embeddings = await provider.embedBatch(textsForEmbedding); + const embeddings = await embedChunks(provider, chunks, input); const insertDoc = db.prepare(` INSERT INTO documents (id, source_type, library, version, topic_id, title, content, url, submitted_by, content_hash, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); - const insertChunk = db.prepare(` - INSERT INTO chunks (id, document_id, content, chunk_index) - VALUES (?, ?, ?, ?) - `); - const insertEmbedding = db.prepare(` - INSERT INTO chunk_embeddings (chunk_id, embedding) - VALUES (?, ?) - `); - const insertMeta = tryPrepareMetaInsert(db); + const writer = createChunkWriter(db, provider); const transaction = db.transaction(() => { insertDoc.run( @@ -473,32 +555,24 @@ export async function indexDocument( contentHash, input.expiresAt ?? null, ); - - for (let i = 0; i < chunks.length; i++) { - const chunkId = randomUUID(); - insertChunk.run(chunkId, docId, chunks[i] ?? "", i); - try { - const vecBuffer = Buffer.from(new Float32Array(embeddings[i] ?? []).buffer); - insertEmbedding.run(chunkId, vecBuffer); - insertMeta?.run(chunkId, provider.name, "unknown"); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - if (!message.includes("no such table")) throw err; - log.debug({ chunkId }, "Skipped vector insertion (sqlite-vec not loaded)"); - } - } + writer.insertChunks(docId, chunks, embeddings); }); transaction(); log.info({ docId, chunkCount: chunks.length }, "Document indexed successfully"); + storeDocumentLinks(db, docId, input.content); + + return { id: docId, chunkCount: chunks.length }; +} + +/** Extract markdown/wiki links from content and store them; failures are logged, not thrown. */ +export function storeDocumentLinks(db: Database.Database, docId: string, content: string): void { try { - extractAndStoreDocumentLinks(db, docId, input.content); + extractAndStoreDocumentLinks(db, docId, content); } catch (err) { - log.warn({ err, docId }, "Failed to extract document links"); + getLogger().warn({ err, docId }, "Failed to extract document links"); } - - return { id: docId, chunkCount: chunks.length }; } export interface IndexFileOptions { diff --git a/src/core/packs.ts b/src/core/packs.ts index aea8d5af..dbb0dc1c 100644 --- a/src/core/packs.ts +++ b/src/core/packs.ts @@ -14,7 +14,12 @@ import { gzipSync, gunzipSync } from "node:zlib"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { ValidationError, FetchError } from "../errors.js"; import { getLogger } from "../logger.js"; -import { chunkContent, chunkContentStreaming, STREAMING_THRESHOLD } from "./indexing.js"; +import { + buildEmbeddingText, + createChunkWriter, + splitIntoChunks, + type ChunkWriter, +} from "./indexing.js"; import { deleteChunkEmbeddings } from "./documents.js"; import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; import { suggestTagsFromText } from "./tags.js"; @@ -341,27 +346,28 @@ type DocChunkInfo = { }; type ResolvedBatch = { docInfos: DocChunkInfo[]; - allChunks: string[]; + /** Text to embed for every chunk in the batch, in docInfos/chunk order. */ + embeddingTexts: string[]; }; /** Chunk a batch's documents on demand, right before embedding. */ function resolveBatch(batchDocs: PackDocument[]): ResolvedBatch { const docInfos: DocChunkInfo[] = []; - const allChunks: string[] = []; + const embeddingTexts: string[] = []; for (const doc of batchDocs) { const contentHash = createHash("sha256").update(doc.content).digest("hex"); - const useStreaming = doc.content.length > STREAMING_THRESHOLD; - const chunks = useStreaming ? chunkContentStreaming(doc.content) : chunkContent(doc.content); + const chunks = splitIntoChunks(doc.content); docInfos.push({ doc, docId: randomUUID(), contentHash, chunks, - chunkOffset: allChunks.length, + chunkOffset: embeddingTexts.length, }); - allChunks.push(...chunks); + // Pack documents are stored with a title only (no library/version). + embeddingTexts.push(...chunks.map((c) => buildEmbeddingText(c, { title: doc.title }))); } - return { docInfos, allChunks }; + return { docInfos, embeddingTexts }; } /** Insert a single resolved batch into the database. Returns number of documents installed. */ @@ -371,10 +377,8 @@ function insertBatchIntoDb( embeddings: number[][], packName: string, insertDoc: Database.Statement, - insertChunk: Database.Statement, - insertEmbedding: Database.Statement, + writer: ChunkWriter, ): number { - const log = getLogger(); const doInsert = db.transaction(() => { for (const info of batch.docInfos) { insertDoc.run( @@ -387,21 +391,11 @@ function insertBatchIntoDb( info.contentHash, packName, ); - for (let j = 0; j < info.chunks.length; j++) { - const chunkId = randomUUID(); - const chunkText = info.chunks[j] ?? ""; - const embedding = embeddings[info.chunkOffset + j] ?? []; - insertChunk.run(chunkId, info.docId, chunkText, j); - try { - const vecBuffer = Buffer.from(new Float32Array(embedding).buffer); - insertEmbedding.run(chunkId, vecBuffer); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - if (!message.includes("no such table")) { - log.warn({ chunkId, err }, "Failed to insert vector embedding"); - } - } - } + writer.insertChunks( + info.docId, + info.chunks, + embeddings.slice(info.chunkOffset, info.chunkOffset + info.chunks.length), + ); } }); doInsert(); @@ -514,14 +508,7 @@ export async function installPack( INSERT INTO documents (id, source_type, title, content, url, submitted_by, content_hash, pack_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?) `); - const insertChunk = db.prepare(` - INSERT INTO chunks (id, document_id, content, chunk_index) - VALUES (?, ?, ?, ?) - `); - const insertEmbedding = db.prepare(` - INSERT INTO chunk_embeddings (chunk_id, embedding) - VALUES (?, ?) - `); + const writer = createChunkWriter(db, provider); type BatchData = { batchDocs: PackDocument[] }; const batches: BatchData[] = []; @@ -549,15 +536,7 @@ export async function installPack( errors += batch.docInfos.length; } else { try { - installed += insertBatchIntoDb( - db, - batch, - embeddings, - pack.name, - insertDoc, - insertChunk, - insertEmbedding, - ); + installed += insertBatchIntoDb(db, batch, embeddings, pack.name, insertDoc, writer); } catch (err) { log.warn( { err, pack: pack.name, batchIndex: i }, @@ -581,7 +560,9 @@ export async function installPack( const resolved = resolveBatch(batches[i]!.batchDocs); try { const embeddings = - resolved.allChunks.length > 0 ? await provider.embedBatch(resolved.allChunks) : []; + resolved.embeddingTexts.length > 0 + ? await provider.embedBatch(resolved.embeddingTexts) + : []; embedResults[i] = { resolved, embeddings, success: true }; } catch (err) { log.warn( diff --git a/src/core/reindex.ts b/src/core/reindex.ts index 199227af..88cb12c5 100644 --- a/src/core/reindex.ts +++ b/src/core/reindex.ts @@ -3,6 +3,7 @@ import type { EmbeddingProvider } from "../providers/embedding.js"; import { DatabaseError } from "../errors.js"; import { getLogger } from "../logger.js"; import { createVectorTable } from "../db/schema.js"; +import { buildEmbeddingText, createChunkWriter } from "./indexing.js"; export interface ReindexOptions { /** Only reindex chunks belonging to these document IDs. */ @@ -36,6 +37,10 @@ export interface ReindexResult { interface ChunkRow { id: string; content: string; + /** Owning document's metadata, used in the embedding text (see buildEmbeddingText). */ + title?: string | null; + library?: string | null; + version?: string | null; } /** @@ -70,8 +75,7 @@ export async function reindex( log.warn("Could not ensure vector table — continuing anyway"); } - const deleteStmt = db.prepare("DELETE FROM chunk_embeddings WHERE chunk_id = ?"); - const insertStmt = db.prepare("INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)"); + const writer = createChunkWriter(db, provider); let completed = 0; let failed = 0; @@ -80,7 +84,7 @@ export async function reindex( for (let i = 0; i < total; i += batchSize) { options.signal?.throwIfAborted(); const batch = chunks.slice(i, i + batchSize); - const texts = batch.map((c) => c.content); + const texts = batch.map((c) => buildEmbeddingText(c.content, c)); const ids = batch.map((c) => c.id); try { @@ -98,9 +102,7 @@ export async function reindex( continue; } try { - deleteStmt.run(chunkId); - const vecBuffer = Buffer.from(new Float32Array(embedding).buffer); - insertStmt.run(chunkId, vecBuffer); + writer.replaceEmbedding(chunkId, embedding); batchSucceeded++; } catch (err) { log.warn({ chunkId, err }, "Failed to update embedding for chunk"); @@ -156,7 +158,7 @@ function queryChunks(db: Database.Database, options: ReindexOptions): ChunkRow[] const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; const sql = ` - SELECT c.id, c.content + SELECT c.id, c.content, d.title, d.library, d.version FROM chunks c JOIN documents d ON c.document_id = d.id ${where} diff --git a/tests/unit/embedding-text-consistency.test.ts b/tests/unit/embedding-text-consistency.test.ts new file mode 100644 index 00000000..9417319c --- /dev/null +++ b/tests/unit/embedding-text-consistency.test.ts @@ -0,0 +1,177 @@ +import { describe, it, expect, beforeAll, afterEach } from "vitest"; +import { writeFileSync, mkdtempSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { initLogger } from "../../src/logger.js"; +import { + buildEmbeddingText, + indexDocument, + type IndexDocumentInput, +} from "../../src/core/indexing.js"; +import { updateDocument } from "../../src/core/documents.js"; +import { reindex } from "../../src/core/reindex.js"; +import { installPack } from "../../src/core/packs.js"; +import { getDocumentLinks } from "../../src/core/links.js"; + +/** Mock provider that records every text it is asked to embed. */ +class CapturingProvider extends MockEmbeddingProvider { + inputs: string[] = []; + + override embedBatch(texts: string[]): Promise<number[][]> { + this.inputs.push(...texts); + return super.embedBatch(texts); + } +} + +const CONTENT = "# Setup\n\nInstall the package.\n\n## Usage\n\nCall the function."; +const DOC: IndexDocumentInput = { + title: "Widget Guide", + content: CONTENT, + sourceType: "library", + library: "widgets", + version: "2.1.0", +}; + +/** Embedding inputs and stored vectors produced by a fresh indexDocument call. */ +async function freshIndex( + input: IndexDocumentInput, +): Promise<{ inputs: string[]; vectors: Buffer[] }> { + const db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const { id } = await indexDocument(db, provider, input); + const vectors = storedVectors(db, id); + db.close(); + return { inputs: provider.inputs, vectors }; +} + +function storedVectors(db: Database.Database, docId: string): Buffer[] { + return ( + db + .prepare( + `SELECT e.embedding FROM chunks c JOIN chunk_embeddings e ON e.chunk_id = c.id + WHERE c.document_id = ? ORDER BY c.chunk_index`, + ) + .all(docId) as Array<{ embedding: Buffer }> + ).map((r) => r.embedding); +} + +describe("embedding text is the same on every write path", () => { + let db: Database.Database | undefined; + + beforeAll(() => { + initLogger("silent"); + }); + + afterEach(() => { + db?.close(); + db = undefined; + }); + + it("buildEmbeddingText prefixes title, library, and version", () => { + expect(buildEmbeddingText("body", { title: "T", library: "L", version: "1" })).toBe( + "T | Library: L | Version: 1\n\nbody", + ); + expect(buildEmbeddingText("body", { title: "T", library: null, version: undefined })).toBe( + "T\n\nbody", + ); + expect(buildEmbeddingText("body", {})).toBe("body"); + }); + + it("reindex embeds the same text and stores the same vectors as a fresh index", async () => { + const fresh = await freshIndex(DOC); + expect(fresh.inputs[0]).toMatch(/^Widget Guide \| Library: widgets \| Version: 2\.1\.0\n\n/); + + db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const { id } = await indexDocument(db, provider, DOC); + provider.inputs = []; + await reindex(db, provider, { documentIds: [id] }); + + expect(provider.inputs).toEqual(fresh.inputs); + expect(storedVectors(db, id)).toEqual(fresh.vectors); + }); + + it("updateDocument with new content embeds the same text as a fresh index", async () => { + const fresh = await freshIndex(DOC); + + db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const { id } = await indexDocument(db, provider, { ...DOC, content: "Old content." }); + provider.inputs = []; + await updateDocument(db, provider, id, { content: CONTENT }); + + expect(provider.inputs).toEqual(fresh.inputs); + expect(storedVectors(db, id)).toEqual(fresh.vectors); + }); + + it("updateDocument with new title/library/version re-embeds to match a fresh index", async () => { + const renamed = { ...DOC, title: "Widget Handbook", library: "gadgets", version: "3.0.0" }; + const fresh = await freshIndex(renamed); + + db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const { id } = await indexDocument(db, provider, DOC); + provider.inputs = []; + await updateDocument(db, provider, id, { + title: renamed.title, + metadata: { library: renamed.library, version: renamed.version }, + }); + + expect(provider.inputs).toEqual(fresh.inputs); + expect(storedVectors(db, id)).toEqual(fresh.vectors); + }); + + it("installPack embeds the same text as a fresh index of the same document", async () => { + const fresh = await freshIndex({ title: DOC.title, content: CONTENT, sourceType: "library" }); + + db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const dir = mkdtempSync(join(tmpdir(), "libscope-embed-text-")); + const packPath = join(dir, "pack.json"); + writeFileSync( + packPath, + JSON.stringify({ + name: "widgets-pack", + version: "1.0.0", + description: "test", + documents: [{ title: DOC.title, content: CONTENT, source: "" }], + metadata: { author: "a", license: "MIT", createdAt: "2024-01-01" }, + }), + ); + await installPack(db, provider, packPath); + const { id } = db + .prepare("SELECT id FROM documents WHERE pack_name = ?") + .get("widgets-pack") as { + id: string; + }; + + expect(provider.inputs).toEqual(fresh.inputs); + expect(storedVectors(db, id)).toEqual(fresh.vectors); + }); + + it("updateDocument with new content extracts document links like indexDocument", async () => { + db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const target = await indexDocument(db, provider, { + title: "Target Doc", + content: "Target body.", + sourceType: "manual", + }); + const { id } = await indexDocument(db, provider, { + title: "Source Doc", + content: "No links yet.", + sourceType: "manual", + }); + expect(getDocumentLinks(db, id).outgoing).toHaveLength(0); + + await updateDocument(db, provider, id, { content: "See [[Target Doc]] for details." }); + + const outgoing = getDocumentLinks(db, id).outgoing; + expect(outgoing).toHaveLength(1); + expect(outgoing[0]!.targetId).toBe(target.id); + expect(outgoing[0]!.linkType).toBe("references"); + }); +}); diff --git a/tests/unit/update-document.test.ts b/tests/unit/update-document.test.ts index 61f9e7e2..4c1628de 100644 --- a/tests/unit/update-document.test.ts +++ b/tests/unit/update-document.test.ts @@ -31,13 +31,26 @@ describe("updateDocument", () => { docId = result.id; }); - it("should update title without re-chunking", async () => { + it("should update title without re-chunking, re-embedding the existing chunks", async () => { + const chunkIdsBefore = db + .prepare("SELECT id FROM chunks WHERE document_id = ? ORDER BY chunk_index") + .all(docId); const batchCallsBefore = provider.embedBatchCallCount; const input: UpdateDocumentInput = { title: "New Title" }; const updated: Document = await updateDocument(db, provider, docId, input); expect(updated.title).toBe("New Title"); expect(updated.content).toBe(baseInput.content); + // The title is part of the embedding text, so vectors are refreshed in place. + expect(provider.embedBatchCallCount).toBe(batchCallsBefore + 1); + expect( + db.prepare("SELECT id FROM chunks WHERE document_id = ? ORDER BY chunk_index").all(docId), + ).toEqual(chunkIdsBefore); + }); + + it("should not re-embed when only url or topic changes", async () => { + const batchCallsBefore = provider.embedBatchCallCount; + await updateDocument(db, provider, docId, { metadata: { url: "https://other.example" } }); expect(provider.embedBatchCallCount).toBe(batchCallsBefore); }); From 6871c428e0b4ca6aff2e334970383c99a7c45951 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:21:51 +0000 Subject: [PATCH 19/83] fix(core): parse import-batch and watch files through the parser registry import-batch (core/batch.ts) and watch (core/watcher.ts plus the CLI's initial scan) read every file as raw UTF-8, so PDF/DOCX/EPUB/PPTX files were indexed as binary garbage. - batchImport now indexes through indexFile (same parser path as `add` and `import`) and honours its sourceType option, which was ignored. Unsupported extensions are skipped with a warning and counted in a new `skipped` field instead of failing. - FileWatcher and the CLI watch scan parse files with the matching parser via readWatchedFile(); .rst stays indexed as plain text, and unsupported extensions are skipped with a warning. - Default extension sets for import, import-batch (--filter) and watch now come from getSupportedExtensions() instead of hand-written lists. - indexFile accepts an optional sourceType override. --- docs/reference/cli.md | 28 ++--- src/cli/index.ts | 42 +++++-- src/core/batch.ts | 33 ++++-- src/core/indexing.ts | 10 +- src/core/watcher.ts | 29 ++++- tests/integration/parsed-file-import.test.ts | 116 +++++++++++++++++++ tests/unit/batch.test.ts | 80 ++++++++++--- tests/unit/watcher.test.ts | 35 +++++- 8 files changed, 314 insertions(+), 59 deletions(-) create mode 100644 tests/integration/parsed-file-import.test.ts diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 94e47ccb..a6d04e17 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -37,27 +37,27 @@ Recursively import files from a directory. libscope import ./wiki/ --topic internal --extensions .md,.mdx,.txt ``` -| Option | Description | -| --------------------- | ------------------------------------------------ | -| `--library <name>` | Assign all docs to a library | -| `--topic <name>` | Assign all docs to a topic | -| `--lib-version <ver>` | Library version tag | -| `--extensions <exts>` | Comma-separated file extensions (default: `.md`) | +| Option | Description | +| --------------------- | ---------------------------------------------------------------- | +| `--library <name>` | Assign all docs to a library | +| `--topic <name>` | Assign all docs to a topic | +| `--lib-version <ver>` | Library version tag | +| `--extensions <exts>` | Comma-separated file extensions (default: all supported formats) | ### `libscope import-batch` -Parallel batch import with progress tracking. +Parallel batch import with progress tracking. Files are parsed by format (PDF, Word, EPUB, PowerPoint, HTML, and so on). Files with an unsupported extension are skipped with a warning. ```bash libscope import-batch ./docs/ --concurrency 10 --filter "**/*.md" --library my-lib ``` -| Option | Description | -| --------------------- | ---------------------------- | -| `--concurrency <n>` | Number of parallel workers | -| `--filter <glob>` | Glob pattern to match files | -| `--library <name>` | Assign all docs to a library | -| `--lib-version <ver>` | Library version tag | +| Option | Description | +| --------------------- | ------------------------------------------------------------ | +| `--concurrency <n>` | Number of parallel workers | +| `--filter <glob>` | Glob pattern to match files (default: all supported formats) | +| `--library <name>` | Assign all docs to a library | +| `--lib-version <ver>` | Library version tag | ### `libscope search` @@ -355,7 +355,7 @@ Creates a git repo with the canonical registry folder structure. See the [Regist | Command | Description | | ------------------------------------- | --------------------------------------- | -| `libscope watch <directory>` | Watch for file changes and auto-reindex | +| `libscope watch <directory>` | Watch for file changes and auto-reindex (all supported formats, plus `.rst`) | | `libscope reindex` | Re-embed chunks with current provider | | `libscope dedupe` | Scan for duplicate documents | | `libscope export <outputPath>` | Export knowledge base to JSON | diff --git a/src/cli/index.ts b/src/cli/index.ts index d71a8f1c..0bbd021b 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -15,7 +15,7 @@ import { createLink, getDocumentLinks, deleteLink, getPrerequisiteChain } from " import type { LinkType } from "../core/links.js"; import { getVersionHistory, rollbackToVersion } from "../core/versioning.js"; import { initLogger, type LogLevel } from "../logger.js"; -import { readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; +import { readdirSync, realpathSync, statSync } from "node:fs"; import { join, extname, basename } from "node:path"; import { fileURLToPath } from "node:url"; import { fetchAndConvert } from "../core/url-fetcher.js"; @@ -64,7 +64,7 @@ import { import { spawnSync } from "node:child_process"; import { createRequire } from "node:module"; -import { FileWatcher, DEFAULT_WATCH_EXTENSIONS } from "../core/watcher.js"; +import { FileWatcher, DEFAULT_WATCH_EXTENSIONS, readWatchedFile } from "../core/watcher.js"; import { indexRepository, parseRepoUrl } from "../core/repo.js"; import { authenticateDeviceCode, @@ -656,7 +656,7 @@ program .option( "--extensions <exts>", "Comma-separated file extensions to include", - ".md,.mdx,.txt,.pdf,.docx,.csv,.yaml,.yml,.json", + getSupportedExtensions().join(","), ) .option("--dedup <mode>", "Dedup mode: skip, warn, or force") .action( @@ -719,7 +719,13 @@ program .command("import-batch <directory>") .description("Batch import files with parallel processing") .option("--concurrency <n>", "Number of parallel imports", "5") - .option("--filter <glob>", "Glob pattern for file selection", "**/*.{md,mdx,txt}") + .option( + "--filter <glob>", + "Glob pattern for file selection", + `**/*.{${getSupportedExtensions() + .map((e) => e.slice(1)) + .join(",")}}`, + ) .option("--dry-run", "Preview files without importing") .option("--topic <topicId>", "Assign all to a topic") .option("--library <name>", "Mark all as library documentation") @@ -766,19 +772,30 @@ program version: opts.libVersion, topicId: opts.topic, onProgress: (progress) => { - const done = progress.completed + progress.failed; + const done = progress.completed + progress.failed + progress.skipped; const file = progress.currentFile ?? ""; console.log(` [${done}/${progress.total}] ${file}`); }, }); const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log(`\nDone: ${result.completed} indexed, ${result.failed} failed in ${elapsed}s`); + console.log( + `\nDone: ${result.completed} indexed, ${result.failed} failed, ${result.skipped} skipped in ${elapsed}s`, + ); + + if (result.skipped > 0) { + console.warn("\nSkipped files (unsupported format):"); + for (const r of result.results) { + if (r.skipped) { + console.warn(` - ${r.file}`); + } + } + } if (result.failed > 0) { console.log("\nFailed files:"); for (const r of result.results) { - if (!r.success) { + if (!r.success && !r.skipped) { console.log(` ✗ ${r.file}: ${r.error}`); } } @@ -1655,7 +1672,12 @@ program let skipped = 0; for (const file of files) { try { - const content = readFileSync(file, "utf-8"); + const content = await readWatchedFile(file); + if (content === null) { + console.warn(` ⚠ Skipping ${file}: unsupported file format`); + skipped++; + continue; + } const title = basename(file).replace(/\.[^.]+$/, ""); const result = await indexDocument(db, provider, { title, @@ -1672,7 +1694,9 @@ program skipped++; } } - console.log(`Initial scan: ${indexed} indexed, ${skipped} skipped (unchanged or failed)`); + console.log( + `Initial scan: ${indexed} indexed, ${skipped} skipped (unchanged, unsupported, or failed)`, + ); console.log(`\nWatching for changes (extensions: ${extensions.join(", ")})...\n`); const watcher = new FileWatcher(db, provider, { diff --git a/src/core/batch.ts b/src/core/batch.ts index 409f64c1..fdb87054 100644 --- a/src/core/batch.ts +++ b/src/core/batch.ts @@ -1,8 +1,8 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { indexDocument } from "./indexing.js"; -import { readFileSync } from "node:fs"; -import { basename } from "node:path"; +import { indexFile } from "./indexing.js"; +import { getParserForFile } from "./parsers/index.js"; +import { basename, extname } from "node:path"; import { getLogger } from "../logger.js"; export interface BatchImportOptions { @@ -18,12 +18,16 @@ export interface BatchProgress { total: number; completed: number; failed: number; + /** Files skipped because no parser handles their extension. */ + skipped: number; currentFile?: string; } export interface BatchFileResult { file: string; success: boolean; + /** True when the file was not imported because its extension is unsupported. */ + skipped?: boolean; chunkCount?: number; documentId?: string; error?: string; @@ -33,6 +37,7 @@ export interface BatchImportResult { total: number; completed: number; failed: number; + skipped: number; results: BatchFileResult[]; } @@ -48,7 +53,7 @@ export async function batchImport( ): Promise<BatchImportResult> { const logger = getLogger(); const concurrency = options.concurrency ?? 5; - const progress: BatchProgress = { total: files.length, completed: 0, failed: 0 }; + const progress: BatchProgress = { total: files.length, completed: 0, failed: 0, skipped: 0 }; const results: BatchFileResult[] = []; logger.debug(`Starting batch import: ${files.length} files, concurrency ${concurrency}`); @@ -75,6 +80,8 @@ export async function batchImport( results[currentIndex] = result; if (result.success) { progress.completed++; + } else if (result.skipped) { + progress.skipped++; } else { progress.failed++; } @@ -91,7 +98,7 @@ export async function batchImport( }) .finally(() => { activeCount--; - if (progress.completed + progress.failed === files.length) { + if (progress.completed + progress.failed + progress.skipped === files.length) { resolve(); } else { runNext(); @@ -107,6 +114,7 @@ export async function batchImport( total: files.length, completed: progress.completed, failed: progress.failed, + skipped: progress.skipped, results, }; } @@ -117,16 +125,17 @@ async function processFile( file: string, options: BatchImportOptions, ): Promise<BatchFileResult> { - const content = readFileSync(file, "utf-8"); - const title = basename(file).replace(/\.[^.]+$/, ""); + if (!getParserForFile(file)) { + const error = `Unsupported file type "${extname(file) || basename(file)}"`; + getLogger().warn({ file }, `Skipping ${file}: ${error}`); + return { file, success: false, skipped: true, error }; + } - const result = await indexDocument(db, provider, { - title, - content, - sourceType: options.library ? "library" : options.topicId ? "topic" : "manual", + const result = await indexFile(db, provider, file, { + sourceType: options.sourceType, library: options.library, version: options.version, - topicId: options.topicId, + topic: options.topicId, }); return { diff --git a/src/core/indexing.ts b/src/core/indexing.ts index 0d660b16..02911795 100644 --- a/src/core/indexing.ts +++ b/src/core/indexing.ts @@ -508,6 +508,14 @@ export interface IndexFileOptions { title?: string | undefined; format?: string | undefined; dedup?: "skip" | "warn" | "force" | undefined; + /** Overrides the source type derived from `library`/`topic`. */ + sourceType?: IndexDocumentInput["sourceType"] | undefined; +} + +function fileSourceType(options: IndexFileOptions): IndexDocumentInput["sourceType"] { + if (options.sourceType) return options.sourceType; + if (options.library) return "library"; + return options.topic ? "topic" : "manual"; } /** @@ -545,7 +553,7 @@ export async function indexFile( return indexDocument(db, provider, { title, content, - sourceType: options.library ? "library" : options.topic ? "topic" : "manual", + sourceType: fileSourceType(options), library: options.library, version: options.version, topicId: options.topic, diff --git a/src/core/watcher.ts b/src/core/watcher.ts index b2d50686..d1363dd2 100644 --- a/src/core/watcher.ts +++ b/src/core/watcher.ts @@ -5,9 +5,30 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { indexDocument } from "./indexing.js"; import { deleteChunkEmbeddings } from "./documents.js"; +import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; import { createChildLogger } from "../logger.js"; -export const DEFAULT_WATCH_EXTENSIONS = [".md", ".mdx", ".txt", ".rst"]; +/** Extensions without a registered parser that the watcher indexes as plain UTF-8 text. */ +const PLAIN_TEXT_WATCH_EXTENSIONS = [".rst"]; + +/** Every extension the parser registry supports, plus the plain-text extras. */ +export const DEFAULT_WATCH_EXTENSIONS = [ + ...getSupportedExtensions(), + ...PLAIN_TEXT_WATCH_EXTENSIONS, +]; + +/** + * Read a watched file and convert it to indexable text with the parser registry. + * Returns null when no parser handles the file's extension. + */ +export async function readWatchedFile(filePath: string): Promise<string | null> { + const parser = getParserForFile(filePath); + if (parser) return parser.parse(readFileSync(filePath)); + if (PLAIN_TEXT_WATCH_EXTENSIONS.includes(extname(filePath).toLowerCase())) { + return readFileSync(filePath, "utf-8"); + } + return null; +} export interface WatchOptions { directory: string; @@ -96,7 +117,11 @@ export class FileWatcher { if (!stat.isFile()) return; - const content = readFileSync(fullPath, "utf-8"); + const content = await readWatchedFile(fullPath); + if (content === null) { + this.log.warn({ path: fullPath }, "No parser for file extension, skipping"); + return; + } const contentHash = createHash("sha256").update(content).digest("hex"); const existing = this.db diff --git a/tests/integration/parsed-file-import.test.ts b/tests/integration/parsed-file-import.test.ts new file mode 100644 index 00000000..dee798a6 --- /dev/null +++ b/tests/integration/parsed-file-import.test.ts @@ -0,0 +1,116 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import PizZip from "pizzip"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { batchImport } from "../../src/core/batch.js"; +import { FileWatcher, readWatchedFile } from "../../src/core/watcher.js"; +import { initLogger } from "../../src/logger.js"; + +const DOCX_TEXT = "Quarterly revenue grew in the northern region"; + +/** Build a minimal but valid .docx (Office Open XML) containing one paragraph. */ +function buildDocx(text: string): Buffer { + const zip = new PizZip(); + zip.file( + "[Content_Types].xml", + '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' + + '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">' + + '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>' + + '<Default Extension="xml" ContentType="application/xml"/>' + + '<Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>' + + "</Types>", + ); + zip.file( + "_rels/.rels", + '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' + + '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' + + '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>' + + "</Relationships>", + ); + zip.file( + "word/document.xml", + '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' + + '<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">' + + `<w:body><w:p><w:r><w:t>${text}</w:t></w:r></w:p></w:body>` + + "</w:document>", + ); + return zip.generate({ type: "nodebuffer" }); +} + +function documentContent(db: Database.Database, id: string): { content: string; source: string } { + return db + .prepare("SELECT content, source_type AS source FROM documents WHERE id = ?") + .get(id) as { content: string; source: string }; +} + +describe("integration: import-batch and watch use the parser registry", () => { + let db: Database.Database; + let provider: MockEmbeddingProvider; + let dir: string; + let docxPath: string; + + beforeEach(() => { + initLogger("silent"); + db = createTestDbWithVec(); + provider = new MockEmbeddingProvider(); + dir = mkdtempSync(join(tmpdir(), "libscope-parsed-import-")); + docxPath = join(dir, "report.docx"); + writeFileSync(docxPath, buildDocx(DOCX_TEXT)); + }); + + afterEach(() => { + db.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + it("batchImport indexes parsed DOCX text and skips unsupported files", async () => { + const pngPath = join(dir, "logo.png"); + writeFileSync(pngPath, Buffer.from([0x89, 0x50, 0x4e, 0x47])); + + const result = await batchImport(db, provider, [docxPath, pngPath], { + sourceType: "model-generated", + }); + + expect(result.completed).toBe(1); + expect(result.failed).toBe(0); + expect(result.skipped).toBe(1); + + const doc = documentContent(db, result.results[0]!.documentId!); + expect(doc.content).toContain(DOCX_TEXT); + expect(doc.content).not.toContain("word/document.xml"); + expect(doc.source).toBe("model-generated"); + }); + + it("readWatchedFile parses binary formats and reads .rst as text", async () => { + const rstPath = join(dir, "guide.rst"); + writeFileSync(rstPath, "Title\n=====\n\nSome reStructuredText."); + + expect((await readWatchedFile(docxPath))?.trim()).toBe(DOCX_TEXT); + expect(await readWatchedFile(rstPath)).toContain("Some reStructuredText."); + expect(await readWatchedFile(join(dir, "logo.png"))).toBeNull(); + }); + + it("FileWatcher indexes the parsed DOCX text for a changed file", async () => { + const indexed: string[] = []; + const watcher = new FileWatcher(db, provider, { + directory: dir, + onIndex: (path): void => { + indexed.push(path); + }, + }); + + // Drive the change handler directly instead of waiting on fs.watch events. + await (watcher as unknown as { processFile(p: string): Promise<void> }).processFile(docxPath); + + expect(indexed).toEqual([docxPath]); + const row = db.prepare("SELECT content FROM documents WHERE url = ?").get(docxPath) as { + content: string; + }; + expect(row.content).toContain(DOCX_TEXT); + expect(row.content).not.toContain("word/document.xml"); + }); +}); diff --git a/tests/unit/batch.test.ts b/tests/unit/batch.test.ts index cb8980bb..b45fd997 100644 --- a/tests/unit/batch.test.ts +++ b/tests/unit/batch.test.ts @@ -1,26 +1,22 @@ -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect, vi, beforeEach } from "vitest"; import { batchImport } from "../../src/core/batch.js"; import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../../src/providers/embedding.js"; // Mock indexing module vi.mock("../../src/core/indexing.js", () => ({ - indexDocument: vi.fn().mockResolvedValue({ id: "doc-1", chunkCount: 3 }), -})); - -// Mock fs -vi.mock("node:fs", () => ({ - readFileSync: vi.fn().mockReturnValue("# Test content\nSome text."), + indexFile: vi.fn().mockResolvedValue({ id: "doc-1", chunkCount: 3 }), })); // Mock logger +const mockLogger = vi.hoisted(() => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +})); vi.mock("../../src/logger.js", () => ({ - getLogger: (): Record<string, ReturnType<typeof vi.fn>> => ({ - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - }), + getLogger: (): typeof mockLogger => mockLogger, })); function createMockDb(): Database.Database { @@ -37,12 +33,17 @@ function createMockProvider(): EmbeddingProvider { } describe("batchImport", () => { + beforeEach(() => { + mockLogger.warn.mockClear(); + }); + it("should return empty results for no files", async () => { const result = await batchImport(createMockDb(), createMockProvider(), []); expect(result.total).toBe(0); expect(result.completed).toBe(0); expect(result.failed).toBe(0); + expect(result.skipped).toBe(0); expect(result.results).toEqual([]); }); @@ -87,8 +88,8 @@ describe("batchImport", () => { }); it("should handle failed files gracefully", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const mockedIndex = vi.mocked(indexDocument); + const { indexFile } = await import("../../src/core/indexing.js"); + const mockedIndex = vi.mocked(indexFile); mockedIndex.mockRejectedValueOnce(new Error("bad file")); mockedIndex.mockResolvedValueOnce({ id: "doc-2", chunkCount: 1 }); @@ -107,8 +108,8 @@ describe("batchImport", () => { }); it("should pass library and topic options through", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const mockedIndex = vi.mocked(indexDocument); + const { indexFile } = await import("../../src/core/indexing.js"); + const mockedIndex = vi.mocked(indexFile); mockedIndex.mockResolvedValue({ id: "doc-3", chunkCount: 2 }); await batchImport(createMockDb(), createMockProvider(), ["test.md"], { @@ -120,15 +121,56 @@ describe("batchImport", () => { expect(mockedIndex).toHaveBeenCalledWith( expect.anything(), expect.anything(), + "test.md", expect.objectContaining({ library: "react", version: "18.0", - topicId: "frontend", - sourceType: "library", + topic: "frontend", }), ); }); + it("should pass the sourceType option through", async () => { + const { indexFile } = await import("../../src/core/indexing.js"); + const mockedIndex = vi.mocked(indexFile); + mockedIndex.mockClear(); + + await batchImport(createMockDb(), createMockProvider(), ["notes.md"], { + sourceType: "model-generated", + }); + + expect(mockedIndex).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + "notes.md", + expect.objectContaining({ sourceType: "model-generated" }), + ); + }); + + it("should skip unsupported file types with a warning", async () => { + const { indexFile } = await import("../../src/core/indexing.js"); + const mockedIndex = vi.mocked(indexFile); + mockedIndex.mockClear(); + + const progressSkipped: number[] = []; + const result = await batchImport(createMockDb(), createMockProvider(), ["logo.png", "a.md"], { + concurrency: 1, + onProgress: (progress) => progressSkipped.push(progress.skipped), + }); + + expect(result.total).toBe(2); + expect(result.completed).toBe(1); + expect(result.failed).toBe(0); + expect(result.skipped).toBe(1); + expect(result.results[0]).toMatchObject({ file: "logo.png", success: false, skipped: true }); + expect(progressSkipped.at(-1)).toBe(1); + expect(mockedIndex).toHaveBeenCalledTimes(1); + expect(mockLogger.warn).toHaveBeenCalledWith( + { file: "logo.png" }, + expect.stringContaining("Unsupported file type"), + ); + }); + it("should default concurrency to 5", async () => { const files = Array.from({ length: 8 }, (_, i) => `file${i}.md`); const result = await batchImport(createMockDb(), createMockProvider(), files); diff --git a/tests/unit/watcher.test.ts b/tests/unit/watcher.test.ts index 394be10a..d5e5b032 100644 --- a/tests/unit/watcher.test.ts +++ b/tests/unit/watcher.test.ts @@ -73,8 +73,39 @@ describe("FileWatcher", () => { vi.restoreAllMocks(); }); - it("should export default watch extensions", () => { - expect(DEFAULT_WATCH_EXTENSIONS).toEqual([".md", ".mdx", ".txt", ".rst"]); + it("should derive default watch extensions from the parser registry", () => { + expect(DEFAULT_WATCH_EXTENSIONS).toEqual( + expect.arrayContaining([".md", ".mdx", ".txt", ".rst", ".pdf", ".docx"]), + ); + }); + + it("should skip files whose extension has no parser without erroring", async () => { + const db = createMockDb(); + const provider = createMockProvider(); + const onIndex = vi.fn(); + const onError = vi.fn(); + + mockStatSync.mockReturnValue({ isFile: () => true } as import("node:fs").Stats); + + const watcher = new FileWatcher(db, provider, { + directory: "/tmp/docs", + extensions: [".rs"], + onIndex, + onError, + }); + + watcher.start(); + watchCallback("change", "main.rs"); + + vi.advanceTimersByTime(500); + await vi.advanceTimersByTimeAsync(0); + + expect(mockStatSync).toHaveBeenCalledTimes(1); + expect(mockReadFileSync).not.toHaveBeenCalled(); + expect(onIndex).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + + watcher.stop(); }); it("should filter files by extension", () => { From 0e6f8d5342635a9b1b71b5e57a4b1985c2040132 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:25:01 +0000 Subject: [PATCH 20/83] fix(cli): make import-batch work on Node 20 with a shared glob matcher import-batch called node:fs globSync, which only exists in Node 22+, so the command crashed on Node 20. Replace it with recursive readdirSync plus a shared globToRegExp helper (supports **, *, ?, {a,b}), and use the same helper for pack-create exclude patterns and the repo indexer's include/exclude globs instead of two separate copies. --- src/api/indexing/repoIndexer.ts | 20 +++++--------- src/cli/index.ts | 10 ++++--- src/core/packs.ts | 16 +++-------- src/utils/glob.ts | 47 +++++++++++++++++++++++++++++++++ tests/unit/glob.test.ts | 47 +++++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 31 deletions(-) create mode 100644 src/utils/glob.ts create mode 100644 tests/unit/glob.test.ts diff --git a/src/api/indexing/repoIndexer.ts b/src/api/indexing/repoIndexer.ts index 86449b32..7a097fb2 100644 --- a/src/api/indexing/repoIndexer.ts +++ b/src/api/indexing/repoIndexer.ts @@ -1,6 +1,7 @@ import { execFileSync } from "node:child_process"; import { readdirSync, readFileSync, rmSync, statSync, existsSync } from "node:fs"; import { join, relative, extname } from "node:path"; +import { globToRegExp, toPosixPath } from "../../utils/glob.js"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { LibScopeLite } from "../../lite/core.js"; @@ -54,20 +55,10 @@ function detectLanguage(filePath: string): string | undefined { return EXTENSION_MAP[ext]; } -/** Convert a glob-style pattern to a RegExp. Handles ** and * wildcards. */ -function globToRegex(pattern: string): RegExp { - const escaped = pattern - .replaceAll(/[.+^${}()|[\]\\]/g, String.raw`\$&`) - .replaceAll("**", "§DOUBLESTAR§") - .replaceAll("*", "[^/]*") - .replaceAll("§DOUBLESTAR§", ".*"); - return new RegExp(`^${escaped}$`); -} - /** Recursively list all files under dir, returning paths relative to dir. */ function walkFiles(dir: string, include?: string[], exclude?: string[]): string[] { - const includeRegexes = include?.map(globToRegex); - const excludeRegexes = exclude?.map(globToRegex); + const includeRegexes = include?.map(globToRegExp); + const excludeRegexes = exclude?.map(globToRegExp); const results: string[] = []; function recurse(current: string): void { @@ -85,8 +76,9 @@ function walkFiles(dir: string, include?: string[], exclude?: string[]): string[ if (entry.name.startsWith(".")) continue; recurse(full); } else if (entry.isFile()) { - if (excludeRegexes?.some((rx) => rx.test(rel))) continue; - if (includeRegexes && !includeRegexes.some((rx) => rx.test(rel))) continue; + const posixRel = toPosixPath(rel); + if (excludeRegexes?.some((rx) => rx.test(posixRel))) continue; + if (includeRegexes && !includeRegexes.some((rx) => rx.test(posixRel))) continue; results.push(rel); } } diff --git a/src/cli/index.ts b/src/cli/index.ts index 0bbd021b..75fdd960 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -17,6 +17,7 @@ import { getVersionHistory, rollbackToVersion } from "../core/versioning.js"; import { initLogger, type LogLevel } from "../logger.js"; import { readdirSync, realpathSync, statSync } from "node:fs"; import { join, extname, basename } from "node:path"; +import { globToRegExp, toPosixPath } from "../utils/glob.js"; import { fileURLToPath } from "node:url"; import { fetchAndConvert } from "../core/url-fetcher.js"; import { spiderUrl } from "../core/spider.js"; @@ -744,10 +745,11 @@ program ) => { const { db, provider } = initializeAppWithEmbedding(); try { - const { globSync } = await import("node:fs"); - const files = globSync(opts.filter, { cwd: directory }).map((f: string) => - join(directory, f), - ); + const matcher = globToRegExp(opts.filter); + const files = readdirSync(directory, { recursive: true, encoding: "utf8" }) + .filter((f) => matcher.test(toPosixPath(f))) + .map((f) => join(directory, f)) + .filter((f) => statSync(f).isFile()); files.sort(); if (files.length === 0) { diff --git a/src/core/packs.ts b/src/core/packs.ts index dbb0dc1c..73a7c1bd 100644 --- a/src/core/packs.ts +++ b/src/core/packs.ts @@ -24,6 +24,7 @@ import { deleteChunkEmbeddings } from "./documents.js"; import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; import { suggestTagsFromText } from "./tags.js"; import { fetchAndConvert } from "./url-fetcher.js"; +import { globToRegExp, toPosixPath } from "../utils/glob.js"; export interface PackDocument { title: string; @@ -709,18 +710,6 @@ export function createPack(db: Database.Database, options: CreatePackOptions): K // Create pack from filesystem / URL sources (no database required) // --------------------------------------------------------------------------- -/** Simple glob-style pattern matching (supports * and ** wildcards). */ -function matchesExcludePattern(relativePath: string, pattern: string): boolean { - // Escape regex special chars except * and ** - // prettier-ignore - const escaped = pattern - .replaceAll(/[.+^${}()|[\]\\]/g, String.raw`\$&`) - .replaceAll("**", "\0") - .replaceAll("*", "[^/]*") - .replaceAll("\0", ".*"); - return new RegExp(`^${escaped}$`).test(relativePath); -} - /** Recursively collect files from a directory. */ function collectFiles( dir: string, @@ -730,6 +719,7 @@ function collectFiles( excludePatterns: string[], ): string[] { const results: string[] = []; + const excludeRegexes = excludePatterns.map(globToRegExp); let entries: string[]; try { entries = readdirSync(dir); @@ -744,7 +734,7 @@ function collectFiles( const rel = relative(rootDir, fullPath); // Check exclude patterns - if (excludePatterns.some((p) => matchesExcludePattern(rel, p))) { + if (excludeRegexes.some((rx) => rx.test(toPosixPath(rel)))) { continue; } diff --git a/src/utils/glob.ts b/src/utils/glob.ts new file mode 100644 index 00000000..5ab7aa43 --- /dev/null +++ b/src/utils/glob.ts @@ -0,0 +1,47 @@ +const REGEX_SPECIAL = new Set([".", "+", "^", "$", "(", ")", "|", "[", "]", "\\", "{", "}"]); + +/** + * Convert a glob to an anchored RegExp that matches `/`-separated relative paths. + * + * Supports `**` (any depth; `**\/` also matches zero directories), `*` (within one + * path segment), `?` (one character), and non-nested `{a,b}` alternation. + */ +export function globToRegExp(glob: string): RegExp { + let source = ""; + let inBraces = false; + + for (let i = 0; i < glob.length; i++) { + const c = glob.charAt(i); + if (c === "*" && glob.charAt(i + 1) === "*") { + i++; + if (glob.charAt(i + 1) === "/") { + i++; + source += "(?:.*/)?"; + } else { + source += ".*"; + } + } else if (c === "*") { + source += "[^/]*"; + } else if (c === "?") { + source += "[^/]"; + } else if (c === "{" && !inBraces) { + inBraces = true; + source += "(?:"; + } else if (c === "}" && inBraces) { + inBraces = false; + source += ")"; + } else if (c === "," && inBraces) { + source += "|"; + } else { + source += REGEX_SPECIAL.has(c) ? `\\${c}` : c; + } + } + if (inBraces) source += ")"; + + return new RegExp(`^${source}$`); +} + +/** Normalize a platform path to `/` separators so it can be matched by {@link globToRegExp}. */ +export function toPosixPath(path: string): string { + return path.split("\\").join("/"); +} diff --git a/tests/unit/glob.test.ts b/tests/unit/glob.test.ts new file mode 100644 index 00000000..36a5a826 --- /dev/null +++ b/tests/unit/glob.test.ts @@ -0,0 +1,47 @@ +import { describe, it, expect } from "vitest"; +import { globToRegExp, toPosixPath } from "../../src/utils/glob.js"; + +describe("globToRegExp", () => { + it("matches files at any depth with **/ including the root", () => { + const rx = globToRegExp("**/*.md"); + expect(rx.test("readme.md")).toBe(true); + expect(rx.test("docs/guide/intro.md")).toBe(true); + expect(rx.test("docs/intro.txt")).toBe(false); + }); + + it("keeps * within a single path segment", () => { + const rx = globToRegExp("docs/*.md"); + expect(rx.test("docs/a.md")).toBe(true); + expect(rx.test("docs/sub/a.md")).toBe(false); + }); + + it("supports brace alternation", () => { + const rx = globToRegExp("**/*.{md,mdx,txt}"); + expect(rx.test("a.md")).toBe(true); + expect(rx.test("x/y/b.mdx")).toBe(true); + expect(rx.test("c.txt")).toBe(true); + expect(rx.test("d.pdf")).toBe(false); + }); + + it("supports ? and trailing **", () => { + expect(globToRegExp("file?.txt").test("file1.txt")).toBe(true); + expect(globToRegExp("file?.txt").test("file12.txt")).toBe(false); + expect(globToRegExp("node_modules/**").test("node_modules/a/b.js")).toBe(true); + }); + + it("escapes regex metacharacters", () => { + const rx = globToRegExp("a+b(1).md"); + expect(rx.test("a+b(1).md")).toBe(true); + expect(rx.test("aab1.md")).toBe(false); + }); + + it("closes an unterminated brace group", () => { + expect(globToRegExp("*.{md,txt").test("a.txt")).toBe(true); + }); +}); + +describe("toPosixPath", () => { + it("converts backslashes to forward slashes", () => { + expect(toPosixPath(String.raw`docs\guide\a.md`)).toBe("docs/guide/a.md"); + }); +}); From 4d28dbbdcb792ecb5a5a5268bc4f8b22b820f114 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:25:48 +0000 Subject: [PATCH 21/83] fix(api,cli): accept every core link type via shared LINK_TYPES The REST API and the CLI link command kept their own link-type lists that omitted "references", which core accepts. Use the exported LINK_TYPES. --- src/api/routes.ts | 3 ++- src/cli/index.ts | 6 +++--- tests/unit/api.test.ts | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/api/routes.ts b/src/api/routes.ts index ffb061b7..6456e5b8 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -36,6 +36,7 @@ import { searchBatch, } from "../core/index.js"; import type { LinkType, BulkSelector, BatchSearchRequest } from "../core/index.js"; +import { LINK_TYPES } from "../core/links.js"; import { loadConfig } from "../config.js"; import { DocumentNotFoundError, FetchError, LibScopeError } from "../errors.js"; import { getLogger } from "../logger.js"; @@ -643,7 +644,7 @@ async function handleCreateDocumentLink(ctx: RouteContext, docId: string): Promi sendError(ctx.res, 400, "VALIDATION_ERROR", "targetId and linkType are required"); return; } - const validLinkTypes = ["see_also", "prerequisite", "supersedes", "related"]; + const validLinkTypes: readonly string[] = LINK_TYPES; if (!validLinkTypes.includes(body.linkType)) { sendError( ctx.res, diff --git a/src/cli/index.ts b/src/cli/index.ts index 75fdd960..2a977564 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -12,7 +12,7 @@ import { getDocumentRatings, listRatings } from "../core/ratings.js"; import { createTopic, listTopics } from "../core/topics.js"; import { getDocument, listDocuments, deleteDocument, updateDocument } from "../core/documents.js"; import { createLink, getDocumentLinks, deleteLink, getPrerequisiteChain } from "../core/links.js"; -import type { LinkType } from "../core/links.js"; +import { LINK_TYPES, type LinkType } from "../core/links.js"; import { getVersionHistory, rollbackToVersion } from "../core/versioning.js"; import { initLogger, type LogLevel } from "../logger.js"; import { readdirSync, realpathSync, statSync } from "node:fs"; @@ -1133,10 +1133,10 @@ ratingsCmd program .command("link <sourceId> <targetId>") .description("Create a cross-reference link between two documents") - .option("--type <type>", "Link type: see_also, prerequisite, supersedes, related", "related") + .option("--type <type>", `Link type: ${LINK_TYPES.join(", ")}`, "related") .option("--label <text>", "Human-readable description of the link") .action((sourceId: string, targetId: string, opts: { type: string; label?: string }) => { - const validTypes = ["see_also", "prerequisite", "supersedes", "related"]; + const validTypes: readonly string[] = LINK_TYPES; if (!validTypes.includes(opts.type)) { console.error(`Invalid link type: ${opts.type}. Must be one of: ${validTypes.join(", ")}`); process.exit(1); diff --git a/tests/unit/api.test.ts b/tests/unit/api.test.ts index a9a42859..53581b76 100644 --- a/tests/unit/api.test.ts +++ b/tests/unit/api.test.ts @@ -1148,4 +1148,27 @@ describe("Links API", () => { await handleRequest(req, res, db, provider); expect(getStatus()).toBe(204); }); + + it("POST /api/v1/documents/:id/links accepts the references link type", async () => { + const doc1 = await indexDocument(db, provider, { + title: "Doc C", + content: "# C\n\nContent C.", + sourceType: "manual", + }); + const doc2 = await indexDocument(db, provider, { + title: "Doc D", + content: "# D\n\nContent D.", + sourceType: "manual", + }); + + const { res, getStatus, getBody } = createMockRes(); + const req = createMockReq("POST", `/api/v1/documents/${doc1.id}/links`, { + targetId: doc2.id, + linkType: "references", + }); + await handleRequest(req, res, db, provider); + expect(getStatus()).toBe(201); + const created = parseResponse(getBody()); + expect((created.data as Record<string, unknown>)?.linkType).toBe("references"); + }); }); From 489ea3c40c1013e80506aca8fe4bf1b6970c35cb Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:26:22 +0000 Subject: [PATCH 22/83] fix(db): one database location rule for CLI, MCP, REST and SDK database.path always had a default (~/.libscope/libscope.db), so the SDK opened that file while CLI/MCP opened the workspace database, the SDK workspace option did nothing, init/stats reported the wrong file, and connect/disconnect onenote built join(wsPath, absolutePath). - resolveDatabasePath(): explicit path (SDK dbPath, or database.path set in a config file, ~ expanded) > workspace (option, else active). - database.path has no default; used by CLI initializeApp, MCP main, LibScope.create (new dbPath option; workspace option now works) and the onenote commands (now use initializeApp*). - init prints db.name; getStats defaults to the open file (CLI stats, REST /health and /stats, SDK stats()). - If the workspace DB does not exist and ~/.libscope/libscope.db does, a single warning names it and how to keep it (config set database.path). Nothing is moved or deleted. --- README.md | 5 +- docs/guide/configuration.md | 16 ++- docs/guide/getting-started.md | 2 +- docs/reference/configuration.md | 12 +- src/LibScope.ts | 19 ++- src/api/routes.ts | 8 +- src/cli/index.ts | 42 ++----- src/config.ts | 35 ++++-- src/core/analytics.ts | 8 +- src/core/workspace.ts | 5 + src/db/connection.ts | 45 ++++++- src/db/index.ts | 1 + src/mcp/server.ts | 4 +- tests/unit/config.test.ts | 2 +- tests/unit/db-location.test.ts | 215 ++++++++++++++++++++++++++++++++ 15 files changed, 340 insertions(+), 79 deletions(-) create mode 100644 tests/unit/db-location.test.ts diff --git a/README.md b/README.md index a77c38da..082c78d8 100644 --- a/README.md +++ b/README.md @@ -356,9 +356,6 @@ API keys use one rule for embeddings and the LLM: `LIBSCOPE_<PROVIDER>_API_KEY`, "provider": "openai", "model": "gpt-4o-mini" }, - "database": { - "path": "~/.libscope/libscope.db" - }, "logging": { "level": "info" }, @@ -585,7 +582,7 @@ There's also a web dashboard at `http://localhost:3377` when you run `libscope s ## How It Works -LibScope stores everything in a local SQLite database (at `~/.libscope/libscope.db` by default): +LibScope stores everything in a local SQLite database, one per workspace (`~/.libscope/workspaces/<workspace>/libscope.db`; the default workspace is `default`). Set `database.path` to use a specific file instead. The CLI, MCP server, REST API, and `LibScope.create()` all open the same file: - Documents are split into chunks by heading boundaries - Each chunk is embedded into a vector using the configured provider diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 3a08716b..75bc8349 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -39,16 +39,24 @@ Example `~/.libscope/config.json`: "provider": "openai", "model": "gpt-4o-mini" }, - "database": { - "path": "~/.libscope/libscope.db" - }, "logging": { "level": "info" } } ``` -You can also create a `.libscope.json` in your project root for per-project settings. This is useful if different projects use different embedding providers or databases. +You can also create a `.libscope.json` in your project root for per-project settings. This is useful if different projects use different embedding providers or databases. In `.libscope.json`, set `"workspace": "<name>"` to use a workspace, or `database.path` to use a specific file. + +## Database Location + +By default the database is the active workspace's file: `~/.libscope/workspaces/<workspace>/libscope.db` (workspace `default` unless you choose another). The CLI, MCP server, REST API, and `LibScope.create()` use the same rule: + +1. `database.path`, if set in `.libscope.json` or `~/.libscope/config.json` (or `dbPath` passed to `LibScope.create()`). A leading `~` is expanded. +2. Otherwise the workspace: `--workspace` / `LibScope.create({ workspace })`, then `LIBSCOPE_WORKSPACE`, then `"workspace"` in `.libscope.json`, then `libscope workspace use`, then `default`. + +`libscope init` and `libscope stats` show the file in use. + +Earlier versions of the SDK stored data in `~/.libscope/libscope.db`. If that file exists and the workspace database does not, LibScope prints a warning once. To keep using the old file, run `libscope config set database.path ~/.libscope/libscope.db`. LibScope does not move or delete it. ## Embedding Providers diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index d332585e..bb9fbc6f 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -20,7 +20,7 @@ Create the database: libscope init ``` -This sets up a SQLite database at `~/.libscope/libscope.db` with vector search and full-text indexing. On first run, it also downloads the local embedding model (~80MB). +This sets up a SQLite database at `~/.libscope/workspaces/default/libscope.db` (the active workspace's database) with vector search and full-text indexing. The command prints the path it used. On first run, it also downloads the local embedding model (~80MB). ## Index Some Documents diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index f36ca501..8987f041 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -35,9 +35,9 @@ Complete reference for all configuration options. ### Database -| Key | Type | Default | Description | -| --------------- | ------ | --------------------------- | --------------------------- | -| `database.path` | string | `"~/.libscope/libscope.db"` | Path to the SQLite database | +| Key | Type | Default | Description | +| --------------- | ------ | ---------------------------------------------------- | -------------------------------------------------------------- | +| `database.path` | string | unset (`~/.libscope/workspaces/<workspace>/libscope.db`) | SQLite file to use instead of the workspace database. `~` is expanded | ### Logging @@ -105,7 +105,8 @@ libscope config set llm.provider openai # openai | ollama | anthropic libscope config set llm.model gpt-4o-mini # Database -libscope config set database.path /custom/path/libscope.db +libscope config set database.path ~/kb/libscope.db # overrides the workspace database +libscope config unset database.path # back to the workspace database # Logging libscope config set logging.level debug # debug | info | warn | error | silent @@ -155,9 +156,6 @@ libscope config set indexing.allowSelfSignedCerts true "provider": "openai", "model": "gpt-4o-mini" }, - "database": { - "path": "~/.libscope/libscope.db" - }, "logging": { "level": "info" }, diff --git a/src/LibScope.ts b/src/LibScope.ts index 5d88dace..5c346730 100644 --- a/src/LibScope.ts +++ b/src/LibScope.ts @@ -2,11 +2,10 @@ import type Database from "better-sqlite3"; import type { LibScopeConfig } from "./config.js"; import type { EmbeddingProvider } from "./providers/embedding.js"; import { loadConfig } from "./config.js"; -import { createDatabase } from "./db/connection.js"; +import { createDatabase, resolveDatabasePath } from "./db/connection.js"; import { runMigrations, createVectorTable } from "./db/schema.js"; import { createEmbeddingProvider } from "./providers/index.js"; import { createLlmProvider } from "./core/rag.js"; -import { getWorkspacePath, DEFAULT_WORKSPACE } from "./core/workspace.js"; import { indexDocument, indexFile, @@ -31,8 +30,14 @@ import { } from "./core/batch-search.js"; export interface LibScopeOptions { - /** Workspace name (default: "default"). */ + /** + * Workspace whose database to open. Default: the active workspace (LIBSCOPE_WORKSPACE, + * `.libscope.json` `workspace`, or `libscope workspace use`), the same one the CLI and MCP server use. + * Ignored when `dbPath` or `database.path` is set. + */ workspace?: string; + /** Explicit SQLite database file. Wins over `workspace` and over `database.path` in config files. */ + dbPath?: string; /** Override config values. */ config?: Partial<LibScopeConfig>; } @@ -65,8 +70,10 @@ export class LibScope { logging: { ...baseConfig.logging, ...options?.config?.logging }, }; - const workspace = options?.workspace ?? DEFAULT_WORKSPACE; - const dbPath = config.database.path ?? getWorkspacePath(workspace); + const dbPath = resolveDatabasePath({ + explicitPath: options?.dbPath ?? config.database.path, + workspace: options?.workspace, + }); const db = createDatabase(dbPath); runMigrations(db); @@ -114,7 +121,7 @@ export class LibScope { /** Get overview stats. */ stats(): OverviewStats { - return getStats(this.db, this.config.database.path); + return getStats(this.db); } /** List documents. */ diff --git a/src/api/routes.ts b/src/api/routes.ts index ffb061b7..ddc7e04c 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -162,9 +162,7 @@ function handleOpenApiSpec(ctx: RouteContext): void { } function handleHealthCheck(ctx: RouteContext): void { - const config = loadConfig(); - const dbPath = config.database?.path; - const stats = getStats(ctx.db, dbPath); + const stats = getStats(ctx.db); sendJson( ctx.res, 200, @@ -544,9 +542,7 @@ function handleScheduleStatus(ctx: RouteContext): void { } function handleStats(ctx: RouteContext): void { - const config = loadConfig(); - const dbPath = config.database?.path; - const stats = getStats(ctx.db, dbPath); + const stats = getStats(ctx.db); sendJson(ctx.res, 200, stats, elapsed(ctx.start)); } diff --git a/src/cli/index.ts b/src/cli/index.ts index 12f4c4a3..84c75a74 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -11,6 +11,7 @@ import { getConfigValue, } from "../config.js"; import { getDatabase, runMigrations, createVectorTable, closeDatabase } from "../db/index.js"; +import { resolveDatabasePath } from "../db/connection.js"; import { createEmbeddingProvider, type EmbeddingProvider } from "../providers/index.js"; import { indexDocument, indexFile } from "../core/indexing.js"; import { getSupportedExtensions } from "../core/parsers/index.js"; @@ -58,7 +59,6 @@ import { listWorkspaces, getActiveWorkspace, setActiveWorkspace, - getWorkspacePath, } from "../core/workspace.js"; import { installPack, @@ -482,7 +482,7 @@ program } catch { console.log(" ℹ Vector table skipped (embedding provider not available)"); } - console.log(`✓ Database initialized at ${config.database.path}`); + console.log(`✓ Database initialized at ${db.name}`); } finally { closeDatabase(); } @@ -1621,8 +1621,10 @@ function initializeApp(): { process.env["LIBSCOPE_WORKSPACE"] = opts.workspace; } - const workspace = getActiveWorkspace(); - const dbPath = getWorkspacePath(workspace); + const dbPath = resolveDatabasePath({ + explicitPath: config.database.path, + warn: (message) => console.error(`⚠ ${message}`), + }); const db = getDatabase(dbPath); runMigrations(db); return { config, db }; @@ -1857,9 +1859,9 @@ statsCmd .command("overview", { isDefault: true }) .description("Show overview dashboard") .action(() => { - const { config, db } = initializeApp(); + const { db } = initializeApp(); try { - const s = getStats(db, config.database.path); + const s = getStats(db); console.log("\n\u{1f4ca} Knowledge Base Overview\n"); console.log(` Documents: ${s.totalDocuments}`); console.log(` Chunks: ${s.totalChunks}`); @@ -2350,20 +2352,7 @@ connectCmd .option("--sync", "Incremental re-sync only") .option("--notebook <name>", "Sync a specific notebook") .action(async (opts: { token?: string; sync?: boolean; notebook?: string }) => { - setupLogging(program.opts<ProgramOpts>()); - const config = loadConfig(); - - const workspace = program.opts().workspace as string | undefined; - if (workspace) { - process.env["LIBSCOPE_WORKSPACE"] = workspace; - } - const wsName = getActiveWorkspace(); - const wsPath = getWorkspacePath(wsName); - const dbPath = join(wsPath, config.database.path); - const db = getDatabase(dbPath); - runMigrations(db); - const provider = createEmbeddingProvider(config); - createVectorTable(db, provider.dimensions); + const { db, provider } = initializeAppWithEmbedding(); try { const connConfig = loadConnectorConfig(); @@ -2429,18 +2418,7 @@ disconnectCmd ) { return; } - setupLogging(program.opts<ProgramOpts>()); - const config = loadConfig(); - - const workspace2 = program.opts().workspace as string | undefined; - if (workspace2) { - process.env["LIBSCOPE_WORKSPACE"] = workspace2; - } - const wsName2 = getActiveWorkspace(); - const wsPath = getWorkspacePath(wsName2); - const dbPath = join(wsPath, config.database.path); - const db = getDatabase(dbPath); - runMigrations(db); + const { db } = initializeApp(); try { const removed = disconnectOneNote(db); diff --git a/src/config.ts b/src/config.ts index 7315a515..5cc27924 100644 --- a/src/config.ts +++ b/src/config.ts @@ -28,7 +28,11 @@ export interface LibScopeConfig { anthropicApiKey?: string; }; database: { - path: string; + /** + * Explicit SQLite file. Unset by default: the active workspace's database + * (~/.libscope/workspaces/<name>/libscope.db) is used. A leading `~` is expanded. + */ + path?: string; }; indexing: { maxDocumentSize: number; @@ -47,9 +51,7 @@ const DEFAULT_CONFIG: LibScopeConfig = { ollamaModel: "nomic-embed-text", openaiModel: "text-embedding-3-small", }, - database: { - path: join(homedir(), ".libscope", "libscope.db"), - }, + database: {}, indexing: { maxDocumentSize: 100 * 1024 * 1024, // 100MB allowPrivateUrls: false, @@ -60,6 +62,13 @@ const DEFAULT_CONFIG: LibScopeConfig = { }, }; +/** Expand a leading `~` (or `~/`) to the user's home directory. */ +export function expandHomeDir(path: string): string { + if (path === "~") return homedir(); + if (path.startsWith("~/") || path.startsWith("~\\")) return join(homedir(), path.slice(2)); + return path; +} + function getConfigDir(): string { return join(homedir(), ".libscope"); } @@ -356,6 +365,15 @@ export function invalidateConfigCache(): void { _configCacheAt = 0; } +/** database.path only when a config file sets it (project wins over user), with `~` expanded. */ +function resolveDatabaseSection( + userConfig: ConfigLayer, + projectConfig: ConfigLayer, +): LibScopeConfig["database"] { + const path = projectConfig.database?.path ?? userConfig.database?.path; + return path ? { path: expandHomeDir(path) } : {}; +} + /** Load config with precedence: env > project > user > defaults. Result is cached for 30 s. */ export function loadConfig(): LibScopeConfig { const now = Date.now(); @@ -391,11 +409,7 @@ export function loadConfig(): LibScopeConfig { ...projectConfig.llm, ...envOverrides.llm, }, - database: { - ...DEFAULT_CONFIG.database, - ...userConfig.database, - ...projectConfig.database, - }, + database: resolveDatabaseSection(userConfig, projectConfig), indexing: { ...DEFAULT_CONFIG.indexing, ...userConfig.indexing, @@ -441,8 +455,9 @@ function validateProviderConfig(config: LibScopeConfig, warnings: string[]): voi } } -/** Check that the database directory is writable or can be created. */ +/** Check that an explicit database directory is writable or can be created. */ function validateDatabasePath(config: LibScopeConfig, warnings: string[]): void { + if (!config.database.path) return; // workspace default — created on demand const dbDir = dirname(config.database.path); try { if (existsSync(dbDir)) { diff --git a/src/core/analytics.ts b/src/core/analytics.ts index b2263405..f1bce173 100644 --- a/src/core/analytics.ts +++ b/src/core/analytics.ts @@ -73,7 +73,7 @@ export function logSearch(db: Database.Database, entry: SearchLogEntry): string return id; } -/** Return overview stats for the knowledge base. */ +/** Return overview stats for the knowledge base. `dbPath` defaults to the open database file (`db.name`). */ export function getStats(db: Database.Database, dbPath?: string): OverviewStats { const StatsRowSchema = z.object({ doc_count: z.number(), @@ -99,10 +99,12 @@ export function getStats(db: Database.Database, dbPath?: string): OverviewStats "getStats.row", ); + // Default to the file this connection actually opened. + const sizePath = dbPath ?? (db.memory ? undefined : db.name); let databaseSizeBytes = 0; - if (dbPath) { + if (sizePath) { try { - databaseSizeBytes = statSync(dbPath).size; + databaseSizeBytes = statSync(sizePath).size; } catch { // Inaccessible paths report 0 } diff --git a/src/core/workspace.ts b/src/core/workspace.ts index cfb2c142..0e113757 100644 --- a/src/core/workspace.ts +++ b/src/core/workspace.ts @@ -133,6 +133,11 @@ export function getWorkspacePath(name: string): string { return join(getWorkspacesDir(), name, "libscope.db"); } +/** Database location used by older releases (and by the SDK before workspaces applied to it). */ +export function getLegacyDatabasePath(): string { + return join(homedir(), ".libscope", "libscope.db"); +} + /** * Resolve the active workspace with precedence: * 1. LIBSCOPE_WORKSPACE env var diff --git a/src/db/connection.ts b/src/db/connection.ts index ba49aed9..355ee84c 100644 --- a/src/db/connection.ts +++ b/src/db/connection.ts @@ -4,17 +4,56 @@ import { createRequire } from "node:module"; import { dirname } from "node:path"; import { DatabaseError } from "../errors.js"; import { getLogger } from "../logger.js"; -import { getActiveWorkspace, getWorkspacePath } from "../core/workspace.js"; +import { expandHomeDir } from "../config.js"; +import { getActiveWorkspace, getLegacyDatabasePath, getWorkspacePath } from "../core/workspace.js"; const require = createRequire(import.meta.url); let db: Database.Database | null = null; let cachedPath: string | null = null; +let legacyWarningShown = false; + +export interface ResolveDatabasePathOptions { + /** Explicit database file (SDK `dbPath`, or `database.path` set in a config file). Wins over workspaces. */ + explicitPath?: string | undefined; + /** Workspace name. Defaults to the active workspace (LIBSCOPE_WORKSPACE, project file, `workspace use`). */ + workspace?: string | undefined; + /** Where to report the legacy-database notice (default: logger.warn). Called at most once per process. */ + warn?: ((message: string) => void) | undefined; +} + +/** + * The one rule for where the database lives, used by the CLI, MCP server, REST API and SDK: + * explicit path > workspace (option, else the active workspace). + * + * If no explicit path is set, the workspace database does not exist yet, and a database exists at + * the pre-workspace location (~/.libscope/libscope.db), a single warning names that file and how to + * keep using it. Nothing is moved or deleted. + */ +export function resolveDatabasePath(options: ResolveDatabasePathOptions = {}): string { + if (options.explicitPath) return expandHomeDir(options.explicitPath); + + const resolved = getWorkspacePath(options.workspace ?? getActiveWorkspace()); + const legacy = getLegacyDatabasePath(); + if (!legacyWarningShown && !existsSync(resolved) && existsSync(legacy)) { + legacyWarningShown = true; + const message = + `Using a new database at ${resolved}. An existing database was found at ${legacy} ` + + `(the location used by earlier versions). To keep using it, run: ` + + `libscope config set database.path ${legacy}`; + (options.warn ?? ((m: string): void => getLogger().warn(m)))(message); + } + return resolved; +} /** Resolve the database path, falling back to the active workspace. */ export function resolveDbPath(dbPath?: string): string { - if (dbPath) return dbPath; - return getWorkspacePath(getActiveWorkspace()); + return resolveDatabasePath({ explicitPath: dbPath }); +} + +/** Reset the once-per-process legacy-database warning (for tests). */ +export function resetLegacyDatabaseWarning(): void { + legacyWarningShown = false; } /** Get or create the database connection. */ diff --git a/src/db/index.ts b/src/db/index.ts index 6cc81d94..b0e453d3 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -4,5 +4,6 @@ export { resetDatabase, createDatabase, resolveDbPath, + resolveDatabasePath, } from "./connection.js"; export { runMigrations, createVectorTable } from "./schema.js"; diff --git a/src/mcp/server.ts b/src/mcp/server.ts index f3868197..1bfc2d95 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -3,7 +3,7 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js" import { z } from "zod"; import { loadConfig } from "../config.js"; import { getDatabase, runMigrations, createVectorTable } from "../db/index.js"; -import { getActiveWorkspace, getWorkspacePath } from "../core/workspace.js"; +import { resolveDatabasePath } from "../db/connection.js"; import { createEmbeddingProvider } from "../providers/index.js"; import { searchDocuments, getRelatedChunks } from "../core/search.js"; import { @@ -237,7 +237,7 @@ async function main(): Promise<void> { let db; try { - db = getDatabase(getWorkspacePath(getActiveWorkspace())); + db = getDatabase(resolveDatabasePath({ explicitPath: config.database.path })); runMigrations(db); } catch (err) { console.error("Failed to initialize database:", err instanceof Error ? err.message : err); diff --git a/tests/unit/config.test.ts b/tests/unit/config.test.ts index 788dd36c..10709014 100644 --- a/tests/unit/config.test.ts +++ b/tests/unit/config.test.ts @@ -21,7 +21,7 @@ describe("config", () => { expect(config.embedding.provider).toBe("local"); expect(config.logging.level).toBe("info"); - expect(config.database.path).toContain("libscope.db"); + // database.path defaults are covered with a temp HOME in db-location.test.ts }); it("should return cached config on repeated calls", () => { diff --git a/tests/unit/db-location.test.ts b/tests/unit/db-location.test.ts new file mode 100644 index 00000000..c446b836 --- /dev/null +++ b/tests/unit/db-location.test.ts @@ -0,0 +1,215 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + resolveDatabasePath, + resolveDbPath, + resetLegacyDatabaseWarning, + createDatabase, +} from "../../src/db/connection.js"; +import { runMigrations } from "../../src/db/schema.js"; +import { getStats } from "../../src/core/analytics.js"; +import { loadConfig, invalidateConfigCache, expandHomeDir } from "../../src/config.js"; +import { LibScope } from "../../src/LibScope.js"; +import { initLogger } from "../../src/logger.js"; +import * as loggerModule from "../../src/logger.js"; + +const ENV_KEYS = ["HOME", "LIBSCOPE_WORKSPACE", "LIBSCOPE_EMBEDDING_PROVIDER"]; + +describe("database location (temp HOME)", () => { + let tempHome: string; + let tempCwd: string; + const savedEnv: Record<string, string | undefined> = {}; + + const wsDb = (name: string): string => + join(tempHome, ".libscope", "workspaces", name, "libscope.db"); + const legacyDb = (): string => join(tempHome, ".libscope", "libscope.db"); + + function touch(path: string): void { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, ""); + } + + function writeUserConfig(data: unknown): void { + mkdirSync(join(tempHome, ".libscope"), { recursive: true }); + writeFileSync(join(tempHome, ".libscope", "config.json"), JSON.stringify(data)); + } + + beforeEach(() => { + initLogger("silent"); + for (const key of ENV_KEYS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + tempHome = mkdtempSync(join(tmpdir(), "libscope-dbloc-home-")); + tempCwd = mkdtempSync(join(tmpdir(), "libscope-dbloc-cwd-")); + process.env["HOME"] = tempHome; + vi.spyOn(process, "cwd").mockReturnValue(tempCwd); + resetLegacyDatabaseWarning(); + invalidateConfigCache(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + for (const [key, val] of Object.entries(savedEnv)) { + if (val === undefined) delete process.env[key]; + else process.env[key] = val; + } + invalidateConfigCache(); + rmSync(tempHome, { recursive: true, force: true }); + rmSync(tempCwd, { recursive: true, force: true }); + }); + + describe("resolveDatabasePath", () => { + it("defaults to the active workspace database", () => { + expect(resolveDatabasePath()).toBe(wsDb("default")); + }); + + it("uses the workspace option", () => { + expect(resolveDatabasePath({ workspace: "work" })).toBe(wsDb("work")); + }); + + it("uses LIBSCOPE_WORKSPACE when no option is given", () => { + process.env["LIBSCOPE_WORKSPACE"] = "env-ws"; + expect(resolveDatabasePath()).toBe(wsDb("env-ws")); + }); + + it("explicit path wins over workspace", () => { + expect(resolveDatabasePath({ explicitPath: "/data/x.db", workspace: "work" })).toBe( + "/data/x.db", + ); + }); + + it("expands a leading ~ in an explicit path", () => { + expect(resolveDatabasePath({ explicitPath: "~/kb/libscope.db" })).toBe( + join(tempHome, "kb", "libscope.db"), + ); + expect(expandHomeDir("~")).toBe(tempHome); + expect(expandHomeDir("/abs/~/x")).toBe("/abs/~/x"); + }); + + it("resolveDbPath keeps its old contract", () => { + expect(resolveDbPath()).toBe(wsDb("default")); + expect(resolveDbPath("/tmp/explicit.db")).toBe("/tmp/explicit.db"); + }); + }); + + describe("legacy ~/.libscope/libscope.db notice", () => { + it("warns once, naming the legacy file and database.path, without touching it", () => { + touch(legacyDb()); + const warn = vi.fn(); + expect(resolveDatabasePath({ warn })).toBe(wsDb("default")); + resolveDatabasePath({ warn }); + expect(warn).toHaveBeenCalledTimes(1); + const message = String(warn.mock.calls[0]?.[0]); + expect(message).toContain(legacyDb()); + expect(message).toContain("libscope config set database.path"); + expect(existsSync(legacyDb())).toBe(true); + }); + + it("uses the logger when no warn callback is given", () => { + touch(legacyDb()); + const loggerWarn = vi.fn(); + vi.spyOn(loggerModule, "getLogger").mockReturnValue({ + warn: loggerWarn, + } as unknown as ReturnType<typeof loggerModule.getLogger>); + resolveDatabasePath(); + expect(loggerWarn).toHaveBeenCalledTimes(1); + }); + + it("does not warn when the workspace database already exists", () => { + touch(legacyDb()); + touch(wsDb("default")); + const warn = vi.fn(); + resolveDatabasePath({ warn }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("does not warn when an explicit path is set", () => { + touch(legacyDb()); + const warn = vi.fn(); + resolveDatabasePath({ explicitPath: legacyDb(), warn }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("does not warn when there is no legacy database", () => { + const warn = vi.fn(); + resolveDatabasePath({ warn }); + expect(warn).not.toHaveBeenCalled(); + }); + }); + + describe("config database.path", () => { + it("is unset by default so the workspace database is used", () => { + expect(loadConfig().database.path).toBeUndefined(); + }); + + it("expands ~ from the user config file", () => { + writeUserConfig({ database: { path: "~/custom/libscope.db" } }); + expect(loadConfig().database.path).toBe(join(tempHome, "custom", "libscope.db")); + }); + + it("project file wins over user file", () => { + writeUserConfig({ database: { path: "/user/libscope.db" } }); + writeFileSync( + join(tempCwd, ".libscope.json"), + JSON.stringify({ database: { path: "/project/libscope.db" } }), + ); + expect(loadConfig().database.path).toBe("/project/libscope.db"); + }); + }); + + describe("LibScope.create", () => { + const local = { embedding: { provider: "local" } } as const; + + it("opens the active workspace database (same as CLI/MCP)", () => { + const ls = LibScope.create({ config: local }); + try { + expect(existsSync(wsDb("default"))).toBe(true); + expect(existsSync(legacyDb())).toBe(false); + const stats = ls.stats(); + expect(stats.databaseSizeBytes).toBe(statSync(wsDb("default")).size); + expect(stats.databaseSizeBytes).toBeGreaterThan(0); + } finally { + ls.close(); + } + }); + + it("honours the workspace option", () => { + const ls = LibScope.create({ workspace: "sdk-ws", config: local }); + ls.close(); + expect(existsSync(wsDb("sdk-ws"))).toBe(true); + }); + + it("honours dbPath over workspace", () => { + const dbPath = join(tempHome, "explicit", "kb.db"); + const ls = LibScope.create({ dbPath, workspace: "ignored", config: local }); + ls.close(); + expect(existsSync(dbPath)).toBe(true); + expect(existsSync(wsDb("ignored"))).toBe(false); + }); + + it("honours database.path from a config file", () => { + writeUserConfig({ database: { path: "~/from-config.db" } }); + const ls = LibScope.create({ config: local }); + ls.close(); + expect(existsSync(join(tempHome, "from-config.db"))).toBe(true); + }); + }); + + describe("getStats", () => { + it("measures the open database file by default", () => { + const path = join(tempHome, "stats.db"); + const db = createDatabase(path); + try { + runMigrations(db); + const stats = getStats(db); + expect(stats.databaseSizeBytes).toBe(statSync(path).size); + expect(stats.databaseSizeBytes).toBeGreaterThan(0); + } finally { + db.close(); + } + }); + }); +}); From d078997837a84ff06a7e1a16a85eb3ca94b7c72d Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:26:44 +0000 Subject: [PATCH 23/83] fix(connectors): make every connector schedulable with one sync row per run Only 'connect slack' saved a named config, so 'schedule set notion' (the command's own help example) always failed, and scheduled runs wrote two connector_syncs rows (scheduler + connector) with drifted stats mappings. - Every 'connect <type>' saves a named config (connectors/<name>.json, 0600, default name = type, --name to override) with the credentials a later run needs and connectorType; an existing schedule is kept. - --sync on every connect command re-runs the saved config (obsidian's vault path is now optional with --sync). - Connectors record the run (new trackSync helper, optional syncName); the scheduler no longer tracks, so each run writes exactly one row under the configured name with the connector's own stats. A config that cannot be loaded records one failed row. - OneNote token refresh moved into connectors/onenote.ts (ensureOneNoteAccessToken), so scheduled runs refresh expired tokens and the new tokens are saved. Old connectors.json OneNote credentials are still read for --sync. - Schedule entries resolve the type from connectorType, so Confluence's cloud/server 'type' field no longer hides the connector type. - Slack: send lastSync as Unix seconds in 'oldest' (saved configs store ISO-8601, which Slack does not accept). - disconnect <type> also deletes the saved config (--name). Connector function signatures only gain an optional options argument. The DB connector_configs functions stay: core/index.ts exports them. --- README.md | 24 +- docs/guide/connectors.md | 48 ++- src/cli/index.ts | 412 ++++++++++++---------- src/connectors/confluence.ts | 121 ++++--- src/connectors/docs.ts | 203 +++++------ src/connectors/index.ts | 46 ++- src/connectors/notion.ts | 98 ++--- src/connectors/obsidian.ts | 103 +++--- src/connectors/onenote.ts | 143 +++++--- src/connectors/saved-config.ts | 208 +++++++++++ src/connectors/slack.ts | 113 +++--- src/connectors/sync-tracker.ts | 31 ++ src/core/scheduler.ts | 113 +----- tests/unit/connector-saved-config.test.ts | 303 ++++++++++++++++ tests/unit/scheduler.test.ts | 219 ++++-------- tests/unit/slack.test.ts | 12 +- 16 files changed, 1384 insertions(+), 813 deletions(-) create mode 100644 src/connectors/saved-config.ts create mode 100644 tests/unit/connector-saved-config.test.ts diff --git a/README.md b/README.md index ab1ec232..89a34701 100644 --- a/README.md +++ b/README.md @@ -112,10 +112,12 @@ Once connected, your assistant can search docs, submit new documents, rate conte LibScope can pull documentation from several platforms. Each connector handles incremental syncing so re-runs only process what changed. +Each `connect` command saves its settings, including credentials, to `~/.libscope/connectors/<name>.json` (mode 0600; the name defaults to the connector type, set it with `--name`). `--sync` re-runs a saved config, and `libscope schedule set <name> "<cron>"` syncs it on a schedule while `libscope serve --api` runs. + ```bash # Obsidian — parses wikilinks, frontmatter, embeds, tags libscope connect obsidian /path/to/vault -libscope connect obsidian /path/to/vault --sync # incremental re-sync +libscope connect obsidian --sync # re-sync the saved vault config # Notion libscope connect notion --token secret_abc123 @@ -136,22 +138,27 @@ libscope connect onenote # GitHub / GitLab repos libscope add-repo https://github.com/org/repo --branch main --path docs/ -# Remove a connector's data +# Sync a saved connector every 6 hours (runs inside `libscope serve --api`) +libscope schedule set notion "0 */6 * * *" + +# Remove a connector's data and its saved config libscope disconnect obsidian /path/to/vault ``` <details> <summary>Connector options reference</summary> -**Obsidian:** `--topic-mapping frontmatter`, `--exclude "templates/*" "daily/*"`, `--sync` +**All connect commands:** `--sync` (re-run the saved config), `--name <name>` (saved config name; default: connector type) + +**Obsidian:** `--topic-mapping frontmatter`, `--exclude "templates/*" "daily/*"` -**Notion:** `--exclude page-id-1 db-id-2`, `--sync` +**Notion:** `--exclude page-id-1 db-id-2` -**Confluence:** `--spaces ENG,DEVOPS`, `--exclude-spaces ARCHIVE` +**Confluence:** `--spaces ENG,DEVOPS`, `--exclude-spaces ARCHIVE`, `--type server` -**Slack:** `--thread-mode aggregate|separate`, `--sync` +**Slack:** `--thread-mode aggregate|separate`, `--exclude random` -**OneNote:** `--notebook "Work Notes"`, `--sync` +**OneNote:** `--notebook "Work Notes"`, `--token <access-token>` **GitHub/GitLab:** `--token`, `--branch`, `--path`, `--extensions .md,.mdx,.rst` @@ -534,13 +541,14 @@ There's also a web dashboard at `http://localhost:3377` when you run `libscope s | Command | Description | | ---------------------------------- | -------------------------- | -| `libscope connect obsidian <path>` | Sync Obsidian vault | +| `libscope connect obsidian [path]` | Sync Obsidian vault | | `libscope connect onenote` | Sync OneNote | | `libscope connect notion` | Sync Notion | | `libscope connect confluence` | Sync Confluence | | `libscope connect slack` | Sync Slack | | `libscope add-repo <url>` | Index a GitHub/GitLab repo | | `libscope disconnect <name>` | Remove connector data | +| `libscope schedule set <name> <cron>` | Schedule a saved connector | **Registries** diff --git a/docs/guide/connectors.md b/docs/guide/connectors.md index 53c0e388..ecd73ca2 100644 --- a/docs/guide/connectors.md +++ b/docs/guide/connectors.md @@ -2,6 +2,38 @@ Connectors pull documents from external tools into your LibScope knowledge base. Each connector handles authentication, pagination, and incremental sync so you don't have to think about it. +## Saved Configs, Re-sync and Schedules + +Every `libscope connect <type>` command saves its settings as a named connector config in `~/.libscope/connectors/<name>.json` (file mode 0600). The default name is the connector type. Use `--name` to keep more than one config of the same type: + +```bash +libscope connect slack --token xoxb-work-token --name work-slack +``` + +A saved config holds what a later run needs, including credentials: + +| Connector | Saved fields | +| ---------- | ---------------------------------------------------------------------------------------------------- | +| Notion | `token`, `excludePages`, `lastSync` | +| Slack | `token`, `channels`, `excludeChannels`, `threadMode`, `lastSync` | +| Confluence | `baseUrl`, `type` (cloud/server), `email`, `token`, `spaces`, `excludeSpaces`, `lastSync` | +| Obsidian | `vaultPath`, `topicMapping`, `excludePatterns`, `lastSync` | +| OneNote | `clientId`, `tenantId`, `accessToken`, `refreshToken`, `tokenExpiry`, `notebooks`, `excludeSections`, `lastSync` | + +Each file also stores `connectorType` and, after `libscope schedule set`, a `schedule`. + +`--sync` on any connect command re-runs the saved config (`--name` selects which one). Other options on the command line are ignored with `--sync`. After each successful sync, `lastSync` is set to the time the sync started. + +To sync a saved config on a schedule, set a cron expression. Schedules run while the API server is running (`libscope serve --api`): + +```bash +libscope schedule set notion "0 */6 * * *" +libscope schedule list +libscope schedule remove notion +``` + +Each run, scheduled or manual, writes one entry to the connector sync history under the config name. `libscope disconnect <type>` also deletes the saved config (`--name` selects which one). + ## Obsidian Sync an entire Obsidian vault. Parses frontmatter, wikilinks, embeds, and tags. Folder structure maps to topics. @@ -13,8 +45,8 @@ libscope connect obsidian /path/to/vault # Map topics from frontmatter instead of folder structure libscope connect obsidian /path/to/vault --topic-mapping frontmatter -# Incremental re-sync (only changed files) -libscope connect obsidian /path/to/vault --sync +# Re-sync the saved vault config (only changed files are re-indexed) +libscope connect obsidian --sync # Exclude folders libscope connect obsidian /path/to/vault --exclude "templates/*" "daily/*" @@ -34,7 +66,7 @@ libscope connect notion --token secret_abc123 # Exclude specific pages or databases libscope connect notion --token $NOTION_TOKEN --exclude page-id-1 db-id-2 -# Re-sync (uses stored token) +# Re-sync with the saved config (pages edited since the last sync) libscope connect notion --sync # Disconnect @@ -60,6 +92,9 @@ libscope connect confluence \ --spaces ENG,DEVOPS \ --exclude-spaces ARCHIVE +# Re-sync with the saved config +libscope connect confluence --sync + # Disconnect libscope disconnect confluence ``` @@ -82,7 +117,7 @@ libscope connect slack \ # aggregate — combines thread replies into one document (default) # separate — one document per reply -# Re-sync +# Re-sync with the saved config (messages since the last sync) libscope connect slack --sync # Disconnect @@ -103,9 +138,12 @@ libscope connect onenote # Sync a specific notebook libscope connect onenote --notebook "Work Notes" -# Re-sync with token refresh +# Re-sync with the saved config (refreshes the access token when it has expired) libscope connect onenote --sync +# Use an existing access token instead of signing in (it cannot be refreshed) +libscope connect onenote --token <access-token> + # Disconnect libscope disconnect onenote ``` diff --git a/src/cli/index.ts b/src/cli/index.ts index 51abf5cf..7761910b 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -16,7 +16,7 @@ import type { LinkType } from "../core/links.js"; import { getVersionHistory, rollbackToVersion } from "../core/versioning.js"; import { initLogger, type LogLevel } from "../logger.js"; import { readFileSync, readdirSync, statSync } from "node:fs"; -import { join, extname, basename } from "node:path"; +import { join, extname, basename, resolve as resolvePath } from "node:path"; import { fetchAndConvert } from "../core/url-fetcher.js"; import { spiderUrl } from "../core/spider.js"; import type { SpiderOptions } from "../core/spider.js"; @@ -67,20 +67,30 @@ import { FileWatcher, DEFAULT_WATCH_EXTENSIONS } from "../core/watcher.js"; import { indexRepository, parseRepoUrl } from "../core/repo.js"; import { authenticateDeviceCode, - refreshAccessToken, syncOneNote, disconnectOneNote, + type OneNoteConfig, } from "../connectors/onenote.js"; import { - loadConnectorConfig, - saveConnectorConfig, saveNamedConnectorConfig, loadNamedConnectorConfig, hasNamedConnectorConfig, + deleteNamedConnectorConfig, } from "../connectors/index.js"; +import type { ConnectorSyncOptions } from "../connectors/sync-tracker.js"; +import { + findSavedConnectorConfig, + findSavedOneNoteConfig, + runAndSaveConnector, + saveConnectorSettings, + resolveConnectorType, + type ConnectorType, +} from "../connectors/saved-config.js"; import { syncNotion, disconnectNotion } from "../connectors/notion.js"; import type { NotionConfig } from "../connectors/notion.js"; import { syncSlack, disconnectSlack, type SlackConfig } from "../connectors/slack.js"; +import { syncConfluence, type ConfluenceConfig } from "../connectors/confluence.js"; +import { syncObsidianVault, type ObsidianConfig } from "../connectors/obsidian.js"; import { createSavedSearch, listSavedSearches, @@ -2254,53 +2264,100 @@ packCmd }, ); -/** Perform device-code auth for OneNote and persist credentials. */ -async function onenoteDeviceAuth( - onenoteConf: Record<string, unknown>, - connConfig: Record<string, unknown>, -): Promise<{ token: string; conf: Record<string, unknown> }> { - const clientId = (onenoteConf.clientId as string | undefined) ?? process.env.ONENOTE_CLIENT_ID; +/** Description for the `--name` option of connect/disconnect commands. */ +const CONNECTOR_NAME_HELP = "Saved connector config name (default: the connector type)"; + +/** Return the saved config for `connect <type> --sync`, or exit with a hint. */ +function requireSavedConfig<T>(saved: T | undefined, type: ConnectorType, name: string): T { + if (saved === undefined) { + console.error( + `No saved ${type} connector named "${name}". Run 'libscope connect ${type}' without --sync first.`, + ); + process.exit(1); + } + return saved; +} + +/** Sync with `config` and save it as connector `name` (with the new lastSync). */ +function runConnector<C extends { lastSync?: string | undefined }, R>( + type: ConnectorType, + name: string, + config: C, + sync: (config: C, options: ConnectorSyncOptions) => Promise<R>, +): Promise<R> { + return runAndSaveConnector(type, name, config, (c) => sync(c, { syncName: name })); +} + +/** Device-code sign-in for OneNote. Returns `base` with the new client and token fields. */ +async function onenoteDeviceAuth(base: Partial<OneNoteConfig>): Promise<OneNoteConfig> { + const clientId = base.clientId ?? process.env.ONENOTE_CLIENT_ID; if (!clientId) { console.error("Error: No client ID. Set ONENOTE_CLIENT_ID env var or provide --token."); process.exit(1); } - const tenantId = - (onenoteConf.tenantId as string | undefined) ?? process.env.ONENOTE_TENANT_ID ?? "common"; + const tenantId = base.tenantId ?? process.env.ONENOTE_TENANT_ID ?? "common"; console.log("Starting device code authentication..."); const auth = await authenticateDeviceCode(clientId, tenantId); - const updated = { - ...onenoteConf, + console.log("\u2713 Authenticated successfully"); + return { + notebooks: ["all"], + excludeSections: [], + ...base, clientId, tenantId, accessToken: auth.accessToken, refreshToken: auth.refreshToken, tokenExpiry: auth.expiresAt, }; - connConfig.onenote = updated; - saveConnectorConfig(connConfig); - console.log("\u2713 Authenticated successfully"); - return { token: auth.accessToken, conf: updated }; } -/** Refresh an existing OneNote access token and persist credentials. */ -async function onenoteRefreshAuth( - onenoteConf: Record<string, unknown>, - connConfig: Record<string, unknown>, -): Promise<string | undefined> { - const clientId = onenoteConf.clientId as string | undefined; - const refreshTok = onenoteConf.refreshToken as string | undefined; - if (!clientId || !refreshTok) { - return onenoteConf.accessToken as string | undefined; +/** + * Delete saved connector config `name` (it may hold credentials) when it belongs to `type` + * and passes the optional `matches` check. + */ +function forgetConnector( + type: ConnectorType, + name: string, + matches: (saved: Record<string, unknown>) => boolean = () => true, +): void { + if (!hasNamedConnectorConfig(name)) return; + const saved = loadNamedConnectorConfig<Record<string, unknown>>(name); + if (resolveConnectorType(name, saved) !== type || !matches(saved)) return; + deleteNamedConnectorConfig(name); + console.log(` Removed saved connector config "${name}".`); +} + +/** Absolute, normalised vault path (relative paths resolve against the working directory). */ +function resolveVaultPath(vaultPath: string): string { + return resolvePath(vaultPath); +} + +/** Build the OneNote config for a first-time (non --sync) connect. */ +async function buildOneNoteConnectConfig( + name: string, + opts: { token?: string; notebook?: string }, +): Promise<OneNoteConfig> { + const saved: Partial<OneNoteConfig> = findSavedOneNoteConfig(name) ?? {}; + const notebooks = opts.notebook ? [opts.notebook] : ["all"]; + if (opts.token) { + // A manually supplied token cannot be refreshed; drop any stored refresh token. + return { + clientId: "", + tenantId: "common", + excludeSections: [], + ...saved, + notebooks, + accessToken: opts.token, + refreshToken: undefined, + tokenExpiry: undefined, + lastSync: undefined, + }; } - const tenantId = (onenoteConf.tenantId as string | undefined) ?? "common"; - const auth = await refreshAccessToken(clientId, refreshTok, tenantId); - onenoteConf.accessToken = auth.accessToken; - onenoteConf.refreshToken = auth.refreshToken; - onenoteConf.tokenExpiry = auth.expiresAt; - connConfig.onenote = onenoteConf; - saveConnectorConfig(connConfig); - return auth.accessToken; + const config = await onenoteDeviceAuth({ ...saved, notebooks, lastSync: undefined }); + // Keep the credentials even if the first sync fails. + saveConnectorSettings("onenote", name, config); + return config; } // connect onenote @@ -2309,10 +2366,11 @@ const connectCmd = program.command("connect").description("Connect external serv connectCmd .command("onenote") .description("Connect and sync OneNote notebooks via Microsoft Graph API") - .option("--token <accessToken>", "Use a pre-existing access token") - .option("--sync", "Incremental re-sync only") + .option("--token <accessToken>", "Use a pre-existing access token (cannot be refreshed)") + .option("--sync", "Re-sync using the saved configuration (refreshes the access token)") .option("--notebook <name>", "Sync a specific notebook") - .action(async (opts: { token?: string; sync?: boolean; notebook?: string }) => { + .option("--name <name>", CONNECTOR_NAME_HELP) + .action(async (opts: { token?: string; sync?: boolean; notebook?: string; name?: string }) => { const config = loadConfig(); setupLogging(program.opts<ProgramOpts>()); @@ -2329,35 +2387,14 @@ connectCmd createVectorTable(db, provider.dimensions); try { - const connConfig = loadConnectorConfig(); - let onenoteConf = (connConfig.onenote ?? {}) as Record<string, unknown>; + const name = opts.name ?? "onenote"; + const oneNoteConfig = opts.sync + ? requireSavedConfig(findSavedOneNoteConfig(name), "onenote", name) + : await buildOneNoteConnectConfig(name, opts); - let accessToken = opts.token; - - if (!accessToken && !opts.sync) { - const result = await onenoteDeviceAuth(onenoteConf, connConfig); - accessToken = result.token; - onenoteConf = result.conf; - } - - if (opts.sync && !accessToken) { - accessToken = await onenoteRefreshAuth(onenoteConf, connConfig); - } - - if (!accessToken) { - console.error("Error: No access token available. Run without --sync to authenticate."); - process.exit(1); - } - - const notebooks = opts.notebook ? [opts.notebook] : ["all"]; - const syncResult = await syncOneNote(db, provider, { - clientId: (onenoteConf.clientId as string) ?? "", - tenantId: (onenoteConf.tenantId as string) ?? "common", - accessToken, - notebooks, - excludeSections: (onenoteConf.excludeSections as string[]) ?? [], - lastSync: opts.sync ? (onenoteConf.lastSync as string | undefined) : undefined, - }); + const syncResult = await runConnector("onenote", name, oneNoteConfig, (c, o) => + syncOneNote(db, provider, c, o), + ); console.log(`\n✓ OneNote sync complete:`); console.log(` Notebooks: ${syncResult.notebooks}`); @@ -2383,7 +2420,8 @@ disconnectCmd .command("onenote") .description("Disconnect OneNote and remove its data") .option("-y, --yes", "Skip confirmation prompt") - .action(async (opts: { yes?: boolean }) => { + .option("--name <name>", CONNECTOR_NAME_HELP) + .action(async (opts: { yes?: boolean; name?: string }) => { if ( !(await confirmOrCancel( "Disconnect OneNote and remove all its data? This cannot be undone.", @@ -2408,46 +2446,54 @@ disconnectCmd try { const removed = disconnectOneNote(db); console.log(`✓ Disconnected OneNote. Removed ${removed} documents.`); + forgetConnector("onenote", opts.name ?? "onenote"); } finally { db.close(); } }); connectCmd - .command("obsidian <vault-path>") + .command("obsidian [vault-path]") .description("Sync an Obsidian vault into the knowledge base") - .option("--sync", "Incremental re-sync (only changed files)") + .option("--sync", "Re-sync using the saved configuration (only changed files are re-indexed)") .option( "--topic-mapping <mode>", "Map topics from 'folder' or 'frontmatter' (default: folder)", "folder", ) .option("--exclude <patterns...>", "Additional exclude patterns") + .option("--name <name>", CONNECTOR_NAME_HELP) .action( async ( - vaultPath: string, - cmdOpts: { sync?: boolean; topicMapping?: string; exclude?: string[] }, + vaultPath: string | undefined, + cmdOpts: { sync?: boolean; topicMapping?: string; exclude?: string[]; name?: string }, ) => { - const { db, provider } = initializeAppWithEmbedding(); - - try { - const { syncObsidianVault } = await import("../connectors/obsidian.js"); - - const topicMapping: "folder" | "frontmatter" = - cmdOpts.topicMapping === "frontmatter" ? "frontmatter" : "folder"; - const obsConfig = { - vaultPath: join(process.cwd(), vaultPath).replace(/\/+$/, ""), - topicMapping, + const name = cmdOpts.name ?? "obsidian"; + let obsConfig: ObsidianConfig; + if (cmdOpts.sync) { + obsConfig = requireSavedConfig( + findSavedConnectorConfig<ObsidianConfig>("obsidian", name), + "obsidian", + name, + ); + } else { + if (!vaultPath) { + console.error("Error: <vault-path> is required (or use --sync with a saved config)."); + process.exit(1); + } + obsConfig = { + vaultPath: resolveVaultPath(vaultPath), + topicMapping: cmdOpts.topicMapping === "frontmatter" ? "frontmatter" : "folder", excludePatterns: cmdOpts.exclude ?? [], }; + } - // Use absolute path if provided - if (vaultPath.startsWith("/")) { - obsConfig.vaultPath = vaultPath; - } - + const { db, provider } = initializeAppWithEmbedding(); + try { console.log(`Syncing Obsidian vault: ${obsConfig.vaultPath}`); - const result = await syncObsidianVault(db, provider, obsConfig); + const result = await runConnector("obsidian", name, obsConfig, (c, o) => + syncObsidianVault(db, provider, c, o), + ); console.log(`✓ Sync complete:`); console.log(` Added: ${result.added}`); @@ -2476,7 +2522,8 @@ connectCmd "Thread handling: aggregate (full thread as one doc) or separate", "aggregate", ) - .option("--sync", "Sync using saved configuration") + .option("--sync", "Re-sync using the saved configuration") + .option("--name <name>", CONNECTOR_NAME_HELP) .action( async (opts: { token?: string; @@ -2484,45 +2531,35 @@ connectCmd exclude?: string; threadMode: string; sync?: boolean; + name?: string; }) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - let slackConfig: SlackConfig; - - if (opts.sync) { - if (!hasNamedConnectorConfig("slack")) { - console.error( - "No Slack configuration found. Run 'libscope connect slack --token ...' first.", - ); - process.exit(1); - } - slackConfig = loadNamedConnectorConfig<SlackConfig>("slack"); - } else { - if (!opts.token) { - console.error("--token is required for initial Slack connection."); - process.exit(1); - } - slackConfig = { - token: opts.token, - channels: splitCsv(opts.channels), - threadMode: opts.threadMode === "separate" ? "separate" : "aggregate", - }; - if (opts.exclude) { - slackConfig = { - ...slackConfig, - excludeChannels: splitCsv(opts.exclude), - }; - } + const name = opts.name ?? "slack"; + let slackConfig: SlackConfig; + if (opts.sync) { + slackConfig = requireSavedConfig( + findSavedConnectorConfig<SlackConfig>("slack", name), + "slack", + name, + ); + } else { + if (!opts.token) { + console.error("--token is required for initial Slack connection."); + process.exit(1); } + slackConfig = { + token: opts.token, + channels: splitCsv(opts.channels), + threadMode: opts.threadMode === "separate" ? "separate" : "aggregate", + ...(opts.exclude ? { excludeChannels: splitCsv(opts.exclude) } : {}), + }; + } + const { db, provider } = initializeAppWithEmbedding(); + try { console.log("Syncing Slack messages..."); - const result = await syncSlack(db, provider, slackConfig); - - const updatedConfig: SlackConfig = { - ...slackConfig, - lastSync: new Date().toISOString(), - }; - saveNamedConnectorConfig("slack", updatedConfig); + const result = await runConnector("slack", name, slackConfig, (c, o) => + syncSlack(db, provider, c, o), + ); console.log(`✓ Slack sync complete:`); console.log(` Channels: ${result.channels}`); @@ -2549,7 +2586,8 @@ connectCmd .option("--token <token>", "API token (Cloud) or Personal Access Token (Server/Data Center)") .option("--spaces <keys>", "Comma-separated space keys, or 'all'", "all") .option("--exclude-spaces <keys>", "Comma-separated space keys to exclude") - .option("--sync", "Sync using previously saved config") + .option("--sync", "Re-sync using the saved configuration") + .option("--name <name>", CONNECTOR_NAME_HELP) .action( async (opts: { url?: string; @@ -2559,26 +2597,33 @@ connectCmd spaces?: string; excludeSpaces?: string; sync?: boolean; + name?: string; }) => { - const { syncConfluence } = await import("../connectors/confluence.js"); + const name = opts.name ?? "confluence"; + let confluenceConfig: ConfluenceConfig; + if (opts.sync) { + confluenceConfig = requireSavedConfig( + findSavedConnectorConfig<ConfluenceConfig>("confluence", name), + "confluence", + name, + ); + } else { + const email = opts.email ?? process.env["CONFLUENCE_EMAIL"]; + confluenceConfig = { + baseUrl: opts.url ?? process.env["CONFLUENCE_URL"] ?? "", + type: opts.type === "server" ? "server" : "cloud", + ...(email ? { email } : {}), + token: opts.token ?? process.env["CONFLUENCE_TOKEN"] ?? "", + spaces: splitCsv(opts.spaces ?? "all"), + excludeSpaces: opts.excludeSpaces ? splitCsv(opts.excludeSpaces) : undefined, + }; + } + const { db, provider } = initializeAppWithEmbedding(); try { - const url = opts.url ?? process.env["CONFLUENCE_URL"] ?? ""; - const confluenceType = opts.type === "server" ? "server" : ("cloud" as "cloud" | "server"); - const email = opts.email ?? process.env["CONFLUENCE_EMAIL"] ?? undefined; - const token = opts.token ?? process.env["CONFLUENCE_TOKEN"] ?? ""; - - const spaces = splitCsv(opts.spaces ?? "all"); - const excludeSpaces = opts.excludeSpaces ? splitCsv(opts.excludeSpaces) : undefined; - - const result = await syncConfluence(db, provider, { - baseUrl: url, - type: confluenceType, - ...(email ? { email } : {}), - token, - spaces, - excludeSpaces, - }); + const result = await runConnector("confluence", name, confluenceConfig, (c, o) => + syncConfluence(db, provider, c, o), + ); console.log(`✓ Confluence sync complete`); console.log(` Spaces: ${result.spaces}`); @@ -2600,7 +2645,8 @@ disconnectCmd .command("obsidian <vault-path>") .description("Remove all documents from an Obsidian vault") .option("-y, --yes", "Skip confirmation prompt") - .action(async (vaultPath: string, opts: { yes?: boolean }) => { + .option("--name <name>", CONNECTOR_NAME_HELP) + .action(async (vaultPath: string, opts: { yes?: boolean; name?: string }) => { if ( !(await confirmOrCancel( `Disconnect Obsidian vault "${vaultPath}" and remove its documents? This cannot be undone.`, @@ -2613,13 +2659,14 @@ disconnectCmd try { const { disconnectVault } = await import("../connectors/obsidian.js"); - let resolvedPath = vaultPath; - if (!vaultPath.startsWith("/")) { - resolvedPath = join(process.cwd(), vaultPath); - } - + const resolvedPath = resolveVaultPath(vaultPath); const removed = disconnectVault(db, resolvedPath); console.log(`✓ Disconnected vault. Removed ${removed} documents.`); + forgetConnector( + "obsidian", + opts.name ?? "obsidian", + (saved) => saved["vaultPath"] === resolvedPath, + ); } finally { closeDatabase(); } @@ -2628,41 +2675,44 @@ disconnectCmd connectCmd .command("notion") .description("Connect and sync Notion pages and databases") - .option("--token <token>", "Notion integration token (secret_...)") - .option("--sync", "Sync pages using a previously stored token") + .option("--token <token>", "Notion integration token (secret_... or ntn_...; or NOTION_TOKEN)") + .option("--sync", "Re-sync using the saved configuration (pages edited since the last sync)") .option("--exclude <ids...>", "Page/database IDs to exclude") - .action(async (opts: { token?: string; sync?: boolean; exclude?: string[] }) => { - const { db, provider } = initializeAppWithEmbedding(); - let token = opts.token; - - if (opts.sync && !token) { - token = process.env["NOTION_TOKEN"]; + .option("--name <name>", CONNECTOR_NAME_HELP) + .action(async (opts: { token?: string; sync?: boolean; exclude?: string[]; name?: string }) => { + const name = opts.name ?? "notion"; + let config: NotionConfig; + if (opts.sync) { + config = requireSavedConfig( + findSavedConnectorConfig<NotionConfig>("notion", name), + "notion", + name, + ); + } else { + const token = opts.token ?? process.env["NOTION_TOKEN"]; if (!token) { - console.error("Error: --token is required, or set NOTION_TOKEN environment variable."); + console.error("Error: --token <token> is required, or set NOTION_TOKEN."); process.exitCode = 1; return; } + config = { token, ...(opts.exclude ? { excludePages: opts.exclude } : {}) }; } - if (!token) { - console.error("Error: --token <token> is required."); - process.exitCode = 1; - return; - } - - const config: NotionConfig = { token }; - if (opts.exclude) { - config.excludePages = opts.exclude; - } - - console.log("Syncing Notion..."); - const result = await syncNotion(db, provider, config); - console.log( - `✓ Synced: ${result.pagesIndexed} pages, ${result.databasesIndexed} databases` + - (result.errors.length > 0 ? `, ${result.errors.length} errors` : ""), - ); - for (const err of result.errors) { - console.log(` ⚠ ${err.page}: ${err.error}`); + const { db, provider } = initializeAppWithEmbedding(); + try { + console.log("Syncing Notion..."); + const result = await runConnector("notion", name, config, (c, o) => + syncNotion(db, provider, c, o), + ); + console.log( + `✓ Synced: ${result.pagesIndexed} pages, ${result.databasesIndexed} databases` + + (result.errors.length > 0 ? `, ${result.errors.length} errors` : ""), + ); + for (const err of result.errors) { + console.log(` ⚠ ${err.page}: ${err.error}`); + } + } finally { + closeDatabase(); } }); @@ -2670,7 +2720,8 @@ disconnectCmd .command("notion") .description("Remove all Notion documents") .option("-y, --yes", "Skip confirmation prompt") - .action(async (opts: { yes?: boolean }) => { + .option("--name <name>", CONNECTOR_NAME_HELP) + .action(async (opts: { yes?: boolean; name?: string }) => { if ( !(await confirmOrCancel( "Disconnect Notion and remove all its documents? This cannot be undone.", @@ -2683,6 +2734,7 @@ disconnectCmd try { const removed = await disconnectNotion(db); console.log(`✓ Removed ${removed} Notion documents.`); + forgetConnector("notion", opts.name ?? "notion"); } finally { closeDatabase(); } @@ -2692,7 +2744,8 @@ disconnectCmd .command("slack") .description("Remove all Slack data from the knowledge base") .option("-y, --yes", "Skip confirmation prompt") - .action(async (opts: { yes?: boolean }) => { + .option("--name <name>", CONNECTOR_NAME_HELP) + .action(async (opts: { yes?: boolean; name?: string }) => { if ( !(await confirmOrCancel( "Disconnect Slack and remove all its data? This cannot be undone.", @@ -2705,6 +2758,7 @@ disconnectCmd try { const count = disconnectSlack(db); console.log(`✓ Removed ${count} Slack documents from the knowledge base.`); + forgetConnector("slack", opts.name ?? "slack"); } finally { closeDatabase(); } @@ -2714,7 +2768,8 @@ disconnectCmd .command("confluence") .description("Remove all Confluence-synced content") .option("-y, --yes", "Skip confirmation prompt") - .action(async (opts: { yes?: boolean }) => { + .option("--name <name>", CONNECTOR_NAME_HELP) + .action(async (opts: { yes?: boolean; name?: string }) => { if ( !(await confirmOrCancel( "Disconnect Confluence and remove all synced content? This cannot be undone.", @@ -2728,6 +2783,7 @@ disconnectCmd try { const removed = disconnectConfluence(db); console.log(`✓ Removed ${removed} Confluence documents`); + forgetConnector("confluence", opts.name ?? "confluence"); } finally { closeDatabase(); } diff --git a/src/connectors/confluence.ts b/src/connectors/confluence.ts index 4a4f508e..dab05708 100644 --- a/src/connectors/confluence.ts +++ b/src/connectors/confluence.ts @@ -8,7 +8,7 @@ import { deleteDocument } from "../core/documents.js"; import { getLogger } from "../logger.js"; import { FetchError, ValidationError } from "../errors.js"; import { fetchWithRetry } from "./http-utils.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; export interface ConfluenceConfig { baseUrl: string; @@ -509,10 +509,37 @@ function validateConfluenceConfig(config: ConfluenceConfig): void { } } +/** Strip trailing slashes from a Confluence base URL (loop instead of a regex). */ +function trimBaseUrl(baseUrl: string): string { + let base = baseUrl; + while (base.endsWith("/")) base = base.slice(0, -1); + return base; +} + export async function syncConfluence( db: Database.Database, provider: EmbeddingProvider, config: ConfluenceConfig, + options: ConnectorSyncOptions = {}, +): Promise<ConfluenceSyncResult> { + return trackSync( + db, + "confluence", + options.syncName ?? trimBaseUrl(config.baseUrl), + () => runConfluenceSync(db, provider, config), + (result) => ({ + added: result.pagesIndexed, + updated: result.pagesUpdated, + deleted: 0, + errored: result.errors.length, + }), + ); +} + +async function runConfluenceSync( + db: Database.Database, + provider: EmbeddingProvider, + config: ConfluenceConfig, ): Promise<ConfluenceSyncResult> { const log = getLogger(); @@ -520,67 +547,53 @@ export async function syncConfluence( const confluenceType = config.type ?? "cloud"; const auth = buildAuthHeader(confluenceType, config.email, config.token); - let base = config.baseUrl; - while (base.endsWith("/")) base = base.slice(0, -1); + const base = trimBaseUrl(config.baseUrl); const urls = getApiUrls(base, confluenceType); - const syncId = startSync(db, "confluence", base); - try { - const result: ConfluenceSyncResult = { - spaces: 0, - pagesIndexed: 0, - pagesUpdated: 0, - errors: [], - }; - - log.info({ baseUrl: base }, "Starting Confluence sync"); - - const allSpaces = await fetchAllPages<ConfluenceSpace>(urls.spaces, base, auth); - const excludeSet = new Set(config.excludeSpaces ?? []); - const requestedAll = config.spaces.length === 1 && config.spaces[0] === "all"; - const spacesToSync = allSpaces.filter((s) => { - if (excludeSet.has(s.key)) return false; - return requestedAll || config.spaces.includes(s.key); - }); + const result: ConfluenceSyncResult = { + spaces: 0, + pagesIndexed: 0, + pagesUpdated: 0, + errors: [], + }; - result.spaces = spacesToSync.length; - log.info({ spaceCount: spacesToSync.length }, "Spaces to sync"); + log.info({ baseUrl: base }, "Starting Confluence sync"); - for (const space of spacesToSync) { - await syncConfluenceSpace({ - db, - provider, - space, - confluenceType, - base, - urls, - auth, - result, - }); - } + const allSpaces = await fetchAllPages<ConfluenceSpace>(urls.spaces, base, auth); + const excludeSet = new Set(config.excludeSpaces ?? []); + const requestedAll = config.spaces.length === 1 && config.spaces[0] === "all"; + const spacesToSync = allSpaces.filter((s) => { + if (excludeSet.has(s.key)) return false; + return requestedAll || config.spaces.includes(s.key); + }); - log.info( - { - spaces: result.spaces, - indexed: result.pagesIndexed, - updated: result.pagesUpdated, - errors: result.errors.length, - }, - "Confluence sync complete", - ); + result.spaces = spacesToSync.length; + log.info({ spaceCount: spacesToSync.length }, "Spaces to sync"); + + for (const space of spacesToSync) { + await syncConfluenceSpace({ + db, + provider, + space, + confluenceType, + base, + urls, + auth, + result, + }); + } - completeSync(db, syncId, { - added: result.pagesIndexed, + log.info( + { + spaces: result.spaces, + indexed: result.pagesIndexed, updated: result.pagesUpdated, - deleted: 0, - errored: result.errors.length, - }); + errors: result.errors.length, + }, + "Confluence sync complete", + ); - return result; - } catch (err) { - failSync(db, syncId, err instanceof Error ? err.message : String(err)); - throw err; - } + return result; } export function disconnectConfluence(db: Database.Database): number { diff --git a/src/connectors/docs.ts b/src/connectors/docs.ts index fc8865e7..b596703f 100644 --- a/src/connectors/docs.ts +++ b/src/connectors/docs.ts @@ -13,7 +13,7 @@ import { fetchRaw } from "../core/url-fetcher.js"; import type { FetchOptions } from "../core/url-fetcher.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; import type { EmbeddingProvider } from "../providers/embedding.js"; // Source type used to tag all docs-connector documents. @@ -632,6 +632,26 @@ export async function syncDocSite( db: Database.Database, provider: EmbeddingProvider, config: DocSiteConfig, + options: ConnectorSyncOptions = {}, +): Promise<DocSiteSyncResult> { + return trackSync( + db, + CONNECTOR_TYPE, + options.syncName ?? config.url, + () => runDocSiteSync(db, provider, config), + (result) => ({ + added: result.pagesIndexed, + updated: result.pagesUpdated, + deleted: 0, + errored: result.errors.length, + }), + ); +} + +async function runDocSiteSync( + db: Database.Database, + provider: EmbeddingProvider, + config: DocSiteConfig, ): Promise<DocSiteSyncResult> { const log = getLogger(); @@ -660,116 +680,99 @@ export async function syncDocSite( errors: [], }; - const syncId = startSync(db, CONNECTOR_TYPE, config.url); + // --- Fetch root page --- + log.info({ url: config.url }, "Fetching documentation root page"); + let rootHtml: string; try { - // --- Fetch root page --- - log.info({ url: config.url }, "Fetching documentation root page"); + const raw = await fetchRaw(config.url, fetchOptions); + rootHtml = raw.body; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new Error(`Failed to fetch root page: ${msg}`); + } - let rootHtml: string; - try { - const raw = await fetchRaw(config.url, fetchOptions); - rootHtml = raw.body; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - throw new Error(`Failed to fetch root page: ${msg}`); - } + // --- Detect site type --- + result.detectedType = + config.type !== undefined && config.type !== "auto" ? config.type : detectDocSiteType(rootHtml); - // --- Detect site type --- - result.detectedType = - config.type !== undefined && config.type !== "auto" - ? config.type - : detectDocSiteType(rootHtml); - - log.info({ type: result.detectedType, url: config.url }, "Documentation site type"); - - // --- URL discovery --- - const visited = new Set<string>(); - const queue: Array<{ url: string; depth: number }> = []; - const rootNormalised = normalizeUrl(config.url); - visited.add(rootNormalised); - - await discoverUrls(config, baseUrl, rootHtml, pathPrefix, fetchOptions, visited, queue); - - // --- Build existing-URL index for update tracking --- - const existingUrls = loadExistingDocUrls(db, config.library); - - const ctx: PageContext = { - siteType: result.detectedType, - db, - provider, - config, - existingUrls, - result, - }; - - // --- Process the root page first --- - await processPage(rootNormalised, rootHtml, ctx); - - // --- BFS crawl --- - while (queue.length > 0 && visited.size <= maxPages) { - const batch = queue.splice(0, concurrency); - - await Promise.allSettled( - batch.map(async ({ url, depth }) => { - if (visited.size > maxPages) return; - - let html: string; - let contentType: string; - try { - const raw = await fetchRaw(url, fetchOptions); - html = raw.body; - contentType = raw.contentType; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - log.warn({ url, error: msg }, "Failed to fetch documentation page"); - result.errors.push({ url, error: msg }); - return; - } + log.info({ type: result.detectedType, url: config.url }, "Documentation site type"); - // Only process HTML pages (skip binary/asset responses that slipped through) - if (!contentType.includes("text/html") && !contentType.includes("text/plain")) { - return; - } + // --- URL discovery --- + const visited = new Set<string>(); + const queue: Array<{ url: string; depth: number }> = []; + const rootNormalised = normalizeUrl(config.url); + visited.add(rootNormalised); + + await discoverUrls(config, baseUrl, rootHtml, pathPrefix, fetchOptions, visited, queue); - await processPage(url, html, ctx); + // --- Build existing-URL index for update tracking --- + const existingUrls = loadExistingDocUrls(db, config.library); - // Continue link discovery if within depth budget - if (depth < maxDepth) { - for (const link of extractDocLinks(html, url, pathPrefix)) { - if (!visited.has(link)) { - visited.add(link); - queue.push({ url: link, depth: depth + 1 }); - } + const ctx: PageContext = { + siteType: result.detectedType, + db, + provider, + config, + existingUrls, + result, + }; + + // --- Process the root page first --- + await processPage(rootNormalised, rootHtml, ctx); + + // --- BFS crawl --- + while (queue.length > 0 && visited.size <= maxPages) { + const batch = queue.splice(0, concurrency); + + await Promise.allSettled( + batch.map(async ({ url, depth }) => { + if (visited.size > maxPages) return; + + let html: string; + let contentType: string; + try { + const raw = await fetchRaw(url, fetchOptions); + html = raw.body; + contentType = raw.contentType; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + log.warn({ url, error: msg }, "Failed to fetch documentation page"); + result.errors.push({ url, error: msg }); + return; + } + + // Only process HTML pages (skip binary/asset responses that slipped through) + if (!contentType.includes("text/html") && !contentType.includes("text/plain")) { + return; + } + + await processPage(url, html, ctx); + + // Continue link discovery if within depth budget + if (depth < maxDepth) { + for (const link of extractDocLinks(html, url, pathPrefix)) { + if (!visited.has(link)) { + visited.add(link); + queue.push({ url: link, depth: depth + 1 }); } } - }), - ); - } - - completeSync(db, syncId, { - added: result.pagesIndexed, - updated: result.pagesUpdated, - deleted: 0, - errored: result.errors.length, - }); - - log.info( - { - pagesIndexed: result.pagesIndexed, - pagesUpdated: result.pagesUpdated, - pagesSkipped: result.pagesSkipped, - errors: result.errors.length, - }, - "Documentation site sync complete", + } + }), ); - - return result; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - failSync(db, syncId, msg); - throw err; } + + log.info( + { + pagesIndexed: result.pagesIndexed, + pagesUpdated: result.pagesUpdated, + pagesSkipped: result.pagesSkipped, + errors: result.errors.length, + }, + "Documentation site sync complete", + ); + + return result; } /** diff --git a/src/connectors/index.ts b/src/connectors/index.ts index a35fdd59..2e5a9b4b 100644 --- a/src/connectors/index.ts +++ b/src/connectors/index.ts @@ -1,4 +1,4 @@ -import { readFileSync, writeFileSync, mkdirSync, existsSync, chmodSync } from "node:fs"; +import { readFileSync, writeFileSync, mkdirSync, existsSync, chmodSync, unlinkSync } from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; import { ConfigError } from "../errors.js"; @@ -108,20 +108,30 @@ export function deleteDbConnectorConfig(db: Database.Database, type: string): bo return result.changes > 0; } -const CONNECTORS_DIR = join(homedir(), ".libscope", "connectors"); +/** Directory holding named connector configs (~/.libscope/connectors). */ +export function getConnectorsDir(): string { + return join(homedir(), ".libscope", "connectors"); +} -function ensureConnectorsDir(): void { - if (existsSync(CONNECTORS_DIR)) { +function ensureConnectorsDir(): string { + const dir = getConnectorsDir(); + if (existsSync(dir)) { // Remediate existing directories that may have permissive permissions - restrictPermissions(CONNECTORS_DIR, 0o700); + restrictPermissions(dir, 0o700); } else { - mkdirSync(CONNECTORS_DIR, { recursive: true, mode: 0o700 }); + mkdirSync(dir, { recursive: true, mode: 0o700 }); } try { - chmodSync(CONNECTORS_DIR, 0o700); + chmodSync(dir, 0o700); } catch { // chmod may fail in test environments or non-POSIX systems } + return dir; +} + +function namedConfigPath(name: string): string { + validateConnectorName(name); + return join(getConnectorsDir(), `${name}.json`); } function validateConnectorName(name: string): void { @@ -133,16 +143,14 @@ function validateConnectorName(name: string): void { /** Save a named connector config to ~/.libscope/connectors/<name>.json */ export function saveNamedConnectorConfig(name: string, config: object): void { validateConnectorName(name); - ensureConnectorsDir(); - const filePath = join(CONNECTORS_DIR, `${name}.json`); + const filePath = join(ensureConnectorsDir(), `${name}.json`); writeRestrictedFile(filePath, JSON.stringify(config, null, 2)); getLogger().info({ connector: name }, "Connector config saved"); } /** Load a named connector config from ~/.libscope/connectors/<name>.json */ export function loadNamedConnectorConfig<T>(name: string): T { - validateConnectorName(name); - const filePath = join(CONNECTORS_DIR, `${name}.json`); + const filePath = namedConfigPath(name); if (!existsSync(filePath)) { throw new ConfigError( `No connector config found for "${name}". Run 'libscope connect ${name}' first.`, @@ -158,9 +166,16 @@ export function loadNamedConnectorConfig<T>(name: string): T { /** Check if a named connector config exists */ export function hasNamedConnectorConfig(name: string): boolean { - validateConnectorName(name); - const filePath = join(CONNECTORS_DIR, `${name}.json`); - return existsSync(filePath); + return existsSync(namedConfigPath(name)); +} + +/** Delete a named connector config. Returns false when none existed. */ +export function deleteNamedConnectorConfig(name: string): boolean { + const filePath = namedConfigPath(name); + if (!existsSync(filePath)) return false; + unlinkSync(filePath); + getLogger().info({ connector: name }, "Connector config deleted"); + return true; } /** @@ -216,8 +231,9 @@ export { failSync, getConnectorStatus, getSyncHistory, + trackSync, } from "./sync-tracker.js"; -export type { SyncStats, ConnectorSyncRow } from "./sync-tracker.js"; +export type { SyncStats, ConnectorSyncRow, ConnectorSyncOptions } from "./sync-tracker.js"; export { syncNotion, convertNotionBlocks, disconnectNotion } from "./notion.js"; export type { NotionConfig, NotionSyncResult, NotionBlock } from "./notion.js"; diff --git a/src/connectors/notion.ts b/src/connectors/notion.ts index 2b1ede86..ea6fe00c 100644 --- a/src/connectors/notion.ts +++ b/src/connectors/notion.ts @@ -5,7 +5,7 @@ import { ValidationError, FetchError } from "../errors.js"; import { fetchWithRetry } from "./http-utils.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; const NOTION_API_BASE = "https://api.notion.com"; const NOTION_VERSION = "2022-06-28"; @@ -505,6 +505,26 @@ export async function syncNotion( db: Database.Database, provider: EmbeddingProvider, config: NotionConfig, + options: ConnectorSyncOptions = {}, +): Promise<NotionSyncResult> { + return trackSync( + db, + "notion", + options.syncName ?? "notion", + () => runNotionSync(db, provider, config), + (result) => ({ + added: result.pagesIndexed + result.databasesIndexed, + updated: 0, + deleted: 0, + errored: result.errors.length, + }), + ); +} + +async function runNotionSync( + db: Database.Database, + provider: EmbeddingProvider, + config: NotionConfig, ): Promise<NotionSyncResult> { const log = getLogger(); @@ -512,58 +532,44 @@ export async function syncNotion( throw new ValidationError("Notion token must start with 'secret_' or 'ntn_'"); } - const syncId = startSync(db, "notion", "notion"); - - try { - const result: NotionSyncResult = { - pagesIndexed: 0, - databasesIndexed: 0, - errors: [], - }; - - const excludeSet = new Set(config.excludePages ?? []); + const result: NotionSyncResult = { + pagesIndexed: 0, + databasesIndexed: 0, + errors: [], + }; - log.info({ lastSync: config.lastSync }, "Starting Notion sync"); - const searchResults = await searchNotion(config.token, config.lastSync); - log.info({ count: searchResults.length }, "Found Notion objects"); + const excludeSet = new Set(config.excludePages ?? []); - for (const item of searchResults) { - if (excludeSet.has(item.id)) { - log.debug({ id: item.id }, "Skipping excluded page"); - continue; - } + log.info({ lastSync: config.lastSync }, "Starting Notion sync"); + const searchResults = await searchNotion(config.token, config.lastSync); + log.info({ count: searchResults.length }, "Found Notion objects"); - try { - await syncNotionItem(db, provider, config.token, item, excludeSet, result); - } catch (err) { - const title = extractTitle(item); - const message = err instanceof Error ? err.message : String(err); - result.errors.push({ page: title, error: message }); - log.warn({ id: item.id, err }, "Failed to index Notion item"); - } + for (const item of searchResults) { + if (excludeSet.has(item.id)) { + log.debug({ id: item.id }, "Skipping excluded page"); + continue; } - log.info( - { - pagesIndexed: result.pagesIndexed, - databasesIndexed: result.databasesIndexed, - errors: result.errors.length, - }, - "Notion sync complete", - ); + try { + await syncNotionItem(db, provider, config.token, item, excludeSet, result); + } catch (err) { + const title = extractTitle(item); + const message = err instanceof Error ? err.message : String(err); + result.errors.push({ page: title, error: message }); + log.warn({ id: item.id, err }, "Failed to index Notion item"); + } + } - completeSync(db, syncId, { - added: result.pagesIndexed + result.databasesIndexed, - updated: 0, - deleted: 0, - errored: result.errors.length, - }); + log.info( + { + pagesIndexed: result.pagesIndexed, + databasesIndexed: result.databasesIndexed, + errors: result.errors.length, + }, + "Notion sync complete", + ); - return result; - } catch (err) { - failSync(db, syncId, err instanceof Error ? err.message : String(err)); - throw err; - } + return result; } /** Remove all Notion-sourced documents from the knowledge base. */ diff --git a/src/connectors/obsidian.ts b/src/connectors/obsidian.ts index 308fd6ea..7ec6a91b 100644 --- a/src/connectors/obsidian.ts +++ b/src/connectors/obsidian.ts @@ -11,7 +11,7 @@ import { createLink, resolveDocumentByTitle } from "../core/links.js"; import { getLogger } from "../logger.js"; import { ValidationError } from "../errors.js"; import { loadConnectorConfig, saveConnectorConfig } from "./index.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; export interface ObsidianConfig { vaultPath: string; @@ -456,6 +456,26 @@ export async function syncObsidianVault( db: Database.Database, provider: EmbeddingProvider, config: ObsidianConfig, + options: ConnectorSyncOptions = {}, +): Promise<SyncResult> { + return trackSync( + db, + "obsidian", + options.syncName ?? config.vaultPath, + () => runObsidianSync(db, provider, config), + (result) => ({ + added: result.added, + updated: result.updated, + deleted: result.deleted, + errored: result.errors.length, + }), + ); +} + +async function runObsidianSync( + db: Database.Database, + provider: EmbeddingProvider, + config: ObsidianConfig, ): Promise<SyncResult> { const log = getLogger(); const result: SyncResult = { added: 0, updated: 0, deleted: 0, errors: [] }; @@ -464,60 +484,43 @@ export async function syncObsidianVault( throw new ValidationError("Vault path is required"); } - const syncId = startSync(db, "obsidian", config.vaultPath); + const excludePatterns = [...DEFAULT_EXCLUDE, ...config.excludePatterns]; + const vaultFiles = findMarkdownFiles(config.vaultPath, excludePatterns); - try { - const excludePatterns = [...DEFAULT_EXCLUDE, ...config.excludePatterns]; - const vaultFiles = findMarkdownFiles(config.vaultPath, excludePatterns); - - log.info( - { vaultPath: config.vaultPath, fileCount: vaultFiles.length }, - "Syncing Obsidian vault", - ); - - const connectorConfig = loadConnectorConfig(); - const vaultKey = `obsidian:${config.vaultPath}`; - const existingState = connectorConfig[vaultKey] as VaultState | undefined; - const trackedFiles = existingState?.files ?? {}; - const newTrackedFiles: Record<string, VaultFileEntry> = {}; - const currentFileSet = new Set(vaultFiles); - - await syncVaultFiles(db, provider, config, vaultFiles, trackedFiles, newTrackedFiles, result); - - result.deleted = deleteRemovedFiles(db, trackedFiles, currentFileSet); - - connectorConfig[vaultKey] = { - type: "obsidian", - vaultPath: config.vaultPath, - lastSync: new Date().toISOString(), - topicMapping: config.topicMapping, - excludePatterns: config.excludePatterns, - files: newTrackedFiles, - } satisfies VaultState; - saveConnectorConfig(connectorConfig); - - log.info( - { - added: result.added, - updated: result.updated, - deleted: result.deleted, - errors: result.errors.length, - }, - "Obsidian vault sync complete", - ); - - completeSync(db, syncId, { + log.info({ vaultPath: config.vaultPath, fileCount: vaultFiles.length }, "Syncing Obsidian vault"); + + const connectorConfig = loadConnectorConfig(); + const vaultKey = `obsidian:${config.vaultPath}`; + const existingState = connectorConfig[vaultKey] as VaultState | undefined; + const trackedFiles = existingState?.files ?? {}; + const newTrackedFiles: Record<string, VaultFileEntry> = {}; + const currentFileSet = new Set(vaultFiles); + + await syncVaultFiles(db, provider, config, vaultFiles, trackedFiles, newTrackedFiles, result); + + result.deleted = deleteRemovedFiles(db, trackedFiles, currentFileSet); + + connectorConfig[vaultKey] = { + type: "obsidian", + vaultPath: config.vaultPath, + lastSync: new Date().toISOString(), + topicMapping: config.topicMapping, + excludePatterns: config.excludePatterns, + files: newTrackedFiles, + } satisfies VaultState; + saveConnectorConfig(connectorConfig); + + log.info( + { added: result.added, updated: result.updated, deleted: result.deleted, - errored: result.errors.length, - }); + errors: result.errors.length, + }, + "Obsidian vault sync complete", + ); - return result; - } catch (err) { - failSync(db, syncId, err instanceof Error ? err.message : String(err)); - throw err; - } + return result; } export function disconnectVault(db: Database.Database, vaultPath: string): number { diff --git a/src/connectors/onenote.ts b/src/connectors/onenote.ts index feb3eb41..ef9cca1e 100644 --- a/src/connectors/onenote.ts +++ b/src/connectors/onenote.ts @@ -8,7 +8,7 @@ import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; import { createTopic } from "../core/topics.js"; import { loadConnectorConfig, saveConnectorConfig } from "./index.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; // --------------------------------------------------------------------------- // Types @@ -549,76 +549,109 @@ function deleteStaleOneNoteDocs(db: Database.Database, seenSourceUrls: Set<strin return deleted; } +const TOKEN_REFRESH_MARGIN_MS = 5 * 60_000; + +/** True when the access token is missing, has no known expiry, or expires soon. */ +function accessTokenNeedsRefresh(config: OneNoteConfig): boolean { + if (!config.accessToken || !config.tokenExpiry) return true; + const expiresAt = Date.parse(config.tokenExpiry); + return !Number.isNaN(expiresAt) && expiresAt - Date.now() < TOKEN_REFRESH_MARGIN_MS; +} + +/** + * Refresh `config.accessToken` in place when it is missing, has no recorded expiry, or is + * about to expire, and the config holds a client ID and refresh token. Updates accessToken, + * refreshToken and tokenExpiry on the config object; callers that persist the config should + * save it after the sync. Returns true when the tokens were refreshed. + */ +export async function ensureOneNoteAccessToken(config: OneNoteConfig): Promise<boolean> { + if (!config.clientId || !config.refreshToken || !accessTokenNeedsRefresh(config)) { + return false; + } + const tenantId = config.tenantId === "" ? undefined : config.tenantId; + const auth = await refreshAccessToken(config.clientId, config.refreshToken, tenantId); + config.accessToken = auth.accessToken; + config.refreshToken = auth.refreshToken; + config.tokenExpiry = auth.expiresAt; + getLogger().info("Refreshed OneNote access token"); + return true; +} + +/** + * Sync OneNote notebooks. Refreshes an expired or missing access token first when the + * config carries a refresh token (see ensureOneNoteAccessToken; the config is updated). + */ export async function syncOneNote( db: Database.Database, provider: EmbeddingProvider, config: OneNoteConfig, + options: ConnectorSyncOptions = {}, +): Promise<OneNoteSyncResult> { + return trackSync( + db, + "onenote", + options.syncName ?? "onenote", + () => runOneNoteSync(db, provider, config), + (result) => ({ + added: result.pagesAdded, + updated: result.pagesUpdated, + deleted: result.pagesDeleted, + errored: result.errors.length, + }), + ); +} + +async function runOneNoteSync( + db: Database.Database, + provider: EmbeddingProvider, + config: OneNoteConfig, ): Promise<OneNoteSyncResult> { const log = getLogger(); + await ensureOneNoteAccessToken(config); const token = config.accessToken; if (!token) { throw new LibScopeError("No access token provided", "ONENOTE_AUTH_ERROR"); } - const syncId = startSync(db, "onenote", "onenote"); + const result: OneNoteSyncResult = { + notebooks: 0, + sections: 0, + pagesAdded: 0, + pagesUpdated: 0, + pagesDeleted: 0, + errors: [], + }; - try { - const result: OneNoteSyncResult = { - notebooks: 0, - sections: 0, - pagesAdded: 0, - pagesUpdated: 0, - pagesDeleted: 0, - errors: [], - }; - - log.info("Starting OneNote sync"); - - const allNotebooks = await listNotebooks(token); - const targetNotebooks = - config.notebooks.length === 1 && config.notebooks[0] === "all" - ? allNotebooks - : allNotebooks.filter((nb) => config.notebooks.includes(nb.displayName)); - - result.notebooks = targetNotebooks.length; - const seenSourceUrls = new Set<string>(); - - for (const notebook of targetNotebooks) { - await syncOneNoteNotebook(db, provider, token, notebook, config, seenSourceUrls, result); - } + log.info("Starting OneNote sync"); - result.pagesDeleted = deleteStaleOneNoteDocs(db, seenSourceUrls); - - const connConfig = loadConnectorConfig(); - const onenoteConf = (connConfig.onenote ?? {}) as Record<string, unknown>; - onenoteConf.lastSync = new Date().toISOString(); - connConfig.onenote = onenoteConf; - saveConnectorConfig(connConfig); - - log.info( - { - notebooks: result.notebooks, - sections: result.sections, - pagesAdded: result.pagesAdded, - pagesUpdated: result.pagesUpdated, - pagesDeleted: result.pagesDeleted, - errors: result.errors.length, - }, - "OneNote sync complete", - ); + const allNotebooks = await listNotebooks(token); + const targetNotebooks = + config.notebooks.length === 1 && config.notebooks[0] === "all" + ? allNotebooks + : allNotebooks.filter((nb) => config.notebooks.includes(nb.displayName)); - completeSync(db, syncId, { - added: result.pagesAdded, - updated: result.pagesUpdated, - deleted: result.pagesDeleted, - errored: result.errors.length, - }); + result.notebooks = targetNotebooks.length; + const seenSourceUrls = new Set<string>(); - return result; - } catch (err) { - failSync(db, syncId, err instanceof Error ? err.message : String(err)); - throw err; + for (const notebook of targetNotebooks) { + await syncOneNoteNotebook(db, provider, token, notebook, config, seenSourceUrls, result); } + + result.pagesDeleted = deleteStaleOneNoteDocs(db, seenSourceUrls); + + log.info( + { + notebooks: result.notebooks, + sections: result.sections, + pagesAdded: result.pagesAdded, + pagesUpdated: result.pagesUpdated, + pagesDeleted: result.pagesDeleted, + errors: result.errors.length, + }, + "OneNote sync complete", + ); + + return result; } // --------------------------------------------------------------------------- diff --git a/src/connectors/saved-config.ts b/src/connectors/saved-config.ts new file mode 100644 index 00000000..f5097d05 --- /dev/null +++ b/src/connectors/saved-config.ts @@ -0,0 +1,208 @@ +/** + * Saved connector configs: one file per named connector in ~/.libscope/connectors/<name>.json + * (mode 0600). `libscope connect <type>` writes them, and `connect <type> --sync`, the + * scheduler and the MCP sync tools read them. + */ +import type Database from "better-sqlite3"; +import type { EmbeddingProvider } from "../providers/embedding.js"; +import { ConfigError } from "../errors.js"; +import { + hasNamedConnectorConfig, + loadConnectorConfig, + loadNamedConnectorConfig, + saveNamedConnectorConfig, +} from "./index.js"; +import { startSync, failSync, type ConnectorSyncOptions } from "./sync-tracker.js"; +import { syncNotion, type NotionConfig } from "./notion.js"; +import { syncSlack, type SlackConfig } from "./slack.js"; +import { syncConfluence, type ConfluenceConfig } from "./confluence.js"; +import { syncObsidianVault, type ObsidianConfig } from "./obsidian.js"; +import { syncOneNote, type OneNoteConfig } from "./onenote.js"; + +/** Connector types that can be saved, re-synced and scheduled. */ +export const CONNECTOR_TYPES = ["notion", "slack", "confluence", "obsidian", "onenote"] as const; +export type ConnectorType = (typeof CONNECTOR_TYPES)[number]; + +export function isConnectorType(value: unknown): value is ConnectorType { + return typeof value === "string" && (CONNECTOR_TYPES as readonly string[]).includes(value); +} + +/** Fields stored next to the connector's own settings. */ +export interface SavedConnectorFields { + /** Connector type the config belongs to. */ + connectorType?: ConnectorType | undefined; + /** Cron schedule set by `libscope schedule set`. */ + schedule?: { cronExpression: string } | undefined; +} + +interface HasLastSync { + lastSync?: string | undefined; +} + +/** + * Connector type a saved config declares: its `connectorType` field, else a legacy `type` + * field that names a connector (Confluence's own `type` is "cloud"/"server"). + */ +function declaredConnectorType(raw: Record<string, unknown>): ConnectorType | undefined { + if (isConnectorType(raw["connectorType"])) return raw["connectorType"]; + if (isConnectorType(raw["type"])) return raw["type"]; + return undefined; +} + +/** Connector type of a saved config: the type it declares, else the config name. */ +export function resolveConnectorType(name: string, raw: Record<string, unknown>): string { + return declaredConnectorType(raw) ?? name; +} + +/** + * Load the saved config `name` for `type`. Throws ConfigError if it is missing or declares + * a different connector type. + */ +export function loadSavedConnectorConfig<T>(type: ConnectorType, name: string = type): T { + const raw = loadNamedConnectorConfig<Record<string, unknown>>(name); + const declared = declaredConnectorType(raw); + if (declared !== undefined && declared !== type) { + throw new ConfigError(`Connector config "${name}" is a ${declared} connector, not ${type}`); + } + return raw as T; +} + +/** Like loadSavedConnectorConfig, but returns undefined when no config named `name` exists. */ +export function findSavedConnectorConfig<T>( + type: ConnectorType, + name: string = type, +): T | undefined { + return hasNamedConnectorConfig(name) ? loadSavedConnectorConfig<T>(type, name) : undefined; +} + +/** + * Saved OneNote config `name`. For the default name, falls back to the credentials that + * older versions stored under the "onenote" key of ~/.libscope/connectors.json. + */ +export function findSavedOneNoteConfig(name: string = "onenote"): OneNoteConfig | undefined { + const saved = findSavedConnectorConfig<OneNoteConfig>("onenote", name); + if (saved || name !== "onenote") return saved; + const legacy = loadConnectorConfig()["onenote"] as Partial<OneNoteConfig> | undefined; + if (!legacy?.accessToken && !legacy?.refreshToken) return undefined; + return { + clientId: legacy.clientId ?? "", + tenantId: legacy.tenantId ?? "common", + accessToken: legacy.accessToken, + refreshToken: legacy.refreshToken, + tokenExpiry: legacy.tokenExpiry, + lastSync: legacy.lastSync, + notebooks: legacy.notebooks ?? ["all"], + excludeSections: legacy.excludeSections ?? [], + }; +} + +/** + * Save connector settings as config `name`, tagged with the connector type. A schedule + * already stored under that name is kept. + */ +export function saveConnectorSettings(type: ConnectorType, name: string, settings: object): void { + const existing = hasNamedConnectorConfig(name) + ? loadNamedConnectorConfig<SavedConnectorFields>(name) + : undefined; + saveNamedConnectorConfig(name, { + ...settings, + connectorType: type, + ...(existing?.schedule ? { schedule: existing.schedule } : {}), + }); +} + +/** + * Run `sync` with `config`, then save the config as `name` with lastSync set to the time + * the sync started. Changes the connector made to the config object (refreshed OneNote + * tokens) are saved too. + */ +export async function runAndSaveConnector<C extends HasLastSync, R>( + type: ConnectorType, + name: string, + config: C, + sync: (config: C) => Promise<R>, +): Promise<R> { + const startedAt = new Date().toISOString(); + const result = await sync(config); + saveConnectorSettings(type, name, { ...config, lastSync: startedAt }); + return result; +} + +/** Record one failed connector_syncs row for a run that never reached the connector. */ +export function recordFailedSync( + db: Database.Database, + connectorType: string, + connectorName: string, + err: unknown, +): void { + failSync( + db, + startSync(db, connectorType, connectorName), + err instanceof Error ? err.message : String(err), + ); +} + +/** + * Re-sync saved connector `name` with its saved config and save the new lastSync. + * The connector records the run in connector_syncs under `name`; when the config cannot be + * loaded, one failed row is recorded here instead. Errors are rethrown. + */ +export async function syncSavedConnector<C extends HasLastSync, R>( + db: Database.Database, + type: ConnectorType, + name: string, + sync: (config: C) => Promise<R>, +): Promise<R> { + let config: C; + try { + config = loadSavedConnectorConfig<C>(type, name); + } catch (err) { + recordFailedSync(db, type, name, err); + throw err; + } + return runAndSaveConnector(type, name, config, sync); +} + +type SavedSyncRunner = ( + db: Database.Database, + provider: EmbeddingProvider, + name: string, + options: ConnectorSyncOptions, +) => Promise<unknown>; + +const SAVED_SYNC_RUNNERS: Record<ConnectorType, SavedSyncRunner> = { + notion: (db, provider, name, options) => + syncSavedConnector<NotionConfig, unknown>(db, "notion", name, (c) => + syncNotion(db, provider, c, options), + ), + slack: (db, provider, name, options) => + syncSavedConnector<SlackConfig, unknown>(db, "slack", name, (c) => + syncSlack(db, provider, c, options), + ), + confluence: (db, provider, name, options) => + syncSavedConnector<ConfluenceConfig, unknown>(db, "confluence", name, (c) => + syncConfluence(db, provider, c, options), + ), + obsidian: (db, provider, name, options) => + syncSavedConnector<ObsidianConfig, unknown>(db, "obsidian", name, (c) => + syncObsidianVault(db, provider, c, options), + ), + onenote: (db, provider, name, options) => + syncSavedConnector<OneNoteConfig, unknown>(db, "onenote", name, (c) => + syncOneNote(db, provider, c, options), + ), +}; + +/** + * Re-sync saved connector `name` of type `type` (used by the scheduler). Writes exactly one + * connector_syncs row, recorded under `name`. + */ +export async function runSavedConnectorSync( + db: Database.Database, + provider: EmbeddingProvider, + type: ConnectorType, + name: string, + options: ConnectorSyncOptions = {}, +): Promise<void> { + await SAVED_SYNC_RUNNERS[type](db, provider, name, { ...options, syncName: name }); +} diff --git a/src/connectors/slack.ts b/src/connectors/slack.ts index 02b04071..aed6264d 100644 --- a/src/connectors/slack.ts +++ b/src/connectors/slack.ts @@ -5,7 +5,7 @@ import { indexDocument } from "../core/indexing.js"; import { getLogger } from "../logger.js"; import { LibScopeError, ValidationError } from "../errors.js"; import { fetchWithRetry } from "./http-utils.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; import { deleteDocumentRows } from "./index.js"; export interface SlackConfig { @@ -443,6 +443,17 @@ function classifyMessages(messages: SlackMessage[]): { return { threadParents, standaloneMessages }; } +/** + * Convert a lastSync value to Slack's `oldest` format (Unix seconds). Saved configs store + * lastSync as an ISO-8601 string; values that are already numeric are passed through. + */ +export function toSlackTimestamp(lastSync: string | undefined): string | undefined { + if (!lastSync) return undefined; + if (Number.isFinite(Number(lastSync))) return lastSync; + const ms = Date.parse(lastSync); + return Number.isNaN(ms) ? undefined : String(ms / 1000); +} + /** Process a single channel: fetch messages, resolve users, and index documents. */ async function syncChannel( db: Database.Database, @@ -451,7 +462,7 @@ async function syncChannel( channel: SlackChannel, result: SlackSyncResult, ): Promise<void> { - const oldest = config.lastSync ?? undefined; + const oldest = toSlackTimestamp(config.lastSync); const messages = await fetchMessages(config.token, channel.id, oldest); const { threadParents, standaloneMessages } = classifyMessages(messages); @@ -500,6 +511,26 @@ export async function syncSlack( db: Database.Database, provider: EmbeddingProvider, config: SlackConfig, + options: ConnectorSyncOptions = {}, +): Promise<SlackSyncResult> { + return trackSync( + db, + "slack", + options.syncName ?? "slack", + () => runSlackSync(db, provider, config), + (result) => ({ + added: result.messagesIndexed + result.threadsIndexed, + updated: 0, + deleted: 0, + errored: result.errors.length, + }), + ); +} + +async function runSlackSync( + db: Database.Database, + provider: EmbeddingProvider, + config: SlackConfig, ): Promise<SlackSyncResult> { const log = getLogger(); @@ -510,57 +541,43 @@ export async function syncSlack( throw new ValidationError("At least one channel must be specified"); } - const syncId = startSync(db, "slack", "slack"); - - try { - userCache.clear(); + userCache.clear(); - const result: SlackSyncResult = { - channels: 0, - messagesIndexed: 0, - threadsIndexed: 0, - errors: [], - }; + const result: SlackSyncResult = { + channels: 0, + messagesIndexed: 0, + threadsIndexed: 0, + errors: [], + }; - log.info("Fetching Slack channel list"); - const allChannels = await listChannels(config.token); - const channels = filterChannels(allChannels, config.channels, config.excludeChannels); - result.channels = channels.length; - - log.info({ channelCount: channels.length }, "Processing Slack channels"); - - for (const channel of channels) { - try { - log.info({ channel: channel.name }, "Syncing channel"); - await syncChannel(db, provider, config, channel, result); - } catch (err) { - const errMsg = err instanceof Error ? err.message : String(err); - log.error({ channel: channel.name, err }, "Error syncing Slack channel"); - result.errors.push({ channel: channel.name, error: errMsg }); - } + log.info("Fetching Slack channel list"); + const allChannels = await listChannels(config.token); + const channels = filterChannels(allChannels, config.channels, config.excludeChannels); + result.channels = channels.length; + + log.info({ channelCount: channels.length }, "Processing Slack channels"); + + for (const channel of channels) { + try { + log.info({ channel: channel.name }, "Syncing channel"); + await syncChannel(db, provider, config, channel, result); + } catch (err) { + const errMsg = err instanceof Error ? err.message : String(err); + log.error({ channel: channel.name, err }, "Error syncing Slack channel"); + result.errors.push({ channel: channel.name, error: errMsg }); } + } - log.info( - { - channels: result.channels, - messages: result.messagesIndexed, - threads: result.threadsIndexed, - }, - "Slack sync complete", - ); - - completeSync(db, syncId, { - added: result.messagesIndexed + result.threadsIndexed, - updated: 0, - deleted: 0, - errored: result.errors.length, - }); + log.info( + { + channels: result.channels, + messages: result.messagesIndexed, + threads: result.threadsIndexed, + }, + "Slack sync complete", + ); - return result; - } catch (err) { - failSync(db, syncId, err instanceof Error ? err.message : String(err)); - throw err; - } + return result; } export function disconnectSlack(db: Database.Database): number { diff --git a/src/connectors/sync-tracker.ts b/src/connectors/sync-tracker.ts index 50e2c4a7..6ed67c2c 100644 --- a/src/connectors/sync-tracker.ts +++ b/src/connectors/sync-tracker.ts @@ -116,3 +116,34 @@ export function getSyncHistory( .prepare(`SELECT * FROM connector_syncs ORDER BY started_at DESC, id DESC LIMIT ?`) .all(limit) as ConnectorSyncRow[]; } + +/** Optional per-call settings accepted by every connector sync function. */ +export interface ConnectorSyncOptions { + /** + * Name recorded in connector_syncs for this run (the saved connector config name). + * Defaults to the connector's historical name when omitted. + */ + syncName?: string | undefined; +} + +/** + * Run `run` and record exactly one connector_syncs row for it: "completed" with the + * stats from `toStats`, or "failed" with the error message (the error is rethrown). + */ +export async function trackSync<R>( + db: Database.Database, + connectorType: string, + connectorName: string, + run: () => Promise<R>, + toStats: (result: R) => SyncStats, +): Promise<R> { + const syncId = startSync(db, connectorType, connectorName); + try { + const result = await run(); + completeSync(db, syncId, toStats(result)); + return result; + } catch (err) { + failSync(db, syncId, err instanceof Error ? err.message : String(err)); + throw err; + } +} diff --git a/src/core/scheduler.ts b/src/core/scheduler.ts index 213c8a83..1a41a3dc 100644 --- a/src/core/scheduler.ts +++ b/src/core/scheduler.ts @@ -1,30 +1,19 @@ import type Database from "better-sqlite3"; import { readdirSync, readFileSync, existsSync } from "node:fs"; import { join } from "node:path"; -import { homedir } from "node:os"; // NOTE: @types/node-cron v3 is used with node-cron v4 — no v4 types are published yet. // The schedule() and ScheduledTask.stop() APIs are compatible across versions. import cron from "node-cron"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; +import { getConnectorsDir } from "../connectors/index.js"; import { - loadNamedConnectorConfig, - saveNamedConnectorConfig, - startSync, - completeSync, - failSync, -} from "../connectors/index.js"; -import { syncNotion } from "../connectors/notion.js"; -import type { NotionConfig } from "../connectors/notion.js"; -import { syncSlack } from "../connectors/slack.js"; -import type { SlackConfig } from "../connectors/slack.js"; -import { syncConfluence } from "../connectors/confluence.js"; -import type { ConfluenceConfig } from "../connectors/confluence.js"; -import { syncObsidianVault } from "../connectors/obsidian.js"; -import type { ObsidianConfig } from "../connectors/obsidian.js"; -import { syncOneNote } from "../connectors/onenote.js"; -import type { OneNoteConfig } from "../connectors/onenote.js"; + isConnectorType, + recordFailedSync, + resolveConnectorType, + runSavedConnectorSync, +} from "../connectors/saved-config.js"; export interface ScheduleConfig { cronExpression: string; @@ -175,16 +164,18 @@ export class ConnectorScheduler { job.running = true; } - const syncId = startSync(this.db, connectorType, connectorName); log.info({ connector: connectorName, type: connectorType }, "Starting scheduled sync"); try { - const stats = await this.executeSync(connectorType, connectorName); - completeSync(this.db, syncId, stats); - log.info({ connector: connectorName, ...stats }, "Scheduled sync completed"); + // The connector records the run (one connector_syncs row, under connectorName). + if (!isConnectorType(connectorType)) { + const err = new ValidationError(`Unknown connector type: ${connectorType}`); + recordFailedSync(this.db, connectorType, connectorName, err); + throw err; + } + await runSavedConnectorSync(this.db, this.provider, connectorType, connectorName); + log.info({ connector: connectorName }, "Scheduled sync completed"); } catch (err) { - const message = err instanceof Error ? err.message : String(err); - failSync(this.db, syncId, message); log.error({ connector: connectorName, err }, "Scheduled sync failed"); } finally { if (job) { @@ -193,76 +184,6 @@ export class ConnectorScheduler { } } } - - private async executeSync( - connectorType: string, - connectorName: string, - ): Promise<{ added: number; updated: number; deleted: number; errored: number }> { - switch (connectorType) { - case "notion": { - const config = loadNamedConnectorConfig<NotionConfig>(connectorName); - const result = await syncNotion(this.db, this.provider, config); - config.lastSync = new Date().toISOString(); - saveNamedConnectorConfig(connectorName, config); - return { - added: result.pagesIndexed, - updated: 0, - deleted: 0, - errored: result.errors.length, - }; - } - case "slack": { - const config = loadNamedConnectorConfig<SlackConfig>(connectorName); - const result = await syncSlack(this.db, this.provider, config); - config.lastSync = new Date().toISOString(); - saveNamedConnectorConfig(connectorName, config); - return { - added: result.messagesIndexed + result.threadsIndexed, - updated: 0, - deleted: 0, - errored: result.errors.length, - }; - } - case "confluence": { - const config = loadNamedConnectorConfig<ConfluenceConfig>(connectorName); - const result = await syncConfluence(this.db, this.provider, config); - config.lastSync = new Date().toISOString(); - saveNamedConnectorConfig(connectorName, config); - return { - added: result.pagesIndexed, - updated: result.pagesUpdated, - deleted: 0, - errored: result.errors.length, - }; - } - case "obsidian": { - const config = loadNamedConnectorConfig<ObsidianConfig>(connectorName); - const result = await syncObsidianVault(this.db, this.provider, config); - config.lastSync = new Date().toISOString(); - saveNamedConnectorConfig(connectorName, config); - return { - added: result.added, - updated: result.updated, - deleted: result.deleted, - errored: result.errors.length, - }; - } - case "onenote": { - const config = loadNamedConnectorConfig<OneNoteConfig>(connectorName); - const result = await syncOneNote(this.db, this.provider, config); - config.lastSync = new Date().toISOString(); - saveNamedConnectorConfig(connectorName, config); - return { - added: result.pagesAdded, - updated: result.pagesUpdated, - deleted: 0, - errored: result.errors.length, - }; - } - default: - throw new ValidationError(`Unknown connector type: ${connectorType}`); - } - } } /** @@ -273,7 +194,7 @@ export function loadScheduleEntries(): ConnectorScheduleEntry[] { const log = getLogger(); const entries: ConnectorScheduleEntry[] = []; - const connectorsDir = join(homedir(), ".libscope", "connectors"); + const connectorsDir = getConnectorsDir(); if (!existsSync(connectorsDir)) { return entries; } @@ -286,8 +207,8 @@ export function loadScheduleEntries(): ConnectorScheduleEntry[] { const schedule = config.schedule as { cronExpression?: string } | undefined; if (schedule?.cronExpression) { - const connectorName = file.replace(/\.json$/, ""); - const connectorType = (config.type as string | undefined) ?? connectorName; + const connectorName = file.slice(0, -".json".length); + const connectorType = resolveConnectorType(connectorName, config); entries.push({ connectorType, connectorName, diff --git a/tests/unit/connector-saved-config.test.ts b/tests/unit/connector-saved-config.test.ts new file mode 100644 index 00000000..a4cae217 --- /dev/null +++ b/tests/unit/connector-saved-config.test.ts @@ -0,0 +1,303 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdirSync, writeFileSync, readFileSync, rmSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { initLogger } from "../../src/logger.js"; +import { ConfigError } from "../../src/errors.js"; + +// Every config file lives under a temp HOME — never the real ~/.libscope. +let tempHome = join(tmpdir(), `libscope-saved-init-${process.pid}`); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const cronCallbacks = new Map<string, () => void>(); +vi.mock("node-cron", () => ({ + default: { + validate: (): boolean => true, + schedule: vi.fn((expr: string, callback: () => void) => { + cronCallbacks.set(expr, callback); + return { stop: vi.fn() }; + }), + }, +})); + +const { + saveConnectorSettings, + loadSavedConnectorConfig, + findSavedOneNoteConfig, + resolveConnectorType, + runSavedConnectorSync, +} = await import("../../src/connectors/saved-config.js"); +const { saveConnectorConfig, getConnectorsDir } = await import("../../src/connectors/index.js"); +const { ConnectorScheduler, loadScheduleEntries } = await import("../../src/core/scheduler.js"); +const { _resetRateLimiter } = await import("../../src/connectors/onenote.js"); + +interface SyncRow { + connector_type: string; + connector_name: string; + status: string; + docs_added: number; + docs_updated: number; + docs_deleted: number; + error_message: string | null; +} + +function syncRows(db: Database.Database): SyncRow[] { + return db.prepare("SELECT * FROM connector_syncs ORDER BY id").all() as SyncRow[]; +} + +function readSaved(name: string): Record<string, unknown> { + return JSON.parse(readFileSync(join(getConnectorsDir(), `${name}.json`), "utf-8")) as Record< + string, + unknown + >; +} + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +/** Notion API: one page with one paragraph. */ +function mockNotionApi(): ReturnType<typeof vi.fn> { + const fetchMock = vi.fn((url: string) => { + if (url.endsWith("/v1/search")) { + return Promise.resolve( + json({ + results: [ + { + object: "page", + id: "page-1", + last_edited_time: "2026-01-01T00:00:00.000Z", + properties: { Name: { type: "title", title: [{ plain_text: "Runbook" }] } }, + }, + ], + has_more: false, + next_cursor: null, + }), + ); + } + return Promise.resolve( + json({ + results: [ + { + id: "b1", + type: "paragraph", + paragraph: { rich_text: [{ plain_text: "Restart the service." }] }, + }, + ], + has_more: false, + next_cursor: null, + }), + ); + }); + vi.stubGlobal("fetch", fetchMock); + return fetchMock; +} + +describe("saved connector configs", () => { + let db: Database.Database; + const provider = new MockEmbeddingProvider(); + + beforeEach(() => { + initLogger("silent"); + tempHome = join(tmpdir(), `libscope-saved-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + db = createTestDbWithVec(); + cronCallbacks.clear(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + db.close(); + rmSync(tempHome, { recursive: true, force: true }); + }); + + describe("saveConnectorSettings", () => { + it("tags the connector type, writes mode 0600 and keeps an existing schedule", () => { + saveConnectorSettings("notion", "team", { token: "secret_a" }); + const path = join(getConnectorsDir(), "team.json"); + expect(statSync(path).mode & 0o777).toBe(0o600); + + const withSchedule = { ...readSaved("team"), schedule: { cronExpression: "0 * * * *" } }; + writeFileSync(path, JSON.stringify(withSchedule)); + saveConnectorSettings("notion", "team", { token: "secret_b" }); + + expect(readSaved("team")).toEqual({ + token: "secret_b", + connectorType: "notion", + schedule: { cronExpression: "0 * * * *" }, + }); + }); + }); + + describe("resolveConnectorType / loadSavedConnectorConfig", () => { + it("prefers connectorType over Confluence's own cloud/server type field", () => { + expect(resolveConnectorType("x", { connectorType: "confluence", type: "cloud" })).toBe( + "confluence", + ); + expect(resolveConnectorType("x", { type: "notion" })).toBe("notion"); + expect(resolveConnectorType("slack", { token: "t" })).toBe("slack"); + }); + + it("rejects a config that declares a different connector type", () => { + saveConnectorSettings("slack", "work", { token: "xoxb", channels: ["all"] }); + expect(() => loadSavedConnectorConfig("notion", "work")).toThrow(ConfigError); + expect(loadSavedConnectorConfig("slack", "work")).toMatchObject({ token: "xoxb" }); + }); + + it("schedules a saved Confluence config as confluence, not as its cloud type", () => { + saveConnectorSettings("confluence", "wiki", { + baseUrl: "https://acme.atlassian.net", + type: "cloud", + token: "t", + spaces: ["all"], + }); + const path = join(getConnectorsDir(), "wiki.json"); + writeFileSync( + path, + JSON.stringify({ ...readSaved("wiki"), schedule: { cronExpression: "0 0 * * *" } }), + ); + expect(loadScheduleEntries()).toEqual([ + { connectorType: "confluence", connectorName: "wiki", cronExpression: "0 0 * * *" }, + ]); + }); + }); + + describe("runSavedConnectorSync", () => { + it("writes exactly one sync row under the configured name and saves lastSync", async () => { + mockNotionApi(); + saveConnectorSettings("notion", "team-notion", { token: "secret_abc" }); + + const before = new Date().toISOString(); + await runSavedConnectorSync(db, provider, "notion", "team-notion"); + + const rows = syncRows(db); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + connector_type: "notion", + connector_name: "team-notion", + status: "completed", + docs_added: 1, + }); + const saved = readSaved("team-notion"); + expect(saved["connectorType"]).toBe("notion"); + expect(String(saved["lastSync"]) >= before).toBe(true); + }); + + it("records one failed row when the connector rejects the saved config", async () => { + saveConnectorSettings("notion", "bad", { token: "not-a-notion-token" }); + await expect(runSavedConnectorSync(db, provider, "notion", "bad")).rejects.toThrow( + "Notion token must start with", + ); + const rows = syncRows(db); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ connector_name: "bad", status: "failed" }); + expect(readSaved("bad")["lastSync"]).toBeUndefined(); + }); + + it("records one failed row when the saved config is missing", async () => { + await expect(runSavedConnectorSync(db, provider, "slack", "nope")).rejects.toThrow( + ConfigError, + ); + const rows = syncRows(db); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + connector_type: "slack", + connector_name: "nope", + status: "failed", + }); + }); + + it("refreshes an expired OneNote token, saves the new tokens and maps deletions", async () => { + _resetRateLimiter(); + db.prepare( + "INSERT INTO documents (id, source_type, title, content, url) VALUES ('old', 'topic', 'Old', 'Body', 'onenote://NB/Sec/Old')", + ).run(); + saveConnectorSettings("onenote", "onenote", { + clientId: "client-1", + tenantId: "common", + accessToken: "expired-access", + refreshToken: "refresh-1", + tokenExpiry: "2000-01-01T00:00:00.000Z", + notebooks: ["all"], + excludeSections: [], + }); + + const fetchMock = vi.fn((url: string, init?: RequestInit) => { + if (url.includes("/oauth2/v2.0/token")) { + expect((init?.body as URLSearchParams).get("refresh_token")).toBe("refresh-1"); + return Promise.resolve( + json({ access_token: "new-access", refresh_token: "refresh-2", expires_in: 3600 }), + ); + } + const auth = (init?.headers as Record<string, string>)["Authorization"]; + expect(auth).toBe("Bearer new-access"); + return Promise.resolve(json({ value: [] })); + }); + vi.stubGlobal("fetch", fetchMock); + + await runSavedConnectorSync(db, provider, "onenote", "onenote"); + + const saved = readSaved("onenote"); + expect(saved["accessToken"]).toBe("new-access"); + expect(saved["refreshToken"]).toBe("refresh-2"); + expect(Date.parse(String(saved["tokenExpiry"]))).toBeGreaterThan(Date.now()); + const rows = syncRows(db); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ connector_name: "onenote", docs_deleted: 1 }); + }); + }); + + describe("findSavedOneNoteConfig", () => { + it("falls back to credentials stored by older versions in connectors.json", () => { + saveConnectorConfig({ + onenote: { clientId: "c", tenantId: "t", refreshToken: "r", lastSync: "2026-01-01" }, + }); + expect(findSavedOneNoteConfig()).toEqual({ + clientId: "c", + tenantId: "t", + accessToken: undefined, + refreshToken: "r", + tokenExpiry: undefined, + lastSync: "2026-01-01", + notebooks: ["all"], + excludeSections: [], + }); + expect(findSavedOneNoteConfig("other")).toBeUndefined(); + }); + }); + + describe("scheduler end to end", () => { + it("runs a scheduled Notion sync from a saved config and records one row", async () => { + mockNotionApi(); + saveConnectorSettings("notion", "notion", { token: "secret_abc" }); + const path = join(getConnectorsDir(), "notion.json"); + writeFileSync( + path, + JSON.stringify({ ...readSaved("notion"), schedule: { cronExpression: "*/5 * * * *" } }), + ); + + const scheduler = new ConnectorScheduler(db, provider); + scheduler.start(loadScheduleEntries()); + cronCallbacks.get("*/5 * * * *")!(); + await vi.waitFor(() => { + expect(scheduler.getStatus().jobs[0]!.lastRun).toBeDefined(); + }); + await scheduler.stop(); + + const rows = syncRows(db); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ connector_name: "notion", status: "completed" }); + expect(readSaved("notion")["schedule"]).toEqual({ cronExpression: "*/5 * * * *" }); + }); + }); +}); diff --git a/tests/unit/scheduler.test.ts b/tests/unit/scheduler.test.ts index d5a9c435..47c4310d 100644 --- a/tests/unit/scheduler.test.ts +++ b/tests/unit/scheduler.test.ts @@ -29,6 +29,7 @@ vi.mock("../../src/connectors/index.js", async (importOriginal) => { ...orig, loadNamedConnectorConfig: vi.fn().mockReturnValue({ token: "test-token" }), saveNamedConnectorConfig: vi.fn(), + hasNamedConnectorConfig: vi.fn().mockReturnValue(false), startSync: vi.fn().mockReturnValue("sync-id-1"), completeSync: vi.fn(), failSync: vi.fn(), @@ -57,7 +58,8 @@ vi.mock("../../src/connectors/onenote.js", () => ({ // Import after mocks are set up const { ConnectorScheduler, loadScheduleEntries } = await import("../../src/core/scheduler.js"); -const { startSync, completeSync, failSync } = await import("../../src/connectors/index.js"); +const { startSync, completeSync, failSync, saveNamedConnectorConfig } = + await import("../../src/connectors/index.js"); const { syncNotion } = await import("../../src/connectors/notion.js"); const { syncSlack } = await import("../../src/connectors/slack.js"); const { syncConfluence } = await import("../../src/connectors/confluence.js"); @@ -149,132 +151,46 @@ describe("ConnectorScheduler", () => { }); describe("runSync via cron callback", () => { - it("runs a notion sync successfully", async () => { - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { connectorType: "notion", connectorName: "my-notion", cronExpression: "0 */6 * * *" }, - ]); - - const callback = cronCallbacks.get("0 */6 * * *"); - expect(callback).toBeDefined(); - callback!(); - // Allow async runSync to complete - await vi.waitFor(() => { - expect(completeSync).toHaveBeenCalled(); - }); - - expect(startSync).toHaveBeenCalledWith(db, "notion", "my-notion"); - expect(syncNotion).toHaveBeenCalled(); - expect(completeSync).toHaveBeenCalledWith(db, "sync-id-1", { - added: 5, - updated: 0, - deleted: 0, - errored: 0, - }); - - const status = scheduler.getStatus(); - expect(status.jobs[0]!.lastRun).toBeDefined(); - expect(status.jobs[0]!.running).toBe(false); - await scheduler.stop(); - }); - - it("runs a slack sync successfully", async () => { - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { connectorType: "slack", connectorName: "my-slack", cronExpression: "0 0 * * *" }, - ]); - - cronCallbacks.get("0 0 * * *")!(); - await vi.waitFor(() => { - expect(completeSync).toHaveBeenCalled(); - }); - - expect(syncSlack).toHaveBeenCalled(); - expect(completeSync).toHaveBeenCalledWith(db, "sync-id-1", { - added: 12, - updated: 0, - deleted: 0, - errored: 0, - }); - await scheduler.stop(); - }); - - it("runs a confluence sync successfully", async () => { - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { - connectorType: "confluence", - connectorName: "my-confluence", - cronExpression: "0 */6 * * *", - }, - ]); - - cronCallbacks.get("0 */6 * * *")!(); - await vi.waitFor(() => { - expect(completeSync).toHaveBeenCalled(); - }); - - expect(syncConfluence).toHaveBeenCalled(); - expect(completeSync).toHaveBeenCalledWith(db, "sync-id-1", { - added: 3, - updated: 1, - deleted: 0, - errored: 0, - }); - await scheduler.stop(); - }); - - it("runs an obsidian sync successfully", async () => { - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { - connectorType: "obsidian", - connectorName: "my-obsidian", - cronExpression: "0 */6 * * *", - }, - ]); - - cronCallbacks.get("0 */6 * * *")!(); - await vi.waitFor(() => { - expect(completeSync).toHaveBeenCalled(); - }); - - expect(syncObsidianVault).toHaveBeenCalled(); - expect(completeSync).toHaveBeenCalledWith(db, "sync-id-1", { - added: 4, - updated: 2, - deleted: 1, - errored: 0, - }); - await scheduler.stop(); - }); - - it("runs a onenote sync successfully", async () => { - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { - connectorType: "onenote", - connectorName: "my-onenote", - cronExpression: "0 */6 * * *", - }, - ]); - - cronCallbacks.get("0 */6 * * *")!(); - await vi.waitFor(() => { - expect(completeSync).toHaveBeenCalled(); - }); - - expect(syncOneNote).toHaveBeenCalled(); - expect(completeSync).toHaveBeenCalledWith(db, "sync-id-1", { - added: 6, - updated: 0, - deleted: 0, - errored: 0, - }); - await scheduler.stop(); - }); + const cases = [ + ["notion", syncNotion], + ["slack", syncSlack], + ["confluence", syncConfluence], + ["obsidian", syncObsidianVault], + ["onenote", syncOneNote], + ] as const; + + it.each(cases)( + "delegates a %s run to the connector under the configured name", + async (type, syncFn) => { + const scheduler = new ConnectorScheduler(db, provider); + scheduler.start([ + { connectorType: type, connectorName: `my-${type}`, cronExpression: "0 */6 * * *" }, + ]); + + cronCallbacks.get("0 */6 * * *")!(); + await vi.waitFor(() => { + expect(scheduler.getStatus().jobs[0]!.lastRun).toBeDefined(); + }); + + expect(syncFn).toHaveBeenCalledWith( + db, + provider, + expect.objectContaining({ token: "test-token" }), + { syncName: `my-${type}` }, + ); + // The connector records the run; the scheduler must not add a second row. + expect(startSync).not.toHaveBeenCalled(); + expect(completeSync).not.toHaveBeenCalled(); + const [savedName, savedConfig] = vi.mocked(saveNamedConnectorConfig).mock.calls[0]!; + expect(savedName).toBe(`my-${type}`); + expect(savedConfig).toMatchObject({ connectorType: type }); + expect(typeof (savedConfig as { lastSync?: unknown }).lastSync).toBe("string"); + expect(scheduler.getStatus().jobs[0]!.running).toBe(false); + await scheduler.stop(); + }, + ); - it("handles sync failure and calls failSync", async () => { + it("does not save lastSync or write a sync row when the connector fails", async () => { vi.mocked(syncNotion).mockRejectedValueOnce(new Error("Network error")); const scheduler = new ConnectorScheduler(db, provider); @@ -284,17 +200,17 @@ describe("ConnectorScheduler", () => { cronCallbacks.get("*/5 * * * *")!(); await vi.waitFor(() => { - expect(failSync).toHaveBeenCalled(); + expect(scheduler.getStatus().jobs[0]!.lastRun).toBeDefined(); }); - expect(failSync).toHaveBeenCalledWith(db, "sync-id-1", "Network error"); - const status = scheduler.getStatus(); - expect(status.jobs[0]!.running).toBe(false); - expect(status.jobs[0]!.lastRun).toBeDefined(); + expect(saveNamedConnectorConfig).not.toHaveBeenCalled(); + expect(startSync).not.toHaveBeenCalled(); + expect(failSync).not.toHaveBeenCalled(); + expect(scheduler.getStatus().jobs[0]!.running).toBe(false); await scheduler.stop(); }); - it("handles unknown connector type", async () => { + it("records one failed row for an unknown connector type", async () => { const scheduler = new ConnectorScheduler(db, provider); scheduler.start([ { @@ -306,14 +222,22 @@ describe("ConnectorScheduler", () => { cronCallbacks.get("0 */6 * * *")!(); await vi.waitFor(() => { - expect(failSync).toHaveBeenCalled(); + expect(scheduler.getStatus().jobs[0]!.lastRun).toBeDefined(); }); - expect(failSync).toHaveBeenCalledWith( - db, - "sync-id-1", - "Unknown connector type: unknown-type", - ); + const rows = db.prepare("SELECT * FROM connector_syncs").all() as Array<{ + connector_type: string; + connector_name: string; + status: string; + error_message: string; + }>; + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + connector_type: "unknown-type", + connector_name: "my-unknown", + status: "failed", + error_message: "Unknown connector type: unknown-type", + }); await scheduler.stop(); }); @@ -323,7 +247,7 @@ describe("ConnectorScheduler", () => { vi.mocked(syncNotion).mockImplementationOnce( () => new Promise((resolve) => { - resolveSync = () => resolve({ pagesIndexed: 1, errors: [] }); + resolveSync = () => resolve({ pagesIndexed: 1, databasesIndexed: 0, errors: [] }); }), ); @@ -354,23 +278,6 @@ describe("ConnectorScheduler", () => { expect(syncNotion).toHaveBeenCalledTimes(1); await scheduler.stop(); }); - - it("handles non-Error thrown values in sync", async () => { - vi.mocked(syncNotion).mockRejectedValueOnce("string error"); - - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { connectorType: "notion", connectorName: "str-err", cronExpression: "0 */6 * * *" }, - ]); - - cronCallbacks.get("0 */6 * * *")!(); - await vi.waitFor(() => { - expect(failSync).toHaveBeenCalled(); - }); - - expect(failSync).toHaveBeenCalledWith(db, "sync-id-1", "string error"); - await scheduler.stop(); - }); }); }); diff --git a/tests/unit/slack.test.ts b/tests/unit/slack.test.ts index 79a53d5b..445c1c6a 100644 --- a/tests/unit/slack.test.ts +++ b/tests/unit/slack.test.ts @@ -4,6 +4,7 @@ import { syncSlack, disconnectSlack, _setRateLimitDelay, + toSlackTimestamp, _clearUserCache, } from "../../src/connectors/slack.js"; import type { SlackConfig } from "../../src/connectors/slack.js"; @@ -230,7 +231,7 @@ describe("syncSlack", () => { expect(result.messagesIndexed).toBe(2); }); - it("uses oldest param for incremental sync", async () => { + it("sends lastSync to Slack as Unix seconds in the oldest param", async () => { const syncTime = "2024-01-01T00:00:00.000Z"; setupChannelList([{ id: "C001", name: "general" }]); setupMessages([]); @@ -242,7 +243,14 @@ describe("syncSlack", () => { (call: unknown[]) => typeof call[0] === "string" && call[0].includes("conversations.history"), ); expect(historyCall).toBeDefined(); - expect(String(historyCall?.[0])).toContain("oldest=" + encodeURIComponent(syncTime)); + expect(new URL(String(historyCall?.[0])).searchParams.get("oldest")).toBe("1704067200"); + }); + + it("toSlackTimestamp passes numeric values through and drops invalid ones", () => { + expect(toSlackTimestamp("1704067200.5")).toBe("1704067200.5"); + expect(toSlackTimestamp("2024-01-01T00:00:01.500Z")).toBe("1704067201.5"); + expect(toSlackTimestamp("not a date")).toBeUndefined(); + expect(toSlackTimestamp(undefined)).toBeUndefined(); }); it("resolves user mentions in message text", async () => { From 165e4951cdfc677405978a0e43fc968b49630307 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:30:55 +0000 Subject: [PATCH 24/83] fix(embeddings): resolve model dimensions and track the model that built the index Ollama and OpenAI dimensions were hardcoded (768 and 1536), so any other model failed every embed. Nothing recorded which model built the vector table, and reindex kept the old table, so a model switch could only be fixed by deleting the database (the docs pointed to a nonexistent `libscope db reset`). - Providers take the vector size from the new optional `embedding.dimensions` config key, else a table of common OpenAI and Ollama models, else the first embedding returned. Their size check uses that value. For text-embedding-3 models the override is also sent as the API `dimensions` parameter. - Migration 18 adds an index_meta table. createVectorTable records the provider, model and vector size when it creates chunk_embeddings, and accepts a provider as well as a bare size. When the configured model or size differs from the index, it throws a ConfigError that names both and tells the user to run `libscope reindex --rebuild`. - `libscope reindex --rebuild` embeds a probe text, drops and recreates chunk_embeddings with the model's size, records the model, and then re-embeds every chunk. - Search falls back to keyword results on a sqlite-vec dimension mismatch and logs the same guidance. - Docs: replace `libscope db reset` with the real procedures and document `embedding.dimensions` and `--rebuild`. --- README.md | 6 +- docs/guide/configuration.md | 8 +- docs/guide/troubleshooting.md | 25 ++- docs/reference/configuration.md | 1 + src/cli/index.ts | 30 ++- src/config.ts | 2 + src/core/search.ts | 24 ++- src/db/index-meta.ts | 210 +++++++++++++++++++++ src/db/schema.ts | 55 ++++-- src/providers/dimensions.ts | 77 ++++++++ src/providers/embedding.ts | 8 +- src/providers/index.ts | 13 +- src/providers/local.ts | 3 +- src/providers/ollama.ts | 32 ++-- src/providers/openai.ts | 36 ++-- tests/unit/embedding-dimensions.test.ts | 190 +++++++++++++++++++ tests/unit/index-meta.test.ts | 231 ++++++++++++++++++++++++ tests/unit/schema-migrations.test.ts | 12 ++ 18 files changed, 904 insertions(+), 59 deletions(-) create mode 100644 src/db/index-meta.ts create mode 100644 src/providers/dimensions.ts create mode 100644 tests/unit/embedding-dimensions.test.ts create mode 100644 tests/unit/index-meta.test.ts diff --git a/README.md b/README.md index 92d7c76b..ae0b421f 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,8 @@ export LIBSCOPE_EMBEDDING_PROVIDER=openai export LIBSCOPE_OPENAI_API_KEY=sk-... ``` +The vector index is tied to the model that built it. After you change the provider or model, run `libscope reindex --rebuild`. For a model whose vector size LibScope does not know, set `embedding.dimensions` in the config file. + ### LLM for RAG The `ask` command and `ask-question` MCP tool need an LLM. Configure one with: @@ -412,8 +414,8 @@ LibScope ships with a few more utilities beyond the core index-and-search loop: # Watch a directory and auto-reindex on changes libscope watch ./docs/ -# Re-embed everything after switching embedding providers -libscope reindex +# Rebuild the vector index after switching embedding provider or model +libscope reindex --rebuild # Find duplicate documents libscope dedupe diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 7a47ed87..f8b11447 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -61,7 +61,13 @@ libscope config set embedding.provider ollama libscope config set embedding.provider openai ``` -If you switch providers after indexing documents, run `libscope reindex` to re-embed existing chunks with the new model. +LibScope records the provider, model, and vector size that built the vector index. If you change the provider, the model, or `embedding.dimensions` after you index documents, LibScope stops with an error that names the old and the new model. To rebuild the vector index with the new model, run: + +```bash +libscope reindex --rebuild +``` + +LibScope knows the vector size of common models (for example `nomic-embed-text`, `mxbai-embed-large`, `all-minilm`, `text-embedding-3-small`, and `text-embedding-3-large`). For a different Ollama or OpenAI model, set `embedding.dimensions` to the vector size of the model. If you do not set it, `libscope reindex --rebuild` embeds one probe text to find the size. For `text-embedding-3-*` models, `embedding.dimensions` also asks OpenAI for shorter vectors. ## LLM Configuration diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index d382bd25..cfed01c5 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -34,15 +34,23 @@ If you're on an unsupported platform, LibScope will fall back to keyword-only se ### Embedding dimension mismatch after switching providers -**Symptom:** Error like `expected N dimensions, got M` when searching after changing the embedding provider. +**Symptom:** LibScope stops with `The vector index (<old model>) does not match the configured embedding model (<new model>)`. Or search logs `Vector index does not match the embedding model, falling back to keyword search`. -**Cause:** Changing embedding providers produces vectors of different dimensions. Existing embeddings in the database are incompatible. +**Cause:** LibScope records the provider, model, and vector size that built the vector index. Different models produce different vectors, so the existing vectors cannot be used with the new model. -**Fix:** Re-index all documents after switching providers: +**Fix:** Rebuild the vector index with the configured model: ```bash -libscope db reset # clears all indexed content -libscope pack install ... # re-install packs +libscope reindex --rebuild ``` +This command drops and recreates the vector table with the size of the new model. Then it re-embeds every chunk. Your documents, topics, and tags are kept. + +### `Expected embedding dimension N, got M` + +**Symptom:** Indexing or search fails with `Expected embedding dimension N, got M from model "<model>"`. + +**Cause:** The model returns vectors of a different size than LibScope expects. LibScope knows the size of common OpenAI and Ollama models. For other models, it uses `embedding.dimensions` from the config, or the size of the first vector the model returns. + +**Fix:** Set `embedding.dimensions` to M in `~/.libscope/config.json` or `.libscope.json`, then run `libscope reindex --rebuild`. --- @@ -87,10 +95,15 @@ LibScope uses SQLite WAL mode which supports concurrent reads but only one write ### How to reset the database +There is no reset command. To remove all indexed content and keep your config, stop all libscope processes (CLI, MCP server, API server). Then delete the database files of the workspace: + ```bash -libscope db reset # removes all indexed content (keeps config) +rm ~/.libscope/workspaces/default/libscope.db* # replace "default" with your workspace name +libscope init ``` +To replace only the vectors (for example, after you change the embedding model), use `libscope reindex --rebuild`. + --- ## Connector Issues diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 30e06aa7..4eaf8293 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -21,6 +21,7 @@ Complete reference for all configuration options. | `embedding.ollamaUrl` | string | `"http://localhost:11434"` | Ollama server URL | | `embedding.ollamaModel` | string | `"nomic-embed-text"` | Ollama embedding model | | `embedding.openaiModel` | string | `"text-embedding-3-small"` | OpenAI embedding model | +| `embedding.dimensions` | number | — | Vector size of the Ollama or OpenAI model. Needed only for models whose size LibScope does not know. For `text-embedding-3-*`, it also requests shorter vectors. After you change it, run `libscope reindex --rebuild`. | ### LLM (for RAG) diff --git a/src/cli/index.ts b/src/cli/index.ts index 00556add..627e764a 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1705,6 +1705,14 @@ program }); // reindex +interface ReindexCliOptions { + doc?: string[]; + since?: string; + before?: string; + batchSize: string; + rebuild?: boolean; +} + program .command("reindex") .description("Re-embed all chunks with the current embedding provider") @@ -1712,10 +1720,28 @@ program .option("--since <date>", "Only reindex documents created on or after this ISO-8601 date") .option("--before <date>", "Only reindex documents created on or before this ISO-8601 date") .option("--batch-size <n>", "Chunks per embedding batch", "50") - .action(async (opts: { doc?: string[]; since?: string; before?: string; batchSize: string }) => { + .option( + "--rebuild", + "Drop and recreate the vector table for the configured embedding model, then re-embed every chunk (use after changing provider, model or dimensions)", + ) + .action(async (opts: ReindexCliOptions) => { + if (opts.rebuild && (opts.doc ?? opts.since ?? opts.before)) { + console.error( + "Error: --rebuild re-embeds every chunk; it cannot be combined with --doc, --since or --before.", + ); + process.exit(1); + } const { reindex } = await import("../core/reindex.js"); - const { db, provider } = initializeAppWithEmbedding(); + const { rebuildVectorTable, describeEmbeddingIdentity } = await import("../db/index-meta.js"); + const { config, db } = initializeApp(); try { + const provider = createEmbeddingProvider(config); + if (opts.rebuild) { + const identity = await rebuildVectorTable(db, provider); + console.log(`✓ Vector table recreated (${describeEmbeddingIdentity(identity)})`); + } else { + createVectorTable(db, provider); + } console.log("Re-embedding chunks..."); const startTime = Date.now(); diff --git a/src/config.ts b/src/config.ts index 02926c78..89754013 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,6 +11,8 @@ export interface LibScopeConfig { ollamaModel?: string; openaiApiKey?: string; openaiModel?: string; + /** Vector size override for models whose dimension is not built in. */ + dimensions?: number; }; llm?: { provider?: "openai" | "ollama" | "anthropic" | "passthrough"; diff --git a/src/core/search.ts b/src/core/search.ts index 3de6762d..57ca7227 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import { withCorrelationId, createChildLogger } from "../logger.js"; import { validateCountRow } from "../utils/db-validation.js"; import { validateRow, validateRows } from "../db/validate.js"; +import { REBUILD_VECTOR_INDEX_HINT } from "../db/index-meta.js"; import { logSearch, recordSearchQuery } from "./analytics.js"; import { performance } from "node:perf_hooks"; @@ -37,6 +38,15 @@ function isVectorTableError(err: unknown): boolean { return VECTOR_TABLE_MISSING_PATTERNS.some((p) => msg.includes(p)); } +/** + * sqlite-vec error raised when the query vector size differs from the vector table's + * (the embedding model changed after the index was built). Both spellings occur in sqlite-vec. + */ +function isVectorDimensionError(err: unknown): boolean { + const msg = (err instanceof Error ? err.message : String(err)).toLowerCase(); + return msg.includes("dimension mismatch") || msg.includes("dimension mistmatch"); +} + /** Escape LIKE special characters so user input is treated literally. */ export function escapeLikePattern(input: string): string { // prettier-ignore @@ -555,9 +565,17 @@ export async function searchDocuments( recordAnalytics(db, options, response, searchMethod, startTime, analyticsEnabled); return finalizeResponse(db, response, options); } catch (err) { - if (!isVectorTableError(err)) throw err; - - log.warn({ err }, "Vector table missing, falling back to keyword search"); + const dimensionMismatch = isVectorDimensionError(err); + if (!dimensionMismatch && !isVectorTableError(err)) throw err; + + if (dimensionMismatch) { + log.warn( + { err }, + `Vector index does not match the embedding model, falling back to keyword search. ${REBUILD_VECTOR_INDEX_HINT}`, + ); + } else { + log.warn({ err }, "Vector table missing, falling back to keyword search"); + } const response = keywordSearch(db, options, limit, offset); response.results = postProcessResults(response.results, options); diff --git a/src/db/index-meta.ts b/src/db/index-meta.ts new file mode 100644 index 00000000..0f35e237 --- /dev/null +++ b/src/db/index-meta.ts @@ -0,0 +1,210 @@ +import type Database from "better-sqlite3"; +import { ConfigError, DatabaseError, EmbeddingError } from "../errors.js"; +import { getLogger } from "../logger.js"; +import { MAX_EMBEDDING_DIMENSIONS } from "../providers/dimensions.js"; +import type { EmbeddingProvider } from "../providers/embedding.js"; + +/** Key/value table that records facts about how the indexes were built. */ +export const INDEX_META_DDL = ` + CREATE TABLE IF NOT EXISTS index_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) +`; + +/** The guidance shown whenever the vector index does not match the embedding model. */ +export const REBUILD_VECTOR_INDEX_HINT = + "Run `libscope reindex --rebuild` to rebuild the vector index with the configured embedding model."; + +/** The embedding provider, model and vector size that built (or will build) the vector index. */ +export interface EmbeddingIndexIdentity { + provider?: string | undefined; + model?: string | undefined; + dimensions?: number | undefined; +} + +type IdentityKey = keyof EmbeddingIndexIdentity; + +const META_KEYS: Record<IdentityKey, string> = { + provider: "embedding_provider", + model: "embedding_model", + dimensions: "embedding_dimensions", +}; + +const IDENTITY_KEYS: IdentityKey[] = ["provider", "model", "dimensions"]; + +/** A vector size, or undefined when it is 0, negative or not a number (unknown). */ +function knownSize(dimensions: number | undefined): number | undefined { + return dimensions !== undefined && dimensions > 0 ? dimensions : undefined; +} + +/** Build an identity from a provider, or from a bare vector size. */ +export function toEmbeddingIdentity( + source: number | Pick<EmbeddingProvider, "name" | "model" | "dimensions">, +): EmbeddingIndexIdentity { + if (typeof source === "number") return { dimensions: source }; + return { provider: source.name, model: source.model, dimensions: source.dimensions }; +} + +/** Read the stored identity. Returns an empty object when nothing is recorded. */ +export function readEmbeddingIdentity(db: Database.Database): EmbeddingIndexIdentity { + let rows: Array<{ key: string; value: string }>; + try { + rows = db.prepare("SELECT key, value FROM index_meta").all() as Array<{ + key: string; + value: string; + }>; + } catch { + return {}; // index_meta does not exist before migration 18 + } + const values = new Map(rows.map((r) => [r.key, r.value])); + return { + provider: values.get(META_KEYS.provider), + model: values.get(META_KEYS.model), + dimensions: knownSize(Number(values.get(META_KEYS.dimensions))), + }; +} + +/** + * Store an identity. Fields that are undefined are removed when `replace` is true and + * left as they are otherwise. Only changed rows are written. + */ +export function writeEmbeddingIdentity( + db: Database.Database, + identity: EmbeddingIndexIdentity, + replace: boolean, +): void { + const stored = readEmbeddingIdentity(db); + const changed = IDENTITY_KEYS.filter( + (k) => identity[k] !== stored[k] && (replace || identity[k] !== undefined), + ); + if (changed.length === 0) return; + db.exec(INDEX_META_DDL); + const upsert = db.prepare( + "INSERT INTO index_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ); + const remove = db.prepare("DELETE FROM index_meta WHERE key = ?"); + for (const k of changed) { + const value = identity[k]; + if (value === undefined) remove.run(META_KEYS[k]); + else upsert.run(META_KEYS[k], String(value)); + } +} + +/** + * Vector size of the existing chunk_embeddings table: undefined when the table does + * not exist, 0 when it exists but its size cannot be read from its definition. + */ +export function getVectorTableDimensions(db: Database.Database): number | undefined { + const row = db.prepare("SELECT sql FROM sqlite_master WHERE name = 'chunk_embeddings'").get() as + | { sql: string | null } + | undefined; + if (!row) return undefined; + const match = /float\[(\d+)\]/.exec(row.sql ?? ""); + return match ? Number(match[1]) : 0; +} + +/** Human-readable form, for example `ollama/nomic-embed-text, 768 dimensions`. */ +export function describeEmbeddingIdentity(identity: EmbeddingIndexIdentity): string { + const parts = [[identity.provider, identity.model].filter(Boolean).join("/")]; + if (identity.dimensions) parts.push(`${identity.dimensions} dimensions`); + return parts.filter(Boolean).join(", ") || "unknown"; +} + +/** True when a field is known on both sides and differs. */ +function identitiesConflict(a: EmbeddingIndexIdentity, b: EmbeddingIndexIdentity): boolean { + return IDENTITY_KEYS.some((k) => { + const left = a[k]; + const right = b[k]; + return Boolean(left) && Boolean(right) && left !== right; + }); +} + +/** + * Check the configured provider/model/vector size against the existing vector index and + * record what is known. Throws a ConfigError that names both when they differ. + */ +export function checkVectorIndexIdentity( + db: Database.Database, + configured: EmbeddingIndexIdentity, + tableDimensions: number, +): void { + const stored = readEmbeddingIdentity(db); + const index: EmbeddingIndexIdentity = { + ...stored, + dimensions: knownSize(tableDimensions) ?? stored.dimensions, + }; + if (identitiesConflict(index, configured)) { + throw new ConfigError( + `The vector index (${describeEmbeddingIdentity(index)}) does not match ` + + `the configured embedding model (${describeEmbeddingIdentity(configured)}). ` + + REBUILD_VECTOR_INDEX_HINT, + ); + } + writeEmbeddingIdentity( + db, + { + provider: configured.provider ?? index.provider, + model: configured.model ?? index.model, + dimensions: index.dimensions ?? knownSize(configured.dimensions), + }, + false, + ); +} + +/** Throw a DatabaseError unless `dimensions` is a usable vector size. */ +export function assertValidVectorDimensions(dimensions: number): void { + if (!Number.isInteger(dimensions) || dimensions <= 0 || dimensions > MAX_EMBEDDING_DIMENSIONS) { + throw new DatabaseError( + `Invalid vector dimensions: must be a positive integer <= ${MAX_EMBEDDING_DIMENSIONS}`, + ); + } +} + +/** Create chunk_embeddings with the given vector size (no-op if it already exists). */ +export function execCreateVectorTable(db: Database.Database, dimensions: number): void { + assertValidVectorDimensions(dimensions); + // dimensions is validated as a positive integer above, so interpolation is safe here + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS chunk_embeddings USING vec0( + chunk_id TEXT PRIMARY KEY, + embedding float[${dimensions}] + ); + `); +} + +/** + * Drop and recreate chunk_embeddings for the given provider, and record the provider, + * model and vector size. All stored vectors are removed; re-embed the chunks afterwards. + * One probe text is embedded first to learn the vector size and to confirm that the model + * works, so a misconfigured model fails before the existing vectors are dropped. + */ +export async function rebuildVectorTable( + db: Database.Database, + provider: EmbeddingProvider, +): Promise<EmbeddingIndexIdentity> { + const dimensions = (await provider.embed("dimension probe")).length; + if (provider.dimensions && provider.dimensions !== dimensions) { + throw new EmbeddingError( + `Expected embedding dimension ${provider.dimensions}, got ${dimensions} from ${describeEmbeddingIdentity(toEmbeddingIdentity(provider))}.`, + ); + } + assertValidVectorDimensions(dimensions); + const identity: EmbeddingIndexIdentity = { ...toEmbeddingIdentity(provider), dimensions }; + + const rebuild = db.transaction(() => { + db.exec("DROP TABLE IF EXISTS chunk_embeddings"); + execCreateVectorTable(db, dimensions); + writeEmbeddingIdentity(db, identity, true); + }); + try { + rebuild(); + } catch (err) { + throw new DatabaseError( + "Could not rebuild the vector table. The sqlite-vec extension must be available.", + err, + ); + } + getLogger().info({ ...identity }, "Vector table rebuilt"); + return identity; +} diff --git a/src/db/schema.ts b/src/db/schema.ts index ca240cf2..a5651429 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,6 +1,16 @@ import type Database from "better-sqlite3"; import { DatabaseError } from "../errors.js"; import { getLogger } from "../logger.js"; +import type { EmbeddingProvider } from "../providers/embedding.js"; +import { + INDEX_META_DDL, + assertValidVectorDimensions, + checkVectorIndexIdentity, + execCreateVectorTable, + getVectorTableDimensions, + toEmbeddingIdentity, + writeEmbeddingIdentity, +} from "./index-meta.js"; const SCHEMA_VERSION = 18; @@ -313,6 +323,9 @@ const MIGRATIONS: Record<number, string> = { -- Serves the title + length dedup lookup that runs on every index call. CREATE INDEX IF NOT EXISTS idx_documents_title ON documents(title); + -- Records the embedding provider, model and vector size of chunk_embeddings. + ${INDEX_META_DDL}; + INSERT INTO schema_version (version) VALUES (18); `, }; @@ -382,20 +395,40 @@ export function runMigrations(db: Database.Database, targetVersion: number = SCH } } -/** Create the virtual table for vector search (requires sqlite-vec). */ -export function createVectorTable(db: Database.Database, dimensions: number): void { - if (!Number.isInteger(dimensions) || dimensions <= 0 || dimensions > 10000) { - throw new DatabaseError("Invalid vector dimensions: must be a positive integer <= 10000"); +/** + * Create the vector table (requires sqlite-vec) and record which embedding model it is for. + * Pass the provider rather than a bare vector size so that a change of provider or model + * is detected, not only a change of vector size. + * + * @throws ConfigError when the existing vector table was built for a different provider, + * model or vector size. The message tells the user to run `libscope reindex --rebuild`. + * @throws DatabaseError when the vector size is invalid, or unknown (0) and no table exists. + */ +export function createVectorTable( + db: Database.Database, + source: number | Pick<EmbeddingProvider, "name" | "model" | "dimensions">, +): void { + const configured = toEmbeddingIdentity(source); + const dimensions = configured.dimensions ?? 0; + // 0 means the provider learns its vector size from its first embedding. + if (dimensions !== 0) assertValidVectorDimensions(dimensions); + + const tableDimensions = getVectorTableDimensions(db); + if (tableDimensions !== undefined) { + checkVectorIndexIdentity(db, configured, tableDimensions); + return; } + if (dimensions === 0) { + throw new DatabaseError( + "Invalid vector dimensions: the vector size of the embedding model is not known yet. " + + "Set embedding.dimensions in the config, or run `libscope reindex --rebuild` to detect it.", + ); + } + const log = getLogger(); try { - // dimensions is validated as a positive integer above, so interpolation is safe here - db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS chunk_embeddings USING vec0( - chunk_id TEXT PRIMARY KEY, - embedding float[${dimensions}] - ); - `); + execCreateVectorTable(db, dimensions); + writeEmbeddingIdentity(db, configured, true); log.info({ dimensions }, "Vector table ready"); } catch (err) { log.warn({ err }, "Could not create vector table — vector search unavailable"); diff --git a/src/providers/dimensions.ts b/src/providers/dimensions.ts new file mode 100644 index 00000000..16fdbe55 --- /dev/null +++ b/src/providers/dimensions.ts @@ -0,0 +1,77 @@ +import { ConfigError, EmbeddingError } from "../errors.js"; + +/** Output dimensions of common embedding models, keyed by model name (without a tag). */ +const KNOWN_MODEL_DIMENSIONS: ReadonlyMap<string, number> = new Map([ + // OpenAI + ["text-embedding-3-small", 1536], + ["text-embedding-3-large", 3072], + ["text-embedding-ada-002", 1536], + // Ollama + ["nomic-embed-text", 768], + ["mxbai-embed-large", 1024], + ["all-minilm", 384], + ["bge-m3", 1024], + ["bge-large", 1024], + ["snowflake-arctic-embed2", 1024], + ["paraphrase-multilingual", 768], +]); + +/** Largest dimension count accepted for the vector table. */ +export const MAX_EMBEDDING_DIMENSIONS = 10000; + +/** + * Look up the output dimension of a known model. Ollama tags (`name:tag`) are ignored, + * because every tag of the models in the table has the same dimension. + */ +export function knownModelDimensions(model: string): number | undefined { + return KNOWN_MODEL_DIMENSIONS.get(model) ?? KNOWN_MODEL_DIMENSIONS.get(model.split(":")[0] ?? ""); +} + +/** Throw a ConfigError unless `value` is a usable `embedding.dimensions` value. */ +export function validateDimensionsOverride(value: unknown): number | undefined { + if (value === undefined) return undefined; + if ( + typeof value !== "number" || + !Number.isInteger(value) || + value <= 0 || + value > MAX_EMBEDDING_DIMENSIONS + ) { + throw new ConfigError( + `embedding.dimensions must be a positive integer <= ${MAX_EMBEDDING_DIMENSIONS}, got ${JSON.stringify(value)}`, + ); + } + return value; +} + +/** + * Tracks the expected vector size for a provider. The expected size comes from the + * `embedding.dimensions` override, else the known-model table, else the first response. + */ +export class EmbeddingDimensions { + private expected: number | undefined; + + constructor( + private readonly model: string, + override?: number, + ) { + this.expected = override ?? knownModelDimensions(model); + } + + /** Expected dimension, or 0 while it is unknown (before the first embedding). */ + get value(): number { + return this.expected ?? 0; + } + + /** Check every vector against the expected size; learn the size from the first vector. */ + check(vectors: number[][]): void { + for (const vector of vectors) { + this.expected ??= vector.length; + if (vector.length !== this.expected) { + throw new EmbeddingError( + `Expected embedding dimension ${this.expected}, got ${vector.length} from model "${this.model}". ` + + `If this model produces ${vector.length}-dimensional vectors, set embedding.dimensions to ${vector.length}.`, + ); + } + } + } +} diff --git a/src/providers/embedding.ts b/src/providers/embedding.ts index 7c5652df..8bc30d02 100644 --- a/src/providers/embedding.ts +++ b/src/providers/embedding.ts @@ -6,7 +6,13 @@ export interface EmbeddingProvider { /** Provider name for display/logging. */ readonly name: string; - /** Dimensionality of the output vectors. */ + /** Model identifier, recorded with the vector index to detect model changes. */ + readonly model?: string | undefined; + + /** + * Dimensionality of the output vectors. + * 0 means not yet known: the provider learns it from the first embedding it returns. + */ readonly dimensions: number; /** Generate an embedding vector for a single text input. */ diff --git a/src/providers/index.ts b/src/providers/index.ts index 4676b7e3..3092083c 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -5,6 +5,7 @@ export { OpenAIEmbeddingProvider } from "./openai.js"; import type { LibScopeConfig } from "../config.js"; import { ConfigError } from "../errors.js"; +import { validateDimensionsOverride } from "./dimensions.js"; import type { EmbeddingProvider } from "./embedding.js"; import { LocalEmbeddingProvider } from "./local.js"; import { OllamaEmbeddingProvider } from "./ollama.js"; @@ -41,7 +42,11 @@ export function createEmbeddingProvider(config: LibScopeConfig): EmbeddingProvid "Ollama model is required. Set LIBSCOPE_OLLAMA_MODEL or configure in ~/.libscope/config.json", ); } - return new OllamaEmbeddingProvider(config.embedding.ollamaUrl, config.embedding.ollamaModel); + return new OllamaEmbeddingProvider( + config.embedding.ollamaUrl, + config.embedding.ollamaModel, + validateDimensionsOverride(config.embedding.dimensions), + ); } case "openai": { const apiKey = config.embedding.openaiApiKey; @@ -50,7 +55,11 @@ export function createEmbeddingProvider(config: LibScopeConfig): EmbeddingProvid "OpenAI API key is required. Set LIBSCOPE_OPENAI_API_KEY or configure in ~/.libscope/config.json", ); } - return new OpenAIEmbeddingProvider(apiKey, config.embedding.openaiModel); + return new OpenAIEmbeddingProvider( + apiKey, + config.embedding.openaiModel, + validateDimensionsOverride(config.embedding.dimensions), + ); } default: throw new ConfigError(`Unknown embedding provider: ${String(config.embedding.provider)}`); diff --git a/src/providers/local.ts b/src/providers/local.ts index 37391b32..57f8effa 100644 --- a/src/providers/local.ts +++ b/src/providers/local.ts @@ -19,6 +19,7 @@ type FeatureExtractionPipeline = ( */ export class LocalEmbeddingProvider implements EmbeddingProvider { readonly name = "local"; + readonly model = "Xenova/all-MiniLM-L6-v2"; readonly dimensions = 384; private pipeline: FeatureExtractionPipeline | null = null; @@ -38,7 +39,7 @@ export class LocalEmbeddingProvider implements EmbeddingProvider { // Cast to the typed interface; @xenova/transformers lacks precise TS generics for pipeline output this.pipeline = (await pipeline( "feature-extraction", - "Xenova/all-MiniLM-L6-v2", + this.model, )) as unknown as FeatureExtractionPipeline; log.info("Local embedding model loaded successfully"); } catch (err) { diff --git a/src/providers/ollama.ts b/src/providers/ollama.ts index 79d2b32d..8bc48b1b 100644 --- a/src/providers/ollama.ts +++ b/src/providers/ollama.ts @@ -1,6 +1,7 @@ import { EmbeddingError } from "../errors.js"; import { createChildLogger } from "../logger.js"; import { withRetry } from "../utils/retry.js"; +import { EmbeddingDimensions } from "./dimensions.js"; import type { EmbeddingProvider } from "./embedding.js"; /** @@ -9,14 +10,23 @@ import type { EmbeddingProvider } from "./embedding.js"; */ export class OllamaEmbeddingProvider implements EmbeddingProvider { readonly name = "ollama"; - readonly dimensions: number; + private readonly expectedDimensions: EmbeddingDimensions; + /** + * @param dimensions - Override for the vector size. When omitted, known models use their + * documented size and other models use the size of the first embedding returned. + */ constructor( private readonly baseUrl: string = "http://localhost:11434", - private readonly model: string = "nomic-embed-text", - dimensions: number = 768, + readonly model: string = "nomic-embed-text", + dimensions?: number, ) { - this.dimensions = dimensions; + this.expectedDimensions = new EmbeddingDimensions(model, dimensions); + } + + /** Vector size, or 0 until the first embedding of an unknown model. */ + get dimensions(): number { + return this.expectedDimensions.value; } async embed(text: string): Promise<number[]> { @@ -62,11 +72,7 @@ export class OllamaEmbeddingProvider implements EmbeddingProvider { if (!embedding) { throw new EmbeddingError("Ollama returned empty embeddings"); } - if (embedding.length !== this.dimensions) { - throw new EmbeddingError( - `Expected embedding dimension ${this.dimensions}, got ${embedding.length}`, - ); - } + this.expectedDimensions.check([embedding]); return embedding; }); } catch (err) { @@ -129,13 +135,7 @@ export class OllamaEmbeddingProvider implements EmbeddingProvider { `Ollama returned ${embeddings.length} embeddings for ${texts.length} inputs`, ); } - for (const emb of embeddings) { - if (emb.length !== this.dimensions) { - throw new EmbeddingError( - `Expected embedding dimension ${this.dimensions}, got ${emb.length}`, - ); - } - } + this.expectedDimensions.check(embeddings); return embeddings; }); } catch (err) { diff --git a/src/providers/openai.ts b/src/providers/openai.ts index b74ad93d..24232df4 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -2,6 +2,7 @@ import OpenAI from "openai"; import { EmbeddingError } from "../errors.js"; import { createChildLogger } from "../logger.js"; import { withRetry } from "../utils/retry.js"; +import { EmbeddingDimensions } from "./dimensions.js"; import type { EmbeddingProvider } from "./embedding.js"; /** @@ -10,15 +11,30 @@ import type { EmbeddingProvider } from "./embedding.js"; */ export class OpenAIEmbeddingProvider implements EmbeddingProvider { readonly name = "openai"; - readonly dimensions = 1536; private readonly client: OpenAI; + private readonly expectedDimensions: EmbeddingDimensions; + /** Sent as the API `dimensions` parameter (text-embedding-3 models can shorten vectors). */ + private readonly requestDimensions: { dimensions?: number }; + /** + * @param dimensions - Override for the vector size. For text-embedding-3 models it is also + * sent to the API to request shortened vectors. + */ constructor( apiKey: string, - private readonly model: string = "text-embedding-3-small", + readonly model: string = "text-embedding-3-small", + dimensions?: number, ) { this.client = new OpenAI({ apiKey, timeout: 30_000 }); + this.expectedDimensions = new EmbeddingDimensions(model, dimensions); + this.requestDimensions = + dimensions !== undefined && model.startsWith("text-embedding-3") ? { dimensions } : {}; + } + + /** Vector size, or 0 until the first embedding of an unknown model. */ + get dimensions(): number { + return this.expectedDimensions.value; } async embed(text: string): Promise<number[]> { @@ -31,16 +47,13 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { const response = await this.client.embeddings.create({ model: this.model, input: text, + ...this.requestDimensions, }); const embedding = response.data[0]?.embedding; if (!embedding) { throw new EmbeddingError("OpenAI returned empty embedding"); } - if (embedding.length !== this.dimensions) { - throw new EmbeddingError( - `Expected embedding dimension ${this.dimensions}, got ${embedding.length}`, - ); - } + this.expectedDimensions.check([embedding]); return embedding; }); } catch (err) { @@ -67,6 +80,7 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { const response = await this.client.embeddings.create({ model: this.model, input: texts, + ...this.requestDimensions, }); if (response.data.length !== texts.length) { throw new EmbeddingError( @@ -74,13 +88,7 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { ); } const embeddings = response.data.map((d) => d.embedding); - for (const emb of embeddings) { - if (emb.length !== this.dimensions) { - throw new EmbeddingError( - `Expected embedding dimension ${this.dimensions}, got ${emb.length}`, - ); - } - } + this.expectedDimensions.check(embeddings); return embeddings; }); } catch (err) { diff --git a/tests/unit/embedding-dimensions.test.ts b/tests/unit/embedding-dimensions.test.ts new file mode 100644 index 00000000..2b43c392 --- /dev/null +++ b/tests/unit/embedding-dimensions.test.ts @@ -0,0 +1,190 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +const openaiCreate = vi.fn(); + +vi.mock("openai", () => ({ + default: class { + embeddings = { create: openaiCreate }; + }, +})); + +vi.mock("../../src/utils/retry.js", () => ({ + withRetry: <T>(fn: () => Promise<T>): Promise<T> => fn(), +})); + +import { + EmbeddingDimensions, + knownModelDimensions, + validateDimensionsOverride, +} from "../../src/providers/dimensions.js"; +import { OllamaEmbeddingProvider } from "../../src/providers/ollama.js"; +import { OpenAIEmbeddingProvider } from "../../src/providers/openai.js"; +import { createEmbeddingProvider } from "../../src/providers/index.js"; +import type { LibScopeConfig } from "../../src/config.js"; +import { ConfigError, EmbeddingError } from "../../src/errors.js"; +import { initLogger } from "../../src/logger.js"; + +function vector(size: number): number[] { + return Array.from({ length: size }, () => 0.1); +} + +function config(embedding: LibScopeConfig["embedding"]): LibScopeConfig { + return { + embedding, + database: { path: ":memory:" }, + indexing: { maxDocumentSize: 1024, allowPrivateUrls: false, allowSelfSignedCerts: false }, + logging: { level: "silent" }, + }; +} + +/** Make fetch return an Ollama /api/embed response with vectors of the given sizes. */ +function stubOllama(...sizes: number[]): void { + vi.stubGlobal( + "fetch", + vi.fn(() => + Promise.resolve( + new Response(JSON.stringify({ embeddings: sizes.map((s) => vector(s)) }), { status: 200 }), + ), + ), + ); +} + +beforeEach(() => { + initLogger("silent"); + openaiCreate.mockReset(); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("knownModelDimensions", () => { + it("knows common OpenAI and Ollama models", () => { + expect(knownModelDimensions("text-embedding-3-small")).toBe(1536); + expect(knownModelDimensions("text-embedding-3-large")).toBe(3072); + expect(knownModelDimensions("nomic-embed-text")).toBe(768); + expect(knownModelDimensions("mxbai-embed-large")).toBe(1024); + expect(knownModelDimensions("all-minilm")).toBe(384); + }); + + it("ignores Ollama tags", () => { + expect(knownModelDimensions("nomic-embed-text:v1.5")).toBe(768); + expect(knownModelDimensions("mxbai-embed-large:latest")).toBe(1024); + }); + + it("returns undefined for unknown models", () => { + expect(knownModelDimensions("my-custom-embedder")).toBeUndefined(); + }); +}); + +describe("validateDimensionsOverride", () => { + it("accepts undefined and positive integers", () => { + expect(validateDimensionsOverride(undefined)).toBeUndefined(); + expect(validateDimensionsOverride(1024)).toBe(1024); + }); + + it.each([0, -1, 1.5, 10001, "512"])("rejects %j", (value) => { + expect(() => validateDimensionsOverride(value)).toThrow(ConfigError); + }); +}); + +describe("EmbeddingDimensions", () => { + it("learns the size from the first vector and checks later vectors against it", () => { + const dims = new EmbeddingDimensions("custom"); + expect(dims.value).toBe(0); + dims.check([vector(5)]); + expect(dims.value).toBe(5); + expect(() => dims.check([vector(4)])).toThrow("Expected embedding dimension 5, got 4"); + }); + + it("prefers the override over the known size", () => { + expect(new EmbeddingDimensions("text-embedding-3-large", 256).value).toBe(256); + }); +}); + +describe("OllamaEmbeddingProvider dimensions", () => { + it("uses the known size of the model", () => { + expect(new OllamaEmbeddingProvider("http://x", "mxbai-embed-large").dimensions).toBe(1024); + }); + + it("learns the size of an unknown model from the first response", async () => { + const provider = new OllamaEmbeddingProvider("http://x", "custom-embed"); + expect(provider.dimensions).toBe(0); + stubOllama(640); + await provider.embed("hello"); + expect(provider.dimensions).toBe(640); + }); + + it("checks batch responses against the learned size", async () => { + const provider = new OllamaEmbeddingProvider("http://x", "custom-embed"); + stubOllama(32, 16); + await expect(provider.embedBatch(["a", "b"])).rejects.toThrow(EmbeddingError); + }); + + it("explains how to fix a wrong override", async () => { + const provider = new OllamaEmbeddingProvider("http://x", "custom-embed", 512); + stubOllama(1024); + await expect(provider.embed("hello")).rejects.toThrow( + /Expected embedding dimension 512, got 1024.*set embedding\.dimensions to 1024/, + ); + }); +}); + +describe("OpenAIEmbeddingProvider dimensions", () => { + it("uses the known size of the model", () => { + expect(new OpenAIEmbeddingProvider("k", "text-embedding-3-large").dimensions).toBe(3072); + }); + + it("sends the override as the dimensions parameter for text-embedding-3 models", async () => { + openaiCreate.mockResolvedValue({ data: [{ embedding: vector(256) }] }); + const provider = new OpenAIEmbeddingProvider("k", "text-embedding-3-large", 256); + + await provider.embed("hello"); + + expect(openaiCreate).toHaveBeenCalledWith( + expect.objectContaining({ model: "text-embedding-3-large", dimensions: 256 }), + ); + expect(provider.dimensions).toBe(256); + }); + + it("does not send a dimensions parameter without an override", async () => { + openaiCreate.mockResolvedValue({ + data: [{ embedding: vector(1536) }, { embedding: vector(1536) }], + }); + const provider = new OpenAIEmbeddingProvider("k", "text-embedding-3-small"); + + await provider.embedBatch(["a", "b"]); + + expect(openaiCreate.mock.calls[0]?.[0]).not.toHaveProperty("dimensions"); + }); +}); + +describe("createEmbeddingProvider dimensions", () => { + it("passes embedding.dimensions to the Ollama provider", () => { + const provider = createEmbeddingProvider( + config({ + provider: "ollama", + ollamaUrl: "http://x", + ollamaModel: "custom-embed", + dimensions: 2048, + }), + ); + expect(provider.dimensions).toBe(2048); + expect(provider.model).toBe("custom-embed"); + }); + + it("uses the known size for the configured OpenAI model", () => { + const provider = createEmbeddingProvider( + config({ provider: "openai", openaiApiKey: "k", openaiModel: "text-embedding-3-large" }), + ); + expect(provider.dimensions).toBe(3072); + }); + + it("rejects an invalid embedding.dimensions", () => { + expect(() => + createEmbeddingProvider( + config({ provider: "ollama", ollamaUrl: "http://x", ollamaModel: "m", dimensions: -5 }), + ), + ).toThrow(ConfigError); + }); +}); diff --git a/tests/unit/index-meta.test.ts b/tests/unit/index-meta.test.ts new file mode 100644 index 00000000..9c31f47e --- /dev/null +++ b/tests/unit/index-meta.test.ts @@ -0,0 +1,231 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import type Database from "better-sqlite3"; +import DatabaseConstructor from "better-sqlite3"; +import { createDatabase } from "../../src/db/connection.js"; +import { runMigrations, createVectorTable } from "../../src/db/schema.js"; +import { + getVectorTableDimensions, + readEmbeddingIdentity, + rebuildVectorTable, +} from "../../src/db/index-meta.js"; +import { searchDocuments } from "../../src/core/search.js"; +import { ConfigError, DatabaseError } from "../../src/errors.js"; +import { initLogger } from "../../src/logger.js"; +import type { EmbeddingProvider } from "../../src/providers/embedding.js"; + +/** Provider stub that returns constant vectors of a fixed size. */ +class FixedProvider implements EmbeddingProvider { + embedCalls = 0; + constructor( + readonly name: string, + readonly model: string, + private size: number, + private readonly reportedSize: number = size, + ) {} + + get dimensions(): number { + return this.reportedSize === 0 && this.embedCalls > 0 ? this.size : this.reportedSize; + } + + embed(_text: string): Promise<number[]> { + this.embedCalls++; + return Promise.resolve(Array.from({ length: this.size }, (_, i) => (i + 1) / this.size)); + } + + async embedBatch(texts: string[]): Promise<number[][]> { + return Promise.all(texts.map((t) => this.embed(t))); + } +} + +function hasSqliteVec(): boolean { + const db = createDatabase(":memory:"); + try { + db.prepare("SELECT vec_version()").get(); + return true; + } catch { + return false; + } finally { + db.close(); + } +} + +const vecAvailable = hasSqliteVec(); + +function vecRowCount(db: Database.Database): number { + return (db.prepare("SELECT COUNT(*) AS n FROM chunk_embeddings").get() as { n: number }).n; +} + +describe.skipIf(!vecAvailable)("vector index metadata (sqlite-vec)", () => { + let db: Database.Database; + + beforeEach(() => { + initLogger("silent"); + db = createDatabase(":memory:"); + }); + + afterEach(() => { + db.close(); + }); + + it("records provider, model and dimensions when it creates the vector table", () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("ollama", "nomic-embed-text", 768)); + + expect(getVectorTableDimensions(db)).toBe(768); + expect(readEmbeddingIdentity(db)).toEqual({ + provider: "ollama", + model: "nomic-embed-text", + dimensions: 768, + }); + }); + + it("populates metadata for a vector table built before schema v18", () => { + runMigrations(db, 17); + db.exec( + "CREATE VIRTUAL TABLE chunk_embeddings USING vec0(chunk_id TEXT PRIMARY KEY, embedding float[4])", + ); + runMigrations(db); + expect(readEmbeddingIdentity(db)).toEqual({ + provider: undefined, + model: undefined, + dimensions: undefined, + }); + + createVectorTable(db, 4); + expect(readEmbeddingIdentity(db).dimensions).toBe(4); + + createVectorTable(db, new FixedProvider("mock", "m1", 4)); + expect(readEmbeddingIdentity(db)).toEqual({ provider: "mock", model: "m1", dimensions: 4 }); + }); + + it("rejects a provider whose vector size differs from the index", () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("ollama", "nomic-embed-text", 768)); + + const change = (): void => + createVectorTable(db, new FixedProvider("ollama", "mxbai-embed-large", 1024)); + expect(change).toThrow(ConfigError); + expect(change).toThrow(/vector index \(ollama\/nomic-embed-text, 768 dimensions\)/); + expect(change).toThrow( + /configured embedding model \(ollama\/mxbai-embed-large, 1024 dimensions\)/, + ); + expect(change).toThrow(/libscope reindex --rebuild/); + // A bare vector size is checked against the table too. + expect(() => createVectorTable(db, 1024)).toThrow( + /does not match the configured embedding model \(1024 dimensions\)/, + ); + }); + + it("rejects a different model with the same vector size", () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("openai", "text-embedding-3-small", 1536)); + + expect(() => + createVectorTable(db, new FixedProvider("openai", "text-embedding-ada-002", 1536)), + ).toThrow(/text-embedding-3-small.*text-embedding-ada-002/); + }); + + it("accepts the same provider again and a provider whose size is not known yet", () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("ollama", "custom", 512)); + + expect(() => createVectorTable(db, new FixedProvider("ollama", "custom", 512))).not.toThrow(); + expect(() => + createVectorTable(db, new FixedProvider("ollama", "custom", 512, 0)), + ).not.toThrow(); + expect(() => createVectorTable(db, 512)).not.toThrow(); + }); + + it("refuses to create a table when the vector size is unknown", () => { + runMigrations(db); + const unknown = (): void => + createVectorTable(db, new FixedProvider("ollama", "custom", 512, 0)); + expect(unknown).toThrow(DatabaseError); + expect(unknown).toThrow(/embedding\.dimensions/); + }); + + it("rebuildVectorTable recreates the table with the new size and records it", async () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("ollama", "nomic-embed-text", 4)); + db.prepare("INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)").run( + "c1", + Buffer.from(new Float32Array([1, 0, 0, 0]).buffer), + ); + + const next = new FixedProvider("ollama", "mxbai-embed-large", 8); + const identity = await rebuildVectorTable(db, next); + + expect(identity).toEqual({ provider: "ollama", model: "mxbai-embed-large", dimensions: 8 }); + expect(getVectorTableDimensions(db)).toBe(8); + expect(vecRowCount(db)).toBe(0); + expect(readEmbeddingIdentity(db)).toEqual(identity); + expect(() => createVectorTable(db, next)).not.toThrow(); + }); + + it("rebuildVectorTable embeds a probe text when the provider size is unknown", async () => { + runMigrations(db); + const provider = new FixedProvider("ollama", "custom", 6, 0); + + const identity = await rebuildVectorTable(db, provider); + + expect(provider.embedCalls).toBe(1); + expect(identity.dimensions).toBe(6); + expect(getVectorTableDimensions(db)).toBe(6); + }); + + it("rebuildVectorTable keeps the old table when the model output does not match its size", async () => { + runMigrations(db); + createVectorTable(db, 4); + // Reports 12 dimensions but returns 40-dimension vectors (a wrong embedding.dimensions). + const provider = new FixedProvider("ollama", "custom", 40, 12); + + await expect(rebuildVectorTable(db, provider)).rejects.toThrow( + "Expected embedding dimension 12, got 40", + ); + expect(getVectorTableDimensions(db)).toBe(4); + }); + + it("search falls back to keyword results when the query vector size does not match", async () => { + runMigrations(db); + createVectorTable(db, 4); + db.prepare( + "INSERT INTO documents (id, source_type, title, content) VALUES ('d1', 'manual', 'Doc', 'x')", + ).run(); + db.prepare( + "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES ('c1', 'd1', 'kiwi orchard', 0)", + ).run(); + db.prepare("INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)").run( + "c1", + Buffer.from(new Float32Array([1, 0, 0, 0]).buffer), + ); + + const { results } = await searchDocuments(db, new FixedProvider("mock", "m", 3), { + query: "kiwi", + }); + + expect(results.map((r) => r.chunkId)).toEqual(["c1"]); + expect(results[0]?.scoreExplanation.method).not.toBe("vector"); + }); +}); + +describe("vector index metadata (no sqlite-vec)", () => { + beforeEach(() => { + initLogger("silent"); + }); + + it("rebuildVectorTable reports that sqlite-vec is required", async () => { + const db = new DatabaseConstructor(":memory:"); + runMigrations(db); + await expect(rebuildVectorTable(db, new FixedProvider("mock", "m", 4))).rejects.toThrow( + /sqlite-vec/, + ); + db.close(); + }); + + it("readEmbeddingIdentity returns nothing before migration 18", () => { + const db = new DatabaseConstructor(":memory:"); + runMigrations(db, 17); + expect(readEmbeddingIdentity(db)).toEqual({}); + db.close(); + }); +}); diff --git a/tests/unit/schema-migrations.test.ts b/tests/unit/schema-migrations.test.ts index 60aa0ac6..c86e7a1e 100644 --- a/tests/unit/schema-migrations.test.ts +++ b/tests/unit/schema-migrations.test.ts @@ -190,4 +190,16 @@ describe("schema migrations", () => { expect(plan.map((p) => p.detail).join(" ")).toContain("idx_documents_title"); }); }); + + describe("migration 18: index_meta table", () => { + it("creates an empty index_meta table on upgrade", () => { + const db = newDb(); + runMigrations(db, 17); + seedChunks(db, 1, 1); + + runMigrations(db); + + expect(count(db, "index_meta")).toBe(0); + }); + }); }); From 9c263709db1383b978a85aa65d8e79bbf48fc27d Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:34:01 +0000 Subject: [PATCH 25/83] fix(mcp): make connector syncs cancellable and stop requiring secrets cancel-task did nothing for the sync-* tools: connectors ignored the task's abort signal. Add an optional signal to every connector sync (notion, slack, confluence, obsidian, onenote, docs), checked between pages and items. An aborted sync throws, records its run as failed and skips end-of-sync deletions. The five sync-* MCP tools pass the task signal. The sync-* tools also required raw secrets as parameters and ignored saved configs. Their secret/location params are now optional and fall back to the saved named config (new 'name' param, default = connector type); explicit params still work and override for that call. OneNote tokens refreshed during an MCP sync are saved back to the config. --- docs/guide/connectors.md | 2 + docs/reference/mcp-tools.md | 65 ++++--- src/connectors/confluence.ts | 21 ++- src/connectors/docs.ts | 4 +- src/connectors/notion.ts | 21 ++- src/connectors/obsidian.ts | 37 ++-- src/connectors/onenote.ts | 31 ++- src/connectors/saved-config.ts | 65 ++++++- src/connectors/slack.ts | 18 +- src/connectors/sync-tracker.ts | 5 + src/mcp/server.ts | 201 +++++++++++++++----- tests/unit/connector-abort.test.ts | 290 +++++++++++++++++++++++++++++ tests/unit/docs-connector.test.ts | 29 +++ 13 files changed, 678 insertions(+), 111 deletions(-) create mode 100644 tests/unit/connector-abort.test.ts diff --git a/docs/guide/connectors.md b/docs/guide/connectors.md index ecd73ca2..3bdf9e5c 100644 --- a/docs/guide/connectors.md +++ b/docs/guide/connectors.md @@ -176,4 +176,6 @@ All connectors are also available as MCP tools, so your AI assistant can trigger - `sync-slack` - `sync-onenote` +Tokens and other settings are optional when a saved config exists: the tools read the config named by their `name` parameter (default: the connector type), so the assistant does not need secrets in its context. Run a sync with `async: true` to get a task ID; `cancel-task` stops the sync between pages or items. + See the [MCP Tools Reference](/reference/mcp-tools) for parameter details. diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 55db89b8..d37b9c76 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -151,55 +151,70 @@ Re-embed all document chunks with the current embedding provider. Use after swit | `before` | string | | Only reindex docs created before this ISO-8601 date | | `documentIds` | string[] | | Only reindex specific documents | +## Connector sync tools + +The five `sync-*` tools share these behaviors: + +- `name` (string, optional) selects a saved connector config (default: the connector type). Configs are saved by `libscope connect <type>`. Any parameter you omit, including tokens, comes from that config, so an agent does not need secrets in its context. Parameters you pass apply to that call only and are not saved. +- `async` (boolean, optional) runs the sync in the background and returns a task ID. `cancel-task` stops the sync between pages or items, and the run is recorded as failed. +- Each call writes one entry to the connector sync history under `name`. + ## sync-obsidian-vault Sync an Obsidian vault into the knowledge base. Parses wikilinks, frontmatter, embeds, and tags. -| Parameter | Type | Required | Description | -| ----------- | ------ | -------- | ------------------------------------ | -| `vaultPath` | string | ✅ | Absolute path to the vault directory | +| Parameter | Type | Required | Description | +| ----------- | ------ | --------------- | ------------------------------------ | +| `vaultPath` | string | unless saved | Absolute path to the vault directory | +| `name` | string | | Saved config name (default: `obsidian`) | ## sync-notion Sync Notion pages and databases. -| Parameter | Type | Required | Description | -| -------------- | -------- | -------- | --------------------------------------- | -| `token` | string | ✅ | Notion integration token | -| `excludePages` | string[] | | Page/database IDs to exclude | -| `lastSync` | string | | ISO-8601 timestamp for incremental sync | +| Parameter | Type | Required | Description | +| -------------- | -------- | ------------ | --------------------------------------- | +| `token` | string | unless saved | Notion integration token | +| `excludePages` | string[] | | Page/database IDs to exclude | +| `lastSync` | string | | ISO-8601 timestamp for incremental sync | +| `name` | string | | Saved config name (default: `notion`) | ## sync-confluence Sync Confluence spaces and pages. -| Parameter | Type | Required | Description | -| --------------- | -------- | -------- | --------------------------------- | -| `baseUrl` | string | ✅ | Confluence base URL | -| `email` | string | ✅ | User email | -| `token` | string | ✅ | API token | -| `spaces` | string[] | | Space keys to sync (default: all) | -| `excludeSpaces` | string[] | | Space keys to exclude | +| Parameter | Type | Required | Description | +| --------------- | -------- | ------------ | --------------------------------------- | +| `baseUrl` | string | unless saved | Confluence base URL | +| `email` | string | Cloud only | User email | +| `token` | string | unless saved | API token or PAT | +| `spaces` | string[] | | Space keys to sync (default: all) | +| `excludeSpaces` | string[] | | Space keys to exclude | +| `name` | string | | Saved config name (default: `confluence`) | ## sync-slack Sync Slack channel messages and threads. -| Parameter | Type | Required | Description | -| ----------------- | -------- | -------- | ----------------------------------- | -| `token` | string | ✅ | Slack bot token | -| `channels` | string[] | ✅ | Channel names/IDs, or `["all"]` | -| `excludeChannels` | string[] | | Channels to exclude | -| `threadMode` | string | | `aggregate` (default) or `separate` | +| Parameter | Type | Required | Description | +| ----------------- | -------- | ------------ | -------------------------------------------- | +| `token` | string | unless saved | Slack bot token | +| `channels` | string[] | | Channel names/IDs, or `["all"]` (default: all) | +| `excludeChannels` | string[] | | Channels to exclude | +| `threadMode` | string | | `aggregate` (default) or `separate` | +| `name` | string | | Saved config name (default: `slack`) | ## sync-onenote Sync OneNote notebooks via Microsoft Graph API. -| Parameter | Type | Required | Description | -| -------------- | ------ | -------- | -------------------------------- | -| `accessToken` | string | ✅ | Microsoft Graph API access token | -| `notebookName` | string | | Specific notebook (default: all) | +| Parameter | Type | Required | Description | +| -------------- | ------ | ------------ | ------------------------------------------------------------------------ | +| `accessToken` | string | unless saved | Microsoft Graph API access token, used as given | +| `notebookName` | string | | Specific notebook (default: all) | +| `name` | string | | Saved config name (default: `onenote`) | + +Without `accessToken`, the saved config's refresh token is used to get a new access token when needed, and the new tokens are saved. ## install-pack diff --git a/src/connectors/confluence.ts b/src/connectors/confluence.ts index dab05708..1f3a41c9 100644 --- a/src/connectors/confluence.ts +++ b/src/connectors/confluence.ts @@ -112,11 +112,17 @@ async function confluenceFetch<T>(url: string, auth: string): Promise<T> { return (await response.json()) as T; } -async function fetchAllPages<T>(initialUrl: string, baseUrl: string, auth: string): Promise<T[]> { +async function fetchAllPages<T>( + initialUrl: string, + baseUrl: string, + auth: string, + signal?: AbortSignal, +): Promise<T[]> { const all: T[] = []; let url: string | undefined = initialUrl; while (url) { + signal?.throwIfAborted(); const resp: PaginatedResponse<T> = await confluenceFetch<PaginatedResponse<T>>(url, auth); all.push(...resp.results); const next: string | undefined = resp._links?.next; @@ -449,19 +455,21 @@ interface SyncConfluenceSpaceOptions { urls: ApiUrls; auth: string; result: ConfluenceSyncResult; + signal: AbortSignal | undefined; } /** Sync all pages within a single Confluence space. */ async function syncConfluenceSpace(options: SyncConfluenceSpaceOptions): Promise<void> { - const { db, provider, space, confluenceType, base, urls, auth, result } = options; + const { db, provider, space, confluenceType, base, urls, auth, result, signal } = options; const log = getLogger(); const topic = createTopic(db, { name: space.name }); let pages: ConfluencePage[]; try { const spaceRef = confluenceType === "server" ? space.key : space.id; - pages = await fetchAllPages<ConfluencePage>(urls.spacePages(spaceRef), base, auth); + pages = await fetchAllPages<ConfluencePage>(urls.spacePages(spaceRef), base, auth, signal); } catch (err) { + if (signal?.aborted) throw err; const msg = err instanceof Error ? err.message : String(err); log.error({ spaceKey: space.key, err }, "Failed to fetch pages for space"); result.errors.push({ page: `space:${space.key}`, error: msg }); @@ -469,6 +477,7 @@ async function syncConfluenceSpace(options: SyncConfluenceSpaceOptions): Promise } for (const page of pages) { + signal?.throwIfAborted(); try { const outcome = await indexConfluencePage({ db, @@ -526,7 +535,7 @@ export async function syncConfluence( db, "confluence", options.syncName ?? trimBaseUrl(config.baseUrl), - () => runConfluenceSync(db, provider, config), + () => runConfluenceSync(db, provider, config, options.signal), (result) => ({ added: result.pagesIndexed, updated: result.pagesUpdated, @@ -540,6 +549,7 @@ async function runConfluenceSync( db: Database.Database, provider: EmbeddingProvider, config: ConfluenceConfig, + signal: AbortSignal | undefined, ): Promise<ConfluenceSyncResult> { const log = getLogger(); @@ -559,7 +569,7 @@ async function runConfluenceSync( log.info({ baseUrl: base }, "Starting Confluence sync"); - const allSpaces = await fetchAllPages<ConfluenceSpace>(urls.spaces, base, auth); + const allSpaces = await fetchAllPages<ConfluenceSpace>(urls.spaces, base, auth, signal); const excludeSet = new Set(config.excludeSpaces ?? []); const requestedAll = config.spaces.length === 1 && config.spaces[0] === "all"; const spacesToSync = allSpaces.filter((s) => { @@ -580,6 +590,7 @@ async function runConfluenceSync( urls, auth, result, + signal, }); } diff --git a/src/connectors/docs.ts b/src/connectors/docs.ts index b596703f..76a7f383 100644 --- a/src/connectors/docs.ts +++ b/src/connectors/docs.ts @@ -638,7 +638,7 @@ export async function syncDocSite( db, CONNECTOR_TYPE, options.syncName ?? config.url, - () => runDocSiteSync(db, provider, config), + () => runDocSiteSync(db, provider, config, options.signal), (result) => ({ added: result.pagesIndexed, updated: result.pagesUpdated, @@ -652,6 +652,7 @@ async function runDocSiteSync( db: Database.Database, provider: EmbeddingProvider, config: DocSiteConfig, + signal: AbortSignal | undefined, ): Promise<DocSiteSyncResult> { const log = getLogger(); @@ -723,6 +724,7 @@ async function runDocSiteSync( // --- BFS crawl --- while (queue.length > 0 && visited.size <= maxPages) { + signal?.throwIfAborted(); const batch = queue.splice(0, concurrency); await Promise.allSettled( diff --git a/src/connectors/notion.ts b/src/connectors/notion.ts index ea6fe00c..5bbc5096 100644 --- a/src/connectors/notion.ts +++ b/src/connectors/notion.ts @@ -106,7 +106,11 @@ async function notionFetch<T>( return (await response.json()) as T; } -async function searchNotion(token: string, lastSync?: string): Promise<NotionSearchResult[]> { +async function searchNotion( + token: string, + lastSync: string | undefined, + signal: AbortSignal | undefined, +): Promise<NotionSearchResult[]> { const log = getLogger(); const allResults: NotionSearchResult[] = []; let cursor: string | null = null; @@ -114,6 +118,7 @@ async function searchNotion(token: string, lastSync?: string): Promise<NotionSea const MAX_PAGES = 10_000; while (hasMore) { + signal?.throwIfAborted(); const body: Record<string, unknown> = { page_size: 100 }; if (cursor) body["start_cursor"] = cursor; if (lastSync) { @@ -460,12 +465,14 @@ async function syncNotionDatabase( token: string, item: NotionSearchResult, excludeSet: Set<string>, + signal: AbortSignal | undefined, ): Promise<void> { const log = getLogger(); const dbTitle = extractTitle(item); const rows = await queryDatabase(token, item.id); for (const row of rows) { + signal?.throwIfAborted(); if (excludeSet.has(row.id)) continue; const rowTitle = extractTitle(row); @@ -490,12 +497,13 @@ async function syncNotionItem( item: NotionSearchResult, excludeSet: Set<string>, result: NotionSyncResult, + signal: AbortSignal | undefined, ): Promise<void> { if (item.object === "page") { const indexed = await syncNotionPage(db, provider, token, item); if (indexed) result.pagesIndexed++; } else if (item.object === "database") { - await syncNotionDatabase(db, provider, token, item, excludeSet); + await syncNotionDatabase(db, provider, token, item, excludeSet, signal); result.databasesIndexed++; } } @@ -511,7 +519,7 @@ export async function syncNotion( db, "notion", options.syncName ?? "notion", - () => runNotionSync(db, provider, config), + () => runNotionSync(db, provider, config, options.signal), (result) => ({ added: result.pagesIndexed + result.databasesIndexed, updated: 0, @@ -525,6 +533,7 @@ async function runNotionSync( db: Database.Database, provider: EmbeddingProvider, config: NotionConfig, + signal: AbortSignal | undefined, ): Promise<NotionSyncResult> { const log = getLogger(); @@ -541,18 +550,20 @@ async function runNotionSync( const excludeSet = new Set(config.excludePages ?? []); log.info({ lastSync: config.lastSync }, "Starting Notion sync"); - const searchResults = await searchNotion(config.token, config.lastSync); + const searchResults = await searchNotion(config.token, config.lastSync, signal); log.info({ count: searchResults.length }, "Found Notion objects"); for (const item of searchResults) { + signal?.throwIfAborted(); if (excludeSet.has(item.id)) { log.debug({ id: item.id }, "Skipping excluded page"); continue; } try { - await syncNotionItem(db, provider, config.token, item, excludeSet, result); + await syncNotionItem(db, provider, config.token, item, excludeSet, result, signal); } catch (err) { + if (signal?.aborted) throw err; const title = extractTitle(item); const message = err instanceof Error ? err.message : String(err); result.errors.push({ page: title, error: message }); diff --git a/src/connectors/obsidian.ts b/src/connectors/obsidian.ts index 7ec6a91b..fca324bf 100644 --- a/src/connectors/obsidian.ts +++ b/src/connectors/obsidian.ts @@ -416,19 +416,24 @@ function applyVaultFileOutcome( } } +interface SyncVaultFilesContext { + db: Database.Database; + provider: EmbeddingProvider; + config: ObsidianConfig; + vaultFiles: string[]; + trackedFiles: Record<string, VaultFileEntry>; + newTrackedFiles: Record<string, VaultFileEntry>; + result: SyncResult; + signal: AbortSignal | undefined; +} + /** Sync all vault files, populating result and newTrackedFiles. */ -async function syncVaultFiles( - db: Database.Database, - provider: EmbeddingProvider, - config: ObsidianConfig, - vaultFiles: string[], - trackedFiles: Record<string, VaultFileEntry>, - newTrackedFiles: Record<string, VaultFileEntry>, - result: SyncResult, -): Promise<void> { +async function syncVaultFiles(ctx: SyncVaultFilesContext): Promise<void> { + const { db, provider, config, vaultFiles, trackedFiles, newTrackedFiles, result, signal } = ctx; const log = getLogger(); const fileMap = buildVaultFileMap(vaultFiles); for (const relPath of vaultFiles) { + signal?.throwIfAborted(); try { const outcome = await processVaultFile( db, @@ -462,7 +467,7 @@ export async function syncObsidianVault( db, "obsidian", options.syncName ?? config.vaultPath, - () => runObsidianSync(db, provider, config), + () => runObsidianSync(db, provider, config, options.signal), (result) => ({ added: result.added, updated: result.updated, @@ -476,6 +481,7 @@ async function runObsidianSync( db: Database.Database, provider: EmbeddingProvider, config: ObsidianConfig, + signal: AbortSignal | undefined, ): Promise<SyncResult> { const log = getLogger(); const result: SyncResult = { added: 0, updated: 0, deleted: 0, errors: [] }; @@ -496,7 +502,16 @@ async function runObsidianSync( const newTrackedFiles: Record<string, VaultFileEntry> = {}; const currentFileSet = new Set(vaultFiles); - await syncVaultFiles(db, provider, config, vaultFiles, trackedFiles, newTrackedFiles, result); + await syncVaultFiles({ + db, + provider, + config, + vaultFiles, + trackedFiles, + newTrackedFiles, + result, + signal, + }); result.deleted = deleteRemovedFiles(db, trackedFiles, currentFileSet); diff --git a/src/connectors/onenote.ts b/src/connectors/onenote.ts index ef9cca1e..98808539 100644 --- a/src/connectors/onenote.ts +++ b/src/connectors/onenote.ts @@ -441,6 +441,7 @@ interface SyncOneNoteSectionOptions { config: OneNoteConfig; seenSourceUrls: Set<string>; result: OneNoteSyncResult; + signal: AbortSignal | undefined; } /** Sync all pages within a single section. */ @@ -455,6 +456,7 @@ async function syncOneNoteSection(options: SyncOneNoteSectionOptions): Promise<v config, seenSourceUrls, result, + signal, } = options; const log = getLogger(); const sectionTopicId = ensureOrCreateTopic(db, section.displayName, notebookTopicId); @@ -472,6 +474,7 @@ async function syncOneNoteSection(options: SyncOneNoteSectionOptions): Promise<v } for (const page of pages) { + signal?.throwIfAborted(); const sourceUrl = buildSourceUrl(notebookName, section.displayName, page.title); seenSourceUrls.add(sourceUrl); @@ -497,14 +500,11 @@ async function syncOneNoteSection(options: SyncOneNoteSectionOptions): Promise<v /** Sync all sections within a single notebook. */ async function syncOneNoteNotebook( - db: Database.Database, - provider: EmbeddingProvider, - token: string, - notebook: GraphNotebook, - config: OneNoteConfig, - seenSourceUrls: Set<string>, - result: OneNoteSyncResult, + options: Omit<SyncOneNoteSectionOptions, "notebookName" | "section" | "notebookTopicId"> & { + notebook: GraphNotebook; + }, ): Promise<void> { + const { db, provider, token, notebook, config, seenSourceUrls, result, signal } = options; const notebookTopicId = ensureOrCreateTopic(db, notebook.displayName); let sections: GraphSection[]; @@ -520,6 +520,7 @@ async function syncOneNoteNotebook( result.sections += filteredSections.length; for (const section of filteredSections) { + signal?.throwIfAborted(); await syncOneNoteSection({ db, provider, @@ -530,6 +531,7 @@ async function syncOneNoteNotebook( config, seenSourceUrls, result, + signal, }); } } @@ -591,7 +593,7 @@ export async function syncOneNote( db, "onenote", options.syncName ?? "onenote", - () => runOneNoteSync(db, provider, config), + () => runOneNoteSync(db, provider, config, options.signal), (result) => ({ added: result.pagesAdded, updated: result.pagesUpdated, @@ -605,6 +607,7 @@ async function runOneNoteSync( db: Database.Database, provider: EmbeddingProvider, config: OneNoteConfig, + signal: AbortSignal | undefined, ): Promise<OneNoteSyncResult> { const log = getLogger(); await ensureOneNoteAccessToken(config); @@ -634,7 +637,17 @@ async function runOneNoteSync( const seenSourceUrls = new Set<string>(); for (const notebook of targetNotebooks) { - await syncOneNoteNotebook(db, provider, token, notebook, config, seenSourceUrls, result); + signal?.throwIfAborted(); + await syncOneNoteNotebook({ + db, + provider, + token, + notebook, + config, + seenSourceUrls, + result, + signal, + }); } result.pagesDeleted = deleteStaleOneNoteDocs(db, seenSourceUrls); diff --git a/src/connectors/saved-config.ts b/src/connectors/saved-config.ts index f5097d05..7c0daeea 100644 --- a/src/connectors/saved-config.ts +++ b/src/connectors/saved-config.ts @@ -5,7 +5,7 @@ */ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { ConfigError } from "../errors.js"; +import { ConfigError, ValidationError } from "../errors.js"; import { hasNamedConnectorConfig, loadConnectorConfig, @@ -206,3 +206,66 @@ export async function runSavedConnectorSync( ): Promise<void> { await SAVED_SYNC_RUNNERS[type](db, provider, name, { ...options, syncName: name }); } + +/** Per-call values for resolveSyncConfig; undefined means "not given". */ +export type SyncConfigOverrides<C> = { [K in keyof C]?: C[K] | undefined }; + +/** Settings for resolveSyncConfig. */ +export interface ResolveSyncConfigOptions<C> { + /** Values used when neither the saved config nor the call provides one. */ + defaults?: Partial<C> | undefined; + /** Fields that must be non-empty after merging. */ + required?: ReadonlyArray<keyof C & string> | undefined; +} + +/** + * Config for an on-demand sync (the MCP sync tools): `defaults`, then saved config `name` + * when it exists, then every defined value in `overrides`. Lets callers omit secrets that + * are already saved. Throws ValidationError naming required fields that are still empty. + */ +export function resolveSyncConfig<C extends object>( + type: ConnectorType, + name: string, + overrides: SyncConfigOverrides<C>, + options: ResolveSyncConfigOptions<C> = {}, +): { config: C; saved: C | undefined } { + const saved = + type === "onenote" + ? (findSavedOneNoteConfig(name) as C | undefined) + : findSavedConnectorConfig<C>(type, name); + const defined = Object.fromEntries( + Object.entries(overrides).filter(([, value]) => value !== undefined), + ) as Partial<C>; + const config: Partial<C> = { ...options.defaults, ...saved, ...defined }; + const missing = (options.required ?? []).filter((key) => { + const value = config[key]; + return value === undefined || value === ""; + }); + if (missing.length > 0) { + throw new ValidationError( + `Missing ${missing.join(", ")} for ${type} sync. Pass the value as a parameter, ` + + `or save a connector named "${name}" with 'libscope connect ${type}'.`, + ); + } + return { config: config as C, saved }; +} + +/** + * After a OneNote sync that used saved config `name`, save tokens the connector refreshed + * so a rotated refresh token is not lost. Does nothing when no refresh happened or the + * caller supplied its own access token (no refresh token in `used`). + */ +export function saveRefreshedOneNoteTokens( + name: string, + saved: OneNoteConfig | undefined, + used: OneNoteConfig, +): void { + if (!saved || !used.refreshToken) return; + if (used.accessToken === saved.accessToken && used.refreshToken === saved.refreshToken) return; + saveConnectorSettings("onenote", name, { + ...saved, + accessToken: used.accessToken, + refreshToken: used.refreshToken, + tokenExpiry: used.tokenExpiry, + }); +} diff --git a/src/connectors/slack.ts b/src/connectors/slack.ts index aed6264d..d0cbef36 100644 --- a/src/connectors/slack.ts +++ b/src/connectors/slack.ts @@ -185,11 +185,12 @@ export async function resolveUserMentions(text: string, token: string): Promise< return result; } -async function listChannels(token: string): Promise<SlackChannel[]> { +async function listChannels(token: string, signal?: AbortSignal): Promise<SlackChannel[]> { const channels: SlackChannel[] = []; let cursor: string | undefined; do { + signal?.throwIfAborted(); const params: Record<string, string> = { types: "public_channel,private_channel", limit: "200", @@ -232,11 +233,13 @@ async function fetchMessages( token: string, channelId: string, oldest?: string, + signal?: AbortSignal, ): Promise<SlackMessage[]> { const messages: SlackMessage[] = []; let cursor: string | undefined; do { + signal?.throwIfAborted(); const params: Record<string, string> = { channel: channelId, limit: "200", @@ -461,9 +464,10 @@ async function syncChannel( config: SlackConfig, channel: SlackChannel, result: SlackSyncResult, + signal: AbortSignal | undefined, ): Promise<void> { const oldest = toSlackTimestamp(config.lastSync); - const messages = await fetchMessages(config.token, channel.id, oldest); + const messages = await fetchMessages(config.token, channel.id, oldest, signal); const { threadParents, standaloneMessages } = classifyMessages(messages); @@ -480,6 +484,7 @@ async function syncChannel( await batchResolveUsers(config.token, allUserIds); for (const msg of standaloneMessages) { + signal?.throwIfAborted(); const indexed = await indexStandaloneMessage(db, provider, config.token, channel.name, msg); if (indexed) result.messagesIndexed++; } @@ -517,7 +522,7 @@ export async function syncSlack( db, "slack", options.syncName ?? "slack", - () => runSlackSync(db, provider, config), + () => runSlackSync(db, provider, config, options.signal), (result) => ({ added: result.messagesIndexed + result.threadsIndexed, updated: 0, @@ -531,6 +536,7 @@ async function runSlackSync( db: Database.Database, provider: EmbeddingProvider, config: SlackConfig, + signal: AbortSignal | undefined, ): Promise<SlackSyncResult> { const log = getLogger(); @@ -551,17 +557,19 @@ async function runSlackSync( }; log.info("Fetching Slack channel list"); - const allChannels = await listChannels(config.token); + const allChannels = await listChannels(config.token, signal); const channels = filterChannels(allChannels, config.channels, config.excludeChannels); result.channels = channels.length; log.info({ channelCount: channels.length }, "Processing Slack channels"); for (const channel of channels) { + signal?.throwIfAborted(); try { log.info({ channel: channel.name }, "Syncing channel"); - await syncChannel(db, provider, config, channel, result); + await syncChannel(db, provider, config, channel, result, signal); } catch (err) { + if (signal?.aborted) throw err; const errMsg = err instanceof Error ? err.message : String(err); log.error({ channel: channel.name, err }, "Error syncing Slack channel"); result.errors.push({ channel: channel.name, error: errMsg }); diff --git a/src/connectors/sync-tracker.ts b/src/connectors/sync-tracker.ts index 6ed67c2c..54fca089 100644 --- a/src/connectors/sync-tracker.ts +++ b/src/connectors/sync-tracker.ts @@ -124,6 +124,11 @@ export interface ConnectorSyncOptions { * Defaults to the connector's historical name when omitted. */ syncName?: string | undefined; + /** + * Cancels the sync. Checked between pages and items; an aborted sync throws the + * signal's reason and is recorded as failed. + */ + signal?: AbortSignal | undefined; } /** diff --git a/src/mcp/server.ts b/src/mcp/server.ts index f07b8cb1..7642980f 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -36,6 +36,11 @@ import { ConfigError, ValidationError } from "../errors.js"; import { errorResponse, textResult, withErrorHandling, type ToolResult } from "./errors.js"; export { errorResponse, withErrorHandling, type ToolResult } from "./errors.js"; import { taskRegistry } from "./tasks.js"; +import type { NotionConfig } from "../connectors/notion.js"; +import type { SlackConfig } from "../connectors/slack.js"; +import type { ConfluenceConfig } from "../connectors/confluence.js"; +import type { ObsidianConfig } from "../connectors/obsidian.js"; +import type { OneNoteConfig } from "../connectors/onenote.js"; import type { Task, TaskType } from "./tasks.js"; /** Build SpiderOptions from submit-document params. */ @@ -837,10 +842,16 @@ async function main(): Promise<void> { "sync-slack", "Sync Slack channel messages and threads into the knowledge base", { - token: z.string().describe("Slack bot token (xoxb-...) or user token (xoxp-...)"), + token: z + .string() + .optional() + .describe( + "Slack bot token (xoxb-...) or user token (xoxp-...). Optional when a saved Slack connector exists.", + ), channels: z .array(z.string()) - .describe("Channel names or IDs to sync, or ['all'] for all channels"), + .optional() + .describe("Channel names or IDs to sync, or ['all'] (default: saved config, else ['all'])"), excludeChannels: z .array(z.string()) .optional() @@ -851,6 +862,12 @@ async function main(): Promise<void> { .describe( "Thread handling: aggregate (default) combines thread into one doc, separate creates one doc per reply", ), + name: z + .string() + .optional() + .describe( + 'Saved connector config name (default: "slack"). Values saved by the CLI connect command fill in any parameter you omit.', + ), async: z .boolean() .optional() @@ -860,16 +877,23 @@ async function main(): Promise<void> { }, withErrorHandling(async (params) => { const { syncSlack: doSyncSlack } = await import("../connectors/slack.js"); + const { resolveSyncConfig } = await import("../connectors/saved-config.js"); - const slackConfig = { - token: params.token, - channels: params.channels, - excludeChannels: params.excludeChannels, - threadMode: params.threadMode ?? ("aggregate" as const), - }; + const name = params.name ?? "slack"; + const { config: slackConfig } = resolveSyncConfig<SlackConfig>( + "slack", + name, + { + token: params.token, + channels: params.channels, + excludeChannels: params.excludeChannels, + threadMode: params.threadMode, + }, + { defaults: { channels: ["all"], threadMode: "aggregate" }, required: ["token"] }, + ); - return runMaybeAsync(params.async, "sync_connector", async () => { - const result = await doSyncSlack(db, provider, slackConfig); + return runMaybeAsync(params.async, "sync_connector", async (signal) => { + const result = await doSyncSlack(db, provider, slackConfig, { syncName: name, signal }); const slackErrorLines = result.errors.map((e) => ` #${e.channel}: ${e.error}`).join("\n"); const slackErrors = result.errors.length > 0 ? `\nErrors:\n${slackErrorLines}` : ""; return ( @@ -959,8 +983,19 @@ async function main(): Promise<void> { "sync-onenote", "Sync OneNote notebooks via Microsoft Graph API", { - accessToken: z.string().describe("Microsoft Graph API access token"), + accessToken: z + .string() + .optional() + .describe( + "Microsoft Graph API access token. Optional when a saved OneNote connector exists (its refresh token is used).", + ), notebookName: z.string().optional().describe("Specific notebook name to sync (default: all)"), + name: z + .string() + .optional() + .describe( + 'Saved connector config name (default: "onenote"). Values saved by the CLI connect command fill in any parameter you omit.', + ), async: z .boolean() .optional() @@ -970,17 +1005,33 @@ async function main(): Promise<void> { }, withErrorHandling(async (params) => { const { syncOneNote } = await import("../connectors/onenote.js"); + const { resolveSyncConfig, saveRefreshedOneNoteTokens } = + await import("../connectors/saved-config.js"); - const oneNoteConfig = { - clientId: "", - tenantId: "common", - accessToken: params.accessToken, - notebooks: params.notebookName ? [params.notebookName] : ["all"], - excludeSections: [] as string[], - }; + const name = params.name ?? "onenote"; + const { config: oneNoteConfig, saved } = resolveSyncConfig<OneNoteConfig>( + "onenote", + name, + { + accessToken: params.accessToken, + notebooks: params.notebookName ? [params.notebookName] : undefined, + }, + { defaults: { clientId: "", tenantId: "common", notebooks: ["all"], excludeSections: [] } }, + ); + if (params.accessToken) { + // Use the caller's token as given; never replace it with a refreshed one. + oneNoteConfig.refreshToken = undefined; + oneNoteConfig.tokenExpiry = undefined; + } + if (!oneNoteConfig.accessToken && !oneNoteConfig.refreshToken) { + throw new ValidationError( + `Missing accessToken for onenote sync. Pass it as a parameter, or save a connector named "${name}" with 'libscope connect onenote'.`, + ); + } - return runMaybeAsync(params.async, "sync_connector", async () => { - const result = await syncOneNote(db, provider, oneNoteConfig); + return runMaybeAsync(params.async, "sync_connector", async (signal) => { + const result = await syncOneNote(db, provider, oneNoteConfig, { syncName: name, signal }); + saveRefreshedOneNoteTokens(name, saved, oneNoteConfig); const oneNoteErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); const oneNoteErrors = result.errors.length > 0 ? `\nErrors: ${oneNoteErrorLines}` : ""; return ( @@ -1001,7 +1052,12 @@ async function main(): Promise<void> { "sync-notion", "Sync pages and databases from a connected Notion workspace into the knowledge base", { - token: z.string().describe("Notion integration token (secret_... or ntn_...)"), + token: z + .string() + .optional() + .describe( + "Notion integration token (secret_... or ntn_...). Optional when a saved Notion connector exists.", + ), lastSync: z .string() .optional() @@ -1010,6 +1066,12 @@ async function main(): Promise<void> { .array(z.string()) .optional() .describe("List of Notion page/database IDs to exclude from sync"), + name: z + .string() + .optional() + .describe( + 'Saved connector config name (default: "notion"). Values saved by the CLI connect command fill in any parameter you omit.', + ), async: z .boolean() .optional() @@ -1019,15 +1081,18 @@ async function main(): Promise<void> { }, withErrorHandling(async (params) => { const { syncNotion } = await import("../connectors/notion.js"); + const { resolveSyncConfig } = await import("../connectors/saved-config.js"); - const notionConfig = { - token: params.token, - lastSync: params.lastSync, - excludePages: params.excludePages, - }; + const name = params.name ?? "notion"; + const { config: notionConfig } = resolveSyncConfig<NotionConfig>( + "notion", + name, + { token: params.token, lastSync: params.lastSync, excludePages: params.excludePages }, + { required: ["token"] }, + ); - return runMaybeAsync(params.async, "sync_connector", async () => { - const result = await syncNotion(db, provider, notionConfig); + return runMaybeAsync(params.async, "sync_connector", async (signal) => { + const result = await syncNotion(db, provider, notionConfig, { syncName: name, signal }); const notionErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); const notionErrors = result.errors.length > 0 ? `\nErrors: ${notionErrorLines}` : ""; return ( @@ -1045,7 +1110,18 @@ async function main(): Promise<void> { "sync-obsidian-vault", "Sync an Obsidian vault into the knowledge base. Parses wikilinks, frontmatter, embeds, and tags with incremental sync support.", { - vaultPath: z.string().describe("Absolute path to the Obsidian vault directory"), + vaultPath: z + .string() + .optional() + .describe( + "Absolute path to the Obsidian vault directory. Optional when a saved Obsidian connector exists.", + ), + name: z + .string() + .optional() + .describe( + 'Saved connector config name (default: "obsidian"). Values saved by the CLI connect command fill in any parameter you omit.', + ), async: z .boolean() .optional() @@ -1055,15 +1131,21 @@ async function main(): Promise<void> { }, withErrorHandling(async (params) => { const { syncObsidianVault } = await import("../connectors/obsidian.js"); + const { resolveSyncConfig } = await import("../connectors/saved-config.js"); - const obsidianConfig = { - vaultPath: params.vaultPath, - topicMapping: "folder" as const, - excludePatterns: [] as string[], - }; + const name = params.name ?? "obsidian"; + const { config: obsidianConfig } = resolveSyncConfig<ObsidianConfig>( + "obsidian", + name, + { vaultPath: params.vaultPath }, + { defaults: { topicMapping: "folder", excludePatterns: [] }, required: ["vaultPath"] }, + ); - return runMaybeAsync(params.async, "sync_connector", async () => { - const result = await syncObsidianVault(db, provider, obsidianConfig); + return runMaybeAsync(params.async, "sync_connector", async (signal) => { + const result = await syncObsidianVault(db, provider, obsidianConfig, { + syncName: name, + signal, + }); const obsidianErrorLines = result.errors.map((e) => `${e.file}: ${e.error}`).join(", "); const obsidianErrors = result.errors.length > 0 ? `\nErrors: ${obsidianErrorLines}` : ""; return ( @@ -1082,14 +1164,25 @@ async function main(): Promise<void> { "sync-confluence", "Sync Confluence spaces and pages into the knowledge base", { - baseUrl: z.string().describe("Confluence base URL (e.g. https://acme.atlassian.net)"), - email: z.string().describe("Confluence user email"), - token: z.string().describe("API token or PAT"), + baseUrl: z + .string() + .optional() + .describe( + "Confluence base URL (e.g. https://acme.atlassian.net). Optional when a saved Confluence connector exists.", + ), + email: z.string().optional().describe("Confluence user email (Cloud). Optional when saved."), + token: z.string().optional().describe("API token or PAT. Optional when saved."), spaces: z .array(z.string()) .optional() - .describe("Space keys to sync, or ['all'] (default: ['all'])"), + .describe("Space keys to sync, or ['all'] (default: saved config, else ['all'])"), excludeSpaces: z.array(z.string()).optional().describe("Space keys to exclude"), + name: z + .string() + .optional() + .describe( + 'Saved connector config name (default: "confluence"). Values saved by the CLI connect command fill in any parameter you omit.', + ), async: z .boolean() .optional() @@ -1099,17 +1192,27 @@ async function main(): Promise<void> { }, withErrorHandling(async (params) => { const { syncConfluence } = await import("../connectors/confluence.js"); + const { resolveSyncConfig } = await import("../connectors/saved-config.js"); - const confluenceConfig = { - baseUrl: params.baseUrl, - email: params.email, - token: params.token, - spaces: params.spaces ?? ["all"], - excludeSpaces: params.excludeSpaces, - }; + const name = params.name ?? "confluence"; + const { config: confluenceConfig } = resolveSyncConfig<ConfluenceConfig>( + "confluence", + name, + { + baseUrl: params.baseUrl, + email: params.email, + token: params.token, + spaces: params.spaces, + excludeSpaces: params.excludeSpaces, + }, + { defaults: { spaces: ["all"] }, required: ["baseUrl", "token"] }, + ); - return runMaybeAsync(params.async, "sync_connector", async () => { - const result = await syncConfluence(db, provider, confluenceConfig); + return runMaybeAsync(params.async, "sync_connector", async (signal) => { + const result = await syncConfluence(db, provider, confluenceConfig, { + syncName: name, + signal, + }); const confluenceErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join(", "); const confluenceErrors = result.errors.length > 0 ? `\nErrors: ${confluenceErrorLines}` : ""; diff --git a/tests/unit/connector-abort.test.ts b/tests/unit/connector-abort.test.ts new file mode 100644 index 00000000..0f3a44d5 --- /dev/null +++ b/tests/unit/connector-abort.test.ts @@ -0,0 +1,290 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdirSync, writeFileSync, rmSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { initLogger } from "../../src/logger.js"; +import { ValidationError } from "../../src/errors.js"; + +// Connector state and saved configs live under a temp HOME — never the real ~/.libscope. +let tempHome = join(tmpdir(), `libscope-abort-init-${process.pid}`); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const { syncNotion } = await import("../../src/connectors/notion.js"); +const { syncSlack, _setRateLimitDelay } = await import("../../src/connectors/slack.js"); +const { syncConfluence } = await import("../../src/connectors/confluence.js"); +const { syncObsidianVault } = await import("../../src/connectors/obsidian.js"); +const { syncOneNote, _resetRateLimiter } = await import("../../src/connectors/onenote.js"); +const { resolveSyncConfig, saveConnectorSettings, saveRefreshedOneNoteTokens } = + await import("../../src/connectors/saved-config.js"); +const { getConnectorsDir } = await import("../../src/connectors/index.js"); + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +function onlySyncRow(db: Database.Database): { status: string; connector_name: string } { + const rows = db.prepare("SELECT status, connector_name FROM connector_syncs").all() as Array<{ + status: string; + connector_name: string; + }>; + expect(rows).toHaveLength(1); + return rows[0]!; +} + +describe("connector abort signals", () => { + let db: Database.Database; + const provider = new MockEmbeddingProvider(); + + beforeEach(() => { + initLogger("silent"); + tempHome = join(tmpdir(), `libscope-abort-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + db = createTestDbWithVec(); + _setRateLimitDelay(0); + _resetRateLimiter(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + db.close(); + rmSync(tempHome, { recursive: true, force: true }); + }); + + it("notion stops between items and records the run as failed", async () => { + const controller = new AbortController(); + const fetchMock = vi.fn((url: string) => { + if (url.endsWith("/v1/search")) { + return Promise.resolve( + json({ + results: ["p1", "p2"].map((id) => ({ + object: "page", + id, + last_edited_time: "2026-01-01T00:00:00.000Z", + properties: { Name: { type: "title", title: [{ plain_text: id }] } }, + })), + has_more: false, + next_cursor: null, + }), + ); + } + controller.abort(); // cancelled while the first page is being fetched + return Promise.resolve( + json({ + results: [ + { id: "b", type: "paragraph", paragraph: { rich_text: [{ plain_text: "x" }] } }, + ], + has_more: false, + next_cursor: null, + }), + ); + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + syncNotion(db, provider, { token: "secret_t" }, { signal: controller.signal, syncName: "n" }), + ).rejects.toThrow(); + + const urls = fetchMock.mock.calls.map((c) => c[0]); + expect(urls.some((u) => u.includes("/blocks/p1/"))).toBe(true); + expect(urls.some((u) => u.includes("/blocks/p2/"))).toBe(false); + expect(onlySyncRow(db)).toEqual({ status: "failed", connector_name: "n" }); + }); + + it("slack stops before the next channel", async () => { + const controller = new AbortController(); + const fetchMock = vi.fn((url: string) => { + if (url.includes("conversations.list")) { + return Promise.resolve( + json({ + ok: true, + channels: [ + { id: "C1", name: "one" }, + { id: "C2", name: "two" }, + ], + }), + ); + } + controller.abort(); + return Promise.resolve(json({ ok: true, messages: [] })); + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + syncSlack( + db, + provider, + { token: "xoxb", channels: ["all"], threadMode: "aggregate" }, + { signal: controller.signal }, + ), + ).rejects.toThrow(); + + const historyCalls = fetchMock.mock.calls.filter((c) => c[0].includes("conversations.history")); + expect(historyCalls).toHaveLength(1); + expect(onlySyncRow(db).status).toBe("failed"); + }); + + it("confluence does not call the API when already aborted", async () => { + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + const controller = new AbortController(); + controller.abort(); + + await expect( + syncConfluence( + db, + provider, + { baseUrl: "https://acme.atlassian.net", email: "e", token: "t", spaces: ["all"] }, + { signal: controller.signal }, + ), + ).rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); + expect(onlySyncRow(db).status).toBe("failed"); + }); + + it("obsidian indexes nothing and keeps no vault state when aborted", async () => { + const vault = join(tempHome, "vault"); + mkdirSync(vault); + writeFileSync(join(vault, "a.md"), "# A\n\nAlpha note."); + writeFileSync(join(vault, "b.md"), "# B\n\nBeta note."); + const controller = new AbortController(); + controller.abort(); + + await expect( + syncObsidianVault( + db, + provider, + { vaultPath: vault, topicMapping: "folder", excludePatterns: [] }, + { signal: controller.signal }, + ), + ).rejects.toThrow(); + expect(db.prepare("SELECT COUNT(*) AS n FROM documents").get()).toEqual({ n: 0 }); + expect(onlySyncRow(db).status).toBe("failed"); + }); + + it("onenote does not delete unseen pages when aborted mid-sync", async () => { + db.prepare( + "INSERT INTO documents (id, source_type, title, content, url) VALUES ('keep', 'topic', 'K', 'Body', 'onenote://NB/S/K')", + ).run(); + const controller = new AbortController(); + vi.stubGlobal( + "fetch", + vi.fn(() => { + controller.abort(); + return Promise.resolve(json({ value: [{ id: "nb1", displayName: "NB" }] })); + }), + ); + + await expect( + syncOneNote( + db, + provider, + { + clientId: "", + tenantId: "common", + accessToken: "a", + tokenExpiry: "2999-01-01T00:00:00.000Z", + notebooks: ["all"], + excludeSections: [], + }, + { signal: controller.signal }, + ), + ).rejects.toThrow(); + expect(db.prepare("SELECT id FROM documents").all()).toEqual([{ id: "keep" }]); + expect(onlySyncRow(db).status).toBe("failed"); + }); +}); + +describe("resolveSyncConfig (MCP sync tools)", () => { + beforeEach(() => { + initLogger("silent"); + tempHome = join(tmpdir(), `libscope-resolve-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + }); + + afterEach(() => { + rmSync(tempHome, { recursive: true, force: true }); + }); + + it("fills omitted parameters from the saved config, then defaults", () => { + saveConnectorSettings("slack", "slack", { token: "xoxb-saved", channels: ["eng"] }); + const { config, saved } = resolveSyncConfig<{ + token: string; + channels: string[]; + threadMode: string; + }>( + "slack", + "slack", + { token: undefined, channels: undefined, threadMode: undefined }, + { defaults: { channels: ["all"], threadMode: "aggregate" }, required: ["token"] }, + ); + expect(config).toMatchObject({ + token: "xoxb-saved", + channels: ["eng"], + threadMode: "aggregate", + }); + expect(saved).toBeDefined(); + }); + + it("lets explicit parameters override the saved config", () => { + saveConnectorSettings("notion", "team", { token: "secret_saved" }); + const { config } = resolveSyncConfig<{ token: string }>("notion", "team", { + token: "secret_given", + }); + expect(config.token).toBe("secret_given"); + }); + + it("names the missing fields and the config name when nothing is saved", () => { + expect(() => + resolveSyncConfig<{ baseUrl: string; token: string }>( + "confluence", + "wiki", + { baseUrl: undefined, token: undefined }, + { required: ["baseUrl", "token"] }, + ), + ).toThrow(ValidationError); + expect(() => + resolveSyncConfig<{ token: string }>("notion", "absent", {}, { required: ["token"] }), + ).toThrow(/token.*"absent"/); + }); + + it("saves refreshed OneNote tokens only when the connector refreshed them", () => { + const saved = { + clientId: "c", + tenantId: "common", + accessToken: "old", + refreshToken: "r1", + notebooks: ["all"], + excludeSections: [], + }; + saveConnectorSettings("onenote", "onenote", saved); + const path = join(getConnectorsDir(), "onenote.json"); + + saveRefreshedOneNoteTokens("onenote", saved, { + ...saved, + accessToken: "x", + refreshToken: undefined, + }); + expect(JSON.parse(readFileSync(path, "utf-8"))).toMatchObject({ accessToken: "old" }); + + saveRefreshedOneNoteTokens("onenote", saved, { + ...saved, + accessToken: "new", + refreshToken: "r2", + tokenExpiry: "2999-01-01T00:00:00.000Z", + }); + expect(JSON.parse(readFileSync(path, "utf-8"))).toMatchObject({ + accessToken: "new", + refreshToken: "r2", + }); + }); +}); diff --git a/tests/unit/docs-connector.test.ts b/tests/unit/docs-connector.test.ts index 679b16ef..4f164781 100644 --- a/tests/unit/docs-connector.test.ts +++ b/tests/unit/docs-connector.test.ts @@ -820,6 +820,35 @@ describe("syncDocSite — mocked fetch", () => { expect(result.pagesIndexed).toBe(0); }); + it("stops crawling when the abort signal fires", async () => { + const controller = new AbortController(); + const root = SPHINX_ROOT_SIMPLE.replace( + "</body>", + '<a href="https://docs.example.com/docs/p1">P1</a></body>', + ); + mockFetch + .mockImplementationOnce(() => { + controller.abort(); + return Promise.resolve(htmlResponse(root)); + }) + .mockResolvedValueOnce(notFoundResponse()); // sitemap.xml + + await expect( + syncDocSite( + db, + provider, + { url: "https://docs.example.com/docs/" }, + { + signal: controller.signal, + }, + ), + ).rejects.toThrow(); + + expect(mockFetch).toHaveBeenCalledTimes(2); // root + sitemap, never p1 + const row = db.prepare("SELECT status FROM connector_syncs").get() as { status: string }; + expect(row.status).toBe("failed"); + }); + it("records sync history in the connector_syncs table", async () => { mockFetch .mockResolvedValueOnce(htmlResponse(SPHINX_ROOT_SIMPLE)) // root From 84eaefa70b6988b8d54e86de9438d9248a0082e7 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 15:35:49 +0000 Subject: [PATCH 26/83] fix(embeddings): check the model at every vector-table open and explain size mismatches - Pass the provider (not just its vector size) to createVectorTable from the CLI, MCP server, SDK, lite, repo indexer and reindex, so switching to a different model with the same vector size is detected too. - lite and reindex no longer swallow the model-mismatch ConfigError. - A vector whose size does not match the index now raises an EmbeddingError that points to `libscope reindex --rebuild` instead of a raw sqlite-vec error. - Remove the per-chunk chunk_embedding_metadata write: no migration creates that table and index_meta records the model for the whole index. - Docs: reindex --rebuild, config get/unset/path and key list, schema v18. --- docs/guide/architecture.md | 40 ++++++++--------- docs/reference/cli.md | 78 +++++++++++++++++---------------- src/LibScope.ts | 2 +- src/api/indexing/repoConfig.ts | 2 +- src/cli/index.ts | 4 +- src/core/documents.ts | 4 +- src/core/indexing.ts | 43 +++++------------- src/core/packs.ts | 2 +- src/core/reindex.ts | 11 ++--- src/core/search.ts | 11 +---- src/db/index-meta.ts | 9 ++++ src/lite/core.ts | 9 ++-- src/mcp/server.ts | 2 +- tests/unit/chunk-writer.test.ts | 57 ++++++++++++++++++++++++ 14 files changed, 159 insertions(+), 115 deletions(-) create mode 100644 tests/unit/chunk-writer.test.ts diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 006e5be1..8f2e11db 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -272,26 +272,26 @@ The LLM integration in `src/core/rag.ts` supports OpenAI, Ollama, Anthropic, and ## Database Schema -Key tables (schema version 17): - -| Table | Purpose | -| ------------------ | -------------------------------------------------- | -| `documents` | Document metadata: title, content, library, topic | -| `chunks` | Document chunks: content, chunk_index, document_id | -| `chunk_embeddings` | Vector table (sqlite-vec): embedding per chunk | -| `chunks_fts` | FTS5 virtual table: full-text search index | -| `topics` | Topic hierarchy (id, name, parent_id) | -| `tags` | Tag definitions | -| `document_tags` | Many-to-many document ↔ tag | -| `ratings` | Document and chunk ratings (1–5) | -| `document_versions`| Version history for rollback | -| `document_links` | Typed cross-references between documents | -| `search_log` | Query analytics | -| `document_hits` | Per-document result hit analytics | -| `saved_searches` | Named query persistence | -| `connector_configs`| Connector state (tokens, last sync) | -| `webhooks` | Event webhook configuration | -| `schema_version` | Current migration version | +Key tables (schema version 18): + +| Table | Purpose | +| ------------------- | -------------------------------------------------- | +| `documents` | Document metadata: title, content, library, topic | +| `chunks` | Document chunks: content, chunk_index, document_id | +| `chunk_embeddings` | Vector table (sqlite-vec): embedding per chunk | +| `chunks_fts` | FTS5 virtual table: full-text search index | +| `topics` | Topic hierarchy (id, name, parent_id) | +| `tags` | Tag definitions | +| `document_tags` | Many-to-many document ↔ tag | +| `ratings` | Document and chunk ratings (1–5) | +| `document_versions` | Version history for rollback | +| `document_links` | Typed cross-references between documents | +| `search_log` | Query analytics | +| `document_hits` | Per-document result hit analytics | +| `saved_searches` | Named query persistence | +| `connector_configs` | Connector state (tokens, last sync) | +| `webhooks` | Event webhook configuration | +| `schema_version` | Current migration version | ## How to Add a New CLI Command diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a6d04e17..a2281b81 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -265,15 +265,15 @@ libscope pack install react-docs --registry official ## Pack Registries -| Command | Description | -| ---------------------------------------------------------- | -------------------------------------- | -| `libscope registry add <url> [-n <alias>]` | Register a git repo as a pack registry | -| `libscope registry remove <name> [-y]` | Unregister a registry | -| `libscope registry list` | List configured registries | -| `libscope registry sync [<name>]` | Sync one or all registries | -| `libscope registry search <query> [-r <name>]` | Search registry pack indexes | -| `libscope registry create <path>` | Initialize a new registry repo | -| `libscope registry publish <file> -r <name>` | Publish a pack file to a registry | +| Command | Description | +| ----------------------------------------------------------------- | ------------------------------------------------------- | +| `libscope registry add <url> [-n <alias>]` | Register a git repo as a pack registry | +| `libscope registry remove <name> [-y]` | Unregister a registry | +| `libscope registry list` | List configured registries | +| `libscope registry sync [<name>]` | Sync one or all registries | +| `libscope registry search <query> [-r <name>]` | Search registry pack indexes | +| `libscope registry create <path>` | Initialize a new registry repo | +| `libscope registry publish <file> -r <name>` | Publish a pack file to a registry | | `libscope registry unpublish <pack> -r <name> --pack-version <v>` | Remove a pack version from a registry (or `<pack>@<v>`) | ### `libscope registry add` @@ -284,11 +284,11 @@ libscope registry add git@github.com:team/packs.git --name team --priority 5 libscope registry add https://github.com/org/registry.git --sync-interval 86400 --no-sync ``` -| Option | Description | -| ---------------------------- | -------------------------------------------------------- | -| `-n, --name <alias>` | Short name for this registry (default: inferred from URL)| -| `--priority <n>` | Conflict resolution priority — lower wins (default: 10) | -| `--sync-interval <seconds>` | Auto-sync interval in seconds, 0 = manual (default: 0) | +| Option | Description | +| --------------------------- | --------------------------------------------------------- | +| `-n, --name <alias>` | Short name for this registry (default: inferred from URL) | +| `--priority <n>` | Conflict resolution priority — lower wins (default: 10) | +| `--sync-interval <seconds>` | Auto-sync interval in seconds, 0 = manual (default: 0) | | `--no-sync` | Skip initial sync after adding | ### `libscope registry search` @@ -311,12 +311,12 @@ libscope registry publish ./my-pack.json -r my-registry libscope registry publish ./my-pack.json -r community --submit ``` -| Option | Description | -| ------------------------ | -------------------------------------------------------- | -| `-r, --registry <name>` | Target registry (required) | -| `--pack-version <semver>` | Version to publish as (default: auto-bump patch) | -| `-m, --message <msg>` | Git commit message | -| `--submit` | Push to a feature branch instead of main | +| Option | Description | +| ------------------------- | ------------------------------------------------ | +| `-r, --registry <name>` | Target registry (required) | +| `--pack-version <semver>` | Version to publish as (default: auto-bump patch) | +| `-m, --message <msg>` | Git commit message | +| `--submit` | Push to a feature branch instead of main | ### `libscope registry unpublish` @@ -324,12 +324,12 @@ libscope registry publish ./my-pack.json -r community --submit libscope registry unpublish my-pack -r my-registry --pack-version 1.0.0 ``` -| Option | Description | -| ------------------------ | --------------------------------- | -| `-r, --registry <name>` | Target registry (required) | +| Option | Description | +| ------------------------- | --------------------------------------------------------------- | +| `-r, --registry <name>` | Target registry (required) | | `--pack-version <semver>` | Version to remove (required unless given as `<pack>@<version>`) | -| `-m, --message <msg>` | Git commit message | -| `-y, --yes` | Skip confirmation prompt | +| `-m, --message <msg>` | Git commit message | +| `-y, --yes` | Skip confirmation prompt | ### `libscope registry create` @@ -353,13 +353,14 @@ Creates a git repo with the canonical registry folder structure. See the [Regist ## Developer Tools -| Command | Description | -| ------------------------------------- | --------------------------------------- | -| `libscope watch <directory>` | Watch for file changes and auto-reindex (all supported formats, plus `.rst`) | -| `libscope reindex` | Re-embed chunks with current provider | -| `libscope dedupe` | Scan for duplicate documents | -| `libscope export <outputPath>` | Export knowledge base to JSON | -| `libscope import-backup <backupPath>` | Import from a backup file | +| Command | Description | +| ------------------------------------- | --------------------------------------------------------------------------------------------- | +| `libscope watch <directory>` | Watch for file changes and auto-reindex (all supported formats, plus `.rst`) | +| `libscope reindex` | Re-embed chunks with current provider | +| `libscope reindex --rebuild` | Recreate the vector index for a new embedding model or vector size, then re-embed every chunk | +| `libscope dedupe` | Scan for duplicate documents | +| `libscope export <outputPath>` | Export knowledge base to JSON | +| `libscope import-backup <backupPath>` | Import from a backup file | ## Analytics @@ -372,12 +373,15 @@ Creates a git repo with the canonical registry folder structure. See the [Regist ## Configuration -| Command | Description | -| ----------------------------------- | -------------------------- | -| `libscope config set <key> <value>` | Set a configuration value | -| `libscope config show` | Show current configuration | +| Command | Description | +| ----------------------------------- | --------------------------------------------- | +| `libscope config set <key> <value>` | Set a configuration value | +| `libscope config get <key>` | Show one configuration value (secrets masked) | +| `libscope config unset <key>` | Remove a value from the user config file | +| `libscope config show` | Show current configuration (secrets masked) | +| `libscope config path` | Print the user config file path | -Supported config keys for `set`: `embedding.provider`, `embedding.ollamaUrl`, `embedding.ollamaModel`, `embedding.openaiModel`, `llm.provider`, `llm.model`, `database.path`, `logging.level`, `indexing.allowPrivateUrls`, `indexing.allowSelfSignedCerts`. +Supported config keys: `embedding.provider`, `embedding.ollamaUrl`, `embedding.ollamaModel`, `embedding.openaiModel`, `embedding.dimensions`, `llm.provider`, `llm.model`, `llm.ollamaUrl`, `database.path`, `indexing.maxDocumentSize`, `indexing.allowPrivateUrls`, `indexing.allowSelfSignedCerts`, `logging.level`. API keys cannot be set with `config set`; use the `LIBSCOPE_OPENAI_API_KEY` / `OPENAI_API_KEY` and `LIBSCOPE_ANTHROPIC_API_KEY` / `ANTHROPIC_API_KEY` environment variables. ## Global Options diff --git a/src/LibScope.ts b/src/LibScope.ts index 5c346730..08efbe7c 100644 --- a/src/LibScope.ts +++ b/src/LibScope.ts @@ -79,7 +79,7 @@ export class LibScope { runMigrations(db); const embeddingProvider = createEmbeddingProvider(config); - createVectorTable(db, embeddingProvider.dimensions); + createVectorTable(db, embeddingProvider); return new LibScope(db, embeddingProvider, config); } diff --git a/src/api/indexing/repoConfig.ts b/src/api/indexing/repoConfig.ts index 1d114733..bee2b670 100644 --- a/src/api/indexing/repoConfig.ts +++ b/src/api/indexing/repoConfig.ts @@ -41,7 +41,7 @@ export function createRepoLibScope(repoSlug: string): LibScopeLite { const db = createDatabase(dbPath); runMigrations(db); try { - createVectorTable(db, provider.dimensions); + createVectorTable(db, provider); } catch { /* sqlite-vec not loaded */ } diff --git a/src/cli/index.ts b/src/cli/index.ts index 43d07c24..a4c0880e 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -480,7 +480,7 @@ program // Only create vector table if provider can be initialized without download try { const provider = createEmbeddingProvider(config); - createVectorTable(db, provider.dimensions); + createVectorTable(db, provider); } catch { console.log(" ℹ Vector table skipped (embedding provider not available)"); } @@ -1658,7 +1658,7 @@ function initializeAppWithEmbedding(): { } { const { config, db } = initializeApp(); const provider = createEmbeddingProvider(config); - createVectorTable(db, provider.dimensions); + createVectorTable(db, provider); return { config, db, provider }; } diff --git a/src/core/documents.ts b/src/core/documents.ts index f2098ebf..12c99247 100644 --- a/src/core/documents.ts +++ b/src/core/documents.ts @@ -217,7 +217,7 @@ export async function updateDocument( const chunks = splitIntoChunks(newContent); const embeddings = await embedChunks(provider, chunks, newMeta); - const writer = createChunkWriter(db, provider); + const writer = createChunkWriter(db); const transaction = db.transaction(() => { saveVersion(db, documentId); @@ -291,7 +291,7 @@ async function embedExistingChunks( rows.map((r) => r.content), meta, ); - const writer = createChunkWriter(db, provider); + const writer = createChunkWriter(db); return { write: (): void => { rows.forEach((row, i) => writer.replaceEmbedding(row.id, embeddings[i] ?? [])); diff --git a/src/core/indexing.ts b/src/core/indexing.ts index 450b1b2c..80c9ef2a 100644 --- a/src/core/indexing.ts +++ b/src/core/indexing.ts @@ -4,7 +4,8 @@ import { readFileSync } from "node:fs"; import { basename } from "node:path"; import type { Readable } from "node:stream"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { ValidationError } from "../errors.js"; +import { EmbeddingError, ValidationError } from "../errors.js"; +import { REBUILD_VECTOR_INDEX_HINT, isVectorDimensionError } from "../db/index-meta.js"; import { getLogger } from "../logger.js"; import { checkDuplicate } from "./dedup.js"; import type { DedupOptions } from "./dedup.js"; @@ -429,7 +430,7 @@ function isNoSuchTableError(err: unknown): boolean { export interface ChunkWriter { /** Insert chunk rows for a document with their embeddings (matched by array index). */ insertChunks(documentId: string, chunks: string[], embeddings: ReadonlyArray<number[]>): void; - /** Replace the stored embedding (and embedding metadata row) of an existing chunk. */ + /** Replace the stored embedding of an existing chunk. */ replaceEmbedding(chunkId: string, embedding: number[]): void; } @@ -437,15 +438,11 @@ export interface ChunkWriter { * Create a {@link ChunkWriter}. Writes are synchronous; callers wrap them in a transaction. * When the vector table does not exist (sqlite-vec not loaded), vectors are skipped. */ -export function createChunkWriter( - db: Database.Database, - provider: Pick<EmbeddingProvider, "name">, -): ChunkWriter { +export function createChunkWriter(db: Database.Database): ChunkWriter { const log = getLogger(); const insertChunk = db.prepare( "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", ); - const insertMeta = tryPrepareMetaInsert(db); let insertVec: Database.Statement | undefined; let deleteVec: Database.Statement | undefined; @@ -457,8 +454,13 @@ export function createChunkWriter( } insertVec ??= db.prepare("INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)"); insertVec.run(chunkId, Buffer.from(new Float32Array(embedding).buffer)); - insertMeta?.run(chunkId, provider.name, "unknown"); } catch (err) { + if (isVectorDimensionError(err)) { + throw new EmbeddingError( + `The embedding has ${embedding.length} dimensions, which does not match the vector index. ${REBUILD_VECTOR_INDEX_HINT}`, + err, + ); + } if (!isNoSuchTableError(err)) throw err; log.debug({ chunkId }, "Skipped vector insertion (sqlite-vec not loaded)"); } @@ -478,29 +480,6 @@ export function createChunkWriter( }; } -/** Try to prepare an insert statement for chunk_embedding_metadata if the table exists. */ -function tryPrepareMetaInsert( - db: Database.Database, -): Database.Statement<[string, string, string]> | null { - const log = getLogger(); - try { - const exists = db - .prepare( - "SELECT name FROM sqlite_master WHERE type='table' AND name='chunk_embedding_metadata'", - ) - .get(); - if (exists) { - return db.prepare(` - INSERT OR REPLACE INTO chunk_embedding_metadata (chunk_id, embedding_provider, embedding_model) - VALUES (?, ?, ?) - `); - } - } catch (err: unknown) { - log.debug({ err }, "Skipped chunk_embedding_metadata check"); - } - return null; -} - /** Index a document: validate, chunk, embed, and store. */ export async function indexDocument( db: Database.Database, @@ -539,7 +518,7 @@ export async function indexDocument( INSERT INTO documents (id, source_type, library, version, topic_id, title, content, url, submitted_by, content_hash, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); - const writer = createChunkWriter(db, provider); + const writer = createChunkWriter(db); const transaction = db.transaction(() => { insertDoc.run( diff --git a/src/core/packs.ts b/src/core/packs.ts index 73a7c1bd..edf3fb41 100644 --- a/src/core/packs.ts +++ b/src/core/packs.ts @@ -509,7 +509,7 @@ export async function installPack( INSERT INTO documents (id, source_type, title, content, url, submitted_by, content_hash, pack_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?) `); - const writer = createChunkWriter(db, provider); + const writer = createChunkWriter(db); type BatchData = { batchDocs: PackDocument[] }; const batches: BatchData[] = []; diff --git a/src/core/reindex.ts b/src/core/reindex.ts index 88cb12c5..4b83c9c7 100644 --- a/src/core/reindex.ts +++ b/src/core/reindex.ts @@ -1,6 +1,6 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { DatabaseError } from "../errors.js"; +import { ConfigError, DatabaseError } from "../errors.js"; import { getLogger } from "../logger.js"; import { createVectorTable } from "../db/schema.js"; import { buildEmbeddingText, createChunkWriter } from "./indexing.js"; @@ -70,12 +70,13 @@ export async function reindex( // Ensure the vector table exists with the correct dimensions for this provider. // Delegates to schema.createVectorTable() — single source of truth for the DDL. try { - createVectorTable(db, provider.dimensions); - } catch { - log.warn("Could not ensure vector table — continuing anyway"); + createVectorTable(db, provider); + } catch (err) { + if (err instanceof ConfigError) throw err; + log.warn({ err }, "Could not ensure vector table — continuing anyway"); } - const writer = createChunkWriter(db, provider); + const writer = createChunkWriter(db); let completed = 0; let failed = 0; diff --git a/src/core/search.ts b/src/core/search.ts index 57ca7227..4ac1f0b9 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -4,7 +4,7 @@ import { z } from "zod"; import { withCorrelationId, createChildLogger } from "../logger.js"; import { validateCountRow } from "../utils/db-validation.js"; import { validateRow, validateRows } from "../db/validate.js"; -import { REBUILD_VECTOR_INDEX_HINT } from "../db/index-meta.js"; +import { REBUILD_VECTOR_INDEX_HINT, isVectorDimensionError } from "../db/index-meta.js"; import { logSearch, recordSearchQuery } from "./analytics.js"; import { performance } from "node:perf_hooks"; @@ -38,15 +38,6 @@ function isVectorTableError(err: unknown): boolean { return VECTOR_TABLE_MISSING_PATTERNS.some((p) => msg.includes(p)); } -/** - * sqlite-vec error raised when the query vector size differs from the vector table's - * (the embedding model changed after the index was built). Both spellings occur in sqlite-vec. - */ -function isVectorDimensionError(err: unknown): boolean { - const msg = (err instanceof Error ? err.message : String(err)).toLowerCase(); - return msg.includes("dimension mismatch") || msg.includes("dimension mistmatch"); -} - /** Escape LIKE special characters so user input is treated literally. */ export function escapeLikePattern(input: string): string { // prettier-ignore diff --git a/src/db/index-meta.ts b/src/db/index-meta.ts index 0f35e237..0b6de705 100644 --- a/src/db/index-meta.ts +++ b/src/db/index-meta.ts @@ -16,6 +16,15 @@ export const INDEX_META_DDL = ` export const REBUILD_VECTOR_INDEX_HINT = "Run `libscope reindex --rebuild` to rebuild the vector index with the configured embedding model."; +/** + * sqlite-vec error raised when a vector's size differs from the vector table's + * (the embedding model changed after the index was built). Both spellings occur in sqlite-vec. + */ +export function isVectorDimensionError(err: unknown): boolean { + const msg = (err instanceof Error ? err.message : String(err)).toLowerCase(); + return msg.includes("dimension mismatch") || msg.includes("dimension mistmatch"); +} + /** The embedding provider, model and vector size that built (or will build) the vector index. */ export interface EmbeddingIndexIdentity { provider?: string | undefined; diff --git a/src/lite/core.ts b/src/lite/core.ts index f3711125..43f55e66 100644 --- a/src/lite/core.ts +++ b/src/lite/core.ts @@ -5,6 +5,7 @@ import type { EmbeddingProvider } from "../providers/embedding.js"; import { LocalEmbeddingProvider } from "../providers/local.js"; import { createDatabase } from "../db/connection.js"; import { runMigrations, createVectorTable } from "../db/schema.js"; +import { ConfigError } from "../errors.js"; import { indexDocument } from "../core/indexing.js"; import { searchDocuments } from "../core/search.js"; import { bulkDelete } from "../core/bulk.js"; @@ -43,9 +44,11 @@ export class LibScopeLite { runMigrations(this.db); // Create vector table best-effort (requires sqlite-vec to be loaded). try { - createVectorTable(this.db, this.provider.dimensions); - } catch { - /* sqlite-vec not loaded — FTS5 search still works */ + createVectorTable(this.db, this.provider); + } catch (err) { + // A model/index mismatch must surface; anything else means sqlite-vec is + // unavailable and FTS5 search still works. + if (err instanceof ConfigError) throw err; } } else { // Caller-provided DB: skip all setup (migrations, extension loading, vector table). diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 319ede9c..1a448f20 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -227,7 +227,7 @@ async function main(): Promise<void> { let llmProvider: LlmProvider | undefined; try { provider = createEmbeddingProvider(config); - createVectorTable(db, provider.dimensions); + createVectorTable(db, provider); } catch (err) { console.error( "Failed to initialize embedding provider:", diff --git a/tests/unit/chunk-writer.test.ts b/tests/unit/chunk-writer.test.ts new file mode 100644 index 00000000..c7d20961 --- /dev/null +++ b/tests/unit/chunk-writer.test.ts @@ -0,0 +1,57 @@ +import { describe, it, expect, afterEach } from "vitest"; +import type Database from "better-sqlite3"; +import { createDatabase } from "../../src/db/connection.js"; +import { runMigrations, createVectorTable } from "../../src/db/schema.js"; +import { createChunkWriter } from "../../src/core/indexing.js"; +import { EmbeddingError } from "../../src/errors.js"; + +function hasSqliteVec(): boolean { + const db = createDatabase(":memory:"); + try { + db.prepare("SELECT vec_version()").get(); + return true; + } catch { + return false; + } finally { + db.close(); + } +} + +describe.skipIf(!hasSqliteVec())("createChunkWriter (sqlite-vec)", () => { + let db: Database.Database | undefined; + + afterEach(() => { + db?.close(); + db = undefined; + }); + + function openIndexedDb(): Database.Database { + const database = createDatabase(":memory:"); + runMigrations(database); + createVectorTable(database, 4); + database + .prepare("INSERT INTO documents (id, title, content, source_type) VALUES (?, ?, ?, ?)") + .run("doc-1", "Doc", "content", "manual"); + return database; + } + + it("stores chunks whose vectors match the index size", () => { + db = openIndexedDb(); + createChunkWriter(db).insertChunks("doc-1", ["a"], [[0.1, 0.2, 0.3, 0.4]]); + const row = db.prepare("SELECT COUNT(*) AS n FROM chunk_embeddings").get() as { n: number }; + expect(row.n).toBe(1); + }); + + it("explains how to rebuild when a vector does not match the index size", () => { + db = openIndexedDb(); + const writer = createChunkWriter(db); + let error: unknown; + try { + writer.insertChunks("doc-1", ["a"], [[0.1, 0.2, 0.3]]); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(EmbeddingError); + expect((error as Error).message).toMatch(/3 dimensions.*libscope reindex --rebuild/); + }); +}); From 7c0786c73645cd291453e7d60bff8f96b8bffcaf Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:37:08 +0000 Subject: [PATCH 27/83] feat(config)!: one zod schema for config keys, env vars, and secrets - src/config-schema.ts: ConfigSchema is the single source for keys, defaults, validation, env var names (LIBSCOPE_<SECTION>_<FIELD> plus listed extras), secret flags, and descriptions. getConfigKeyTable() returns the table for generated docs. CONFIG_KEY_NAMES is derived. - Provider-neutral keys: embedding.{provider,model,url,dimensions}, llm.{provider,model,url}. API keys move to openai.apiKey and anthropic.apiKey (env LIBSCOPE_<P>_API_KEY > <P>_API_KEY > ~/.libscope/secrets.json). Keys in config.json/.libscope.json are ignored with a warning. Old keys and old env vars are removed. - config set/unset write secrets to secrets.json (0600), other keys to config.json; invalid file and env values are ignored with a warning. - llm.provider defaults to "auto": passthrough under MCP, else openai with an OpenAI key, else anthropic with an Anthropic key, else ollama when llm.url is set or embedding.provider is ollama, else no LLM. createLlmProvider/isPassthroughMode take { surface }; createLlmProvider throws ConfigError with a hint when no LLM resolves. BREAKING CHANGE: embedding.ollamaUrl/ollamaModel/openaiModel/openaiApiKey, llm.ollamaUrl/openaiApiKey/anthropicApiKey, LIBSCOPE_OLLAMA_URL, LIBSCOPE_OLLAMA_MODEL, LIBSCOPE_ALLOW_PRIVATE_URLS and LIBSCOPE_ALLOW_SELF_SIGNED_CERTS are no longer read. See docs/migration-v2.md. --- src/LibScope.ts | 2 + src/cli/index.ts | 16 +- src/config-schema.ts | 261 +++++++++++ src/config.ts | 572 +++++++++++------------- src/core/rag.ts | 82 ++-- src/core/url-fetcher.ts | 2 +- src/providers/index.ts | 33 +- tests/unit/api.test.ts | 3 +- tests/unit/config-layers.test.ts | 265 +++++++---- tests/unit/config-save.test.ts | 302 +++++++------ tests/unit/config.test.ts | 48 +- tests/unit/embedding-dimensions.test.ts | 11 +- tests/unit/rag.test.ts | 126 +++++- 13 files changed, 1111 insertions(+), 612 deletions(-) create mode 100644 src/config-schema.ts diff --git a/src/LibScope.ts b/src/LibScope.ts index 08efbe7c..7014420f 100644 --- a/src/LibScope.ts +++ b/src/LibScope.ts @@ -65,6 +65,8 @@ export class LibScope { const config: LibScopeConfig = { embedding: { ...baseConfig.embedding, ...options?.config?.embedding }, llm: { ...baseConfig.llm, ...options?.config?.llm }, + openai: { ...baseConfig.openai, ...options?.config?.openai }, + anthropic: { ...baseConfig.anthropic, ...options?.config?.anthropic }, database: { ...baseConfig.database, ...options?.config?.database }, indexing: { ...baseConfig.indexing, ...options?.config?.indexing }, logging: { ...baseConfig.logging, ...options?.config?.logging }, diff --git a/src/cli/index.ts b/src/cli/index.ts index d1dd02e8..767f0273 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -5,6 +5,9 @@ import { loadConfig, maskConfigSecrets, getUserConfigPath, + getSecretsPath, + getConfigFileFor, + maskSecret, CONFIG_KEY_NAMES, setUserConfigValue, unsetUserConfigValue, @@ -1574,12 +1577,15 @@ function runConfigAction(fn: () => void): void { configCmd .command("set <key> <value>") .description( - `Set a value in the user config file. Keys: ${CONFIG_KEY_NAMES.join(", ")}. API keys must be set with environment variables.`, + `Set a value in the user config file. Keys: ${CONFIG_KEY_NAMES.join(", ")}. ` + + `API keys are written to ${getSecretsPath()} (mode 0600), never to config.json.`, ) .action((key: string, value: string) => { runConfigAction(() => { const stored = setUserConfigValue(key, value); - console.log(`✓ ${key} set to: ${String(stored)}`); + const file = getConfigFileFor(key); + const shown = file === getSecretsPath() ? maskSecret(String(stored)) : String(stored); + console.log(`✓ ${key} set to: ${shown} (${file})`); }); }); @@ -1595,11 +1601,13 @@ configCmd configCmd .command("unset <key>") - .description("Remove a key from the user config file (the default or other layers apply)") + .description( + "Remove a key from the user config file, or from the secrets file for API keys (the default or other layers apply)", + ) .action((key: string) => { runConfigAction(() => { const removed = unsetUserConfigValue(key); - console.log(removed ? `✓ ${key} removed` : `${key} is not set in ${getUserConfigPath()}`); + console.log(removed ? `✓ ${key} removed` : `${key} is not set in ${getConfigFileFor(key)}`); }); }); diff --git a/src/config-schema.ts b/src/config-schema.ts new file mode 100644 index 00000000..3d94a2ab --- /dev/null +++ b/src/config-schema.ts @@ -0,0 +1,261 @@ +/** + * The configuration schema: the single source for config keys, defaults, validation, + * environment variable names, secret flags, and descriptions. + * + * Every leaf key `section.field` can be set from the environment variable + * `LIBSCOPE_<SECTION>_<FIELD>` (SCREAMING_SNAKE case), followed by any extra names in its + * metadata. Secret keys are read only from env vars and ~/.libscope/secrets.json. + */ +import { z } from "zod"; + +/** Metadata attached to each leaf of ConfigSchema. */ +export interface ConfigKeyMeta { + /** Extra env var names, checked in order after the canonical LIBSCOPE_<SECTION>_<FIELD>. */ + env?: readonly string[]; + /** Secret values are masked on display and stored in secrets.json, never in config.json. */ + secret?: boolean; + /** + * Config files may hold values outside the enum (names of providers registered at runtime + * with registerProvider). `config set` and env vars accept only the listed values. + */ + openEnum?: boolean; + /** Default text for the docs table when the default depends on other keys. */ + defaultText?: string; +} + +/** Registry that holds the metadata of every leaf schema. */ +export const configKeyMeta = z.registry<ConfigKeyMeta>(); + +function leaf<T extends z.ZodType>(schema: T, meta: ConfigKeyMeta = {}): T { + configKeyMeta.add(schema, meta); + return schema; +} + +export const EMBEDDING_PROVIDERS = ["local", "ollama", "openai"] as const; +export const LLM_PROVIDERS = ["auto", "openai", "anthropic", "ollama", "passthrough"] as const; +export const LOG_LEVELS = ["debug", "info", "warn", "error", "silent"] as const; + +/** Ollama server URL used when embedding.url and llm.url are not set. */ +export const DEFAULT_OLLAMA_URL = "http://localhost:11434"; + +/** Largest accepted `embedding.dimensions` value. */ +const MAX_DIMENSIONS = 10000; + +const nonEmpty = (): z.ZodString => z.string().trim().min(1); + +export const ConfigSchema = z.object({ + embedding: z.object({ + provider: leaf( + z + .enum(EMBEDDING_PROVIDERS) + .default("local") + .describe("Embedding provider. A config file may also name a provider registered in code."), + { openEnum: true }, + ), + model: leaf( + nonEmpty() + .optional() + .describe( + "Embedding model. Ignored by the local provider (always Xenova/all-MiniLM-L6-v2).", + ), + { defaultText: "ollama: nomic-embed-text; openai: text-embedding-3-small" }, + ), + url: leaf( + nonEmpty() + .optional() + .describe("Server URL of the embedding provider. Used by the ollama provider."), + { defaultText: DEFAULT_OLLAMA_URL }, + ), + dimensions: leaf( + z + .number() + .int() + .positive() + .max(MAX_DIMENSIONS) + .optional() + .describe("Vector size, for models whose size is not built in."), + { defaultText: "size of the model" }, + ), + }), + llm: z.object({ + provider: leaf( + z + .enum(LLM_PROVIDERS) + .default("auto") + .describe( + 'LLM for "ask". "auto": passthrough under MCP, else openai if an OpenAI key is set, ' + + "else anthropic if an Anthropic key is set, else ollama if llm.url is set or " + + "embedding.provider is ollama, else none.", + ), + ), + model: leaf(nonEmpty().optional().describe("LLM model."), { + defaultText: "openai: gpt-4o-mini; anthropic: claude-3-5-haiku-20241022; ollama: llama3.2", + }), + url: leaf( + nonEmpty().optional().describe("Server URL of the LLM. Used by the ollama provider."), + { defaultText: `embedding.url, then ${DEFAULT_OLLAMA_URL}` }, + ), + }), + openai: z.object({ + apiKey: leaf(nonEmpty().optional().describe("OpenAI API key (embeddings and LLM)."), { + secret: true, + env: ["OPENAI_API_KEY"], + }), + }), + anthropic: z.object({ + apiKey: leaf(nonEmpty().optional().describe("Anthropic API key (LLM)."), { + secret: true, + env: ["ANTHROPIC_API_KEY"], + }), + }), + database: z.object({ + path: leaf( + nonEmpty() + .optional() + .describe("SQLite database file. A leading ~ is expanded. Unset: the workspace database."), + { defaultText: "~/.libscope/workspaces/<workspace>/libscope.db" }, + ), + }), + indexing: z.object({ + maxDocumentSize: leaf( + z + .number() + .int() + .positive() + .default(100 * 1024 * 1024) + .describe("Largest document to index, in bytes."), + ), + allowPrivateUrls: leaf( + z.boolean().default(false).describe("Allow fetching URLs on private or local networks."), + ), + allowSelfSignedCerts: leaf( + z.boolean().default(false).describe("Accept self-signed TLS certificates when fetching."), + ), + }), + logging: z.object({ + level: leaf(z.enum(LOG_LEVELS).default("info").describe("Log level.")), + }), +}); + +type SchemaOutput = z.output<typeof ConfigSchema>; +type SectionName = keyof SchemaOutput; + +/** Every settable key, as `section.field`, derived from ConfigSchema. */ +export type ConfigKey = { + [S in SectionName]: `${S}.${keyof SchemaOutput[S] & string}`; +}[SectionName]; + +/** One leaf of the schema, with everything needed to read, write, and document it. */ +export interface ConfigKeySpec { + key: ConfigKey; + section: SectionName; + field: string; + schema: z.ZodType; + /** Env var names in priority order (canonical name first). */ + env: readonly string[]; + secret: boolean; + openEnum: boolean; + description: string; + /** Default value, or undefined when the key has no fixed default. */ + defaultValue: unknown; + defaultText: string | undefined; + /** "string", "integer", "boolean", or the enum values. */ + type: "string" | "integer" | "boolean" | readonly string[]; +} + +/** `indexing.allowPrivateUrls` -> `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS`. */ +export function canonicalEnvName(section: string, field: string): string { + const snake = field.replaceAll(/([a-z0-9])([A-Z])/g, "$1_$2").toUpperCase(); + return `LIBSCOPE_${section.toUpperCase()}_${snake}`; +} + +/** Strip optional/default wrappers to reach the base schema. */ +function baseSchema(schema: z.ZodType): z.ZodType { + let current: z.ZodType = schema; + while (current instanceof z.ZodOptional || current instanceof z.ZodDefault) { + current = current.unwrap() as z.ZodType; + } + return current; +} + +function leafType(schema: z.ZodType): ConfigKeySpec["type"] { + const base = baseSchema(schema); + if (base instanceof z.ZodEnum) return base.options.map(String); + if (base instanceof z.ZodBoolean) return "boolean"; + if (base instanceof z.ZodNumber) return "integer"; + return "string"; +} + +function buildSpecs(): ConfigKeySpec[] { + const specs: ConfigKeySpec[] = []; + for (const [section, sectionSchema] of Object.entries(ConfigSchema.shape)) { + for (const [field, schemaValue] of Object.entries(sectionSchema.shape)) { + const schema = schemaValue as z.ZodType; + const meta = configKeyMeta.get(schema) ?? {}; + const parsedDefault = schema.safeParse(undefined); + specs.push({ + key: `${section}.${field}` as ConfigKey, + section: section as SectionName, + field, + schema, + env: [canonicalEnvName(section, field), ...(meta.env ?? [])], + secret: meta.secret ?? false, + openEnum: meta.openEnum ?? false, + description: schema.description ?? "", + defaultValue: parsedDefault.success ? parsedDefault.data : undefined, + defaultText: meta.defaultText, + type: leafType(schema), + }); + } + } + return specs; +} + +/** All leaf specs, in schema order. */ +export const CONFIG_KEY_SPECS: readonly ConfigKeySpec[] = buildSpecs(); + +const SPECS_BY_KEY: ReadonlyMap<string, ConfigKeySpec> = new Map( + CONFIG_KEY_SPECS.map((spec) => [spec.key, spec]), +); + +/** All settable config keys. */ +export const CONFIG_KEY_NAMES: readonly ConfigKey[] = CONFIG_KEY_SPECS.map((s) => s.key); + +/** Spec for a key, or undefined. */ +export function getConfigKeySpec(key: string): ConfigKeySpec | undefined { + return SPECS_BY_KEY.get(key); +} + +/** One row of the generated configuration reference. */ +export interface ConfigKeyTableRow { + key: ConfigKey; + /** "string", "integer", "boolean", or the allowed values joined with " | ". */ + type: string; + /** Default value as text ("" when none). */ + default: string; + /** Env var names in priority order. */ + env: string[]; + secret: boolean; + description: string; +} + +function defaultAsText(spec: ConfigKeySpec): string { + if (spec.defaultText !== undefined) return spec.defaultText; + const value = spec.defaultValue; + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + return String(value); + } + return ""; +} + +/** The key table (for docs and `config` help): key, type, default, env vars, secret, description. */ +export function getConfigKeyTable(): ConfigKeyTableRow[] { + return CONFIG_KEY_SPECS.map((spec) => ({ + key: spec.key, + type: typeof spec.type === "string" ? spec.type : spec.type.join(" | "), + default: defaultAsText(spec), + env: [...spec.env], + secret: spec.secret, + description: spec.description, + })); +} diff --git a/src/config.ts b/src/config.ts index 4aeef75b..e1dfeda0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,24 +11,50 @@ import { join, dirname } from "node:path"; import { homedir } from "node:os"; import { ConfigError, ValidationError } from "./errors.js"; import { getLogger } from "./logger.js"; +import { + CONFIG_KEY_NAMES, + CONFIG_KEY_SPECS, + getConfigKeySpec, + type ConfigKey, + type ConfigKeySpec, + type EMBEDDING_PROVIDERS, + type LLM_PROVIDERS, + type LOG_LEVELS, +} from "./config-schema.js"; + +export { + ConfigSchema, + CONFIG_KEY_NAMES, + DEFAULT_OLLAMA_URL, + getConfigKeyTable, + type ConfigKey, + type ConfigKeyTableRow, +} from "./config-schema.js"; + +/** Value of `llm.provider`. "auto" picks a provider when the LLM is created. */ +export type LlmProviderSetting = (typeof LLM_PROVIDERS)[number]; export interface LibScopeConfig { embedding: { - provider: "local" | "ollama" | "openai" | (string & {}); - ollamaUrl?: string; - ollamaModel?: string; - openaiApiKey?: string; - openaiModel?: string; + provider: (typeof EMBEDDING_PROVIDERS)[number] | (string & {}); + /** Embedding model. Unset: the provider's default model. */ + model?: string | undefined; + /** Server URL of the provider (used by ollama). Unset: http://localhost:11434. */ + url?: string | undefined; /** Vector size override for models whose dimension is not built in. */ - dimensions?: number; + dimensions?: number | undefined; }; llm?: { - provider?: "openai" | "ollama" | "anthropic" | "passthrough"; - model?: string; - ollamaUrl?: string; - openaiApiKey?: string; - anthropicApiKey?: string; + /** Unset means "auto". */ + provider?: LlmProviderSetting | undefined; + model?: string | undefined; + /** Server URL of the LLM (used by ollama). Unset: embedding.url, then http://localhost:11434. */ + url?: string | undefined; }; + /** Read from env (LIBSCOPE_OPENAI_API_KEY, OPENAI_API_KEY) or ~/.libscope/secrets.json. */ + openai?: { apiKey?: string | undefined }; + /** Read from env (LIBSCOPE_ANTHROPIC_API_KEY, ANTHROPIC_API_KEY) or ~/.libscope/secrets.json. */ + anthropic?: { apiKey?: string | undefined }; database: { /** * Explicit SQLite file. Unset by default: the active workspace's database @@ -42,28 +68,10 @@ export interface LibScopeConfig { allowSelfSignedCerts: boolean; }; logging: { - level: "debug" | "info" | "warn" | "error" | "silent"; + level: (typeof LOG_LEVELS)[number]; }; } -const DEFAULT_CONFIG: LibScopeConfig = { - embedding: { - provider: "local", - ollamaUrl: "http://localhost:11434", - ollamaModel: "nomic-embed-text", - openaiModel: "text-embedding-3-small", - }, - database: {}, - indexing: { - maxDocumentSize: 100 * 1024 * 1024, // 100MB - allowPrivateUrls: false, - allowSelfSignedCerts: false, - }, - logging: { - level: "info", - }, -}; - /** Expand a leading `~` (or `~/`) to the user's home directory. */ export function expandHomeDir(path: string): string { if (path === "~") return homedir(); @@ -80,20 +88,19 @@ export function getUserConfigPath(): string { return join(getConfigDir(), "config.json"); } -function getProjectConfigPath(): string { - return join(process.cwd(), ".libscope.json"); +/** Path to the secrets file (~/.libscope/secrets.json, mode 0600). */ +export function getSecretsPath(): string { + return join(getConfigDir(), "secrets.json"); } -/** A partial config layer (file or env): every key optional, only set keys present. */ -export interface ConfigLayer { - embedding?: Partial<LibScopeConfig["embedding"]>; - llm?: LibScopeConfig["llm"]; - database?: Partial<LibScopeConfig["database"]>; - indexing?: Partial<LibScopeConfig["indexing"]>; - logging?: Partial<LibScopeConfig["logging"]>; +function getProjectConfigPath(): string { + return join(process.cwd(), ".libscope.json"); } type RawConfig = Record<string, unknown>; +type ConfigValue = string | number | boolean; +/** A config layer flattened to `section.field` keys. */ +type FlatLayer = Record<string, unknown>; function isPlainObject(value: unknown): value is RawConfig { return typeof value === "object" && value !== null && !Array.isArray(value); @@ -110,8 +117,16 @@ function readJsonObject(path: string): RawConfig { } } -function loadJsonFile(path: string): ConfigLayer { - return readJsonObject(path) as ConfigLayer; +/** Write a JSON file under ~/.libscope (directory 0700, file 0600). */ +function writePrivateJson(path: string, data: RawConfig): void { + const dir = getConfigDir(); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + } + writeFileSync(path, JSON.stringify(data, null, 2) + "\n", { encoding: "utf-8", mode: 0o600 }); + // writeFileSync's mode only applies when the file is created; tighten existing files too. + chmodSync(path, 0o600); + invalidateConfigCache(); } /** @@ -124,156 +139,80 @@ export function readRawUserConfig(): RawConfig { /** Write ~/.libscope/config.json (directory 0700, file 0600). */ export function writeRawUserConfig(config: RawConfig): void { - const dir = getConfigDir(); - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true, mode: 0o700 }); - } - const configPath = getUserConfigPath(); - writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n", { - encoding: "utf-8", - mode: 0o600, - }); - // writeFileSync's mode only applies when the file is created; tighten existing files too. - chmodSync(configPath, 0o600); - invalidateConfigCache(); + writePrivateJson(getUserConfigPath(), config); } -const EMBEDDING_PROVIDERS = ["local", "ollama", "openai"] as const; -const LLM_PROVIDERS = ["openai", "ollama", "anthropic", "passthrough"] as const; -type EmbeddingProviderName = (typeof EMBEDDING_PROVIDERS)[number]; -type LlmProviderName = (typeof LLM_PROVIDERS)[number]; - -function isEmbeddingProvider(value: string | undefined): value is EmbeddingProviderName { - return value !== undefined && (EMBEDDING_PROVIDERS as readonly string[]).includes(value); +function splitKey(key: string): [section: string, field: string] { + const dot = key.indexOf("."); + return dot === -1 ? [key, ""] : [key.slice(0, dot), key.slice(dot + 1)]; } -function isLlmProvider(value: string | undefined): value is LlmProviderName { - return value !== undefined && (LLM_PROVIDERS as readonly string[]).includes(value); +/** Flatten `{ section: { field: v } }` to `{ "section.field": v }` (one level only). */ +function flatten(raw: RawConfig): FlatLayer { + const flat: FlatLayer = {}; + for (const [section, values] of Object.entries(raw)) { + if (!isPlainObject(values)) continue; + for (const [field, value] of Object.entries(values)) { + if (value !== undefined) flat[`${section}.${field}`] = value; + } + } + return flat; } -/** Env-var flag parsing: only "true" and "1" enable a flag. */ -function truthy(value: string | undefined): true | undefined { - return value === "true" || value === "1" ? true : undefined; +function rawSection(raw: RawConfig, section: string): RawConfig { + const existing = raw[section]; + if (isPlainObject(existing)) return existing; + const created: RawConfig = {}; + raw[section] = created; + return created; } -type Compact<T> = { [K in keyof T]?: Exclude<T[K], undefined> }; - -/** Drop undefined and empty-string values so a layer only carries keys that were actually set. */ -function compact<T extends Record<string, unknown>>(obj: T): Compact<T> { - const out: Record<string, unknown> = {}; - for (const [key, value] of Object.entries(obj)) { - if (value !== undefined && value !== "") out[key] = value; +/** Validate a key name; throws ValidationError listing the valid keys. */ +export function parseConfigKey(name: string): ConfigKey { + if (!getConfigKeySpec(name)) { + throw new ValidationError( + `Unknown config key: ${name}. Valid keys: ${CONFIG_KEY_NAMES.join(", ")}`, + ); } - return out as Compact<T>; -} - -/** Return the first value that is set and non-empty. */ -function firstSet(...values: Array<string | undefined>): string | undefined { - return values.find((v) => v !== undefined && v !== ""); + return name as ConfigKey; } -/** - * Collect config values from env vars. Only variables that are set appear in the result. - * - * API keys follow one rule for embedding and LLM providers: - * LIBSCOPE_<P>_API_KEY > <P>_API_KEY > config file. - */ -function getEnvOverrides(): ConfigLayer { - const env = process.env; - const provider = env["LIBSCOPE_EMBEDDING_PROVIDER"]; - const llmProvider = env["LIBSCOPE_LLM_PROVIDER"]; - const openaiApiKey = firstSet(env["LIBSCOPE_OPENAI_API_KEY"], env["OPENAI_API_KEY"]); - const anthropicApiKey = firstSet(env["LIBSCOPE_ANTHROPIC_API_KEY"], env["ANTHROPIC_API_KEY"]); - - return { - embedding: compact({ - provider: isEmbeddingProvider(provider) ? provider : undefined, - openaiApiKey, - ollamaUrl: env["LIBSCOPE_OLLAMA_URL"], - ollamaModel: env["LIBSCOPE_OLLAMA_MODEL"], - }), - llm: compact({ - provider: isLlmProvider(llmProvider) ? llmProvider : undefined, - model: env["LIBSCOPE_LLM_MODEL"], - openaiApiKey, - anthropicApiKey, - }), - indexing: compact({ - allowPrivateUrls: truthy(env["LIBSCOPE_ALLOW_PRIVATE_URLS"]), - allowSelfSignedCerts: truthy(env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"]), - }), - }; +function specFor(name: string): ConfigKeySpec { + const spec = getConfigKeySpec(parseConfigKey(name)); + if (!spec) throw new ValidationError(`Unknown config key: ${name}`); + return spec; } -/** Every settable key, as `section.field`, derived from the LibScopeConfig shape. */ -export type ConfigKey = { - [S in keyof LibScopeConfig]-?: `${S}.${keyof NonNullable<LibScopeConfig[S]> & string}`; -}[keyof LibScopeConfig]; - -type ConfigValue = string | number | boolean; - -interface ConfigKeySpec { - /** "string", "boolean", "positiveInt", or the list of allowed values. */ - type: "string" | "boolean" | "positiveInt" | readonly string[]; - /** For secrets: the env var to use instead of storing the value in a config file. */ - secretEnv?: string; -} - -const CONFIG_KEYS: Record<ConfigKey, ConfigKeySpec> = { - "embedding.provider": { type: EMBEDDING_PROVIDERS }, - "embedding.ollamaUrl": { type: "string" }, - "embedding.ollamaModel": { type: "string" }, - "embedding.openaiApiKey": { type: "string", secretEnv: "LIBSCOPE_OPENAI_API_KEY" }, - "embedding.openaiModel": { type: "string" }, - "embedding.dimensions": { type: "positiveInt" }, - "llm.provider": { type: LLM_PROVIDERS }, - "llm.model": { type: "string" }, - "llm.ollamaUrl": { type: "string" }, - "llm.openaiApiKey": { type: "string", secretEnv: "LIBSCOPE_OPENAI_API_KEY" }, - "llm.anthropicApiKey": { type: "string", secretEnv: "LIBSCOPE_ANTHROPIC_API_KEY" }, - "database.path": { type: "string" }, - "indexing.maxDocumentSize": { type: "positiveInt" }, - "indexing.allowPrivateUrls": { type: "boolean" }, - "indexing.allowSelfSignedCerts": { type: "boolean" }, - "logging.level": { type: ["debug", "info", "warn", "error", "silent"] }, -}; - -/** All settable config keys. */ -export const CONFIG_KEY_NAMES = Object.keys(CONFIG_KEYS) as ConfigKey[]; - -function isConfigKey(key: string): key is ConfigKey { - return Object.hasOwn(CONFIG_KEYS, key); -} - -/** Validate a dotted key name; throws ValidationError listing the valid keys. */ -export function parseConfigKey(key: string): ConfigKey { - if (!isConfigKey(key)) { - throw new ValidationError( - `Unknown config key: ${key}. Valid keys: ${CONFIG_KEY_NAMES.join(", ")}`, - ); +/** Run the key's schema; throws ValidationError with the first issue. */ +function validateWithSchema(spec: ConfigKeySpec, value: unknown): ConfigValue { + const result = spec.schema.safeParse(value); + if (!result.success) { + const issue = result.error.issues[0]?.message ?? "invalid value"; + throw new ValidationError(`${spec.key}: ${issue} (got ${JSON.stringify(value)})`); } - return key; + return result.data as ConfigValue; } -/** Convert a CLI string to the key's type (booleans, positive integers, enums). */ +/** Convert a string (CLI argument or env var) to the key's type and validate it. */ export function coerceConfigValue(key: ConfigKey, raw: string): ConfigValue { - const { type } = CONFIG_KEYS[key]; + const spec = specFor(key); + const { type } = spec; if (type === "boolean") { const lower = raw.trim().toLowerCase(); if (lower === "true" || lower === "1") return true; if (lower === "false" || lower === "0") return false; throw new ValidationError(`${key} must be true or false (got "${raw}")`); } - if (type === "positiveInt") { + if (type === "integer") { const n = Number(raw); - if (!Number.isInteger(n) || n <= 0) { + if (raw.trim() === "" || !Number.isInteger(n) || n <= 0) { throw new ValidationError(`${key} must be a positive integer (got "${raw}")`); } - return n; + return validateWithSchema(spec, n); } if (type === "string") { if (raw.trim() === "") throw new ValidationError(`${key} must not be empty`); - return raw; + return validateWithSchema(spec, raw); } if (!type.includes(raw)) { throw new ValidationError(`${key} must be one of: ${type.join(", ")} (got "${raw}")`); @@ -281,66 +220,72 @@ export function coerceConfigValue(key: ConfigKey, raw: string): ConfigValue { return raw; } -function splitKey(key: ConfigKey): [section: string, field: string] { - const [section = "", field = ""] = key.split("."); - return [section, field]; +function warnIgnored(what: string, err: unknown): void { + const reason = err instanceof Error ? err.message : String(err); + getLogger().warn(`Ignoring ${what}: ${reason}`); } -function rawSection(raw: RawConfig, section: string): RawConfig { - const existing = raw[section]; - if (isPlainObject(existing)) return existing; - const created: RawConfig = {}; - raw[section] = created; - return created; +/** + * Validate a value read from a config file. Invalid values are logged and ignored, so one bad + * key does not stop libscope from starting. Strings are accepted for booleans and integers. + */ +function readFileValue(spec: ConfigKeySpec, value: unknown, source: string): unknown { + if (spec.openEnum && typeof value === "string" && value.trim() !== "") return value; + try { + if (typeof value === "string" && (spec.type === "boolean" || spec.type === "integer")) { + return coerceConfigValue(spec.key, value); + } + return validateWithSchema(spec, value); + } catch (err) { + warnIgnored(`invalid config value in ${source}`, err); + return undefined; + } } /** - * Set one key in ~/.libscope/config.json. Validates and coerces the value and keeps every - * other key in the file (including `registries`). API keys are refused: use env vars. + * Read a config file into a flat layer of known keys. `secrets` selects which keys are kept: + * secret keys (secrets.json) or non-secret keys (config files). Secret keys found in a config + * file are reported and ignored. */ -export function setUserConfigValue(key: string, rawValue: string): ConfigValue { - const configKey = parseConfigKey(key); - const { secretEnv } = CONFIG_KEYS[configKey]; - if (secretEnv) { - throw new ValidationError( - `${configKey} is a secret and is not written to config files. Set the ${secretEnv} environment variable instead.`, +function readFileLayer(path: string, secrets: boolean): FlatLayer { + const layer: FlatLayer = {}; + const ignoredSecrets: string[] = []; + for (const [key, value] of Object.entries(flatten(readJsonObject(path)))) { + const spec = getConfigKeySpec(key); + if (!spec) continue; + if (spec.secret !== secrets) { + if (spec.secret) ignoredSecrets.push(key); + continue; + } + const parsed = readFileValue(spec, value, path); + if (parsed !== undefined) layer[key] = parsed; + } + if (ignoredSecrets.length > 0) { + getLogger().warn( + `Ignoring ${ignoredSecrets.join(", ")} in ${path}: API keys are read only from ` + + `environment variables and ${getSecretsPath()}. Run "libscope config set <key> <value>" to store them there.`, ); } - const value = coerceConfigValue(configKey, rawValue); - const raw = readRawUserConfig(); - const [section, field] = splitKey(configKey); - rawSection(raw, section)[field] = value; - writeRawUserConfig(raw); - return value; -} - -/** Remove one key from ~/.libscope/config.json. Returns false if the file did not set it. */ -export function unsetUserConfigValue(key: string): boolean { - const configKey = parseConfigKey(key); - const raw = readRawUserConfig(); - const [section, field] = splitKey(configKey); - const sectionObj = raw[section]; - if (!isPlainObject(sectionObj) || !Object.hasOwn(sectionObj, field)) return false; - delete sectionObj[field]; - if (Object.keys(sectionObj).length === 0) delete raw[section]; - writeRawUserConfig(raw); - return true; -} - -/** Read the effective value of a key from a loaded config. Secrets are masked. */ -export function getConfigValue(config: LibScopeConfig, key: string): ConfigValue | undefined { - const configKey = parseConfigKey(key); - const [section, field] = splitKey(configKey); - const sections = config as unknown as Record<string, RawConfig | undefined>; - const value = sections[section]?.[field] as ConfigValue | undefined; - if (typeof value === "string" && CONFIG_KEYS[configKey].secretEnv) return maskSecret(value); - return value; + return layer; } -function hasApiKey(layer: ConfigLayer): boolean { - return Boolean( - layer.embedding?.openaiApiKey ?? layer.llm?.openaiApiKey ?? layer.llm?.anthropicApiKey, - ); +/** + * Collect config values from env vars. Only variables that are set appear in the result. + * Each key reads LIBSCOPE_<SECTION>_<FIELD>, then its extra names (e.g. OPENAI_API_KEY). + */ +function readEnvLayer(): FlatLayer { + const layer: FlatLayer = {}; + for (const spec of CONFIG_KEY_SPECS) { + const name = spec.env.find((n) => process.env[n] !== undefined && process.env[n] !== ""); + const value = name === undefined ? undefined : process.env[name]; + if (name === undefined || value === undefined) continue; + try { + layer[spec.key] = coerceConfigValue(spec.key, value); + } catch (err) { + warnIgnored(`env var ${name}`, err); + } + } + return layer; } /** Mask a secret for display: `sk-…abcd` for long values, `****` otherwise. */ @@ -348,13 +293,17 @@ export function maskSecret(value: string): string { return value.length >= 12 ? `${value.slice(0, 3)}…${value.slice(-4)}` : "****"; } -/** Return a copy of the config with every API key masked (for display). */ +/** Return a copy of the config with every secret masked (for display). */ export function maskConfigSecrets(config: LibScopeConfig): LibScopeConfig { const masked: LibScopeConfig = structuredClone(config); - const { embedding, llm } = masked; - if (embedding.openaiApiKey) embedding.openaiApiKey = maskSecret(embedding.openaiApiKey); - if (llm?.openaiApiKey) llm.openaiApiKey = maskSecret(llm.openaiApiKey); - if (llm?.anthropicApiKey) llm.anthropicApiKey = maskSecret(llm.anthropicApiKey); + const sections = masked as unknown as Record<string, RawConfig | undefined>; + for (const spec of CONFIG_KEY_SPECS) { + const section = sections[spec.section]; + const value = section?.[spec.field]; + if (section && spec.secret && typeof value === "string") { + section[spec.field] = maskSecret(value); + } + } return masked; } @@ -368,93 +317,68 @@ export function invalidateConfigCache(): void { _configCacheAt = 0; } -/** database.path only when a config file sets it (project wins over user), with `~` expanded. */ -function resolveDatabaseSection( - userConfig: ConfigLayer, - projectConfig: ConfigLayer, -): LibScopeConfig["database"] { - const path = projectConfig.database?.path ?? userConfig.database?.path; - return path ? { path: expandHomeDir(path) } : {}; -} - -/** Load config with precedence: env > project > user > defaults. Result is cached for 30 s. */ +/** + * Load config. Precedence: env > project .libscope.json > ~/.libscope/config.json > schema + * defaults. Secret keys (API keys): env > ~/.libscope/secrets.json. No network calls are + * made. The result is cached for 30 s. + */ export function loadConfig(): LibScopeConfig { const now = Date.now(); if (_configCache && now - _configCacheAt < CONFIG_CACHE_TTL_MS) { return _configCache; } - const userConfig = loadJsonFile(getUserConfigPath()); - const projectConfig = loadJsonFile(getProjectConfigPath()); - const envOverrides = getEnvOverrides(); - - for (const [path, layer] of [ - [getUserConfigPath(), userConfig], - [getProjectConfigPath(), projectConfig], - ] as const) { - if (hasApiKey(layer)) { - getLogger().warn( - `API keys found in config file ${path}. Prefer environment variables ` + - "(LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY, LIBSCOPE_ANTHROPIC_API_KEY or ANTHROPIC_API_KEY); " + - "they take precedence over keys in config files.", - ); - } + + const merged: FlatLayer = {}; + for (const spec of CONFIG_KEY_SPECS) { + if (spec.defaultValue !== undefined) merged[spec.key] = spec.defaultValue; } + Object.assign( + merged, + readFileLayer(getUserConfigPath(), false), + readFileLayer(getProjectConfigPath(), false), + readFileLayer(getSecretsPath(), true), + readEnvLayer(), + ); - const config: LibScopeConfig = { - embedding: { - ...DEFAULT_CONFIG.embedding, - ...userConfig.embedding, - ...projectConfig.embedding, - ...envOverrides.embedding, - }, - llm: { - ...userConfig.llm, - ...projectConfig.llm, - ...envOverrides.llm, - }, - database: resolveDatabaseSection(userConfig, projectConfig), - indexing: { - ...DEFAULT_CONFIG.indexing, - ...userConfig.indexing, - ...projectConfig.indexing, - ...envOverrides.indexing, - }, - logging: { - ...DEFAULT_CONFIG.logging, - ...userConfig.logging, - ...projectConfig.logging, - }, - }; + const config: RawConfig = {}; + for (const [key, value] of Object.entries(merged)) { + const [section, field] = splitKey(key); + rawSection(config, section)[field] = value; + } + const dbPath = merged["database.path"]; + config["database"] = typeof dbPath === "string" ? { path: expandHomeDir(dbPath) } : {}; - validateConfig(config); + const result = config as unknown as LibScopeConfig; + validateConfig(result); - _configCache = config; + _configCache = result; _configCacheAt = now; - return config; + return result; } -const OPENAI_KEY_HINT = "Set LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY"; +function apiKeyHint(provider: "OPENAI" | "ANTHROPIC"): string { + return ( + `Set LIBSCOPE_${provider}_API_KEY or ${provider}_API_KEY, ` + + `or run "libscope config set ${provider.toLowerCase()}.apiKey <key>".` + ); +} -/** Check embedding and LLM provider configuration for missing keys/URLs. */ +/** Check embedding and LLM provider configuration for missing keys. */ function validateProviderConfig(config: LibScopeConfig, warnings: string[]): void { - if (config.embedding.provider === "openai" && !config.embedding.openaiApiKey) { - warnings.push( - `embedding.provider is "openai" but no API key found. ${OPENAI_KEY_HINT} (or embedding.openaiApiKey in a config file).`, - ); + const openaiKey = config.openai?.apiKey; + if (config.embedding.provider === "openai" && !openaiKey) { + warnings.push(`embedding.provider is "openai" but no API key found. ${apiKeyHint("OPENAI")}`); } - if (config.embedding.provider === "ollama" && !config.embedding.ollamaUrl) { - warnings.push('embedding.provider is "ollama" but embedding.ollamaUrl is not set.'); - } - const llm = config.llm; - if (llm?.provider === "openai" && !(llm.openaiApiKey ?? config.embedding.openaiApiKey)) { + if (config.embedding.provider === "local" && config.embedding.model) { warnings.push( - `llm.provider is "openai" but no API key found. ${OPENAI_KEY_HINT} (or llm.openaiApiKey in a config file).`, + `embedding.model "${config.embedding.model}" is ignored by the local provider (it always uses Xenova/all-MiniLM-L6-v2).`, ); } - if (llm?.provider === "anthropic" && !llm.anthropicApiKey) { - warnings.push( - 'llm.provider is "anthropic" but no API key found. Set LIBSCOPE_ANTHROPIC_API_KEY or ANTHROPIC_API_KEY (or llm.anthropicApiKey in a config file).', - ); + if (config.llm?.provider === "openai" && !openaiKey) { + warnings.push(`llm.provider is "openai" but no API key found. ${apiKeyHint("OPENAI")}`); + } + if (config.llm?.provider === "anthropic" && !config.anthropic?.apiKey) { + warnings.push(`llm.provider is "anthropic" but no API key found. ${apiKeyHint("ANTHROPIC")}`); } } @@ -495,20 +419,68 @@ export function validateConfig(config: LibScopeConfig): string[] { return warnings; } +/** Surfaces that create an LLM provider. "auto" resolves to passthrough under MCP. */ +export type LlmSurface = "cli" | "mcp" | "api" | "sdk"; + +/** LLM provider after "auto" is resolved. */ +export type ResolvedLlmProviderName = Exclude<LlmProviderSetting, "auto">; + /** - * Merge values into ~/.libscope/config.json. Only the keys passed in are written; every - * other key in the file (including `registries` and values added by hand) is kept. - * Security: API keys passed in are never written — use environment variables instead. + * Resolve `llm.provider` ("auto" by default) to a concrete provider, or null when no LLM is + * available. Auto: passthrough under MCP; else openai if an OpenAI key is set; else anthropic + * if an Anthropic key is set; else ollama if llm.url is set or embedding.provider is "ollama"; + * else null. Reads only the config object; no network calls. */ -export function saveUserConfig(config: ConfigLayer): void { - const raw = readRawUserConfig(); - for (const [section, values] of Object.entries(config)) { - if (!isPlainObject(values)) continue; - for (const [field, value] of Object.entries(values)) { - const key = `${section}.${field}`; - if (value === undefined || !isConfigKey(key) || CONFIG_KEYS[key].secretEnv) continue; - rawSection(raw, section)[field] = value; - } - } - writeRawUserConfig(raw); +export function resolveLlmProviderName( + config: LibScopeConfig, + options: { surface?: LlmSurface | undefined } = {}, +): ResolvedLlmProviderName | null { + const setting = config.llm?.provider ?? "auto"; + if (setting !== "auto") return setting; + if (options.surface === "mcp") return "passthrough"; + if (config.openai?.apiKey) return "openai"; + if (config.anthropic?.apiKey) return "anthropic"; + if (config.llm?.url || config.embedding.provider === "ollama") return "ollama"; + return null; +} + +/** The file a key is stored in: secrets.json for secret keys, else config.json. */ +export function getConfigFileFor(key: string): string { + return specFor(key).secret ? getSecretsPath() : getUserConfigPath(); +} + +/** + * Set one key in its file (config.json, or secrets.json for API keys). Validates and coerces + * the value and keeps every other key in the file (including `registries`). + */ +export function setUserConfigValue(key: string, rawValue: string): ConfigValue { + const spec = specFor(key); + const value = coerceConfigValue(spec.key, rawValue); + const path = getConfigFileFor(key); + const raw = readJsonObject(path); + rawSection(raw, spec.section)[spec.field] = value; + writePrivateJson(path, raw); + return value; +} + +/** Remove one key from its file. Returns false if the file did not set it. */ +export function unsetUserConfigValue(key: string): boolean { + const spec = specFor(key); + const path = getConfigFileFor(key); + const raw = readJsonObject(path); + const sectionObj = raw[spec.section]; + if (!isPlainObject(sectionObj) || !Object.hasOwn(sectionObj, spec.field)) return false; + delete sectionObj[spec.field]; + if (Object.keys(sectionObj).length === 0) delete raw[spec.section]; + writePrivateJson(path, raw); + return true; +} + +/** Read the effective value of a key from a loaded config. Secrets are masked. */ +export function getConfigValue(config: LibScopeConfig, key: string): ConfigValue | undefined { + const spec = specFor(key); + const sections = config as unknown as Record<string, RawConfig | undefined>; + const value = sections[spec.section]?.[spec.field] as ConfigValue | undefined; + if (typeof value === "string" && spec.secret) return maskSecret(value); + return value; } diff --git a/src/core/rag.ts b/src/core/rag.ts index 248ddc78..fc5e1015 100644 --- a/src/core/rag.ts +++ b/src/core/rag.ts @@ -1,6 +1,11 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import type { LibScopeConfig } from "../config.js"; +import { + DEFAULT_OLLAMA_URL, + resolveLlmProviderName, + type LibScopeConfig, + type LlmSurface, +} from "../config.js"; import { searchDocuments, type SearchResult } from "./search.js"; import { ConfigError, FetchError } from "../errors.js"; @@ -66,38 +71,60 @@ export function extractSources(results: SearchResult[]): RagSource[] { })); } -interface LlmConfig { - provider?: "openai" | "ollama" | "anthropic" | "passthrough"; - model?: string; - ollamaUrl?: string; - openaiApiKey?: string; - anthropicApiKey?: string; +export { resolveLlmProviderName, type LlmSurface } from "../config.js"; + +type LlmConfig = NonNullable<LibScopeConfig["llm"]>; + +/** Options for resolving `llm.provider` ("auto" depends on the calling surface). */ +export interface LlmResolveOptions { + surface?: LlmSurface | undefined; } -/** Returns true if the config is set to passthrough mode (delegate synthesis to the calling LLM). */ -export function isPassthroughMode(config: LibScopeConfig): boolean { - return config.llm?.provider === "passthrough"; +/** + * Returns true if the LLM step is delegated to the caller: `llm.provider` is "passthrough", + * or it is "auto" and the surface is "mcp" (the calling assistant is the LLM). + */ +export function isPassthroughMode( + config: LibScopeConfig, + options: LlmResolveOptions = {}, +): boolean { + return resolveLlmProviderName(config, options) === "passthrough"; } -/** Create an LLM provider from config. */ -export function createLlmProvider(config: LibScopeConfig): LlmProvider { +const NO_LLM_HINT = + "No LLM provider configured. Set an OpenAI or Anthropic API key (LIBSCOPE_OPENAI_API_KEY, " + + "OPENAI_API_KEY, LIBSCOPE_ANTHROPIC_API_KEY, ANTHROPIC_API_KEY), or run " + + '"libscope config set llm.provider <openai|anthropic|ollama|passthrough>".'; + +/** + * Create an LLM provider from config. `llm.provider` "auto" (the default) is resolved with + * resolveLlmProviderName. Throws ConfigError when no LLM is available, or when the result is + * passthrough (no LLM is created; check isPassthroughMode first). + */ +export function createLlmProvider( + config: LibScopeConfig, + options: LlmResolveOptions = {}, +): LlmProvider { const llmConfig: LlmConfig | undefined = config.llm; - const providerType = llmConfig?.provider; + const providerType = resolveLlmProviderName(config, options); if (providerType === "openai") { - return createOpenAiProvider(config.embedding, llmConfig); + return createOpenAiProvider(config.openai?.apiKey, llmConfig); } if (providerType === "ollama") { return createOllamaProvider(config.embedding, llmConfig); } if (providerType === "anthropic") { - return createAnthropicProvider(llmConfig); + return createAnthropicProvider(config.anthropic?.apiKey, llmConfig); + } + if (providerType === "passthrough") { + throw new ConfigError( + 'llm.provider resolves to "passthrough": no LLM is created and the caller writes the ' + + "answer from the retrieved context.", + ); } - throw new ConfigError( - "No LLM provider configured. Set llm.provider to 'openai', 'ollama', 'anthropic', or 'passthrough' in your config, " + - "or set LIBSCOPE_LLM_PROVIDER environment variable.", - ); + throw new ConfigError(NO_LLM_HINT); } /** Retrieve the top-K search results for a RAG question (default K = 5). */ @@ -138,13 +165,13 @@ export async function getContextForQuestion( } function createOpenAiProvider( - embedding: LibScopeConfig["embedding"], + apiKey: string | undefined, llmConfig: LlmConfig | undefined, ): LlmProvider { - const apiKey = llmConfig?.openaiApiKey ?? embedding.openaiApiKey; if (!apiKey) { throw new ConfigError( - "OpenAI API key is required. Set llm.openaiApiKey or embedding.openaiApiKey in config.", + "OpenAI API key is required. Set LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY, " + + 'or run "libscope config set openai.apiKey <key>".', ); } @@ -220,7 +247,7 @@ function createOllamaProvider( embedding: LibScopeConfig["embedding"], llmConfig: LlmConfig | undefined, ): LlmProvider { - const baseUrl = llmConfig?.ollamaUrl ?? embedding.ollamaUrl ?? "http://localhost:11434"; + const baseUrl = llmConfig?.url ?? embedding.url ?? DEFAULT_OLLAMA_URL; const model = llmConfig?.model ?? "llama3.2"; return { @@ -282,11 +309,14 @@ function createOllamaProvider( }; } -function createAnthropicProvider(llmConfig: LlmConfig | undefined): LlmProvider { - const apiKey = llmConfig?.anthropicApiKey ?? process.env["ANTHROPIC_API_KEY"]; +function createAnthropicProvider( + apiKey: string | undefined, + llmConfig: LlmConfig | undefined, +): LlmProvider { if (!apiKey) { throw new ConfigError( - "Anthropic API key is required. Set llm.anthropicApiKey in config or ANTHROPIC_API_KEY env var.", + "Anthropic API key is required. Set LIBSCOPE_ANTHROPIC_API_KEY or ANTHROPIC_API_KEY, " + + 'or run "libscope config set anthropic.apiKey <key>".', ); } diff --git a/src/core/url-fetcher.ts b/src/core/url-fetcher.ts index 887ed14d..050c6e8c 100644 --- a/src/core/url-fetcher.ts +++ b/src/core/url-fetcher.ts @@ -113,7 +113,7 @@ async function validateUrl(url: string, allowPrivateUrls = false): Promise<strin for (const addr of addresses) { if (!allowPrivateUrls && isPrivateIP(addr)) { throw new FetchError( - `Blocked request to private/internal IP ${addr} (resolved from ${hostname}). Set LIBSCOPE_ALLOW_PRIVATE_URLS=true to allow.`, + `Blocked request to private/internal IP ${addr} (resolved from ${hostname}). Set LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS=true to allow.`, ); } } diff --git a/src/providers/index.ts b/src/providers/index.ts index 3092083c..4d6a05b8 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -3,7 +3,7 @@ export { LocalEmbeddingProvider } from "./local.js"; export { OllamaEmbeddingProvider } from "./ollama.js"; export { OpenAIEmbeddingProvider } from "./openai.js"; -import type { LibScopeConfig } from "../config.js"; +import { DEFAULT_OLLAMA_URL, type LibScopeConfig } from "../config.js"; import { ConfigError } from "../errors.js"; import { validateDimensionsOverride } from "./dimensions.js"; import type { EmbeddingProvider } from "./embedding.js"; @@ -28,37 +28,28 @@ export function createEmbeddingProvider(config: LibScopeConfig): EmbeddingProvid return registered(config); } - switch (config.embedding.provider) { + const { embedding } = config; + switch (embedding.provider) { case "local": return new LocalEmbeddingProvider(); - case "ollama": { - if (!config.embedding.ollamaUrl) { - throw new ConfigError( - "Ollama URL is required. Set LIBSCOPE_OLLAMA_URL or configure in ~/.libscope/config.json", - ); - } - if (!config.embedding.ollamaModel) { - throw new ConfigError( - "Ollama model is required. Set LIBSCOPE_OLLAMA_MODEL or configure in ~/.libscope/config.json", - ); - } + case "ollama": return new OllamaEmbeddingProvider( - config.embedding.ollamaUrl, - config.embedding.ollamaModel, - validateDimensionsOverride(config.embedding.dimensions), + embedding.url ?? DEFAULT_OLLAMA_URL, + embedding.model ?? "nomic-embed-text", + validateDimensionsOverride(embedding.dimensions), ); - } case "openai": { - const apiKey = config.embedding.openaiApiKey; + const apiKey = config.openai?.apiKey; if (!apiKey) { throw new ConfigError( - "OpenAI API key is required. Set LIBSCOPE_OPENAI_API_KEY or configure in ~/.libscope/config.json", + "OpenAI API key is required. Set LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY, " + + 'or run "libscope config set openai.apiKey <key>".', ); } return new OpenAIEmbeddingProvider( apiKey, - config.embedding.openaiModel, - validateDimensionsOverride(config.embedding.dimensions), + embedding.model, + validateDimensionsOverride(embedding.dimensions), ); } default: diff --git a/tests/unit/api.test.ts b/tests/unit/api.test.ts index 53581b76..28ba6a65 100644 --- a/tests/unit/api.test.ts +++ b/tests/unit/api.test.ts @@ -424,7 +424,8 @@ describe("API routes", () => { vi.spyOn(configModule, "loadConfig").mockReturnValue({ embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test" }, + llm: { provider: "openai" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, diff --git a/tests/unit/config-layers.test.ts b/tests/unit/config-layers.test.ts index 20dc4ccc..65803738 100644 --- a/tests/unit/config-layers.test.ts +++ b/tests/unit/config-layers.test.ts @@ -7,25 +7,16 @@ import { invalidateConfigCache, maskSecret, maskConfigSecrets, + getConfigKeyTable, } from "../../src/config.js"; import * as loggerModule from "../../src/logger.js"; import { initLogger } from "../../src/logger.js"; /** Env vars that influence config loading; cleared for every test. */ const CONFIG_ENV_VARS = [ + ...getConfigKeyTable().flatMap((row) => row.env), "HOME", - "LIBSCOPE_EMBEDDING_PROVIDER", - "LIBSCOPE_OPENAI_API_KEY", - "LIBSCOPE_OLLAMA_URL", - "LIBSCOPE_OLLAMA_MODEL", - "LIBSCOPE_LLM_PROVIDER", - "LIBSCOPE_LLM_MODEL", - "LIBSCOPE_ANTHROPIC_API_KEY", - "LIBSCOPE_ALLOW_PRIVATE_URLS", - "LIBSCOPE_ALLOW_SELF_SIGNED_CERTS", "LIBSCOPE_WORKSPACE", - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", ]; describe("config layering (temp HOME and cwd)", () => { @@ -38,6 +29,11 @@ describe("config layering (temp HOME and cwd)", () => { writeFileSync(join(tempHome, ".libscope", "config.json"), JSON.stringify(data), "utf-8"); } + function writeSecrets(data: unknown): void { + mkdirSync(join(tempHome, ".libscope"), { recursive: true }); + writeFileSync(join(tempHome, ".libscope", "secrets.json"), JSON.stringify(data), "utf-8"); + } + function writeProjectConfig(data: unknown): void { writeFileSync(join(tempCwd, ".libscope.json"), JSON.stringify(data), "utf-8"); } @@ -48,6 +44,19 @@ describe("config layering (temp HOME and cwd)", () => { return loadConfig(); } + /** Replace the logger with a spy and return the warn mock. */ + function spyWarn(): ReturnType<typeof vi.fn> { + const warn = vi.fn(); + vi.spyOn(loggerModule, "getLogger").mockReturnValue({ + warn, + } as unknown as ReturnType<typeof loggerModule.getLogger>); + return warn; + } + + function messages(warn: ReturnType<typeof vi.fn>): string[] { + return warn.mock.calls.map((c) => String(c[0])); + } + beforeEach(() => { initLogger("silent"); for (const key of CONFIG_ENV_VARS) { @@ -71,27 +80,46 @@ describe("config layering (temp HOME and cwd)", () => { rmSync(tempCwd, { recursive: true, force: true }); }); + describe("defaults", () => { + it("applies schema defaults when no files and no env vars exist", () => { + const config = load(); + expect(config.embedding).toEqual({ provider: "local" }); + expect(config.llm).toEqual({ provider: "auto" }); + expect(config.indexing).toEqual({ + maxDocumentSize: 100 * 1024 * 1024, + allowPrivateUrls: false, + allowSelfSignedCerts: false, + }); + expect(config.logging.level).toBe("info"); + expect(config.database).toEqual({}); + expect(config.openai).toBeUndefined(); + expect(config.anthropic).toBeUndefined(); + }); + }); + describe("env overrides only replace keys that are set", () => { it("keeps provider=openai from the config file when LIBSCOPE_OPENAI_API_KEY is set", () => { writeUserConfig({ embedding: { provider: "openai" } }); const config = load({ LIBSCOPE_OPENAI_API_KEY: "sk-env" }); expect(config.embedding.provider).toBe("openai"); - expect(config.embedding.openaiApiKey).toBe("sk-env"); + expect(config.openai?.apiKey).toBe("sk-env"); }); - it("keeps file ollamaModel/ollamaUrl when LIBSCOPE_EMBEDDING_PROVIDER is set", () => { + it("keeps file model/url when LIBSCOPE_EMBEDDING_PROVIDER is set", () => { writeUserConfig({ - embedding: { ollamaUrl: "http://gpu-box:11434", ollamaModel: "mxbai-embed-large" }, + embedding: { url: "http://gpu-box:11434", model: "mxbai-embed-large" }, }); const config = load({ LIBSCOPE_EMBEDDING_PROVIDER: "ollama" }); - expect(config.embedding.provider).toBe("ollama"); - expect(config.embedding.ollamaUrl).toBe("http://gpu-box:11434"); - expect(config.embedding.ollamaModel).toBe("mxbai-embed-large"); + expect(config.embedding).toEqual({ + provider: "ollama", + url: "http://gpu-box:11434", + model: "mxbai-embed-large", + }); }); - it("keeps project-level indexing settings when one indexing env flag is set", () => { + it("keeps project-level indexing settings when one indexing env var is set", () => { writeProjectConfig({ indexing: { allowPrivateUrls: true, maxDocumentSize: 1234 } }); - const config = load({ LIBSCOPE_ALLOW_SELF_SIGNED_CERTS: "1" }); + const config = load({ LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS: "1" }); expect(config.indexing.allowSelfSignedCerts).toBe(true); expect(config.indexing.allowPrivateUrls).toBe(true); expect(config.indexing.maxDocumentSize).toBe(1234); @@ -104,90 +132,176 @@ describe("config layering (temp HOME and cwd)", () => { expect(config.llm?.model).toBe("llama3"); }); - it("ignores invalid LIBSCOPE_EMBEDDING_PROVIDER without resetting the file value", () => { + it("project config wins over user config", () => { + writeUserConfig({ embedding: { model: "user-model" } }); + writeProjectConfig({ embedding: { model: "project-model" } }); + expect(load().embedding.model).toBe("project-model"); + }); + + it("ignores an invalid LIBSCOPE_EMBEDDING_PROVIDER without resetting the file value", () => { + const warn = spyWarn(); writeUserConfig({ embedding: { provider: "ollama" } }); const config = load({ LIBSCOPE_EMBEDDING_PROVIDER: "bogus" }); expect(config.embedding.provider).toBe("ollama"); + expect(messages(warn).some((m) => m.includes("LIBSCOPE_EMBEDDING_PROVIDER"))).toBe(true); }); - it("applies defaults when no files and no env vars exist", () => { - const config = load(); - expect(config.embedding.provider).toBe("local"); - expect(config.embedding.ollamaUrl).toBe("http://localhost:11434"); + it("an env value of false overrides true in a config file", () => { + writeUserConfig({ indexing: { allowPrivateUrls: true } }); + const config = load({ LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS: "false" }); expect(config.indexing.allowPrivateUrls).toBe(false); }); }); - describe("API keys: LIBSCOPE_<P>_API_KEY > <P>_API_KEY > config file", () => { - it("OPENAI_API_KEY alone is used for embedding and llm", () => { - const config = load({ OPENAI_API_KEY: "sk-plain" }); - expect(config.embedding.openaiApiKey).toBe("sk-plain"); - expect(config.llm?.openaiApiKey).toBe("sk-plain"); + describe("env var names are LIBSCOPE_<SECTION>_<FIELD>", () => { + it("reads every non-secret key from its env var", () => { + const config = load({ + LIBSCOPE_EMBEDDING_PROVIDER: "ollama", + LIBSCOPE_EMBEDDING_MODEL: "bge-m3", + LIBSCOPE_EMBEDDING_URL: "http://embed:11434", + LIBSCOPE_EMBEDDING_DIMENSIONS: "1024", + LIBSCOPE_LLM_PROVIDER: "ollama", + LIBSCOPE_LLM_MODEL: "llama3", + LIBSCOPE_LLM_URL: "http://llm:11434", + LIBSCOPE_INDEXING_MAX_DOCUMENT_SIZE: "2048", + LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS: "true", + LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS: "1", + LIBSCOPE_LOGGING_LEVEL: "warn", + }); + expect(config.embedding).toEqual({ + provider: "ollama", + model: "bge-m3", + url: "http://embed:11434", + dimensions: 1024, + }); + expect(config.llm).toEqual({ provider: "ollama", model: "llama3", url: "http://llm:11434" }); + expect(config.indexing).toEqual({ + maxDocumentSize: 2048, + allowPrivateUrls: true, + allowSelfSignedCerts: true, + }); + expect(config.logging.level).toBe("warn"); + }); + + it("LIBSCOPE_DATABASE_PATH sets database.path and expands ~", () => { + const config = load({ LIBSCOPE_DATABASE_PATH: "~/data/x.db" }); + expect(config.database.path).toBe(join(tempHome, "data", "x.db")); + }); + + it("old env var names are not read", () => { + const config = load({ + LIBSCOPE_OLLAMA_URL: "http://old:11434", + LIBSCOPE_OLLAMA_MODEL: "old-model", + LIBSCOPE_ALLOW_PRIVATE_URLS: "true", + }); + delete process.env["LIBSCOPE_OLLAMA_URL"]; + delete process.env["LIBSCOPE_OLLAMA_MODEL"]; + delete process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"]; + expect(config.embedding.url).toBeUndefined(); + expect(config.embedding.model).toBeUndefined(); + expect(config.indexing.allowPrivateUrls).toBe(false); + }); + + it("ignores an invalid env value, logs a warning, and keeps the file value", () => { + const warn = spyWarn(); + writeUserConfig({ embedding: { dimensions: 512 } }); + const config = load({ LIBSCOPE_EMBEDDING_DIMENSIONS: "lots" }); + expect(config.embedding.dimensions).toBe(512); + expect(messages(warn).some((m) => m.includes("LIBSCOPE_EMBEDDING_DIMENSIONS"))).toBe(true); + }); + }); + + describe("API keys: LIBSCOPE_<P>_API_KEY > <P>_API_KEY > secrets.json", () => { + it("OPENAI_API_KEY alone sets openai.apiKey", () => { + expect(load({ OPENAI_API_KEY: "sk-plain" }).openai?.apiKey).toBe("sk-plain"); }); it("LIBSCOPE_OPENAI_API_KEY wins over OPENAI_API_KEY", () => { const config = load({ OPENAI_API_KEY: "sk-plain", LIBSCOPE_OPENAI_API_KEY: "sk-scoped" }); - expect(config.embedding.openaiApiKey).toBe("sk-scoped"); - expect(config.llm?.openaiApiKey).toBe("sk-scoped"); + expect(config.openai?.apiKey).toBe("sk-scoped"); }); it("an empty LIBSCOPE_OPENAI_API_KEY falls through to OPENAI_API_KEY", () => { const config = load({ LIBSCOPE_OPENAI_API_KEY: "", OPENAI_API_KEY: "sk-plain" }); - expect(config.embedding.openaiApiKey).toBe("sk-plain"); + expect(config.openai?.apiKey).toBe("sk-plain"); }); - it("env keys win over keys in the config file", () => { - writeUserConfig({ embedding: { openaiApiKey: "sk-file" } }); - const config = load({ OPENAI_API_KEY: "sk-env" }); - expect(config.embedding.openaiApiKey).toBe("sk-env"); + it("LIBSCOPE_ANTHROPIC_API_KEY wins over ANTHROPIC_API_KEY", () => { + const config = load({ + ANTHROPIC_API_KEY: "sk-ant-plain", + LIBSCOPE_ANTHROPIC_API_KEY: "sk-ant-scoped", + }); + expect(config.anthropic?.apiKey).toBe("sk-ant-scoped"); }); - it("config file key is used when no env key is set", () => { - writeUserConfig({ embedding: { provider: "openai", openaiApiKey: "sk-file" } }); + it("secrets.json is used when no env key is set", () => { + writeSecrets({ openai: { apiKey: "sk-file" }, anthropic: { apiKey: "sk-ant-file" } }); const config = load(); - expect(config.embedding.openaiApiKey).toBe("sk-file"); + expect(config.openai?.apiKey).toBe("sk-file"); + expect(config.anthropic?.apiKey).toBe("sk-ant-file"); + }); + + it("env keys win over secrets.json", () => { + writeSecrets({ openai: { apiKey: "sk-file" } }); + expect(load({ OPENAI_API_KEY: "sk-env" }).openai?.apiKey).toBe("sk-env"); }); - it("LIBSCOPE_ANTHROPIC_API_KEY works without LIBSCOPE_LLM_PROVIDER", () => { - writeUserConfig({ llm: { provider: "anthropic" } }); - const config = load({ LIBSCOPE_ANTHROPIC_API_KEY: "sk-ant-scoped" }); - expect(config.llm?.provider).toBe("anthropic"); - expect(config.llm?.anthropicApiKey).toBe("sk-ant-scoped"); + it("secrets.json holds only secrets: other keys there are ignored", () => { + writeSecrets({ logging: { level: "debug" } }); + expect(load().logging.level).toBe("info"); }); - it("ANTHROPIC_API_KEY is used when LIBSCOPE_ANTHROPIC_API_KEY is not set", () => { - const config = load({ ANTHROPIC_API_KEY: "sk-ant-plain" }); - expect(config.llm?.anthropicApiKey).toBe("sk-ant-plain"); + it("API keys in config.json or .libscope.json are ignored with a warning", () => { + const warn = spyWarn(); + writeUserConfig({ openai: { apiKey: "sk-user-file" } }); + writeProjectConfig({ anthropic: { apiKey: "sk-ant-project-file" } }); + const config = load(); + expect(config.openai).toBeUndefined(); + expect(config.anthropic).toBeUndefined(); + const keyWarnings = messages(warn).filter((m) => m.includes("API keys are read only")); + expect(keyWarnings).toHaveLength(2); + expect(keyWarnings.join("\n")).toContain(join(tempCwd, ".libscope.json")); + expect(keyWarnings.join("\n")).toContain("secrets.json"); }); it("validation accepts exactly the keys providers will use", () => { - const warn = vi.fn(); - vi.spyOn(loggerModule, "getLogger").mockReturnValue({ - warn, - } as unknown as ReturnType<typeof loggerModule.getLogger>); + const warn = spyWarn(); writeUserConfig({ embedding: { provider: "openai" }, llm: { provider: "anthropic" } }); - const config = load({ OPENAI_API_KEY: "sk-plain", ANTHROPIC_API_KEY: "sk-ant" }); - expect(config.embedding.openaiApiKey).toBe("sk-plain"); + load({ OPENAI_API_KEY: "sk-plain", ANTHROPIC_API_KEY: "sk-ant" }); expect(warn).not.toHaveBeenCalled(); }); + }); - it("warns about keys in a config file, naming the file and the env vars", () => { - const warn = vi.fn(); - vi.spyOn(loggerModule, "getLogger").mockReturnValue({ - warn, - } as unknown as ReturnType<typeof loggerModule.getLogger>); - writeProjectConfig({ llm: { anthropicApiKey: "sk-ant-file" } }); - load(); - const messages = warn.mock.calls.map((c) => String(c[0])); - const keyWarning = messages.find((m) => m.includes("API keys found")); - expect(keyWarning).toContain(join(tempCwd, ".libscope.json")); - expect(keyWarning).toContain("ANTHROPIC_API_KEY"); - expect(keyWarning).not.toContain("deprecated"); + describe("config file validation", () => { + it("ignores an invalid file value, logs a warning, and keeps the default", () => { + const warn = spyWarn(); + writeUserConfig({ logging: { level: "loud" }, indexing: { maxDocumentSize: -1 } }); + const config = load(); + expect(config.logging.level).toBe("info"); + expect(config.indexing.maxDocumentSize).toBe(100 * 1024 * 1024); + expect(messages(warn).filter((m) => m.includes("invalid config value"))).toHaveLength(2); + }); + + it("accepts string booleans and integers in a file", () => { + writeUserConfig({ indexing: { allowSelfSignedCerts: "true", maxDocumentSize: "4096" } }); + const config = load(); + expect(config.indexing.allowSelfSignedCerts).toBe(true); + expect(config.indexing.maxDocumentSize).toBe(4096); + }); + + it("accepts a custom embedding provider name in a file", () => { + writeUserConfig({ embedding: { provider: "my-plugin" } }); + expect(load().embedding.provider).toBe("my-plugin"); }); - it("LIBSCOPE_OLLAMA_MODEL sets embedding.ollamaModel", () => { - const config = load({ LIBSCOPE_OLLAMA_MODEL: "mxbai-embed-large" }); - expect(config.embedding.ollamaModel).toBe("mxbai-embed-large"); + it("drops unknown and old keys from the loaded config", () => { + writeUserConfig({ + embedding: { ollamaUrl: "http://old:11434", ollamaModel: "old" }, + registries: [], + }); + const config = load(); + expect(config.embedding).toEqual({ provider: "local" }); + expect(config).not.toHaveProperty("registries"); }); }); @@ -201,16 +315,13 @@ describe("config layering (temp HOME and cwd)", () => { }); it("masks every API key in a config without mutating the original", () => { - const config = load({ - LIBSCOPE_OPENAI_API_KEY: "sk-openai-secret-value-9999", - ANTHROPIC_API_KEY: "sk-ant-secret-value-8888", - }); + writeSecrets({ anthropic: { apiKey: "sk-ant-secret-value-8888" } }); + const config = load({ LIBSCOPE_OPENAI_API_KEY: "sk-openai-secret-value-9999" }); const masked = maskConfigSecrets(config); - const text = JSON.stringify(masked); - expect(text).not.toContain("secret-value"); - expect(masked.embedding.openaiApiKey).toBe("sk-…9999"); - expect(masked.llm?.anthropicApiKey).toBe("sk-…8888"); - expect(config.embedding.openaiApiKey).toBe("sk-openai-secret-value-9999"); + expect(JSON.stringify(masked)).not.toContain("secret-value"); + expect(masked.openai?.apiKey).toBe("sk-…9999"); + expect(masked.anthropic?.apiKey).toBe("sk-…8888"); + expect(config.openai?.apiKey).toBe("sk-openai-secret-value-9999"); }); }); }); diff --git a/tests/unit/config-save.test.ts b/tests/unit/config-save.test.ts index bcea2b6d..631fd175 100644 --- a/tests/unit/config-save.test.ts +++ b/tests/unit/config-save.test.ts @@ -1,10 +1,17 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync, chmodSync } from "node:fs"; +import { + existsSync, + mkdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, + chmodSync, +} from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { initLogger } from "../../src/logger.js"; -import type { LibScopeConfig } from "../../src/config.js"; // Create a unique temp HOME for each test run — must be initialized before module load let tempHome: string = join(tmpdir(), `libscope-config-save-test-${process.pid}`); @@ -20,32 +27,42 @@ vi.mock("node:os", async (importOriginal) => { // Dynamic import after mock is set up const { - saveUserConfig, invalidateConfigCache, setUserConfigValue, unsetUserConfigValue, getConfigValue, + getConfigFileFor, getUserConfigPath, + getSecretsPath, loadConfig, CONFIG_KEY_NAMES, + getConfigKeyTable, } = await import("../../src/config.js"); const { saveRegistries, loadRegistries } = await import("../../src/registry/config.js"); +/** Env vars that would leak into loadConfig from the developer's shell. */ +const ENV_VARS = getConfigKeyTable().flatMap((row) => row.env); +const savedEnv: Record<string, string | undefined> = {}; + function configPath(): string { return join(tempHome, ".libscope", "config.json"); } -function readSavedRaw(): Record<string, unknown> { - return JSON.parse(readFileSync(configPath(), "utf-8")) as Record<string, unknown>; +function secretsPath(): string { + return join(tempHome, ".libscope", "secrets.json"); +} + +function readJson(path: string): Record<string, unknown> { + return JSON.parse(readFileSync(path, "utf-8")) as Record<string, unknown>; } -function readSavedConfig(): LibScopeConfig { - return readSavedRaw() as unknown as LibScopeConfig; +function readSavedRaw(): Record<string, unknown> { + return readJson(configPath()); } -function writeExisting(data: unknown): void { +function writeExisting(data: unknown, path = configPath()): void { mkdirSync(join(tempHome, ".libscope"), { recursive: true }); - writeFileSync(configPath(), JSON.stringify(data), "utf-8"); + writeFileSync(path, JSON.stringify(data), "utf-8"); } const REGISTRY = { @@ -58,171 +75,111 @@ const REGISTRY = { beforeEach(() => { initLogger("silent"); + for (const key of ENV_VARS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } tempHome = join(tmpdir(), `libscope-config-save-test-${randomUUID()}`); mkdirSync(tempHome, { recursive: true }); invalidateConfigCache(); }); afterEach(() => { + for (const [key, val] of Object.entries(savedEnv)) { + if (val === undefined) delete process.env[key]; + else process.env[key] = val; + } rmSync(tempHome, { recursive: true, force: true }); }); -describe("saveUserConfig credential stripping", () => { - it("should not persist embedding.openaiApiKey to disk", () => { - saveUserConfig({ embedding: { provider: "openai", openaiApiKey: "sk-test-key" } }); - - const written = readFileSync(configPath(), "utf-8"); - const parsed = readSavedConfig(); - - expect(parsed.embedding.provider).toBe("openai"); - expect(parsed.embedding.openaiApiKey).toBeUndefined(); - expect(written).not.toContain("sk-test-key"); - }); - - it("should not persist llm.openaiApiKey to disk", () => { - saveUserConfig({ llm: { provider: "openai", openaiApiKey: "sk-llm-key" } }); - - const written = readFileSync(configPath(), "utf-8"); - expect(readSavedConfig().llm?.openaiApiKey).toBeUndefined(); - expect(written).not.toContain("sk-llm-key"); - }); - - it("should not persist llm.anthropicApiKey to disk", () => { - saveUserConfig({ llm: { provider: "anthropic", anthropicApiKey: "sk-ant-key" } }); - - const written = readFileSync(configPath(), "utf-8"); - expect(readSavedConfig().llm?.anthropicApiKey).toBeUndefined(); - expect(written).not.toContain("sk-ant-key"); - }); - - it("should strip all credential fields simultaneously", () => { - saveUserConfig({ - embedding: { provider: "openai", openaiApiKey: "sk-embed" }, - llm: { provider: "openai", openaiApiKey: "sk-llm", anthropicApiKey: "sk-ant" }, - }); - - const written = readFileSync(configPath(), "utf-8"); - expect(written).not.toContain("sk-embed"); - expect(written).not.toContain("sk-llm"); - expect(written).not.toContain("sk-ant"); - expect(written).not.toContain("ApiKey"); - }); - - it("should preserve non-credential config fields", () => { - saveUserConfig({ - embedding: { - provider: "openai", - openaiModel: "text-embedding-3-large", - openaiApiKey: "sk-test", - }, - logging: { level: "debug" }, - }); - - const parsed = readSavedConfig(); - expect(parsed.embedding.provider).toBe("openai"); - expect(parsed.embedding.openaiModel).toBe("text-embedding-3-large"); - expect(parsed.logging.level).toBe("debug"); - expect(parsed.embedding.openaiApiKey).toBeUndefined(); - }); -}); - -describe("saveUserConfig read-modify-write", () => { - it("keeps registries and unknown top-level keys", () => { - writeExisting({ registries: [REGISTRY], customTool: { a: 1 } }); - saveUserConfig({ embedding: { provider: "ollama" } }); - - const raw = readSavedRaw(); - expect(raw["registries"]).toEqual([REGISTRY]); - expect(raw["customTool"]).toEqual({ a: 1 }); - }); - - it("writes only the values that were set, not defaults", () => { - saveUserConfig({ logging: { level: "warn" } }); - expect(readSavedRaw()).toEqual({ logging: { level: "warn" } }); - }); - - it("keeps sibling keys in the same section", () => { - writeExisting({ embedding: { ollamaUrl: "http://gpu:11434" } }); - saveUserConfig({ embedding: { provider: "ollama" } }); - expect(readSavedRaw()["embedding"]).toEqual({ - ollamaUrl: "http://gpu:11434", - provider: "ollama", - }); - }); - - it("does not delete an API key the user put in the file by hand", () => { - writeExisting({ embedding: { openaiApiKey: "sk-hand-written" } }); - saveUserConfig({ embedding: { provider: "openai" } }); - expect(readSavedConfig().embedding.openaiApiKey).toBe("sk-hand-written"); - }); - - it("writes the file with mode 0600, also when it already existed", () => { - writeExisting({}); - chmodSync(configPath(), 0o644); - saveUserConfig({ logging: { level: "info" } }); - expect(statSync(configPath()).mode & 0o777).toBe(0o600); - }); - - it("invalidates the config cache so the next load sees the change", () => { - loadConfig(); - saveUserConfig({ logging: { level: "error" } }); - expect(loadConfig().logging.level).toBe("error"); - }); -}); - -describe("setUserConfigValue", () => { - it("covers every key in the LibScopeConfig shape", () => { +describe("config keys", () => { + it("are derived from the schema", () => { expect([...CONFIG_KEY_NAMES].sort()).toEqual( [ + "anthropic.apiKey", "database.path", "embedding.dimensions", - "embedding.ollamaModel", - "embedding.ollamaUrl", - "embedding.openaiApiKey", - "embedding.openaiModel", + "embedding.model", "embedding.provider", + "embedding.url", "indexing.allowPrivateUrls", "indexing.allowSelfSignedCerts", "indexing.maxDocumentSize", - "llm.anthropicApiKey", "llm.model", - "llm.ollamaUrl", - "llm.openaiApiKey", "llm.provider", + "llm.url", "logging.level", + "openai.apiKey", ].sort(), ); }); + it("key table lists type, default, env vars, secret flag, and description", () => { + const table = getConfigKeyTable(); + expect(table.map((r) => r.key)).toEqual([...CONFIG_KEY_NAMES]); + for (const row of table) expect(row.description).not.toBe(""); + + expect(table.find((r) => r.key === "llm.provider")).toMatchObject({ + type: "auto | openai | anthropic | ollama | passthrough", + default: "auto", + env: ["LIBSCOPE_LLM_PROVIDER"], + secret: false, + }); + expect(table.find((r) => r.key === "indexing.allowSelfSignedCerts")).toMatchObject({ + type: "boolean", + default: "false", + env: ["LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS"], + }); + expect(table.find((r) => r.key === "indexing.maxDocumentSize")).toMatchObject({ + type: "integer", + default: String(100 * 1024 * 1024), + }); + expect(table.find((r) => r.key === "openai.apiKey")).toMatchObject({ + env: ["LIBSCOPE_OPENAI_API_KEY", "OPENAI_API_KEY"], + secret: true, + }); + expect(table.find((r) => r.key === "anthropic.apiKey")).toMatchObject({ + env: ["LIBSCOPE_ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY"], + secret: true, + }); + }); +}); + +describe("setUserConfigValue", () => { it.each([ - ["embedding.ollamaModel", "mxbai-embed-large", "mxbai-embed-large"], - ["embedding.openaiModel", "text-embedding-3-large", "text-embedding-3-large"], + ["embedding.provider", "ollama", "ollama"], + ["embedding.model", "mxbai-embed-large", "mxbai-embed-large"], + ["embedding.url", "http://gpu:11434", "http://gpu:11434"], ["embedding.dimensions", "1024", 1024], ["llm.provider", "anthropic", "anthropic"], ["llm.model", "gpt-4o", "gpt-4o"], - ["llm.ollamaUrl", "http://localhost:11434", "http://localhost:11434"], + ["llm.url", "http://localhost:11434", "http://localhost:11434"], ["database.path", "/data/libscope.db", "/data/libscope.db"], ["logging.level", "debug", "debug"], ["indexing.maxDocumentSize", "2048", 2048], ["indexing.allowPrivateUrls", "true", true], ["indexing.allowSelfSignedCerts", "0", false], - ] as const)("sets %s", (key, input, expected) => { + ] as const)("sets %s in config.json", (key, input, expected) => { expect(setUserConfigValue(key, input)).toBe(expected); const [section = "", field = ""] = key.split("."); const sectionObj = readSavedRaw()[section] as Record<string, unknown>; expect(sectionObj[field]).toBe(expected); + expect(existsSync(secretsPath())).toBe(false); }); - it("rejects unknown keys and lists valid keys", () => { + it("rejects unknown and old keys and lists valid keys", () => { expect(() => setUserConfigValue("embedding.nope", "x")).toThrow( /Unknown config key.*llm\.model/, ); + expect(() => setUserConfigValue("embedding.ollamaModel", "x")).toThrow(/Unknown config key/); + expect(() => setUserConfigValue("embedding.openaiApiKey", "x")).toThrow(/Unknown config key/); }); it("rejects values outside an enum", () => { expect(() => setUserConfigValue("embedding.provider", "cohere")).toThrow( /one of: local, ollama, openai/, ); + expect(() => setUserConfigValue("llm.provider", "gemini")).toThrow(/one of: auto/); expect(() => setUserConfigValue("logging.level", "loud")).toThrow(/one of/); }); @@ -230,15 +187,34 @@ describe("setUserConfigValue", () => { expect(() => setUserConfigValue("indexing.allowPrivateUrls", "yes")).toThrow(/true or false/); expect(() => setUserConfigValue("indexing.maxDocumentSize", "-5")).toThrow(/positive integer/); expect(() => setUserConfigValue("indexing.maxDocumentSize", "1.5")).toThrow(/positive integer/); + expect(() => setUserConfigValue("embedding.dimensions", "20000")).toThrow(/dimensions/); + expect(() => setUserConfigValue("llm.model", " ")).toThrow(/must not be empty/); }); - it("refuses API keys and names the env var to use", () => { - expect(() => setUserConfigValue("embedding.openaiApiKey", "sk-x")).toThrow( - /LIBSCOPE_OPENAI_API_KEY/, - ); - expect(() => setUserConfigValue("llm.anthropicApiKey", "sk-x")).toThrow( - /LIBSCOPE_ANTHROPIC_API_KEY/, - ); + it("writes the file with mode 0600, also when it already existed", () => { + writeExisting({}); + chmodSync(configPath(), 0o644); + setUserConfigValue("logging.level", "info"); + expect(statSync(configPath()).mode & 0o777).toBe(0o600); + }); + + it("keeps registries, unknown top-level keys, and sibling keys", () => { + writeExisting({ + registries: [REGISTRY], + customTool: { a: 1 }, + embedding: { url: "http://gpu:11434" }, + }); + setUserConfigValue("embedding.provider", "ollama"); + const raw = readSavedRaw(); + expect(raw["registries"]).toEqual([REGISTRY]); + expect(raw["customTool"]).toEqual({ a: 1 }); + expect(raw["embedding"]).toEqual({ url: "http://gpu:11434", provider: "ollama" }); + }); + + it("invalidates the config cache so the next load sees the change", () => { + loadConfig(); + setUserConfigValue("logging.level", "error"); + expect(loadConfig().logging.level).toBe("error"); }); it("keeps registries written by the registry module", () => { @@ -256,6 +232,42 @@ describe("setUserConfigValue", () => { }); }); +describe("secrets", () => { + it("API keys are written to secrets.json (0600), never to config.json", () => { + writeExisting({ logging: { level: "warn" } }); + expect(setUserConfigValue("openai.apiKey", "sk-openai-test")).toBe("sk-openai-test"); + expect(setUserConfigValue("anthropic.apiKey", "sk-ant-test")).toBe("sk-ant-test"); + + expect(readJson(secretsPath())).toEqual({ + openai: { apiKey: "sk-openai-test" }, + anthropic: { apiKey: "sk-ant-test" }, + }); + expect(statSync(secretsPath()).mode & 0o777).toBe(0o600); + expect(readFileSync(configPath(), "utf-8")).not.toContain("sk-"); + expect(readSavedRaw()).toEqual({ logging: { level: "warn" } }); + }); + + it("tightens an existing secrets.json to 0600", () => { + writeExisting({}, secretsPath()); + chmodSync(secretsPath(), 0o644); + setUserConfigValue("openai.apiKey", "sk-x"); + expect(statSync(secretsPath()).mode & 0o777).toBe(0o600); + }); + + it("keys written by config set are loaded and masked by config get", () => { + setUserConfigValue("openai.apiKey", "sk-proj-abcdefghijkl9876"); + const config = loadConfig(); + expect(config.openai?.apiKey).toBe("sk-proj-abcdefghijkl9876"); + expect(getConfigValue(config, "openai.apiKey")).toBe("sk-…9876"); + }); + + it("getConfigFileFor names the file a key is stored in", () => { + expect(getConfigFileFor("openai.apiKey")).toBe(getSecretsPath()); + expect(getConfigFileFor("llm.model")).toBe(getUserConfigPath()); + expect(getSecretsPath()).toBe(secretsPath()); + }); +}); + describe("unsetUserConfigValue", () => { it("removes the key and drops an empty section", () => { writeExisting({ logging: { level: "debug" }, registries: [REGISTRY] }); @@ -264,8 +276,8 @@ describe("unsetUserConfigValue", () => { }); it("keeps other keys in the section", () => { - writeExisting({ embedding: { provider: "ollama", ollamaModel: "m" } }); - unsetUserConfigValue("embedding.ollamaModel"); + writeExisting({ embedding: { provider: "ollama", model: "m" } }); + unsetUserConfigValue("embedding.model"); expect(readSavedRaw()["embedding"]).toEqual({ provider: "ollama" }); }); @@ -273,10 +285,11 @@ describe("unsetUserConfigValue", () => { expect(unsetUserConfigValue("llm.model")).toBe(false); }); - it("can remove a hand-written API key", () => { - writeExisting({ embedding: { openaiApiKey: "sk-old" } }); - expect(unsetUserConfigValue("embedding.openaiApiKey")).toBe(true); - expect(readFileSync(configPath(), "utf-8")).not.toContain("sk-old"); + it("removes an API key from secrets.json", () => { + setUserConfigValue("openai.apiKey", "sk-old"); + setUserConfigValue("anthropic.apiKey", "sk-ant-keep"); + expect(unsetUserConfigValue("openai.apiKey")).toBe(true); + expect(readJson(secretsPath())).toEqual({ anthropic: { apiKey: "sk-ant-keep" } }); }); it("rejects unknown keys", () => { @@ -290,19 +303,12 @@ describe("getConfigValue", () => { const config = loadConfig(); expect(getConfigValue(config, "indexing.maxDocumentSize")).toBe(4096); expect(getConfigValue(config, "logging.level")).toBe("info"); - }); - - it("masks API keys", () => { - const config = loadConfig(); - const withKey: LibScopeConfig = { - ...config, - embedding: { ...config.embedding, openaiApiKey: "sk-proj-abcdefghijkl9876" }, - }; - expect(getConfigValue(withKey, "embedding.openaiApiKey")).toBe("sk-…9876"); + expect(getConfigValue(config, "llm.provider")).toBe("auto"); }); it("returns undefined for unset optional keys", () => { expect(getConfigValue(loadConfig(), "llm.model")).toBeUndefined(); + expect(getConfigValue(loadConfig(), "openai.apiKey")).toBeUndefined(); }); it("reports the user config path under HOME", () => { diff --git a/tests/unit/config.test.ts b/tests/unit/config.test.ts index 10709014..44a56243 100644 --- a/tests/unit/config.test.ts +++ b/tests/unit/config.test.ts @@ -43,21 +43,21 @@ describe("config", () => { it("should pick up LIBSCOPE_OPENAI_API_KEY", () => { const config = loadConfigWithEnv({ LIBSCOPE_OPENAI_API_KEY: "sk-test123" }); - expect(config.embedding.openaiApiKey).toBe("sk-test123"); + expect(config.openai?.apiKey).toBe("sk-test123"); }); - it("should pick up LIBSCOPE_OLLAMA_URL", () => { - const config = loadConfigWithEnv({ LIBSCOPE_OLLAMA_URL: "http://custom:11434" }); - expect(config.embedding.ollamaUrl).toBe("http://custom:11434"); + it("should pick up LIBSCOPE_EMBEDDING_URL", () => { + const config = loadConfigWithEnv({ LIBSCOPE_EMBEDDING_URL: "http://custom:11434" }); + expect(config.embedding.url).toBe("http://custom:11434"); }); - it("should pick up LIBSCOPE_ALLOW_PRIVATE_URLS", () => { - const config = loadConfigWithEnv({ LIBSCOPE_ALLOW_PRIVATE_URLS: "true" }); + it("should pick up LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS", () => { + const config = loadConfigWithEnv({ LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS: "true" }); expect(config.indexing.allowPrivateUrls).toBe(true); }); - it("should pick up LIBSCOPE_ALLOW_SELF_SIGNED_CERTS", () => { - const config = loadConfigWithEnv({ LIBSCOPE_ALLOW_SELF_SIGNED_CERTS: "1" }); + it("should pick up LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS", () => { + const config = loadConfigWithEnv({ LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS: "1" }); expect(config.indexing.allowSelfSignedCerts).toBe(true); }); @@ -75,15 +75,14 @@ function makeConfig(overrides: Partial<LibScopeConfig> = {}): LibScopeConfig { return { embedding: { provider: "local", - ollamaUrl: "http://localhost:11434", - ollamaModel: "nomic-embed-text", - openaiModel: "text-embedding-3-small", ...overrides.embedding, }, database: { path: "/tmp/test-libscope/libscope.db", ...overrides.database }, indexing: { maxDocumentSize: 100 * 1024 * 1024, ...overrides.indexing }, logging: { level: "info", ...overrides.logging }, ...("llm" in overrides ? { llm: overrides.llm } : {}), + ...("openai" in overrides ? { openai: overrides.openai } : {}), + ...("anthropic" in overrides ? { anthropic: overrides.anthropic } : {}), }; } @@ -138,7 +137,8 @@ describe("validateConfig", () => { it("should not warn when embedding provider is openai with config key", () => { const config = makeConfig({ - embedding: { provider: "openai", openaiApiKey: "sk-test" }, + embedding: { provider: "openai" }, + openai: { apiKey: "sk-test" }, }); const warnings = validateConfig(config); expect(warnings).toHaveLength(0); @@ -158,7 +158,10 @@ describe("validateConfig", () => { }); it("should not warn when llm provider is anthropic with a key", () => { - const config = makeConfig({ llm: { provider: "anthropic", anthropicApiKey: "sk-ant" } }); + const config = makeConfig({ + llm: { provider: "anthropic" }, + anthropic: { apiKey: "sk-ant" }, + }); expect(validateConfig(config)).toHaveLength(0); }); @@ -170,26 +173,29 @@ describe("validateConfig", () => { expect(warnings.some((w) => w.includes("llm.provider"))).toBe(true); }); - it("should not warn when llm provider is openai with embedding key available", () => { + it("should not warn when llm provider is openai with an OpenAI key", () => { const config = makeConfig({ - embedding: { provider: "local", openaiApiKey: "sk-shared" }, llm: { provider: "openai" }, + openai: { apiKey: "sk-shared" }, }); const warnings = validateConfig(config); expect(warnings.some((w) => w.includes("llm.provider"))).toBe(false); }); - it("should warn when ollama provider has no URL", () => { - const config = makeConfig({ - embedding: { provider: "ollama", ollamaUrl: undefined }, - }); + it("should not warn when ollama provider has no URL (the default URL applies)", () => { + const config = makeConfig({ embedding: { provider: "ollama" } }); + expect(validateConfig(config)).toHaveLength(0); + }); + + it("should warn when embedding.model is set for the local provider", () => { + const config = makeConfig({ embedding: { provider: "local", model: "bge-m3" } }); const warnings = validateConfig(config); - expect(warnings.some((w) => w.includes("ollamaUrl"))).toBe(true); + expect(warnings.some((w) => w.includes("ignored by the local provider"))).toBe(true); }); it("should not warn when ollama provider has a URL", () => { const config = makeConfig({ - embedding: { provider: "ollama", ollamaUrl: "http://localhost:11434" }, + embedding: { provider: "ollama", url: "http://localhost:11434" }, }); const warnings = validateConfig(config); expect(warnings).toHaveLength(0); diff --git a/tests/unit/embedding-dimensions.test.ts b/tests/unit/embedding-dimensions.test.ts index 2b43c392..4a871751 100644 --- a/tests/unit/embedding-dimensions.test.ts +++ b/tests/unit/embedding-dimensions.test.ts @@ -28,9 +28,10 @@ function vector(size: number): number[] { return Array.from({ length: size }, () => 0.1); } -function config(embedding: LibScopeConfig["embedding"]): LibScopeConfig { +function config(embedding: LibScopeConfig["embedding"], openaiApiKey?: string): LibScopeConfig { return { embedding, + ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}), database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024, allowPrivateUrls: false, allowSelfSignedCerts: false }, logging: { level: "silent" }, @@ -164,8 +165,8 @@ describe("createEmbeddingProvider dimensions", () => { const provider = createEmbeddingProvider( config({ provider: "ollama", - ollamaUrl: "http://x", - ollamaModel: "custom-embed", + url: "http://x", + model: "custom-embed", dimensions: 2048, }), ); @@ -175,7 +176,7 @@ describe("createEmbeddingProvider dimensions", () => { it("uses the known size for the configured OpenAI model", () => { const provider = createEmbeddingProvider( - config({ provider: "openai", openaiApiKey: "k", openaiModel: "text-embedding-3-large" }), + config({ provider: "openai", model: "text-embedding-3-large" }, "k"), ); expect(provider.dimensions).toBe(3072); }); @@ -183,7 +184,7 @@ describe("createEmbeddingProvider dimensions", () => { it("rejects an invalid embedding.dimensions", () => { expect(() => createEmbeddingProvider( - config({ provider: "ollama", ollamaUrl: "http://x", ollamaModel: "m", dimensions: -5 }), + config({ provider: "ollama", url: "http://x", model: "m", dimensions: -5 }), ), ).toThrow(ConfigError); }); diff --git a/tests/unit/rag.test.ts b/tests/unit/rag.test.ts index d99a7e63..e097b96f 100644 --- a/tests/unit/rag.test.ts +++ b/tests/unit/rag.test.ts @@ -3,6 +3,8 @@ import { buildContextPrompt, extractSources, createLlmProvider, + isPassthroughMode, + resolveLlmProviderName, DEFAULT_SYSTEM_PROMPT, type LlmProvider, } from "../../src/core/rag.js"; @@ -216,7 +218,8 @@ describe("createLlmProvider", () => { it("creates an OpenAI provider when configured", () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test", model: "gpt-4o" }, + llm: { provider: "openai", model: "gpt-4o" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -255,7 +258,8 @@ describe("createLlmProvider", () => { it("uses default model for OpenAI when not specified", () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test" }, + llm: { provider: "openai" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -265,10 +269,11 @@ describe("createLlmProvider", () => { expect(provider.model).toBe("gpt-4o-mini"); }); - it("falls back to embedding API key for OpenAI provider", () => { + it("uses the shared OpenAI key for embeddings and the LLM", () => { const config: LibScopeConfig = { - embedding: { provider: "openai", openaiApiKey: "sk-embed" }, + embedding: { provider: "openai" }, llm: { provider: "openai" }, + openai: { apiKey: "sk-shared" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -399,7 +404,8 @@ describe("createLlmProvider", () => { it("handles OpenAI empty choices response", async () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test" }, + llm: { provider: "openai" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -422,7 +428,8 @@ describe("createLlmProvider", () => { it("handles OpenAI unknown status codes", async () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test" }, + llm: { provider: "openai" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -445,7 +452,8 @@ describe("createLlmProvider", () => { it("handles OpenAI 429 rate limit", async () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test" }, + llm: { provider: "openai" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -480,7 +488,8 @@ describe("createLlmProvider", () => { it("sanitizes OpenAI error messages (does not leak response body)", async () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", model: "gpt-4o", openaiApiKey: "sk-test" }, + llm: { provider: "openai", model: "gpt-4o" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -595,3 +604,104 @@ describe("askQuestionStream", () => { } }); }); + +describe('llm.provider "auto"', () => { + function autoConfig(overrides: Partial<LibScopeConfig> = {}): LibScopeConfig { + return { + embedding: { provider: "local" }, + llm: { provider: "auto" }, + database: {}, + indexing: { maxDocumentSize: 1024, allowPrivateUrls: false, allowSelfSignedCerts: false }, + logging: { level: "silent" }, + ...overrides, + }; + } + + it("resolves to passthrough under MCP, even when API keys are set", () => { + const config = autoConfig({ openai: { apiKey: "sk-x" } }); + expect(resolveLlmProviderName(config, { surface: "mcp" })).toBe("passthrough"); + expect(isPassthroughMode(config, { surface: "mcp" })).toBe(true); + expect(() => createLlmProvider(config, { surface: "mcp" })).toThrow(/passthrough/); + }); + + it.each(["cli", "api", "sdk", undefined] as const)( + "is not passthrough on surface %s", + (surface) => { + expect(isPassthroughMode(autoConfig(), { surface })).toBe(false); + }, + ); + + it("picks openai when an OpenAI key is set", () => { + const config = autoConfig({ + openai: { apiKey: "sk-x" }, + anthropic: { apiKey: "sk-ant" }, + }); + expect(resolveLlmProviderName(config, { surface: "cli" })).toBe("openai"); + expect(createLlmProvider(config).model).toBe("gpt-4o-mini"); + }); + + it("picks anthropic when only an Anthropic key is set", () => { + const config = autoConfig({ anthropic: { apiKey: "sk-ant" } }); + expect(resolveLlmProviderName(config)).toBe("anthropic"); + expect(createLlmProvider(config).model).toBe("claude-3-5-haiku-20241022"); + }); + + it("picks ollama when llm.url is set", () => { + const config = autoConfig({ llm: { provider: "auto", url: "http://gpu:11434" } }); + expect(resolveLlmProviderName(config)).toBe("ollama"); + expect(createLlmProvider(config).model).toBe("llama3.2"); + }); + + it("picks ollama when the embedding provider is ollama", () => { + const config = autoConfig({ embedding: { provider: "ollama" } }); + expect(resolveLlmProviderName(config)).toBe("ollama"); + }); + + it("returns null and createLlmProvider throws a hint when nothing is configured", () => { + const config = autoConfig(); + expect(resolveLlmProviderName(config)).toBeNull(); + expect(() => createLlmProvider(config)).toThrow(/No LLM provider configured.*OPENAI_API_KEY/); + }); + + it("treats a missing llm section as auto", () => { + const config = autoConfig({ openai: { apiKey: "sk-x" } }); + delete config.llm; + expect(resolveLlmProviderName(config)).toBe("openai"); + }); + + it("an explicit provider is used on every surface", () => { + const config = autoConfig({ llm: { provider: "ollama" }, openai: { apiKey: "sk-x" } }); + expect(resolveLlmProviderName(config, { surface: "mcp" })).toBe("ollama"); + expect(isPassthroughMode(autoConfig({ llm: { provider: "passthrough" } }))).toBe(true); + }); + + it("the ollama LLM uses llm.url, then embedding.url", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ response: "hi" }), + }); + vi.stubGlobal("fetch", fetchMock); + try { + const fromEmbedding = createLlmProvider( + autoConfig({ + embedding: { provider: "ollama", url: "http://embed:11434" }, + llm: { provider: "ollama" }, + }), + ); + await fromEmbedding.complete("q"); + const fromLlm = createLlmProvider( + autoConfig({ + embedding: { provider: "ollama", url: "http://embed:11434" }, + llm: { provider: "ollama", url: "http://llm:11434" }, + }), + ); + await fromLlm.complete("q"); + expect(fetchMock.mock.calls.map((c) => String(c[0]))).toEqual([ + "http://embed:11434/api/generate", + "http://llm:11434/api/generate", + ]); + } finally { + vi.unstubAllGlobals(); + } + }); +}); From 73548de1cb1bf1eedcb8a4a55863ae0548ffc14e Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:37:11 +0000 Subject: [PATCH 28/83] docs(config): document the config schema, secrets.json, auto LLM, and v2 renames --- docs/guide/configuration.md | 115 +++++++++++++--------- docs/guide/troubleshooting.md | 4 +- docs/migration-v2.md | 50 ++++++++++ docs/reference/cli.md | 2 +- docs/reference/configuration.md | 166 +++++++++++++++++--------------- 5 files changed, 212 insertions(+), 125 deletions(-) create mode 100644 docs/migration-v2.md diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 8d969da1..028ff141 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1,15 +1,19 @@ # Configuration -LibScope uses a 3-tier config system. Higher tiers override lower ones: +LibScope reads settings from these layers. Higher layers override lower ones: **Environment variables** > **Project `.libscope.json`** > **User `~/.libscope/config.json`** > **Defaults** +API keys are the exception: LibScope reads them only from environment variables and `~/.libscope/secrets.json`. See [API keys](#api-keys). + +Every setting has an environment variable named `LIBSCOPE_<SECTION>_<FIELD>`, for example `embedding.model` → `LIBSCOPE_EMBEDDING_MODEL`. The [configuration reference](../reference/configuration.md) lists all keys. Keys renamed in 2.0 are listed in the [migration guide](../migration-v2.md#configuration). + ## Config File You can set options via the CLI or by editing the config file directly. ```bash -# Set a value (any key from the configuration reference, except API keys) +# Set a value (any key from the configuration reference) libscope config set embedding.provider ollama # Read or remove one value @@ -23,17 +27,16 @@ libscope config path libscope config show ``` -`config set` validates the key and value, and changes only that key in `~/.libscope/config.json`. Other content of the file, such as `registries`, is kept. +`config set` validates the key and value, and changes only that key. Other content of the file, such as `registries`, is kept. API keys are written to `~/.libscope/secrets.json`. All other keys are written to `~/.libscope/config.json`. Example `~/.libscope/config.json`: ```json { "embedding": { - "provider": "local", - "ollamaUrl": "http://localhost:11434", - "ollamaModel": "nomic-embed-text", - "openaiModel": "text-embedding-3-small" + "provider": "ollama", + "url": "http://localhost:11434", + "model": "nomic-embed-text" }, "llm": { "provider": "openai", @@ -66,18 +69,23 @@ Embeddings turn text into vectors for semantic search. LibScope supports three p | -------- | ------- | ---------------------- | --------------------------------------------- | | `local` | ✅ | Nothing | all-MiniLM-L6-v2, ~80MB download on first use | | `ollama` | | Ollama running locally | Uses nomic-embed-text by default | -| `openai` | | API key | Uses text-embedding-3-small | +| `openai` | | API key | Uses text-embedding-3-small by default | The local provider works out of the box — no API keys, no external services. It runs the model in-process using `@xenova/transformers`. ```bash # Switch to Ollama libscope config set embedding.provider ollama +libscope config set embedding.url http://gpu-box:11434 # optional, default http://localhost:11434 +libscope config set embedding.model mxbai-embed-large # optional # Or OpenAI libscope config set embedding.provider openai +libscope config set openai.apiKey sk-... # or export OPENAI_API_KEY ``` +`embedding.model` and `embedding.url` apply to the selected provider. The local provider always uses all-MiniLM-L6-v2 and ignores `embedding.model`. + LibScope records the provider, model, and vector size that built the vector index. If you change the provider, the model, or `embedding.dimensions` after you index documents, LibScope stops with an error that names the old and the new model. To rebuild the vector index with the new model, run: ```bash @@ -88,59 +96,80 @@ LibScope knows the vector size of common models (for example `nomic-embed-text`, ## LLM Configuration -The `ask` command and the `ask-question` MCP tool use an LLM to synthesize answers from search results (RAG). This requires a separate LLM provider: +The `ask` command and the `ask-question` MCP tool use an LLM to write answers from search results (RAG). + +`llm.provider` is `auto` by default. LibScope selects the LLM when `ask` runs, without network calls: + +1. Under the MCP server: `passthrough`. LibScope returns the retrieved context, and the calling assistant writes the answer. +2. Otherwise, `openai` if an OpenAI key is set. +3. Otherwise, `anthropic` if an Anthropic key is set. +4. Otherwise, `ollama` if `llm.url` is set or `embedding.provider` is `ollama`. +5. Otherwise there is no LLM, and `ask` fails with a message that tells you what to set. + +To choose the provider yourself: ```bash # Via config -libscope config set llm.provider openai +libscope config set llm.provider openai # auto | openai | anthropic | ollama | passthrough +libscope config set llm.model gpt-4o-mini # optional # Via environment variables export LIBSCOPE_LLM_PROVIDER=openai export LIBSCOPE_LLM_MODEL=gpt-4o-mini ``` -Supported providers: `openai`, `ollama`, `anthropic`, `passthrough`. +For Ollama, `llm.url` sets the server. If it is not set, LibScope uses `embedding.url`, then `http://localhost:11434`. -The `anthropic` provider uses Anthropic's Claude models. Set the API key with an environment variable (`LIBSCOPE_ANTHROPIC_API_KEY` or `ANTHROPIC_API_KEY`): +The `anthropic` provider uses Anthropic's Claude models: ```bash -export LIBSCOPE_LLM_PROVIDER=anthropic -export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-... +export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-... # or: libscope config set anthropic.apiKey sk-ant-... ``` -The key is read even when `LIBSCOPE_LLM_PROVIDER` is not set, so you can set `llm.provider` to `"anthropic"` in a config file and keep the key in the environment. A `llm.anthropicApiKey` value in a config file also works, but environment variables take precedence. You can optionally set `llm.model` to choose a specific Claude model. +With `llm.provider` set to `auto`, this key alone selects Anthropic (when no OpenAI key is set). You can set `llm.model` to choose a specific Claude model. + +The `passthrough` provider does not call an LLM. The MCP `ask-question` tool then returns the retrieved context, and the calling assistant writes the answer. + +## API keys + +LibScope reads each API key in this order: + +1. `LIBSCOPE_OPENAI_API_KEY` / `LIBSCOPE_ANTHROPIC_API_KEY` +2. `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` +3. `~/.libscope/secrets.json`, written by `libscope config set openai.apiKey <key>` or `libscope config set anthropic.apiKey <key>` (mode `0600`) -The `passthrough` provider is for advanced integrations where you supply your own LLM responses externally. When set, the `ask` command emits an event stream that your application handles rather than calling an LLM directly. +The OpenAI key is used for OpenAI embeddings and the OpenAI LLM. LibScope ignores API keys in `config.json` and `.libscope.json`, and logs a warning. `libscope config show` and `libscope config get` mask keys. ## Environment Variables -| Variable | Description | Default | -| ---------------------------------- | -------------------------------------------------- | ------------------------ | -| `LIBSCOPE_EMBEDDING_PROVIDER` | Embedding provider (`local` / `ollama` / `openai`) | `local` | -| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key (embeddings and LLM) | — | -| `OPENAI_API_KEY` | Used when `LIBSCOPE_OPENAI_API_KEY` is not set | — | -| `LIBSCOPE_OLLAMA_URL` | Ollama server URL | `http://localhost:11434` | -| `LIBSCOPE_OLLAMA_MODEL` | Ollama embedding model | `nomic-embed-text` | -| `LIBSCOPE_LLM_PROVIDER` | LLM provider for RAG (`openai` / `ollama` / `anthropic`) | — | -| `LIBSCOPE_LLM_MODEL` | LLM model override | — | -| `LIBSCOPE_ANTHROPIC_API_KEY` | Anthropic API key (for Claude models) | — | -| `ANTHROPIC_API_KEY` | Used when `LIBSCOPE_ANTHROPIC_API_KEY` is not set | — | -| `LIBSCOPE_ALLOW_PRIVATE_URLS` | Allow fetching from private/internal IPs | `false` | -| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | Accept self-signed TLS certificates | `false` | -| `LIBSCOPE_WORKSPACE` | Active workspace for this shell | `default` | -| `LIBSCOPE_API_KEY` | REST API key (`Authorization: Bearer <key>`) | — | -| `LIBSCOPE_SECRET_KEY` | Encrypts stored webhook secrets | — | -| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (stderr) in the CLI | — | -| `ONENOTE_CLIENT_ID` | Microsoft app registration client ID | — | -| `ONENOTE_TENANT_ID` | Microsoft tenant ID | `common` | -| `NOTION_TOKEN` | Notion integration token | — | -| `CONFLUENCE_URL` | Confluence base URL | — | -| `CONFLUENCE_EMAIL` | Confluence user email | — | -| `CONFLUENCE_TOKEN` | Confluence API token | — | - -Environment variables always take precedence over config files. LibScope does not load `.env` files; export the variables in your shell or MCP client config. - -API keys use one rule for embeddings and the LLM: `LIBSCOPE_<PROVIDER>_API_KEY`, then `<PROVIDER>_API_KEY` (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`), then the key in a config file. `libscope config show` masks keys. +| Variable | Config key | Default | +| ------------------------------------------------- | --------------------------------------------------- | ------------------------ | +| `LIBSCOPE_EMBEDDING_PROVIDER` | `embedding.provider` | `local` | +| `LIBSCOPE_EMBEDDING_MODEL` | `embedding.model` | provider default | +| `LIBSCOPE_EMBEDDING_URL` | `embedding.url` | `http://localhost:11434` | +| `LIBSCOPE_EMBEDDING_DIMENSIONS` | `embedding.dimensions` | size of the model | +| `LIBSCOPE_LLM_PROVIDER` | `llm.provider` | `auto` | +| `LIBSCOPE_LLM_MODEL` | `llm.model` | provider default | +| `LIBSCOPE_LLM_URL` | `llm.url` | `embedding.url` | +| `LIBSCOPE_OPENAI_API_KEY`, `OPENAI_API_KEY` | `openai.apiKey` | — | +| `LIBSCOPE_ANTHROPIC_API_KEY`, `ANTHROPIC_API_KEY` | `anthropic.apiKey` | — | +| `LIBSCOPE_DATABASE_PATH` | `database.path` | workspace database | +| `LIBSCOPE_LOGGING_LEVEL` | `logging.level` | `info` | +| `LIBSCOPE_INDEXING_MAX_DOCUMENT_SIZE` | `indexing.maxDocumentSize` | `104857600` | +| `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS` | `indexing.allowPrivateUrls` | `false` | +| `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS` | `indexing.allowSelfSignedCerts` | `false` | +| `LIBSCOPE_WORKSPACE` | Active workspace for this shell | `default` | +| `LIBSCOPE_API_KEY` | REST API key (`Authorization: Bearer <key>`) | — | +| `LIBSCOPE_SECRET_KEY` | Encrypts stored webhook secrets | — | +| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (stderr) in the CLI | — | +| `ONENOTE_CLIENT_ID` | Microsoft app registration client ID | — | +| `ONENOTE_TENANT_ID` | Microsoft tenant ID | `common` | +| `NOTION_TOKEN` | Notion integration token | — | +| `CONFLUENCE_URL` | Confluence base URL | — | +| `CONFLUENCE_EMAIL` | Confluence user email | — | +| `CONFLUENCE_TOKEN` | Confluence API token | — | + +Environment variables take precedence over config files. LibScope does not load `.env` files; export the variables in your shell or MCP client config. ## Workspaces diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 0f7b5331..d62b43ed 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -198,7 +198,7 @@ If you are fetching from a server with a self-signed certificate: ```bash libscope config set indexing.allowSelfSignedCerts true # or -export LIBSCOPE_ALLOW_SELF_SIGNED_CERTS=true +export LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS=true ``` For internal/private URLs (RFC 1918 address ranges): @@ -206,7 +206,7 @@ For internal/private URLs (RFC 1918 address ranges): ```bash libscope config set indexing.allowPrivateUrls true # or -export LIBSCOPE_ALLOW_PRIVATE_URLS=true +export LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS=true ``` ### Import is very slow for large directories diff --git a/docs/migration-v2.md b/docs/migration-v2.md new file mode 100644 index 00000000..186d566c --- /dev/null +++ b/docs/migration-v2.md @@ -0,0 +1,50 @@ +# Migrating to LibScope 2.0 + +LibScope 2.0 removes old names without aliases. This page lists every removed or renamed name and what to use instead. Your indexed data is not changed by the upgrade. + +## Configuration + +### Config keys + +Edit `~/.libscope/config.json` and `.libscope.json`, or run `libscope config set <new key> <value>`. LibScope 2.0 ignores the old keys. + +| Old key (1.x) | New key (2.0) | Notes | +| ------------------------ | ------------------ | ------------------------------------------------------------------------------------------------------ | +| `embedding.ollamaUrl` | `embedding.url` | | +| `embedding.ollamaModel` | `embedding.model` | `embedding.model` applies to the selected `embedding.provider`. | +| `embedding.openaiModel` | `embedding.model` | Same as above. | +| `embedding.openaiApiKey` | `openai.apiKey` | One OpenAI key for embeddings and the LLM. Stored in `~/.libscope/secrets.json`, not in `config.json`. | +| `llm.ollamaUrl` | `llm.url` | When unset, the Ollama LLM uses `embedding.url`, then `http://localhost:11434`. | +| `llm.openaiApiKey` | `openai.apiKey` | Same key as for embeddings. | +| `llm.anthropicApiKey` | `anthropic.apiKey` | Stored in `~/.libscope/secrets.json`. | + +### Environment variables + +| Old variable (1.x) | New variable (2.0) | +| ---------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `LIBSCOPE_OLLAMA_URL` | `LIBSCOPE_EMBEDDING_URL` (and `LIBSCOPE_LLM_URL` for the LLM, which falls back to `LIBSCOPE_EMBEDDING_URL`) | +| `LIBSCOPE_OLLAMA_MODEL` | `LIBSCOPE_EMBEDDING_MODEL` | +| `LIBSCOPE_ALLOW_PRIVATE_URLS` | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS` | +| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS` | + +These variables do not change: `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_LLM_PROVIDER`, `LIBSCOPE_LLM_MODEL`, `LIBSCOPE_OPENAI_API_KEY`, `OPENAI_API_KEY`, `LIBSCOPE_ANTHROPIC_API_KEY`, `ANTHROPIC_API_KEY`. Every config key now has a variable named `LIBSCOPE_<SECTION>_<FIELD>`. See the [configuration reference](reference/configuration.md). + +### Behavior changes + +| 1.x | 2.0 | +| ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| API keys could be stored in `config.json` or `.libscope.json`. | API keys are read only from environment variables and `~/.libscope/secrets.json` (mode `0600`). Keys in `config.json` or `.libscope.json` are ignored with a warning. Move them with `libscope config set openai.apiKey <key>` or `libscope config set anthropic.apiKey <key>`. | +| `config set` refused API keys. | `config set openai.apiKey` and `config set anthropic.apiKey` write to `~/.libscope/secrets.json`. | +| `llm.provider` had no default; `ask` failed until it was set. | `llm.provider` defaults to `auto`: passthrough under MCP, else OpenAI if an OpenAI key is set, else Anthropic if an Anthropic key is set, else Ollama if `llm.url` is set or the embedding provider is `ollama`. | +| An invalid value in a config file was used as is. | An invalid value is ignored with a warning, and the default (or lower layer) applies. | +| `LIBSCOPE_ALLOW_PRIVATE_URLS=false` did not override `true` in a file. | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS=false` (or `0`) sets `false`. | + +### Library (SDK) code + +| Old (1.x) | New (2.0) | +| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| `LibScopeConfig.embedding.openaiApiKey`, `llm.openaiApiKey` | `LibScopeConfig.openai.apiKey` | +| `LibScopeConfig.llm.anthropicApiKey` | `LibScopeConfig.anthropic.apiKey` | +| `embedding.ollamaUrl`, `ollamaModel`, `openaiModel` | `embedding.url`, `embedding.model` | +| `llm.ollamaUrl` | `llm.url` | +| `createLlmProvider(config)` read `ANTHROPIC_API_KEY` itself | `createLlmProvider(config, { surface })` reads only `config`; `loadConfig()` reads the env vars. | diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a2281b81..a62ef2f7 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -381,7 +381,7 @@ Creates a git repo with the canonical registry folder structure. See the [Regist | `libscope config show` | Show current configuration (secrets masked) | | `libscope config path` | Print the user config file path | -Supported config keys: `embedding.provider`, `embedding.ollamaUrl`, `embedding.ollamaModel`, `embedding.openaiModel`, `embedding.dimensions`, `llm.provider`, `llm.model`, `llm.ollamaUrl`, `database.path`, `indexing.maxDocumentSize`, `indexing.allowPrivateUrls`, `indexing.allowSelfSignedCerts`, `logging.level`. API keys cannot be set with `config set`; use the `LIBSCOPE_OPENAI_API_KEY` / `OPENAI_API_KEY` and `LIBSCOPE_ANTHROPIC_API_KEY` / `ANTHROPIC_API_KEY` environment variables. +Supported config keys: `embedding.provider`, `embedding.model`, `embedding.url`, `embedding.dimensions`, `llm.provider`, `llm.model`, `llm.url`, `openai.apiKey`, `anthropic.apiKey`, `database.path`, `indexing.maxDocumentSize`, `indexing.allowPrivateUrls`, `indexing.allowSelfSignedCerts`, `logging.level`. `openai.apiKey` and `anthropic.apiKey` are written to `~/.libscope/secrets.json` (mode `0600`), never to `config.json`; the `LIBSCOPE_OPENAI_API_KEY` / `OPENAI_API_KEY` and `LIBSCOPE_ANTHROPIC_API_KEY` / `ANTHROPIC_API_KEY` environment variables take precedence. ## Global Options diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 03617963..f7864c5e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1,94 +1,101 @@ # Configuration Reference -Complete reference for all configuration options. +Complete reference for all configuration options. One schema (`src/config-schema.ts`) defines every key, its default, its validation, and its environment variable. Keys renamed in 2.0 are listed in the [migration guide](../migration-v2.md#configuration). ## Config File Locations -| Location | Scope | Precedence | -| ------------------------------- | ------- | ---------- | -| Environment variables | Session | Highest | -| `.libscope.json` (project root) | Project | Medium | -| `~/.libscope/config.json` | User | Low | -| Built-in defaults | Global | Lowest | +| Location | Holds | Precedence | +| ------------------------------- | --------------- | ---------- | +| Environment variables | Any key | Highest | +| `~/.libscope/secrets.json` | API keys only | | +| `.libscope.json` (project root) | Non-secret keys | | +| `~/.libscope/config.json` | Non-secret keys | | +| Built-in defaults | — | Lowest | + +API keys are read only from environment variables and `~/.libscope/secrets.json` (mode `0600`). An API key in `config.json` or `.libscope.json` is ignored, and LibScope logs a warning. + +A value in a config file that fails validation is ignored with a warning, and the next layer applies. An environment variable with an invalid value is also ignored with a warning. ## All Config Keys +Every key `section.field` has the environment variable `LIBSCOPE_<SECTION>_<FIELD>` (for example `embedding.model` → `LIBSCOPE_EMBEDDING_MODEL`). + ### Embedding -| Key | Type | Default | Description | -| ------------------------ | ------ | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `embedding.provider` | string | `"local"` | `local`, `ollama`, or `openai` | -| `embedding.ollamaUrl` | string | `"http://localhost:11434"` | Ollama server URL | -| `embedding.ollamaModel` | string | `"nomic-embed-text"` | Ollama embedding model | -| `embedding.openaiModel` | string | `"text-embedding-3-small"` | OpenAI embedding model | -| `embedding.openaiApiKey` | string | — | OpenAI API key (env vars take precedence, see [API keys](#api-keys)) | -| `embedding.dimensions` | number | — | Vector size of the Ollama or OpenAI model. Needed only for models whose size LibScope does not know. For `text-embedding-3-*`, it also requests shorter vectors. After you change it, run `libscope reindex --rebuild`. | +| Key | Type | Default | Env var | Description | +| ---------------------- | --------------------------- | ------------------------------------------------------------ | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `embedding.provider` | `local`, `ollama`, `openai` | `local` | `LIBSCOPE_EMBEDDING_PROVIDER` | Embedding provider. A config file can also name a provider registered in code with `registerProvider()`. | +| `embedding.model` | string | ollama: `nomic-embed-text`; openai: `text-embedding-3-small` | `LIBSCOPE_EMBEDDING_MODEL` | Embedding model. The local provider always uses `Xenova/all-MiniLM-L6-v2` and ignores this key. | +| `embedding.url` | string | `http://localhost:11434` | `LIBSCOPE_EMBEDDING_URL` | Server URL of the embedding provider. Used by the `ollama` provider. | +| `embedding.dimensions` | integer (1–10000) | size of the model | `LIBSCOPE_EMBEDDING_DIMENSIONS` | Vector size of the Ollama or OpenAI model. Needed only for models whose size LibScope does not know. For `text-embedding-3-*`, it also requests shorter vectors. After you change it, run `libscope reindex --rebuild`. | ### LLM (for RAG) -| Key | Type | Default | Description | -| --------------------- | ------ | ------- | -------------------------------------------------------------------------------- | -| `llm.provider` | string | — | `openai`, `ollama`, `anthropic`, or `passthrough` | -| `llm.model` | string | — | Model name override | -| `llm.ollamaUrl` | string | — | Ollama server URL (overrides embedding URL) | -| `llm.openaiApiKey` | string | — | OpenAI API key for the LLM (env vars take precedence, see [API keys](#api-keys)) | -| `llm.anthropicApiKey` | string | — | Anthropic API key (env vars take precedence, see [API keys](#api-keys)) | +| Key | Type | Default | Env var | Description | +| -------------- | ------------------------------------------------------ | --------------------------------------------------------------------------------- | ----------------------- | ----------------------------------------------------------------------- | +| `llm.provider` | `auto`, `openai`, `anthropic`, `ollama`, `passthrough` | `auto` | `LIBSCOPE_LLM_PROVIDER` | LLM for `ask`. See [Automatic LLM selection](#automatic-llm-selection). | +| `llm.model` | string | openai: `gpt-4o-mini`; anthropic: `claude-3-5-haiku-20241022`; ollama: `llama3.2` | `LIBSCOPE_LLM_MODEL` | LLM model. | +| `llm.url` | string | `embedding.url`, then `http://localhost:11434` | `LIBSCOPE_LLM_URL` | Server URL of the LLM. Used by the `ollama` provider. | + +### API keys (secrets) + +| Key | Type | Default | Env vars (first set wins) | Description | +| ------------------ | ------ | ------- | ------------------------------------------------- | ------------------------------------ | +| `openai.apiKey` | string | — | `LIBSCOPE_OPENAI_API_KEY`, `OPENAI_API_KEY` | OpenAI API key (embeddings and LLM). | +| `anthropic.apiKey` | string | — | `LIBSCOPE_ANTHROPIC_API_KEY`, `ANTHROPIC_API_KEY` | Anthropic API key (LLM). | + +Each API key is read in this order: `LIBSCOPE_<PROVIDER>_API_KEY`, then `<PROVIDER>_API_KEY`, then `~/.libscope/secrets.json`. `libscope config set openai.apiKey <key>` writes the key to `secrets.json`, never to `config.json`. `libscope config show` and `libscope config get` mask keys (for example `sk-…abcd`). ### Database -| Key | Type | Default | Description | -| --------------- | ------ | -------------------------------------------------------- | --------------------------------------------------------------------- | -| `database.path` | string | unset (`~/.libscope/workspaces/<workspace>/libscope.db`) | SQLite file to use instead of the workspace database. `~` is expanded | +| Key | Type | Default | Env var | Description | +| --------------- | ------ | -------------------------------------------------------- | ------------------------ | ---------------------------------------------------------------------- | +| `database.path` | string | unset (`~/.libscope/workspaces/<workspace>/libscope.db`) | `LIBSCOPE_DATABASE_PATH` | SQLite file to use instead of the workspace database. `~` is expanded. | ### Logging -| Key | Type | Default | Description | -| --------------- | ------ | -------- | --------------------------------------------- | -| `logging.level` | string | `"info"` | `debug`, `info`, `warn`, `error`, or `silent` | +| Key | Type | Default | Env var | Description | +| --------------- | ------------------------------------------ | ------- | ------------------------ | ----------- | +| `logging.level` | `debug`, `info`, `warn`, `error`, `silent` | `info` | `LIBSCOPE_LOGGING_LEVEL` | Log level. | ### Indexing -| Key | Type | Default | Description | -| ------------------------------- | ------- | ----------- | ------------------------------------------------- | -| `indexing.maxDocumentSize` | number | `104857600` | Maximum document size in bytes (100 MB) | -| `indexing.allowPrivateUrls` | boolean | `false` | Allow fetching from private/internal IP addresses | -| `indexing.allowSelfSignedCerts` | boolean | `false` | Accept self-signed or untrusted TLS certificates | - -## Environment Variables - -| Variable | Maps to | Default | -| ---------------------------------- | ------------------------------------------------------------------------ | ------------------------ | -| `LIBSCOPE_EMBEDDING_PROVIDER` | `embedding.provider` | `local` | -| `LIBSCOPE_OPENAI_API_KEY` | `embedding.openaiApiKey`, `llm.openaiApiKey` | — | -| `OPENAI_API_KEY` | Same as above, used when `LIBSCOPE_OPENAI_API_KEY` is not set | — | -| `LIBSCOPE_OLLAMA_URL` | `embedding.ollamaUrl` | `http://localhost:11434` | -| `LIBSCOPE_OLLAMA_MODEL` | `embedding.ollamaModel` | `nomic-embed-text` | -| `LIBSCOPE_LLM_PROVIDER` | `llm.provider` | — | -| `LIBSCOPE_LLM_MODEL` | `llm.model` | — | -| `LIBSCOPE_ANTHROPIC_API_KEY` | `llm.anthropicApiKey` | — | -| `ANTHROPIC_API_KEY` | Same as above, used when `LIBSCOPE_ANTHROPIC_API_KEY` is not set | — | -| `LIBSCOPE_ALLOW_PRIVATE_URLS` | `indexing.allowPrivateUrls` | `false` | -| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | `indexing.allowSelfSignedCerts` | `false` | -| `LIBSCOPE_WORKSPACE` | Active workspace (overrides `libscope workspace use`) | `default` | -| `LIBSCOPE_API_KEY` | REST API key. When set, requests must send `Authorization: Bearer <key>` | — | -| `LIBSCOPE_SECRET_KEY` | Key used to encrypt stored webhook secrets | — | -| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (to stderr) in the CLI | — | -| `ONENOTE_CLIENT_ID` | OneNote app client ID | — | -| `ONENOTE_TENANT_ID` | OneNote tenant ID | `common` | -| `NOTION_TOKEN` | Notion integration token | — | -| `CONFLUENCE_URL` | Confluence base URL | — | -| `CONFLUENCE_EMAIL` | Confluence user email | — | -| `CONFLUENCE_TOKEN` | Confluence API token | — | - -### API keys - -OpenAI and Anthropic keys use the same rule for embeddings and for the LLM: - -1. `LIBSCOPE_OPENAI_API_KEY` / `LIBSCOPE_ANTHROPIC_API_KEY` -2. `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` -3. The key in a config file (`embedding.openaiApiKey`, `llm.openaiApiKey`, `llm.anthropicApiKey`) - -`libscope config show` masks keys (for example `sk-…abcd`). +| Key | Type | Default | Env var | Description | +| ------------------------------- | ------- | ----------- | ------------------------------------------- | ----------------------------------------------------- | +| `indexing.maxDocumentSize` | integer | `104857600` | `LIBSCOPE_INDEXING_MAX_DOCUMENT_SIZE` | Largest document to index, in bytes (100 MB). | +| `indexing.allowPrivateUrls` | boolean | `false` | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS` | Allow fetching from private or internal IP addresses. | +| `indexing.allowSelfSignedCerts` | boolean | `false` | `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS` | Accept self-signed or untrusted TLS certificates. | + +Booleans accept `true`, `false`, `1`, and `0` in environment variables and in `config set`. + +## Automatic LLM selection + +When `llm.provider` is `auto` (the default), LibScope selects the LLM when `ask` runs. It does not make network calls to decide: + +1. Under the MCP server: `passthrough`. The calling assistant writes the answer from the retrieved context. +2. Otherwise, `openai` if an OpenAI key is set. +3. Otherwise, `anthropic` if an Anthropic key is set. +4. Otherwise, `ollama` if `llm.url` is set or `embedding.provider` is `ollama`. +5. Otherwise there is no LLM. `ask` fails with a message that tells you which key or setting to add. + +Set `llm.provider` to a specific provider to skip this selection. + +## Other Environment Variables + +These variables are not config keys: + +| Variable | Description | Default | +| --------------------- | ------------------------------------------------------------------------ | --------- | +| `LIBSCOPE_WORKSPACE` | Active workspace (overrides `libscope workspace use`) | `default` | +| `LIBSCOPE_API_KEY` | REST API key. When set, requests must send `Authorization: Bearer <key>` | — | +| `LIBSCOPE_SECRET_KEY` | Key used to encrypt stored webhook secrets | — | +| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (to stderr) in the CLI | — | +| `ONENOTE_CLIENT_ID` | OneNote app client ID | — | +| `ONENOTE_TENANT_ID` | OneNote tenant ID | `common` | +| `NOTION_TOKEN` | Notion integration token | — | +| `CONFLUENCE_URL` | Confluence base URL | — | +| `CONFLUENCE_EMAIL` | Confluence user email | — | +| `CONFLUENCE_TOKEN` | Confluence API token | — | LibScope does not load `.env` files. Export variables in your shell, or set them in the `env` block of your MCP client config. @@ -97,14 +104,17 @@ LibScope does not load `.env` files. Export variables in your shell, or set them ```bash # Embedding libscope config set embedding.provider ollama # local | ollama | openai -libscope config set embedding.ollamaUrl http://localhost:11434 -libscope config set embedding.ollamaModel nomic-embed-text -libscope config set embedding.openaiModel text-embedding-3-small +libscope config set embedding.url http://localhost:11434 +libscope config set embedding.model nomic-embed-text # LLM (for RAG) -libscope config set llm.provider openai # openai | ollama | anthropic | passthrough +libscope config set llm.provider openai # auto | openai | anthropic | ollama | passthrough libscope config set llm.model gpt-4o-mini +# API keys (written to ~/.libscope/secrets.json) +libscope config set openai.apiKey sk-... +libscope config set anthropic.apiKey sk-ant-... + # Database libscope config set database.path ~/kb/libscope.db # overrides the workspace database libscope config unset database.path # back to the workspace database @@ -121,16 +131,14 @@ libscope config set indexing.maxDocumentSize 52428800 # Read, remove, and locate libscope config get embedding.provider # effective value (API keys masked) -libscope config unset llm.model # remove from the user config file +libscope config unset llm.model # remove from the file that holds the key libscope config path # print the user config file path # View current config (API keys masked) libscope config show ``` -`config set` accepts every key listed in [All Config Keys](#all-config-keys) except API keys. It checks the value type (booleans accept `true`/`false`/`1`/`0`, numbers must be positive integers, and enum keys must use a listed value). It writes only the key you set to `~/.libscope/config.json` and keeps all other content of that file, including `registries`. The file is written with mode `0600`. - -API keys (`embedding.openaiApiKey`, `llm.openaiApiKey`, `llm.anthropicApiKey`) are not written by `config set`. Use the environment variables in [API keys](#api-keys). +`config set` accepts every key in [All Config Keys](#all-config-keys). It checks the value type (booleans accept `true`/`false`/`1`/`0`, integers must be positive, and enum keys must use a listed value). It writes only the key you set and keeps all other content of the file, including `registries`. API keys go to `~/.libscope/secrets.json`. All other keys go to `~/.libscope/config.json`. Both files are written with mode `0600`. ## Corporate / Internal Networks @@ -150,8 +158,8 @@ libscope config set indexing.allowSelfSignedCerts true { "embedding": { "provider": "ollama", - "ollamaUrl": "http://localhost:11434", - "ollamaModel": "nomic-embed-text" + "url": "http://localhost:11434", + "model": "nomic-embed-text" }, "llm": { "provider": "openai", From 4f3b8ed584cd27966090774c85c4157dddb05215 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:21:50 +0000 Subject: [PATCH 29/83] feat(errors): add NotFoundError base for every not-found error DocumentNotFoundError, ChunkNotFoundError and TopicNotFoundError now extend NotFoundError (codes unchanged). Missing links, webhooks and saved searches throw NotFoundError instead of ValidationError/DocumentNotFoundError, and getRelatedChunks throws ChunkNotFoundError. REST maps NotFoundError to 404. --- src/api/routes.ts | 4 ++-- src/core/links.ts | 4 ++-- src/core/saved-searches.ts | 6 +++--- src/core/search.ts | 3 ++- src/core/webhooks.ts | 6 +++--- src/errors.ts | 17 ++++++++++++++--- tests/unit/errors.test.ts | 19 +++++++++++++++++++ tests/unit/saved-searches.test.ts | 10 ++++------ tests/unit/webhooks.test.ts | 14 +++++++------- 9 files changed, 56 insertions(+), 27 deletions(-) diff --git a/src/api/routes.ts b/src/api/routes.ts index e2f53cd2..1a6f5690 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -38,7 +38,7 @@ import { import type { LinkType, BulkSelector, BatchSearchRequest } from "../core/index.js"; import { LINK_TYPES } from "../core/links.js"; import { loadConfig } from "../config.js"; -import { DocumentNotFoundError, FetchError, LibScopeError } from "../errors.js"; +import { NotFoundError, FetchError, LibScopeError } from "../errors.js"; import { getLogger } from "../logger.js"; import { parseJsonBody, sendJson, sendError } from "./middleware.js"; import { OPENAPI_SPEC } from "./openapi.js"; @@ -861,7 +861,7 @@ function handleRouteError( const log = getLogger(); log.error({ err, method, pathname }, "API request error"); - if (err instanceof DocumentNotFoundError) { + if (err instanceof NotFoundError) { sendError(res, 404, "NOT_FOUND", err.message); return; } diff --git a/src/core/links.ts b/src/core/links.ts index 6814edfd..79b2cff6 100644 --- a/src/core/links.ts +++ b/src/core/links.ts @@ -1,6 +1,6 @@ import type Database from "better-sqlite3"; import { randomUUID } from "node:crypto"; -import { ValidationError, DocumentNotFoundError } from "../errors.js"; +import { ValidationError, DocumentNotFoundError, NotFoundError } from "../errors.js"; import { createChildLogger } from "../logger.js"; import { extractMarkdownLinks, extractWikilinks } from "./link-extractor.js"; @@ -170,7 +170,7 @@ export function deleteLink(db: Database.Database, linkId: string): void { const log = createChildLogger({ operation: "deleteLink" }); const result = db.prepare("DELETE FROM document_links WHERE id = ?").run(linkId); if (result.changes === 0) { - throw new ValidationError(`Link not found: ${linkId}`); + throw new NotFoundError(`Link not found: ${linkId}`, "LINK_NOT_FOUND"); } log.info({ linkId }, "Document link deleted"); } diff --git a/src/core/saved-searches.ts b/src/core/saved-searches.ts index 42b8ddfa..3b53f625 100644 --- a/src/core/saved-searches.ts +++ b/src/core/saved-searches.ts @@ -3,7 +3,7 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { searchDocuments } from "./search.js"; import type { SearchOptions, SearchResult } from "./search.js"; -import { ValidationError, DocumentNotFoundError } from "../errors.js"; +import { ValidationError, NotFoundError } from "../errors.js"; import { getLogger } from "../logger.js"; export interface SavedSearch { @@ -101,7 +101,7 @@ export function getSavedSearch(db: Database.Database, id: string): SavedSearch { | SavedSearchRow | undefined; if (!row) { - throw new DocumentNotFoundError(id); + throw new NotFoundError(`Saved search not found: ${id}`, "SAVED_SEARCH_NOT_FOUND"); } return rowToSavedSearch(row); } @@ -109,7 +109,7 @@ export function getSavedSearch(db: Database.Database, id: string): SavedSearch { export function deleteSavedSearch(db: Database.Database, id: string): void { const result = db.prepare("DELETE FROM saved_searches WHERE id = ? OR name = ?").run(id, id); if (result.changes === 0) { - throw new DocumentNotFoundError(id); + throw new NotFoundError(`Saved search not found: ${id}`, "SAVED_SEARCH_NOT_FOUND"); } } diff --git a/src/core/search.ts b/src/core/search.ts index 84295c9d..76add8ef 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -8,6 +8,7 @@ import { REBUILD_VECTOR_INDEX_HINT, isVectorDimensionError } from "../db/index-m import { logSearch, recordSearchQuery } from "./analytics.js"; import { emitEvent } from "./events.js"; import { performance } from "node:perf_hooks"; +import { ChunkNotFoundError } from "../errors.js"; /** Build SQL clause and params for AND-logic tag filtering on a document alias. */ function buildTagFilter( @@ -871,7 +872,7 @@ export function getRelatedChunks( "getRelatedChunks.sourceChunk", ); if (!sourceChunkRow) { - throw new Error(`Chunk not found: ${chunkId}`); + throw new ChunkNotFoundError(chunkId); } const sourceChunk = { diff --git a/src/core/webhooks.ts b/src/core/webhooks.ts index 2e951989..29fc1e7a 100644 --- a/src/core/webhooks.ts +++ b/src/core/webhooks.ts @@ -9,7 +9,7 @@ import { import { promises as dns, lookup as dnsLookup } from "node:dns"; import { promisify } from "node:util"; import type Database from "better-sqlite3"; -import { ValidationError } from "../errors.js"; +import { NotFoundError, ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; import { isPrivateIP } from "./url-fetcher.js"; import { LIBSCOPE_EVENTS, onEvent, type LibScopeEvent } from "./events.js"; @@ -245,7 +245,7 @@ export function listWebhooks(db: Database.Database, limit?: number, offset?: num export function getWebhook(db: Database.Database, id: string): Webhook { const row = db.prepare("SELECT * FROM webhooks WHERE id = ?").get(id) as WebhookRow | undefined; if (!row) { - throw new ValidationError(`Webhook not found: ${id}`); + throw new NotFoundError(`Webhook not found: ${id}`, "WEBHOOK_NOT_FOUND"); } return rowToWebhook(row); } @@ -253,7 +253,7 @@ export function getWebhook(db: Database.Database, id: string): Webhook { export function deleteWebhook(db: Database.Database, id: string): void { const result = db.prepare("DELETE FROM webhooks WHERE id = ?").run(id); if (result.changes === 0) { - throw new ValidationError(`Webhook not found: ${id}`); + throw new NotFoundError(`Webhook not found: ${id}`, "WEBHOOK_NOT_FOUND"); } } diff --git a/src/errors.ts b/src/errors.ts index bd141127..4d91985b 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -49,21 +49,32 @@ export class ConfigError extends LibScopeError { } } -export class DocumentNotFoundError extends LibScopeError { +/** + * A requested resource does not exist. REST maps every NotFoundError to HTTP 404. + * Subclasses keep their own `code`; other resources use this class directly with a specific code. + */ +export class NotFoundError extends LibScopeError { + constructor(message: string, code: string = "NOT_FOUND", cause?: unknown) { + super(message, code, cause); + this.name = "NotFoundError"; + } +} + +export class DocumentNotFoundError extends NotFoundError { constructor(documentId: string) { super(`Document not found: ${documentId}`, "DOCUMENT_NOT_FOUND"); this.name = "DocumentNotFoundError"; } } -export class ChunkNotFoundError extends LibScopeError { +export class ChunkNotFoundError extends NotFoundError { constructor(chunkId: string) { super(`Chunk not found: ${chunkId}`, "CHUNK_NOT_FOUND"); this.name = "ChunkNotFoundError"; } } -export class TopicNotFoundError extends LibScopeError { +export class TopicNotFoundError extends NotFoundError { constructor(topicId: string) { super(`Topic not found: ${topicId}`, "TOPIC_NOT_FOUND"); this.name = "TopicNotFoundError"; diff --git a/tests/unit/errors.test.ts b/tests/unit/errors.test.ts index 3170ec72..dced1a45 100644 --- a/tests/unit/errors.test.ts +++ b/tests/unit/errors.test.ts @@ -8,8 +8,27 @@ import { ConfigError, DocumentNotFoundError, ChunkNotFoundError, + TopicNotFoundError, + NotFoundError, } from "../../src/errors.js"; +describe("NotFoundError", () => { + it("is the base class of every not-found error and keeps each code", () => { + const cases: Array<[NotFoundError, string]> = [ + [new DocumentNotFoundError("d"), "DOCUMENT_NOT_FOUND"], + [new ChunkNotFoundError("c"), "CHUNK_NOT_FOUND"], + [new TopicNotFoundError("t"), "TOPIC_NOT_FOUND"], + [new NotFoundError("Link not found: l", "LINK_NOT_FOUND"), "LINK_NOT_FOUND"], + [new NotFoundError("gone"), "NOT_FOUND"], + ]; + for (const [err, code] of cases) { + expect(err).toBeInstanceOf(NotFoundError); + expect(err).toBeInstanceOf(LibScopeError); + expect(err.code).toBe(code); + } + }); +}); + describe("errors", () => { it("should create LibScopeError with code", () => { const err = new LibScopeError("test", "TEST_ERROR"); diff --git a/tests/unit/saved-searches.test.ts b/tests/unit/saved-searches.test.ts index b2eb001f..db01fbf0 100644 --- a/tests/unit/saved-searches.test.ts +++ b/tests/unit/saved-searches.test.ts @@ -9,7 +9,7 @@ import { runSavedSearch, } from "../../src/core/saved-searches.js"; import { indexDocument } from "../../src/core/indexing.js"; -import { ValidationError, DocumentNotFoundError } from "../../src/errors.js"; +import { ValidationError, NotFoundError } from "../../src/errors.js"; import type Database from "better-sqlite3"; import { initLogger } from "../../src/logger.js"; @@ -90,7 +90,7 @@ describe("saved-searches", () => { }); it("should throw for nonexistent search", () => { - expect(() => getSavedSearch(db, "nonexistent")).toThrow(DocumentNotFoundError); + expect(() => getSavedSearch(db, "nonexistent")).toThrow(NotFoundError); }); }); @@ -108,7 +108,7 @@ describe("saved-searches", () => { }); it("should throw for nonexistent search", () => { - expect(() => deleteSavedSearch(db, "nonexistent")).toThrow(DocumentNotFoundError); + expect(() => deleteSavedSearch(db, "nonexistent")).toThrow(NotFoundError); }); }); @@ -168,9 +168,7 @@ describe("saved-searches", () => { }); it("should throw for nonexistent search", async () => { - await expect(runSavedSearch(db, provider, "nonexistent")).rejects.toThrow( - DocumentNotFoundError, - ); + await expect(runSavedSearch(db, provider, "nonexistent")).rejects.toThrow(NotFoundError); }); }); diff --git a/tests/unit/webhooks.test.ts b/tests/unit/webhooks.test.ts index 1c371678..6e92d9c8 100644 --- a/tests/unit/webhooks.test.ts +++ b/tests/unit/webhooks.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, vi, afterEach } from "vitest"; import type Database from "better-sqlite3"; import { createTestDb } from "../fixtures/test-db.js"; import type { WebhookEvent } from "../../src/core/webhooks.js"; -import { ValidationError } from "../../src/errors.js"; +import { NotFoundError, ValidationError } from "../../src/errors.js"; import { initLogger } from "../../src/logger.js"; // Mock DNS resolution so webhook SSRF checks pass in tests @@ -139,8 +139,8 @@ describe("webhooks", () => { expect(fetched.url).toBe(created.url); }); - it("should throw ValidationError for non-existent id", () => { - expect(() => getWebhook(db, "non-existent")).toThrow(ValidationError); + it("should throw NotFoundError for non-existent id", () => { + expect(() => getWebhook(db, "non-existent")).toThrow(NotFoundError); }); }); @@ -151,8 +151,8 @@ describe("webhooks", () => { expect(listWebhooks(db)).toHaveLength(0); }); - it("should throw ValidationError for non-existent id", () => { - expect(() => deleteWebhook(db, "non-existent")).toThrow(ValidationError); + it("should throw NotFoundError for non-existent id", () => { + expect(() => deleteWebhook(db, "non-existent")).toThrow(NotFoundError); }); }); @@ -215,9 +215,9 @@ describe("webhooks", () => { ).rejects.toThrow(ValidationError); }); - it("should throw ValidationError for non-existent id", async () => { + it("should throw NotFoundError for non-existent id", async () => { await expect(updateWebhook(db, "non-existent", { url: "https://new.com" })).rejects.toThrow( - ValidationError, + NotFoundError, ); }); }); From 11a78246394c57a0c40d0235e804c64484e9d004 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:21:55 +0000 Subject: [PATCH 30/83] refactor(core): move the task registry from mcp to core src/mcp/tasks.ts becomes src/core/tasks.ts so MCP, REST and operations share one registry. Adds TaskRegistry.list() and an "operation" task type that records the operation name. --- src/{mcp => core}/tasks.ts | 29 ++++++++++++++++++++++++----- src/mcp/server.ts | 4 ++-- tests/unit/tasks.test.ts | 6 +++--- 3 files changed, 29 insertions(+), 10 deletions(-) rename src/{mcp => core}/tasks.ts (80%) diff --git a/src/mcp/tasks.ts b/src/core/tasks.ts similarity index 80% rename from src/mcp/tasks.ts rename to src/core/tasks.ts index 4c4e365c..aeaf078c 100644 --- a/src/mcp/tasks.ts +++ b/src/core/tasks.ts @@ -1,7 +1,17 @@ +/** + * In-process registry of background tasks (long-running work started by MCP tools, REST + * routes or operations). Tasks are kept for one hour after they finish. + */ import { randomUUID } from "node:crypto"; export type TaskStatus = "pending" | "running" | "completed" | "failed" | "cancelled"; -export type TaskType = "index_document" | "reindex_documents" | "sync_connector" | "install_pack"; +export type TaskType = + | "index_document" + | "reindex_documents" + | "sync_connector" + | "install_pack" + /** Any operation from src/core/operations run in the background; see Task.operation. */ + | "operation"; export interface TaskProgress { current: number; @@ -12,6 +22,8 @@ export interface TaskProgress { export interface Task { id: string; type: TaskType; + /** Operation name when `type` is "operation". */ + operation?: string | undefined; status: TaskStatus; progress?: TaskProgress | undefined; result?: string | undefined; @@ -41,11 +53,12 @@ export class TaskRegistry { private readonly controllers = new Map<string, AbortController>(); /** Create a new task and return it along with its AbortSignal. */ - create(type: TaskType): { task: Task; signal: AbortSignal } { + create(type: TaskType, operation?: string): { task: Task; signal: AbortSignal } { const id = randomUUID(); const task: Task = { id, type, + ...(operation === undefined ? {} : { operation }), status: "pending", createdAt: new Date(), }; @@ -61,8 +74,8 @@ export class TaskRegistry { * to stop it), "cancelled" if it rejects because the task signal was aborted, else "failed". * `done` settles after the final status is recorded; it never rejects. */ - run(type: TaskType, work: TaskWork): { task: Task; done: Promise<void> } { - const { task, signal } = this.create(type); + run(type: TaskType, work: TaskWork, operation?: string): { task: Task; done: Promise<void> } { + const { task, signal } = this.create(type, operation); this.update(task.id, { status: "running", startedAt: new Date() }); const onProgress = (current: number, total: number): void => { this.update(task.id, { progress: { current, total } }); @@ -95,6 +108,12 @@ export class TaskRegistry { return this.tasks.get(id); } + /** All tasks that have not expired, newest first. */ + list(): Task[] { + this.prune(); + return [...this.tasks.values()].sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()); + } + /** Apply partial updates to a task. No-op if task not found. */ update(id: string, updates: Partial<Task>): void { const task = this.tasks.get(id); @@ -138,5 +157,5 @@ export class TaskRegistry { } } -/** Module-level singleton task registry used by the MCP server. */ +/** Module-level singleton task registry shared by the MCP server, REST API and operations. */ export const taskRegistry = new TaskRegistry(); diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 65c7a4f5..120ce102 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -35,8 +35,8 @@ import { ConfigError, ValidationError } from "../errors.js"; import { errorResponse, textResult, withErrorHandling, type ToolResult } from "./errors.js"; export { errorResponse, withErrorHandling, type ToolResult } from "./errors.js"; import { formatDocumentLinks, formatLinkCreated, formatSearchResults } from "./format.js"; -import { taskRegistry } from "./tasks.js"; -import type { TaskType, TaskWork } from "./tasks.js"; +import { taskRegistry } from "../core/tasks.js"; +import type { TaskType, TaskWork } from "../core/tasks.js"; import type { NotionConfig } from "../connectors/notion.js"; import type { SlackConfig } from "../connectors/slack.js"; import type { ConfluenceConfig } from "../connectors/confluence.js"; diff --git a/tests/unit/tasks.test.ts b/tests/unit/tasks.test.ts index c126db56..d200d4af 100644 --- a/tests/unit/tasks.test.ts +++ b/tests/unit/tasks.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; -import { TaskRegistry, isAbortRejection } from "../../src/mcp/tasks.js"; -import type { TaskType } from "../../src/mcp/tasks.js"; +import { TaskRegistry, isAbortRejection } from "../../src/core/tasks.js"; +import type { TaskType } from "../../src/core/tasks.js"; import { reindex } from "../../src/core/reindex.js"; import { initLogger } from "../../src/logger.js"; import { createTestDbWithVec } from "../fixtures/test-db.js"; @@ -341,7 +341,7 @@ describe("TaskRegistry", () => { describe("taskRegistry singleton", () => { it("exports a module-level TaskRegistry instance", async () => { - const { taskRegistry } = await import("../../src/mcp/tasks.js"); + const { taskRegistry } = await import("../../src/core/tasks.js"); expect(taskRegistry).toBeInstanceOf(TaskRegistry); }); }); From 846ace8cd27d6b94da7adcb4fd709dce0bbc7116 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:44:40 +0000 Subject: [PATCH 31/83] refactor(core): shared helpers for the operation layer - listDocuments takes offset, version and tags filters; countDocuments pages it - removeTagsFromDocument(db, id, names) replaces removeTagFromDocument(db, id, tagId); bulk retag and the CLI use it instead of their own name lookups - findTopicId/resolveTopicId accept a topic ID or name - firstChunkId finds content related to a whole document - getOverview: stats, topics, packs, index model and health in one call - getDocumentView: document with tags, links, ratings and paged content - missing packs and document versions throw NotFoundError --- src/cli/index.ts | 10 ++-- src/core/bulk.ts | 16 +----- src/core/document-view.ts | 55 ++++++++++++++++++++ src/core/documents.ts | 104 ++++++++++++++++++++++++------------- src/core/index.ts | 2 +- src/core/overview.ts | 70 +++++++++++++++++++++++++ src/core/packs.ts | 4 +- src/core/saved-searches.ts | 4 ++ src/core/search.ts | 21 +++++++- src/core/tags.ts | 28 ++++++---- src/core/topics.ts | 15 ++++++ src/core/versioning.ts | 7 ++- tests/unit/tags.test.ts | 11 ++-- 13 files changed, 268 insertions(+), 79 deletions(-) create mode 100644 src/core/document-view.ts create mode 100644 src/core/overview.ts diff --git a/src/cli/index.ts b/src/cli/index.ts index 767f0273..26a37f14 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -51,7 +51,7 @@ import { confirmOrCancel } from "./confirm.js"; import { createReporter, isVerbose } from "./reporter.js"; import { addTagsToDocument, - removeTagFromDocument, + removeTagsFromDocument, listTags, getDocumentTags, suggestTags, @@ -1408,14 +1408,12 @@ tagCmd .action((docId: string, tagName: string) => { const { db } = initializeApp(); try { - const docTags = getDocumentTags(db, docId); - const tag = docTags.find((t) => t.name === tagName.trim().toLowerCase()); - if (!tag) { + const [removed] = removeTagsFromDocument(db, docId, [tagName]); + if (!removed) { console.log(`Tag "${tagName}" not found on document ${docId}`); return; } - removeTagFromDocument(db, docId, tag.id); - console.log(`✓ Removed tag "${tag.name}" from document ${docId}`); + console.log(`✓ Removed tag "${removed}" from document ${docId}`); } finally { closeDatabase(); } diff --git a/src/core/bulk.ts b/src/core/bulk.ts index d00b15e8..a2a97a5c 100644 --- a/src/core/bulk.ts +++ b/src/core/bulk.ts @@ -2,12 +2,7 @@ import type Database from "better-sqlite3"; import { getLogger } from "../logger.js"; import { ValidationError } from "../errors.js"; import { deleteDocument, listDocuments } from "./documents.js"; -import { - addTagsToDocument, - removeTagFromDocument, - getDocumentTags, - getDocumentTagsBatch, -} from "./tags.js"; +import { addTagsToDocument, removeTagsFromDocument, getDocumentTagsBatch } from "./tags.js"; export interface BulkSelector { topicId?: string; @@ -132,14 +127,7 @@ export function bulkRetag( addTagsToDocument(db, id, addTags); } if (removeTags && removeTags.length > 0) { - const docTags = getDocumentTags(db, id); - for (const tagName of removeTags) { - const normalized = tagName.trim().toLowerCase(); - const tag = docTags.find((t) => t.name === normalized); - if (tag) { - removeTagFromDocument(db, id, tag.id); - } - } + removeTagsFromDocument(db, id, removeTags); } } }); diff --git a/src/core/document-view.ts b/src/core/document-view.ts new file mode 100644 index 00000000..64dfc27a --- /dev/null +++ b/src/core/document-view.ts @@ -0,0 +1,55 @@ +import type Database from "better-sqlite3"; +import { getDocument, type Document } from "./documents.js"; +import { getDocumentLinks, type DocumentLinks } from "./links.js"; +import { getDocumentRatings, type RatingSummary } from "./ratings.js"; +import { getDocumentTags } from "./tags.js"; + +export interface DocumentView { + /** The document without its content (see `content`). */ + document: Omit<Document, "content" | "id"> & { documentId: string }; + /** The requested slice of the content. */ + content: string; + /** Length of the full content in characters. */ + contentLength: number; + /** Start of this slice. */ + offset: number; + /** Offset of the next slice, or null when this slice reaches the end. */ + nextOffset: number | null; + tags: string[]; + links: DocumentLinks; + ratings: RatingSummary; +} + +export interface DocumentViewOptions { + /** Character offset to start from (default 0). */ + offset?: number | undefined; + /** Maximum characters of content to return (default: all). */ + maxLength?: number | undefined; +} + +/** + * A document with its tags, links and rating summary, and its content optionally paged + * by character offset. Throws DocumentNotFoundError. + */ +export function getDocumentView( + db: Database.Database, + documentId: string, + options: DocumentViewOptions = {}, +): DocumentView { + const { id, content: full, ...rest } = getDocument(db, documentId); + const offset = Math.min(Math.max(0, options.offset ?? 0), full.length); + const end = + options.maxLength === undefined + ? full.length + : Math.min(full.length, offset + options.maxLength); + return { + document: { documentId: id, ...rest }, + content: full.slice(offset, end), + contentLength: full.length, + offset, + nextOffset: end < full.length ? end : null, + tags: getDocumentTags(db, documentId).map((t) => t.name), + links: getDocumentLinks(db, documentId), + ratings: getDocumentRatings(db, documentId), + }; +} diff --git a/src/core/documents.ts b/src/core/documents.ts index 3bdf674b..6c312134 100644 --- a/src/core/documents.ts +++ b/src/core/documents.ts @@ -13,6 +13,7 @@ import { import { getLogger } from "../logger.js"; import { saveVersion } from "./versioning.js"; import { emitEvent } from "./events.js"; +import { buildTagFilter } from "./search.js"; export interface Document { id: string; @@ -117,50 +118,79 @@ export function deleteDocument(db: Database.Database, documentId: string): void emitEvent(db, "document.deleted", { documentId }); } -/** List documents with optional filters. */ -export function listDocuments( - db: Database.Database, - options?: { - library?: string | undefined; - topicId?: string | undefined; - sourceType?: string | undefined; - dateFrom?: string | undefined; - dateTo?: string | undefined; - limit?: number | undefined; - }, -): Document[] { - let sql = `SELECT ${DOC_COLUMNS} FROM documents WHERE 1=1`; - const params: unknown[] = []; - - if (options?.library) { - sql += " AND library = ?"; - params.push(options.library); - } - if (options?.topicId) { - sql += " AND topic_id = ?"; - params.push(options.topicId); - } - if (options?.sourceType) { - sql += " AND source_type = ?"; - params.push(options.sourceType); - } - if (options?.dateFrom) { - sql += " AND created_at >= ?"; - params.push(options.dateFrom); - } - if (options?.dateTo) { - sql += " AND created_at <= ?"; - params.push(options.dateTo); +/** Throw DocumentNotFoundError unless a document with this ID exists. */ +export function assertDocumentExists(db: Database.Database, documentId: string): void { + if (!db.prepare("SELECT 1 FROM documents WHERE id = ?").get(documentId)) { + throw new DocumentNotFoundError(documentId); } +} - sql += " ORDER BY updated_at DESC LIMIT ?"; - params.push(options?.limit ?? 50); +export interface DocumentFilters { + library?: string | undefined; + version?: string | undefined; + topicId?: string | undefined; + sourceType?: string | undefined; + /** Only documents carrying all of these tags. */ + tags?: string[] | undefined; + dateFrom?: string | undefined; + dateTo?: string | undefined; +} - const rows = db.prepare(sql).all(...params) as DocumentRow[]; +export interface ListDocumentsOptions extends DocumentFilters { + /** Default 50. */ + limit?: number | undefined; + offset?: number | undefined; +} + +/** WHERE clause (starting with "WHERE 1=1") and params for DocumentFilters on `documents d`. */ +function documentFilterSql(filters: DocumentFilters | undefined): { + where: string; + params: unknown[]; +} { + let where = "WHERE 1=1"; + const params: unknown[] = []; + const equals: Array<[string, string | undefined]> = [ + ["d.library = ?", filters?.library], + ["d.version = ?", filters?.version], + ["d.topic_id = ?", filters?.topicId], + ["d.source_type = ?", filters?.sourceType], + ["d.created_at >= ?", filters?.dateFrom], + ["d.created_at <= ?", filters?.dateTo], + ]; + for (const [clause, value] of equals) { + if (value) { + where += ` AND ${clause}`; + params.push(value); + } + } + const tagFilter = buildTagFilter(filters?.tags, "d"); + return { where: where + tagFilter.clause, params: [...params, ...tagFilter.params] }; +} + +/** List documents with optional filters, most recently updated first. */ +export function listDocuments(db: Database.Database, options?: ListDocumentsOptions): Document[] { + const { where, params } = documentFilterSql(options); + const columns = DOC_COLUMNS.split(", ") + .map((c) => `d.${c}`) + .join(", "); + const rows = db + .prepare( + `SELECT ${columns} FROM documents d ${where} ORDER BY d.updated_at DESC LIMIT ? OFFSET ?`, + ) + .all(...params, options?.limit ?? 50, options?.offset ?? 0) as DocumentRow[]; return rows.map((row) => rowToDocument(row)); } +/** Number of documents matching the filters (for paging listDocuments). */ +export function countDocuments(db: Database.Database, filters?: DocumentFilters): number { + const { where, params } = documentFilterSql(filters); + const row = db.prepare(`SELECT COUNT(*) AS n FROM documents d ${where}`).get(...params) as { + n: number; + }; + return row.n; +} + export interface UpdateDocumentInput { title?: string | undefined; content?: string | undefined; diff --git a/src/core/index.ts b/src/core/index.ts index 1e956129..e80d2018 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -89,7 +89,7 @@ export { deleteTag, listTags, addTagsToDocument, - removeTagFromDocument, + removeTagsFromDocument, getDocumentTags, getDocumentsByTag, suggestTags, diff --git a/src/core/overview.ts b/src/core/overview.ts new file mode 100644 index 00000000..352d2705 --- /dev/null +++ b/src/core/overview.ts @@ -0,0 +1,70 @@ +import type Database from "better-sqlite3"; +import type { EmbeddingProvider } from "../providers/embedding.js"; +import { + getVectorTableDimensions, + readEmbeddingIdentity, + toEmbeddingIdentity, + type EmbeddingIndexIdentity, +} from "../db/index-meta.js"; +import { getLogger } from "../logger.js"; +import { getStats, type OverviewStats } from "./analytics.js"; +import { listInstalledPacks, type InstalledPack } from "./packs.js"; +import { getTopicStats, type TopicStats } from "./topics.js"; + +export type HealthStatus = "ok" | "error"; + +export interface Overview { + stats: OverviewStats; + /** Every topic with its document count. */ + topics: TopicStats[]; + packs: InstalledPack[]; + index: { + /** Provider/model/vector size recorded for the vector index (empty when none). */ + stored: EmbeddingIndexIdentity; + /** The configured embedding provider, when one was given. */ + configured?: EmbeddingIndexIdentity | undefined; + /** Vector size of chunk_embeddings; undefined when the table does not exist. */ + vectorTableDimensions?: number | undefined; + }; + health: { + database: HealthStatus; + /** Full-text (keyword) index. */ + fts: HealthStatus; + /** True when the vector table exists, so semantic search can run. */ + vectorSearch: boolean; + }; +} + +function check(name: string, fn: () => unknown): HealthStatus { + try { + fn(); + return "ok"; + } catch (err) { + getLogger().warn({ err }, `Health check failed: ${name}`); + return "error"; + } +} + +/** + * Everything a user or assistant needs to orient themselves, computed in one place: + * counts, topics, installed packs, which embedding model built the index, and health. + * Replaces the separate stats / health-check / list-topics / list-packs views. + */ +export function getOverview(db: Database.Database, provider?: EmbeddingProvider): Overview { + const vectorTableDimensions = getVectorTableDimensions(db); + return { + stats: getStats(db), + topics: getTopicStats(db), + packs: listInstalledPacks(db), + index: { + stored: readEmbeddingIdentity(db), + ...(provider ? { configured: toEmbeddingIdentity(provider) } : {}), + ...(vectorTableDimensions === undefined ? {} : { vectorTableDimensions }), + }, + health: { + database: check("database", () => db.prepare("SELECT 1").get()), + fts: check("fts", () => db.prepare("SELECT COUNT(*) FROM chunks_fts").get()), + vectorSearch: vectorTableDimensions !== undefined, + }, + }; +} diff --git a/src/core/packs.ts b/src/core/packs.ts index edf3fb41..3c5f5dc9 100644 --- a/src/core/packs.ts +++ b/src/core/packs.ts @@ -12,7 +12,7 @@ import { } from "node:path"; import { gzipSync, gunzipSync } from "node:zlib"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { ValidationError, FetchError } from "../errors.js"; +import { ValidationError, FetchError, NotFoundError } from "../errors.js"; import { getLogger } from "../logger.js"; import { buildEmbeddingText, @@ -603,7 +603,7 @@ export function removePack(db: Database.Database, packName: string): void { | undefined; if (!existing) { - throw new ValidationError(`Pack "${packName}" is not installed`); + throw new NotFoundError(`Pack "${packName}" is not installed`, "PACK_NOT_FOUND"); } const docIds = db.prepare("SELECT id FROM documents WHERE pack_name = ?").all(packName) as Array<{ diff --git a/src/core/saved-searches.ts b/src/core/saved-searches.ts index 3b53f625..59b0e6d0 100644 --- a/src/core/saved-searches.ts +++ b/src/core/saved-searches.ts @@ -96,6 +96,10 @@ export function listSavedSearches( return rows.map(rowToSavedSearch); } +export function countSavedSearches(db: Database.Database): number { + return (db.prepare("SELECT COUNT(*) AS n FROM saved_searches").get() as { n: number }).n; +} + export function getSavedSearch(db: Database.Database, id: string): SavedSearch { const row = db.prepare("SELECT * FROM saved_searches WHERE id = ? OR name = ?").get(id, id) as | SavedSearchRow diff --git a/src/core/search.ts b/src/core/search.ts index 76add8ef..6ad8fbca 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -8,10 +8,10 @@ import { REBUILD_VECTOR_INDEX_HINT, isVectorDimensionError } from "../db/index-m import { logSearch, recordSearchQuery } from "./analytics.js"; import { emitEvent } from "./events.js"; import { performance } from "node:perf_hooks"; -import { ChunkNotFoundError } from "../errors.js"; +import { ChunkNotFoundError, DocumentNotFoundError } from "../errors.js"; /** Build SQL clause and params for AND-logic tag filtering on a document alias. */ -function buildTagFilter( +export function buildTagFilter( tags: string[] | undefined, docAlias: string, ): { clause: string; params: unknown[] } { @@ -850,6 +850,23 @@ const SourceChunkSchema = z.object({ }); const EmbeddingRowSchema = z.object({ embedding: z.instanceof(Buffer) }); +/** + * ID of a document's first chunk, used to find content related to a whole document. + * Throws DocumentNotFoundError for an unknown document, ChunkNotFoundError when it has no chunks. + */ +export function firstChunkId(db: Database.Database, documentId: string): string { + const row = db + .prepare( + `SELECT d.id AS document_id, c.id AS chunk_id FROM documents d + LEFT JOIN chunks c ON c.document_id = d.id + WHERE d.id = ? ORDER BY c.chunk_index LIMIT 1`, + ) + .get(documentId) as { document_id: string; chunk_id: string | null } | undefined; + if (!row) throw new DocumentNotFoundError(documentId); + if (row.chunk_id === null) throw new ChunkNotFoundError(`first chunk of ${documentId}`); + return row.chunk_id; +} + /** * Find chunks related to a given chunk by vector similarity. * Looks up the source chunk's embedding, then searches for similar chunks diff --git a/src/core/tags.ts b/src/core/tags.ts index 5026d019..9e4548e8 100644 --- a/src/core/tags.ts +++ b/src/core/tags.ts @@ -247,18 +247,28 @@ export function addTagsToDocument( return tags; } -/** Remove a specific tag from a document. */ -export function removeTagFromDocument( +/** + * Remove tags (by name, case-insensitive) from a document. Names the document does not carry + * are ignored. Returns the names that were removed. + */ +export function removeTagsFromDocument( db: Database.Database, documentId: string, - tagId: string, -): void { - const log = createChildLogger({ operation: "removeTagFromDocument" }); - db.prepare("DELETE FROM document_tags WHERE document_id = ? AND tag_id = ?").run( - documentId, - tagId, + tagNames: string[], +): string[] { + const log = createChildLogger({ operation: "removeTagsFromDocument" }); + const remove = db.prepare( + "DELETE FROM document_tags WHERE document_id = ? AND tag_id = (SELECT id FROM tags WHERE name = ?)", ); - log.info({ documentId, tagId }, "Tag removed from document"); + const removed: string[] = []; + const run = db.transaction(() => { + for (const name of new Set(tagNames.map((t) => t.trim().toLowerCase()))) { + if (remove.run(documentId, name).changes > 0) removed.push(name); + } + }); + run(); + log.info({ documentId, removed }, "Tags removed from document"); + return removed; } /** Get all tags for multiple documents in a single query. Returns a Map of documentId → tags. */ diff --git a/src/core/topics.ts b/src/core/topics.ts index 16e08365..e053c543 100644 --- a/src/core/topics.ts +++ b/src/core/topics.ts @@ -138,6 +138,21 @@ export function getTopic(db: Database.Database, topicId: string): Topic { return rowToTopic(row); } +/** ID of the topic whose ID or name (case-insensitive) is `ref`, or undefined. */ +export function findTopicId(db: Database.Database, ref: string): string | undefined { + const row = db + .prepare("SELECT id FROM topics WHERE id = ? OR name = ? COLLATE NOCASE ORDER BY id = ? DESC") + .get(ref, ref, ref) as { id: string } | undefined; + return row?.id; +} + +/** ID of the topic whose ID or name is `ref`. Throws TopicNotFoundError when there is none. */ +export function resolveTopicId(db: Database.Database, ref: string): string { + const id = findTopicId(db, ref); + if (id === undefined) throw new TopicNotFoundError(ref); + return id; +} + export interface GetDocumentsByTopicOptions { limit?: number; offset?: number; diff --git a/src/core/versioning.ts b/src/core/versioning.ts index 364a2735..5cdade69 100644 --- a/src/core/versioning.ts +++ b/src/core/versioning.ts @@ -2,7 +2,7 @@ import type Database from "better-sqlite3"; import { randomUUID } from "node:crypto"; import { z } from "zod"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { DocumentNotFoundError } from "../errors.js"; +import { NotFoundError } from "../errors.js"; import { getDocument, updateDocument } from "./documents.js"; import { getLogger } from "../logger.js"; import { validateRow, validateRows } from "../db/validate.js"; @@ -124,7 +124,10 @@ export function getVersion( .get(documentId, version); if (!raw) { - throw new DocumentNotFoundError(`Version ${version} of document ${documentId}`); + throw new NotFoundError( + `Version ${version} of document ${documentId} not found`, + "VERSION_NOT_FOUND", + ); } const row = validateRow(VersionRowSchema, raw, "getVersion"); diff --git a/tests/unit/tags.test.ts b/tests/unit/tags.test.ts index 2992de52..13e25120 100644 --- a/tests/unit/tags.test.ts +++ b/tests/unit/tags.test.ts @@ -5,7 +5,7 @@ import { deleteTag, listTags, addTagsToDocument, - removeTagFromDocument, + removeTagsFromDocument, getDocumentTags, getDocumentsByTag, suggestTags, @@ -110,13 +110,12 @@ describe("tags", () => { }); }); - describe("removeTagFromDocument", () => { - it("should remove only the specified tag", () => { + describe("removeTagsFromDocument", () => { + it("should remove only the named tags and report them", () => { insertDocument(db, "doc-1", "Test Doc"); - const tags = addTagsToDocument(db, "doc-1", ["keep", "remove"]); - const removeId = tags.find((t) => t.name === "remove")!.id; + addTagsToDocument(db, "doc-1", ["keep", "remove"]); - removeTagFromDocument(db, "doc-1", removeId); + expect(removeTagsFromDocument(db, "doc-1", ["REMOVE", "absent"])).toEqual(["remove"]); const docTags = getDocumentTags(db, "doc-1"); expect(docTags).toHaveLength(1); From 67d364125cd24056491d6056e5a1dce09f1f81b6 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:44:52 +0000 Subject: [PATCH 32/83] feat(core): one ingest entry point and one default source-type rule ingest() adds inline content, a file, a directory (include/exclude globs via utils/glob), a URL, a URL crawl or a GitHub/GitLab repository. spiderAndIndex replaces the three crawl-and-index loops in the CLI, MCP server and REST API. resolveSourceType is the single default: explicit > library > topic > manual. indexRepository accepts library/version/topic/sourceType and an abort signal and returns the indexed documents. --- src/core/indexing.ts | 32 ++- src/core/ingest.ts | 396 ++++++++++++++++++++++++++++++++++++++ src/core/repo.ts | 49 ++++- tests/unit/ingest.test.ts | 228 ++++++++++++++++++++++ 4 files changed, 695 insertions(+), 10 deletions(-) create mode 100644 src/core/ingest.ts create mode 100644 tests/unit/ingest.test.ts diff --git a/src/core/indexing.ts b/src/core/indexing.ts index bb301ec1..2d029f79 100644 --- a/src/core/indexing.ts +++ b/src/core/indexing.ts @@ -17,7 +17,7 @@ import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; export interface IndexDocumentInput { title: string; content: string; - sourceType: "library" | "topic" | "manual" | "model-generated"; + sourceType: SourceType; library?: string | undefined; version?: string | undefined; topicId?: string | undefined; @@ -570,12 +570,31 @@ export interface IndexFileOptions { dedup?: "skip" | "warn" | "force" | undefined; /** Overrides the source type derived from `library`/`topic`. */ sourceType?: IndexDocumentInput["sourceType"] | undefined; + /** ISO 8601 expiry timestamp (see IndexDocumentInput.expiresAt). */ + expiresAt?: string | undefined; } -function fileSourceType(options: IndexFileOptions): IndexDocumentInput["sourceType"] { - if (options.sourceType) return options.sourceType; - if (options.library) return "library"; - return options.topic ? "topic" : "manual"; +/** Document source types. */ +export const SOURCE_TYPES = ["library", "topic", "manual", "model-generated"] as const; +export type SourceType = (typeof SOURCE_TYPES)[number]; + +/** + * The one rule for a document's default source type, used by every write path + * (indexFile, ingest, and through ingest the CLI, MCP, REST and SDK): + * 1. an explicit `sourceType` wins; + * 2. else "library" when a library name is given; + * 3. else "topic" when a topic is given; + * 4. else "manual". + * "model-generated" is never inferred; callers set it explicitly. + */ +export function resolveSourceType(input: { + sourceType?: SourceType | undefined; + library?: string | undefined; + topic?: string | undefined; +}): SourceType { + if (input.sourceType) return input.sourceType; + if (input.library) return "library"; + return input.topic ? "topic" : "manual"; } /** @@ -613,10 +632,11 @@ export async function indexFile( return indexDocument(db, provider, { title, content, - sourceType: fileSourceType(options), + sourceType: resolveSourceType(options), library: options.library, version: options.version, topicId: options.topic, dedup: options.dedup, + expiresAt: options.expiresAt, }); } diff --git a/src/core/ingest.ts b/src/core/ingest.ts new file mode 100644 index 00000000..659a7949 --- /dev/null +++ b/src/core/ingest.ts @@ -0,0 +1,396 @@ +/** + * One entry point for "add this to the knowledge base": inline content, a local file, a + * directory (with include/exclude globs), a single URL, a URL crawl (spider), or a GitHub / + * GitLab repository. The CLI `add`, MCP `submit-document`, REST document routes and the SDK + * all reach indexing through here (via the `add` operation), so they share one set of + * defaults. The default source type comes from {@link resolveSourceType}. + */ +import type Database from "better-sqlite3"; +import { readdirSync, statSync } from "node:fs"; +import { basename, join } from "node:path"; +import type { EmbeddingProvider } from "../providers/embedding.js"; +import { ValidationError } from "../errors.js"; +import { globToRegExp, toPosixPath } from "../utils/glob.js"; +import { + indexDocument, + indexFile, + resolveSourceType, + type IndexDocumentInput, + type IndexedDocument, + type SourceType, +} from "./indexing.js"; +import { getParserForFile } from "./parsers/index.js"; +import { fetchAndConvert, type FetchOptions } from "./url-fetcher.js"; +import { spiderUrl, type SpiderOptions } from "./spider.js"; +import { indexRepository, isRepoUrl } from "./repo.js"; +import { addTagsToDocument } from "./tags.js"; + +export type IngestKind = "content" | "file" | "directory" | "url" | "repo"; + +export interface IngestInput { + /** Local file, directory, URL or repository URL. Omit when `content` is given. */ + source?: string | undefined; + /** How to read `source`. Default "auto": see {@link detectIngestKind}. */ + kind?: IngestKind | "auto" | undefined; + /** Inline document content (markdown). */ + content?: string | undefined; + /** Title. Required with `content`; overrides the detected title for a file or single URL. */ + title?: string | undefined; + /** Canonical URL stored with inline content. */ + url?: string | undefined; + topic?: string | undefined; + library?: string | undefined; + version?: string | undefined; + sourceType?: SourceType | undefined; + /** Tags added to every indexed document. */ + tags?: string[] | undefined; + /** ISO 8601 expiry timestamp; expired documents are removed by pruneExpiredDocuments(). */ + expiresAt?: string | undefined; + dedup?: "skip" | "warn" | "force" | undefined; + /** Force a file format (e.g. ".pdf") instead of using the extension. */ + format?: string | undefined; + /** Directory: globs a file's relative path must match (default: every supported format). */ + include?: string[] | undefined; + /** Directory: globs for files to skip. A glob without "/" also matches the file name. */ + exclude?: string[] | undefined; + /** URL: crawl pages linked from the URL. */ + spider?: boolean | undefined; + maxPages?: number | undefined; + maxDepth?: number | undefined; + sameDomain?: boolean | undefined; + pathPrefix?: string | undefined; + /** URL crawl: globs for URLs to skip. */ + excludePatterns?: string[] | undefined; + /** Repository: branch (default: from the URL, else "main"). */ + branch?: string | undefined; + /** Repository: only index files under these paths. */ + paths?: string[] | undefined; + /** Repository: file extensions to index (default: .md, .mdx, .txt, .rst). */ + extensions?: string[] | undefined; + /** Repository: access token for private repositories. */ + token?: string | undefined; + /** List what would be indexed without indexing anything. */ + dryRun?: boolean | undefined; +} + +export interface IngestContext { + db: Database.Database; + provider: EmbeddingProvider; + /** URL fetch settings (from config.indexing). */ + fetchOptions?: Pick<FetchOptions, "allowPrivateUrls" | "allowSelfSignedCerts"> | undefined; + submittedBy?: IndexDocumentInput["submittedBy"]; + signal?: AbortSignal | undefined; + onProgress?: + | ((p: { done: number; total?: number | undefined; message?: string | undefined }) => void) + | undefined; +} + +export interface IngestedDocument { + documentId: string; + title: string; + chunkCount: number; + /** File path or URL the document came from ("" for inline content). */ + source: string; +} + +export interface IngestResult { + kind: IngestKind; + documents: IngestedDocument[]; + errors: Array<{ source: string; error: string }>; + /** Sources that were not indexed (unsupported format, unchanged content). */ + skipped: Array<{ source: string; reason: string }>; + /** With `dryRun`: the sources that would be indexed. */ + planned?: string[] | undefined; + /** URL crawl statistics (spider only). */ + crawl?: + | { pagesCrawled: number; pagesSkipped: number; abortReason?: string | undefined } + | undefined; +} + +function isHttpUrl(value: string): boolean { + return value.startsWith("http://") || value.startsWith("https://"); +} + +/** + * Decide how to read an ingest input: + * inline `content` -> "content"; http(s) URL -> "repo" for a GitHub/GitLab repository URL + * (unless `spider` is set), else "url"; an existing directory -> "directory"; else "file". + */ +export function detectIngestKind(input: IngestInput): IngestKind { + if (input.kind && input.kind !== "auto") return input.kind; + if (input.content !== undefined) return "content"; + const source = input.source; + if (!source) throw new ValidationError("Provide content, or a file, directory or URL to add"); + if (isHttpUrl(source)) return !input.spider && isRepoUrl(source) ? "repo" : "url"; + let isDirectory = false; + try { + isDirectory = statSync(source).isDirectory(); + } catch { + throw new ValidationError(`File or directory not found: ${source}`); + } + return isDirectory ? "directory" : "file"; +} + +/** Glob matcher for one include/exclude pattern; a pattern without "/" also matches the file name. */ +function globMatcher(pattern: string): (relPath: string) => boolean { + const re = globToRegExp(pattern); + const nameOnly = !pattern.includes("/"); + return (relPath) => re.test(relPath) || (nameOnly && re.test(basename(relPath))); +} + +/** + * Files under `dir` (recursive, sorted, hidden files and directories skipped) that match + * `include` (default: every file) and no `exclude` glob. Paths are joined to `dir`. + */ +export function listDirectoryFiles( + dir: string, + options: { include?: string[] | undefined; exclude?: string[] | undefined } = {}, +): string[] { + const include = options.include?.map(globMatcher); + const exclude = options.exclude?.map(globMatcher) ?? []; + const entries = readdirSync(dir, { recursive: true, encoding: "utf8" }); + const files: string[] = []; + for (const entry of entries) { + const rel = toPosixPath(entry); + if (rel.split("/").some((segment) => segment.startsWith("."))) continue; + if (include && !include.some((m) => m(rel))) continue; + if (exclude.some((m) => m(rel))) continue; + const full = join(dir, entry); + if (statSync(full).isFile()) files.push(full); + } + return files.sort((a, b) => a.localeCompare(b)); +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +/** Shared per-document settings derived once from the input. */ +function documentDefaults( + input: IngestInput, + ctx: IngestContext, +): Pick< + IndexDocumentInput, + "sourceType" | "library" | "version" | "topicId" | "dedup" | "expiresAt" | "submittedBy" +> { + return { + sourceType: resolveSourceType(input), + library: input.library, + version: input.version, + topicId: input.topic, + dedup: input.dedup, + expiresAt: input.expiresAt, + submittedBy: ctx.submittedBy, + }; +} + +function emptyResult(kind: IngestKind): IngestResult { + return { kind, documents: [], errors: [], skipped: [] }; +} + +function record( + ctx: IngestContext, + input: IngestInput, + result: IngestResult, + doc: IndexedDocument, + title: string, + source: string, +): void { + if (input.tags && input.tags.length > 0) addTagsToDocument(ctx.db, doc.id, input.tags); + result.documents.push({ documentId: doc.id, title, chunkCount: doc.chunkCount, source }); +} + +async function ingestContent(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + const result = emptyResult("content"); + if (!input.title?.trim()) { + throw new ValidationError("A title is required when adding content directly"); + } + if (input.dryRun) return { ...result, planned: [input.title] }; + const doc = await indexDocument(ctx.db, ctx.provider, { + ...documentDefaults(input, ctx), + title: input.title, + content: input.content ?? "", + url: input.url, + }); + record(ctx, input, result, doc, input.title, input.url ?? ""); + return result; +} + +function fileTitle(filePath: string): string { + return basename(filePath).replace(/\.[^.]+$/, ""); +} + +async function ingestFile(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + const result = emptyResult("file"); + const path = input.source ?? ""; + if (input.dryRun) return { ...result, planned: [path] }; + const doc = await indexFile(ctx.db, ctx.provider, path, { + title: input.title, + topic: input.topic, + library: input.library, + version: input.version, + format: input.format, + dedup: input.dedup, + sourceType: input.sourceType, + expiresAt: input.expiresAt, + }); + record(ctx, input, result, doc, input.title ?? fileTitle(path), path); + return result; +} + +async function ingestDirectory(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + const result = emptyResult("directory"); + const files = listDirectoryFiles(input.source ?? "", input); + const supported: string[] = []; + for (const file of files) { + if (input.format ?? getParserForFile(file)) supported.push(file); + else result.skipped.push({ source: file, reason: "unsupported file format" }); + } + if (input.dryRun) return { ...result, planned: supported }; + + for (const [i, file] of supported.entries()) { + ctx.signal?.throwIfAborted(); + try { + const doc = await indexFile(ctx.db, ctx.provider, file, { + topic: input.topic, + library: input.library, + version: input.version, + format: input.format, + dedup: input.dedup, + sourceType: input.sourceType, + expiresAt: input.expiresAt, + }); + record(ctx, input, result, doc, fileTitle(file), file); + } catch (err) { + result.errors.push({ source: file, error: errorMessage(err) }); + } + ctx.onProgress?.({ done: i + 1, total: supported.length, message: file }); + } + return result; +} + +async function ingestUrl(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + const url = input.source ?? input.url ?? ""; + if (input.spider) return spiderAndIndex(ctx, url, input); + const result = emptyResult("url"); + if (input.dryRun) return { ...result, planned: [url] }; + const fetched = await fetchAndConvert(url, ctx.fetchOptions); + ctx.signal?.throwIfAborted(); + const title = input.title ?? fetched.title; + const doc = await indexDocument(ctx.db, ctx.provider, { + ...documentDefaults(input, ctx), + title, + content: fetched.content, + url, + }); + record(ctx, input, result, doc, title, url); + return result; +} + +/** Build SpiderOptions from ingest input, copying only the fields that are set. */ +function spiderOptions(ctx: IngestContext, input: IngestInput): SpiderOptions { + const options: SpiderOptions = { fetchOptions: ctx.fetchOptions ?? {} }; + if (input.maxPages !== undefined) options.maxPages = input.maxPages; + if (input.maxDepth !== undefined) options.maxDepth = input.maxDepth; + if (input.sameDomain !== undefined) options.sameDomain = input.sameDomain; + if (input.pathPrefix !== undefined) options.pathPrefix = input.pathPrefix; + if (input.excludePatterns !== undefined) options.excludePatterns = input.excludePatterns; + if (ctx.signal) options.signal = ctx.signal; + return options; +} + +/** + * Crawl `url` (see spiderUrl for limits and robots.txt handling) and index every page. + * A page that fails to index is recorded in `errors`; the crawl continues. + */ +export async function spiderAndIndex( + ctx: IngestContext, + url: string, + input: IngestInput = {}, +): Promise<IngestResult> { + const result = emptyResult("url"); + if (input.dryRun) return { ...result, planned: [url] }; + const defaults = documentDefaults(input, ctx); + const gen = spiderUrl(url, spiderOptions(ctx, input)); + let next = await gen.next(); + while (!next.done) { + const page = next.value; + try { + const doc = await indexDocument(ctx.db, ctx.provider, { + ...defaults, + title: page.title, + content: page.content, + url: page.url, + }); + record(ctx, input, result, doc, page.title, page.url); + } catch (err) { + result.errors.push({ source: page.url, error: errorMessage(err) }); + } + ctx.onProgress?.({ done: result.documents.length + result.errors.length, message: page.url }); + next = await gen.next(); + } + const stats = next.value; + result.crawl = { + pagesCrawled: stats.pagesCrawled, + pagesSkipped: stats.pagesSkipped, + ...(stats.abortReason ? { abortReason: stats.abortReason } : {}), + }; + return result; +} + +async function ingestRepo(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + const result = emptyResult("repo"); + const url = input.source ?? ""; + if (input.dryRun) return { ...result, planned: [url] }; + const repo = await indexRepository( + ctx.db, + ctx.provider, + { + url, + branch: input.branch, + paths: input.paths, + extensions: input.extensions?.map((e) => (e.startsWith(".") ? e : `.${e}`)), + token: input.token, + library: input.library, + version: input.version, + topicId: input.topic, + // Repository files are library docs unless the caller says otherwise. + sourceType: input.sourceType ?? "library", + signal: ctx.signal, + }, + (message) => ctx.onProgress?.({ done: result.documents.length, message }), + ); + for (const doc of repo.documents) { + record( + ctx, + input, + result, + { id: doc.documentId, chunkCount: doc.chunkCount }, + doc.title, + `${url}#${doc.path}`, + ); + } + for (const error of repo.errors) result.errors.push({ source: url, error }); + if (repo.skipped > 0) { + result.skipped.push({ source: url, reason: `${repo.skipped} unchanged file(s)` }); + } + return result; +} + +const INGESTERS: Record< + IngestKind, + (ctx: IngestContext, input: IngestInput) => Promise<IngestResult> +> = { + content: ingestContent, + file: ingestFile, + directory: ingestDirectory, + url: ingestUrl, + repo: ingestRepo, +}; + +/** Add content, a file, a directory, a URL (optionally crawled) or a repository. */ +export async function ingest(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + ctx.signal?.throwIfAborted(); + const kind = detectIngestKind(input); + return INGESTERS[kind](ctx, input); +} diff --git a/src/core/repo.ts b/src/core/repo.ts index 793de555..1bf9ba59 100644 --- a/src/core/repo.ts +++ b/src/core/repo.ts @@ -16,12 +16,44 @@ export interface RepoOptions { paths?: string[] | undefined; extensions?: string[] | undefined; token?: string | undefined; + /** Library name for indexed files (default: "<owner>/<repo>"). */ + library?: string | undefined; + /** Library version for indexed files. */ + version?: string | undefined; + /** Topic ID for indexed files. */ + topicId?: string | undefined; + /** Source type for indexed files (default: "library"). */ + sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; + /** Stops indexing between files; the promise then rejects with the signal's reason. */ + signal?: AbortSignal | undefined; } export interface RepoResult { indexed: number; skipped: number; errors: string[]; + /** Documents indexed by this run. */ + documents: Array<{ documentId: string; title: string; chunkCount: number; path: string }>; +} + +/** + * True when `url` names a GitHub or GitLab repository (or a tree inside one) rather than a + * single page: `https://github.com/owner/repo[.git]`, `.../tree/<branch>[/path]`, + * `https://gitlab.com/owner/repo`, `.../-/tree/<branch>[/path]`. + */ +export function isRepoUrl(url: string): boolean { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return false; + } + const host = parsed.hostname.toLowerCase(); + if (host !== "github.com" && host !== "gitlab.com") return false; + const segments = parsed.pathname.split("/").filter(Boolean); + if (segments.length === 2) return true; + if (host === "github.com") return segments[2] === "tree" && segments.length >= 4; + return segments[2] === "-" && segments[3] === "tree"; } export interface ParsedRepoUrl { @@ -378,14 +410,15 @@ export async function indexRepository( const log = getLogger(); const { owner, repo, branch: urlBranch } = parseRepoUrl(options.url); const branch = options.branch ?? urlBranch ?? "main"; - const library = `${owner}/${repo}`; + const library = options.library ?? `${owner}/${repo}`; log.info({ library, branch }, "Indexing repository"); const files = await fetchRepoContents(options, onProgress); - const result: RepoResult = { indexed: 0, skipped: 0, errors: [] }; + const result: RepoResult = { indexed: 0, skipped: 0, errors: [], documents: [] }; for (let i = 0; i < files.length; i++) { + options.signal?.throwIfAborted(); const file = files[i]!; onProgress?.(`Indexing [${i + 1}/${files.length}] ${file.path}`); @@ -408,16 +441,24 @@ export async function indexRepository( .pop() ?.replace(/\.[^.]+$/, "") ?? file.path; - await indexDocument(db, provider, { + const doc = await indexDocument(db, provider, { title, content: file.content, - sourceType: "library", + sourceType: options.sourceType ?? "library", library, + version: options.version, + topicId: options.topicId, url: fileUrl, submittedBy: "crawler", }); result.indexed++; + result.documents.push({ + documentId: doc.id, + title, + chunkCount: doc.chunkCount, + path: file.path, + }); } catch (err) { const message = `${file.path}: ${err instanceof Error ? err.message : String(err)}`; result.errors.push(message); diff --git a/tests/unit/ingest.test.ts b/tests/unit/ingest.test.ts new file mode 100644 index 00000000..a4b8b31d --- /dev/null +++ b/tests/unit/ingest.test.ts @@ -0,0 +1,228 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type Database from "better-sqlite3"; +import { createTestDb } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { initLogger } from "../../src/logger.js"; +import { ValidationError } from "../../src/errors.js"; + +vi.mock("../../src/core/url-fetcher.js", async (importOriginal) => { + const orig = await importOriginal<Record<string, unknown>>(); + return { + ...orig, + fetchAndConvert: vi.fn((url: string) => + Promise.resolve({ title: `Page ${url}`, content: `Fetched content of ${url}` }), + ), + }; +}); + +vi.mock("../../src/core/spider.js", () => ({ + // eslint-disable-next-line @typescript-eslint/require-await + spiderUrl: async function* ( + url: string, + ): AsyncGenerator< + { url: string; title: string; content: string; depth: number }, + { pagesFetched: number; pagesCrawled: number; pagesSkipped: number; errors: never[] } + > { + yield { url, title: "Seed", content: "Seed page content", depth: 0 }; + yield { url: `${url}/empty`, title: "Empty", content: " ", depth: 1 }; + return { pagesFetched: 2, pagesCrawled: 2, pagesSkipped: 1, errors: [] }; + }, +})); + +vi.mock("../../src/core/repo.js", async (importOriginal) => { + const orig = await importOriginal<Record<string, unknown>>(); + return { + ...orig, + indexRepository: vi.fn(() => + Promise.resolve({ + indexed: 1, + skipped: 2, + errors: ["bad.md: HTTP 404"], + documents: [{ documentId: "repo-doc", title: "README", chunkCount: 3, path: "README.md" }], + }), + ), + }; +}); + +const { detectIngestKind, ingest, listDirectoryFiles } = await import("../../src/core/ingest.js"); +const { resolveSourceType } = await import("../../src/core/indexing.js"); +const { isRepoUrl, indexRepository } = await import("../../src/core/repo.js"); +const { getDocumentTags } = await import("../../src/core/tags.js"); + +initLogger("silent"); + +describe("resolveSourceType", () => { + it("prefers explicit, then library, then topic, then manual", () => { + expect(resolveSourceType({ sourceType: "model-generated", library: "x" })).toBe( + "model-generated", + ); + expect(resolveSourceType({ library: "x", topic: "t" })).toBe("library"); + expect(resolveSourceType({ topic: "t" })).toBe("topic"); + expect(resolveSourceType({})).toBe("manual"); + }); +}); + +describe("isRepoUrl", () => { + it("recognises repository and tree URLs only", () => { + expect(isRepoUrl("https://github.com/owner/repo")).toBe(true); + expect(isRepoUrl("https://github.com/owner/repo.git")).toBe(true); + expect(isRepoUrl("https://github.com/owner/repo/tree/main/docs")).toBe(true); + expect(isRepoUrl("https://gitlab.com/owner/repo/-/tree/main")).toBe(true); + expect(isRepoUrl("https://github.com/owner/repo/blob/main/README.md")).toBe(false); + expect(isRepoUrl("https://github.com/owner")).toBe(false); + expect(isRepoUrl("https://example.com/owner/repo")).toBe(false); + expect(isRepoUrl("not a url")).toBe(false); + }); +}); + +describe("ingest", () => { + let db: Database.Database; + let provider: MockEmbeddingProvider; + let dir: string; + + beforeEach(() => { + db = createTestDb(); + provider = new MockEmbeddingProvider(); + dir = mkdtempSync(join(tmpdir(), "libscope-ingest-")); + }); + + afterEach(() => { + db.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + it("detects the kind of input", () => { + writeFileSync(join(dir, "a.md"), "# A"); + expect(detectIngestKind({ content: "x" })).toBe("content"); + expect(detectIngestKind({ source: "https://github.com/o/r" })).toBe("repo"); + expect(detectIngestKind({ source: "https://github.com/o/r", spider: true })).toBe("url"); + expect(detectIngestKind({ source: "https://example.com/docs" })).toBe("url"); + expect(detectIngestKind({ source: dir })).toBe("directory"); + expect(detectIngestKind({ source: join(dir, "a.md") })).toBe("file"); + expect(() => detectIngestKind({ source: join(dir, "missing.md") })).toThrow(ValidationError); + expect(() => detectIngestKind({})).toThrow(ValidationError); + }); + + it("lists directory files with include/exclude globs and skips hidden entries", () => { + mkdirSync(join(dir, "docs", "api"), { recursive: true }); + mkdirSync(join(dir, ".git")); + writeFileSync(join(dir, "README.md"), "x"); + writeFileSync(join(dir, "docs", "guide.md"), "x"); + writeFileSync(join(dir, "docs", "api", "ref.md"), "x"); + writeFileSync(join(dir, "docs", "app.min.js"), "x"); + writeFileSync(join(dir, ".git", "HEAD.md"), "x"); + writeFileSync(join(dir, ".hidden.md"), "x"); + + const rel = (files: string[]): string[] => files.map((f) => f.slice(dir.length + 1)); + expect(rel(listDirectoryFiles(dir))).toEqual([ + join("docs", "api", "ref.md"), + join("docs", "app.min.js"), + join("docs", "guide.md"), + "README.md", + ]); + expect( + rel(listDirectoryFiles(dir, { include: ["docs/**/*.md"], exclude: ["api/**"] })), + ).toEqual([join("docs", "api", "ref.md"), join("docs", "guide.md")]); + expect(rel(listDirectoryFiles(dir, { exclude: ["*.min.js", "docs/api/**"] }))).toEqual([ + join("docs", "guide.md"), + "README.md", + ]); + }); + + it("indexes a directory, skipping unsupported files, and applies tags", async () => { + writeFileSync(join(dir, "a.md"), "# A\n\nAlpha"); + writeFileSync(join(dir, "b.txt"), "Bravo text"); + writeFileSync(join(dir, "c.bin"), "binary"); + const progress: number[] = []; + const result = await ingest( + { db, provider, onProgress: (p) => progress.push(p.done) }, + { source: dir, library: "lib", tags: ["imported"] }, + ); + expect(result.kind).toBe("directory"); + expect(result.documents.map((d) => d.title)).toEqual(["a", "b"]); + expect(result.skipped).toEqual([ + { source: join(dir, "c.bin"), reason: "unsupported file format" }, + ]); + expect(progress).toEqual([1, 2]); + const id = result.documents[0]!.documentId; + expect(getDocumentTags(db, id).map((t) => t.name)).toEqual(["imported"]); + const row = db.prepare("SELECT source_type, library FROM documents WHERE id = ?").get(id); + expect(row).toEqual({ source_type: "library", library: "lib" }); + }); + + it("plans without indexing on a dry run", async () => { + writeFileSync(join(dir, "a.md"), "# A"); + const result = await ingest({ db, provider }, { source: dir, dryRun: true }); + expect(result.planned).toEqual([join(dir, "a.md")]); + expect(db.prepare("SELECT COUNT(*) AS n FROM documents").get()).toEqual({ n: 0 }); + }); + + it("requires a title for inline content", async () => { + await expect(ingest({ db, provider }, { content: "body" })).rejects.toThrow(ValidationError); + const result = await ingest( + { db, provider }, + { content: "body", title: "T", url: "https://x.dev" }, + ); + expect(result.documents[0]).toMatchObject({ title: "T", source: "https://x.dev" }); + }); + + it("fetches a single URL and records who submitted it", async () => { + const result = await ingest( + { db, provider, submittedBy: "model" }, + { source: "https://example.com/page" }, + ); + expect(result.documents[0]?.title).toBe("Page https://example.com/page"); + const row = db.prepare("SELECT submitted_by, url FROM documents").get(); + expect(row).toEqual({ submitted_by: "model", url: "https://example.com/page" }); + }); + + it("crawls a URL, recording per-page errors and crawl stats", async () => { + const result = await ingest( + { db, provider }, + { source: "https://example.com/docs", spider: true }, + ); + expect(result.documents.map((d) => d.title)).toEqual(["Seed"]); + expect(result.errors.map((e) => e.source)).toEqual(["https://example.com/docs/empty"]); + expect(result.errors[0]?.error).toContain("content"); + expect(result.crawl).toEqual({ pagesCrawled: 2, pagesSkipped: 1 }); + }); + + it("indexes a repository as library docs and reports skipped files and errors", async () => { + const result = await ingest( + { db, provider }, + { source: "https://github.com/o/r", extensions: ["md"], topic: "t1" }, + ); + expect(vi.mocked(indexRepository)).toHaveBeenCalledWith( + db, + provider, + expect.objectContaining({ extensions: [".md"], topicId: "t1", sourceType: "library" }), + expect.any(Function), + ); + expect(result.documents).toEqual([ + { + documentId: "repo-doc", + title: "README", + chunkCount: 3, + source: "https://github.com/o/r#README.md", + }, + ]); + expect(result.errors).toEqual([ + { source: "https://github.com/o/r", error: "bad.md: HTTP 404" }, + ]); + expect(result.skipped).toEqual([ + { source: "https://github.com/o/r", reason: "2 unchanged file(s)" }, + ]); + }); + + it("stops between files when aborted", async () => { + writeFileSync(join(dir, "a.md"), "# A"); + const controller = new AbortController(); + controller.abort(new Error("stop")); + await expect( + ingest({ db, provider, signal: controller.signal }, { source: dir }), + ).rejects.toThrow("stop"); + }); +}); From 6050cf40dcc7d984f32c9544394be03e1c1dfd7b Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:44:57 +0000 Subject: [PATCH 33/83] feat(rag): answer() handles passthrough and LLM answers in one place RAG retrieval also accepts version, source type, tags and minimum rating. --- src/core/rag.ts | 46 ++++++++++++++++++++++ tests/unit/rag-answer.test.ts | 73 +++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 tests/unit/rag-answer.test.ts diff --git a/src/core/rag.ts b/src/core/rag.ts index fc5e1015..cac22216 100644 --- a/src/core/rag.ts +++ b/src/core/rag.ts @@ -18,6 +18,12 @@ export interface RagOptions { topK?: number | undefined; topic?: string | undefined; library?: string | undefined; + version?: string | undefined; + /** Filter by document source type. */ + sourceType?: string | undefined; + /** Only documents carrying all of these tags. */ + tags?: string[] | undefined; + minRating?: number | undefined; systemPrompt?: string | undefined; } @@ -137,6 +143,10 @@ async function retrieveResults( query: options.question, topic: options.topic, library: options.library, + version: options.version, + source: options.sourceType, + tags: options.tags, + minRating: options.minRating, limit: options.topK ?? 5, }); return results; @@ -409,3 +419,39 @@ export async function askQuestion( tokensUsed, }; } + +/** How `answer` should produce its result. */ +export interface AnswerMode { + /** Return the retrieved context instead of calling an LLM (the caller is the LLM). */ + passthrough: boolean; + /** LLM used when not in passthrough mode; null when none is configured. */ + llm: LlmProvider | null; +} + +/** An LLM answer, or (passthrough) the context prompt for the caller to answer from. */ +export type AnswerResult = + | ({ mode: "answer" } & RagResult) + | ({ mode: "context" } & PassthroughResult); + +/** Message for `ask` when no LLM is available. */ +export const NO_LLM_HINT = + "No LLM provider configured. Set one with `libscope config set llm.provider <openai|anthropic|ollama|passthrough>` " + + "or the LIBSCOPE_LLM_PROVIDER environment variable."; + +/** + * Answer a question from the knowledge base. Every surface uses this one function: + * passthrough mode returns the context prompt and sources; otherwise the LLM writes the answer. + * @throws ConfigError when not in passthrough mode and no LLM is configured. + */ +export async function answer( + db: Database.Database, + embeddingProvider: EmbeddingProvider, + mode: AnswerMode, + options: RagOptions, +): Promise<AnswerResult> { + if (mode.passthrough) { + return { mode: "context", ...(await getContextForQuestion(db, embeddingProvider, options)) }; + } + if (!mode.llm) throw new ConfigError(NO_LLM_HINT); + return { mode: "answer", ...(await askQuestion(db, embeddingProvider, mode.llm, options)) }; +} diff --git a/tests/unit/rag-answer.test.ts b/tests/unit/rag-answer.test.ts new file mode 100644 index 00000000..e2cca744 --- /dev/null +++ b/tests/unit/rag-answer.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import type Database from "better-sqlite3"; +import { createTestDb } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { answer, NO_LLM_HINT, type LlmProvider } from "../../src/core/rag.js"; +import { indexDocument } from "../../src/core/indexing.js"; +import { addTagsToDocument } from "../../src/core/tags.js"; +import { ConfigError } from "../../src/errors.js"; +import { initLogger } from "../../src/logger.js"; + +initLogger("silent"); + +describe("answer", () => { + let db: Database.Database; + let provider: MockEmbeddingProvider; + + beforeEach(async () => { + db = createTestDb(); + provider = new MockEmbeddingProvider(); + const tagged = await indexDocument(db, provider, { + title: "Tagged guide", + content: "Deploying the service with docker compose", + sourceType: "library", + library: "svc", + version: "2", + }); + addTagsToDocument(db, tagged.id, ["ops"]); + await indexDocument(db, provider, { + title: "Untagged guide", + content: "Deploying the service by hand", + sourceType: "manual", + }); + }); + + afterEach(() => { + db.close(); + }); + + it("returns the context prompt in passthrough mode without calling an LLM", async () => { + const complete = vi.fn(); + const llm: LlmProvider = { model: "m", complete }; + const result = await answer( + db, + provider, + { passthrough: true, llm }, + { question: "deploying" }, + ); + expect(result.mode).toBe("context"); + expect(complete).not.toHaveBeenCalled(); + if (result.mode === "context") expect(result.sources).toHaveLength(2); + }); + + it("asks the LLM and applies the filters to retrieval", async () => { + const complete = vi.fn((prompt: string) => Promise.resolve({ text: prompt, tokensUsed: 3 })); + const result = await answer( + db, + provider, + { passthrough: false, llm: { model: "m", complete } }, + { question: "deploying", tags: ["ops"], version: "2", sourceType: "library" }, + ); + expect(result.mode).toBe("answer"); + if (result.mode === "answer") { + expect(result.sources.map((s) => s.title)).toEqual(["Tagged guide"]); + expect(result.tokensUsed).toBe(3); + } + }); + + it("throws a ConfigError with setup instructions when no LLM is configured", async () => { + await expect( + answer(db, provider, { passthrough: false, llm: null }, { question: "q" }), + ).rejects.toThrow(new ConfigError(NO_LLM_HINT)); + }); +}); From 04aff818c3ad416e6b7eeea450ebcea618ac665a Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:45:00 +0000 Subject: [PATCH 34/83] feat(core): bootstrap() opens config, database, provider and vector table One startup sequence for CLI, MCP, REST, SDK and lite, with options for an injected database or provider, best-effort vector tables and an optional embedding provider. --- src/core/bootstrap.ts | 148 +++++++++++++++++++++++++++++++++++ tests/unit/bootstrap.test.ts | 117 +++++++++++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 src/core/bootstrap.ts create mode 100644 tests/unit/bootstrap.test.ts diff --git a/src/core/bootstrap.ts b/src/core/bootstrap.ts new file mode 100644 index 00000000..d8b8dd84 --- /dev/null +++ b/src/core/bootstrap.ts @@ -0,0 +1,148 @@ +/** + * One startup sequence for every surface (CLI, MCP server, REST server, SDK, lite): + * load config -> resolve the database path -> open and migrate the database -> + * create the embedding provider -> create or check the vector table. + */ +import type Database from "better-sqlite3"; +import { loadConfig, type LibScopeConfig } from "../config.js"; +import { createDatabase, resolveDatabasePath } from "../db/connection.js"; +import { createVectorTable, runMigrations } from "../db/schema.js"; +import { ConfigError } from "../errors.js"; +import { getLogger } from "../logger.js"; +import { createEmbeddingProvider } from "../providers/index.js"; +import type { EmbeddingProvider } from "../providers/embedding.js"; + +/** Config sections that callers may partially override. */ +export type ConfigOverrides = { [K in keyof LibScopeConfig]?: Partial<LibScopeConfig[K]> }; + +export interface BootstrapOptions { + /** Workspace whose database to open (default: the active workspace). Ignored with `dbPath`. */ + workspace?: string | undefined; + /** Explicit database file. Wins over `workspace` and over `database.path` in config. */ + dbPath?: string | undefined; + /** Use this config instead of loadConfig(). */ + config?: LibScopeConfig | undefined; + /** Per-section values merged over the loaded config (e.g. `{ llm: { model: "x" } }`). */ + configOverrides?: ConfigOverrides | undefined; + /** Use this embedding provider instead of creating one from config. */ + provider?: EmbeddingProvider | undefined; + /** Use an already-open database. It is migrated but not closed by `close()`. */ + db?: Database.Database | undefined; + /** + * Vector table handling: "required" (default) throws on any error; "best-effort" only + * throws when the index belongs to a different embedding model (ConfigError) and otherwise + * continues without vector search; "skip" does not touch it. + */ + vectorTable?: "required" | "best-effort" | "skip" | undefined; + /** + * When false, a provider that cannot be created (e.g. missing API key) does not fail + * startup: commands that do not embed still work, and embedding calls throw the original + * error. Default true. + */ + requireProvider?: boolean | undefined; + /** Where to report the one-time legacy-database notice (default: logger.warn). */ + warn?: ((message: string) => void) | undefined; +} + +export interface Bootstrapped { + db: Database.Database; + provider: EmbeddingProvider; + config: LibScopeConfig; + /** Database file that was opened (":memory:" or the caller's file for an injected db). */ + dbPath: string; + /** Close the database if bootstrap opened it. Safe to call twice. */ + close(): void; +} + +/** Merge per-section overrides over `base` (one level deep). */ +export function mergeConfig(base: LibScopeConfig, overrides?: ConfigOverrides): LibScopeConfig { + if (!overrides) return base; + const merged: Record<string, unknown> = { ...base }; + for (const [section, values] of Object.entries(overrides)) { + if (values === undefined) continue; + const current = (base as unknown as Record<string, unknown>)[section]; + merged[section] = { ...(current as object | undefined), ...values }; + } + return merged as unknown as LibScopeConfig; +} + +/** Stand-in provider used when the configured one cannot be created and it is not required. */ +export class UnavailableEmbeddingProvider implements EmbeddingProvider { + readonly name = "unavailable"; + readonly dimensions = 0; + + constructor(private readonly cause: unknown) {} + + embed(): Promise<number[]> { + return Promise.reject(this.cause instanceof Error ? this.cause : new Error(String(this.cause))); + } + + embedBatch(): Promise<number[][]> { + return Promise.reject(this.cause instanceof Error ? this.cause : new Error(String(this.cause))); + } +} + +function createProvider(config: LibScopeConfig, options: BootstrapOptions): EmbeddingProvider { + if (options.provider) return options.provider; + try { + return createEmbeddingProvider(config); + } catch (err) { + if (options.requireProvider ?? true) throw err; + getLogger().warn({ err }, "Embedding provider unavailable; commands that embed will fail"); + return new UnavailableEmbeddingProvider(err); + } +} + +function setUpVectorTable( + db: Database.Database, + provider: EmbeddingProvider, + mode: NonNullable<BootstrapOptions["vectorTable"]>, +): void { + if (mode === "skip" || provider instanceof UnavailableEmbeddingProvider) return; + try { + createVectorTable(db, provider); + } catch (err) { + if (mode === "required" || err instanceof ConfigError) throw err; + getLogger().warn({ err }, "Vector table unavailable; keyword search only"); + } +} + +/** Open everything a surface needs. Call `close()` when done. */ +export function bootstrap(options: BootstrapOptions = {}): Bootstrapped { + const config = mergeConfig(options.config ?? loadConfig(), options.configOverrides); + + let db: Database.Database; + let dbPath: string; + let ownsDb: boolean; + if (options.db) { + db = options.db; + dbPath = db.name; + ownsDb = false; + } else { + dbPath = resolveDatabasePath({ + explicitPath: options.dbPath ?? config.database.path, + workspace: options.workspace, + warn: options.warn, + }); + db = createDatabase(dbPath); + ownsDb = true; + } + + try { + runMigrations(db); + const provider = createProvider(config, options); + setUpVectorTable(db, provider, options.vectorTable ?? "required"); + return { + db, + provider, + config, + dbPath, + close: (): void => { + if (ownsDb && db.open) db.close(); + }, + }; + } catch (err) { + if (ownsDb) db.close(); + throw err; + } +} diff --git a/tests/unit/bootstrap.test.ts b/tests/unit/bootstrap.test.ts new file mode 100644 index 00000000..370c2aae --- /dev/null +++ b/tests/unit/bootstrap.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { LibScopeConfig } from "../../src/config.js"; +import { bootstrap, mergeConfig, UnavailableEmbeddingProvider } from "../../src/core/bootstrap.js"; +import { getVectorTableDimensions } from "../../src/db/index-meta.js"; +import { ConfigError } from "../../src/errors.js"; +import { initLogger } from "../../src/logger.js"; +import { createTestDb } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; + +initLogger("silent"); + +function config(provider = "local"): LibScopeConfig { + return { + embedding: { provider }, + llm: {}, + database: {}, + indexing: { maxDocumentSize: 1000, allowPrivateUrls: false, allowSelfSignedCerts: false }, + logging: { level: "silent" }, + } as LibScopeConfig; +} + +class EightDimProvider extends MockEmbeddingProvider { + override readonly dimensions = 8 as 4; +} + +describe("bootstrap", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "libscope-bootstrap-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("opens and migrates the database and creates the vector table", () => { + const dbPath = join(dir, "kb.db"); + const app = bootstrap({ dbPath, config: config(), provider: new MockEmbeddingProvider() }); + expect(app.dbPath).toBe(dbPath); + const row = app.db.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { + v: number; + }; + expect(row.v).toBeGreaterThan(0); + expect(getVectorTableDimensions(app.db)).toBe(4); + app.close(); + app.close(); + expect(app.db.open).toBe(false); + }); + + it("uses database.path from config when no dbPath is given", () => { + const dbPath = join(dir, "from-config.db"); + const app = bootstrap({ + config: config(), + configOverrides: { database: { path: dbPath } }, + provider: new MockEmbeddingProvider(), + }); + expect(app.dbPath).toBe(dbPath); + app.close(); + }); + + it("fails on an embedding model mismatch and closes the database it opened", () => { + const dbPath = join(dir, "kb.db"); + bootstrap({ dbPath, config: config(), provider: new MockEmbeddingProvider() }).close(); + expect(() => + bootstrap({ + dbPath, + config: config(), + provider: new EightDimProvider(), + vectorTable: "best-effort", + }), + ).toThrow(ConfigError); + }); + + it("keeps going without an embedding provider when it is not required", async () => { + const app = bootstrap({ + dbPath: join(dir, "kb.db"), + config: config("openai"), + requireProvider: false, + }); + expect(app.provider).toBeInstanceOf(UnavailableEmbeddingProvider); + await expect(app.provider.embed("x")).rejects.toBeInstanceOf(ConfigError); + expect(getVectorTableDimensions(app.db)).toBeUndefined(); + app.close(); + expect(() => bootstrap({ dbPath: join(dir, "kb2.db"), config: config("openai") })).toThrow( + ConfigError, + ); + }); + + it("uses an injected database without closing it", () => { + const db = createTestDb(); + const app = bootstrap({ + db, + config: config(), + provider: new MockEmbeddingProvider(), + vectorTable: "skip", + }); + app.close(); + expect(db.open).toBe(true); + db.close(); + }); +}); + +describe("mergeConfig", () => { + it("merges each section one level deep", () => { + const merged = mergeConfig(config(), { indexing: { allowPrivateUrls: true } }); + expect(merged.indexing).toEqual({ + maxDocumentSize: 1000, + allowPrivateUrls: true, + allowSelfSignedCerts: false, + }); + expect(mergeConfig(config())).toEqual(config()); + }); +}); From 489ed7918af5ebef2810491dcda15e66b8be630a Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:45:10 +0000 Subject: [PATCH 35/83] feat(connectors): connector registry for sync, disconnect and listing CONNECTORS describes every connector type (config schema, secret fields, sync, result summary, disconnect), including the docs-site connector, which can now be saved and scheduled. runSavedConnectorSync moves to registry.ts and returns a summary; syncConnection, listConnections and disconnectConnection work by saved connection name. The scheduler reads saved configs through listNamedConnectorConfigs. schema.ts notes that connector_configs is unused. --- src/connectors/index.ts | 33 +- src/connectors/registry.ts | 357 ++++++++++++++++++++++ src/connectors/saved-config.ts | 62 +--- src/core/scheduler.ts | 40 +-- src/db/schema.ts | 3 + tests/unit/connector-registry.test.ts | 66 ++++ tests/unit/connector-saved-config.test.ts | 2 +- 7 files changed, 479 insertions(+), 84 deletions(-) create mode 100644 src/connectors/registry.ts create mode 100644 tests/unit/connector-registry.test.ts diff --git a/src/connectors/index.ts b/src/connectors/index.ts index 2e5a9b4b..a7ec0968 100644 --- a/src/connectors/index.ts +++ b/src/connectors/index.ts @@ -1,4 +1,12 @@ -import { readFileSync, writeFileSync, mkdirSync, existsSync, chmodSync, unlinkSync } from "node:fs"; +import { + readFileSync, + readdirSync, + writeFileSync, + mkdirSync, + existsSync, + chmodSync, + unlinkSync, +} from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; import { ConfigError } from "../errors.js"; @@ -169,6 +177,29 @@ export function hasNamedConnectorConfig(name: string): boolean { return existsSync(namedConfigPath(name)); } +/** + * Every named connector config in ~/.libscope/connectors, sorted by name. A file that cannot + * be read or parsed is skipped with a warning. + */ +export function listNamedConnectorConfigs(): Array<{ + name: string; + config: Record<string, unknown>; +}> { + const dir = getConnectorsDir(); + if (!existsSync(dir)) return []; + const entries: Array<{ name: string; config: Record<string, unknown> }> = []; + for (const file of readdirSync(dir).filter((f) => f.endsWith(".json"))) { + const name = file.slice(0, -".json".length); + try { + const config = JSON.parse(readFileSync(join(dir, file), "utf-8")) as Record<string, unknown>; + entries.push({ name, config }); + } catch (err) { + getLogger().warn({ file, err }, "Skipping unreadable connector config"); + } + } + return entries.sort((a, b) => a.name.localeCompare(b.name)); +} + /** Delete a named connector config. Returns false when none existed. */ export function deleteNamedConnectorConfig(name: string): boolean { const filePath = namedConfigPath(name); diff --git a/src/connectors/registry.ts b/src/connectors/registry.ts new file mode 100644 index 00000000..20ed67a0 --- /dev/null +++ b/src/connectors/registry.ts @@ -0,0 +1,357 @@ +/** + * Connector registry: one table describing every connector type (its config shape, which + * fields are secrets, how to sync it and how to remove its data). `libscope sync`, + * `libscope disconnect`, the MCP/REST sync operations and the scheduler all go through it. + * Connector settings are the saved configs in ~/.libscope/connectors/<name>.json. + */ +import type Database from "better-sqlite3"; +import { z } from "zod"; +import type { EmbeddingProvider } from "../providers/embedding.js"; +import { NotFoundError, ValidationError } from "../errors.js"; +import { + deleteNamedConnectorConfig, + hasNamedConnectorConfig, + listNamedConnectorConfigs, + loadNamedConnectorConfig, +} from "./index.js"; +import { + isConnectorType, + loadSavedConnectorConfig, + resolveConnectorType, + syncSavedConnector, + type ConnectorType, +} from "./saved-config.js"; +import { getConnectorStatus, type ConnectorSyncOptions } from "./sync-tracker.js"; +import { + syncNotion, + disconnectNotion, + type NotionConfig, + type NotionSyncResult, +} from "./notion.js"; +import { syncSlack, disconnectSlack, type SlackConfig, type SlackSyncResult } from "./slack.js"; +import { + syncConfluence, + disconnectConfluence, + type ConfluenceConfig, + type ConfluenceSyncResult, +} from "./confluence.js"; +import { + syncObsidianVault, + disconnectVault, + type ObsidianConfig, + type SyncResult as ObsidianSyncResult, +} from "./obsidian.js"; +import { + syncOneNote, + disconnectOneNote, + type OneNoteConfig, + type OneNoteSyncResult, +} from "./onenote.js"; +import { + syncDocSite, + disconnectDocSite, + type DocSiteConfig, + type DocSiteSyncResult, +} from "./docs.js"; + +/** Counts from one sync, the same for every connector type. */ +export interface ConnectorSyncSummary { + added: number; + updated: number; + deleted: number; + /** One line per item that failed. */ + errors: string[]; +} + +export interface ConnectorDefinition<C = Record<string, unknown>, R = unknown> { + type: ConnectorType; + /** Human-readable name, e.g. "Confluence". */ + label: string; + /** Shape of the saved config (the connector's own settings). */ + configSchema: z.ZodObject; + /** Config fields that hold credentials; never shown in listings. */ + secretFields: readonly string[]; + sync( + db: Database.Database, + provider: EmbeddingProvider, + config: C, + options: ConnectorSyncOptions, + ): Promise<R>; + summarize(result: R): ConnectorSyncSummary; + /** Remove the documents this connector created. `config` is the saved config, if any. */ + disconnect(db: Database.Database, config: C | undefined): number | Promise<number>; +} + +/** Erase the config/result types so definitions fit in one table. */ +function defineConnector<C, R>(def: ConnectorDefinition<C, R>): ConnectorDefinition { + return def as unknown as ConnectorDefinition; +} + +function requireField<C>(config: C | undefined, field: keyof C & string, type: string): C { + if (!config?.[field]) { + throw new ValidationError(`Disconnecting ${type} needs the saved connection's ${field}`); + } + return config; +} + +const errorLines = <E>(errors: E[], format: (e: E) => string): string[] => errors.map(format); + +// Each sync/disconnect is wrapped in an arrow so connector modules are only touched when used. +export const CONNECTORS: Readonly<Record<ConnectorType, ConnectorDefinition>> = { + notion: defineConnector<NotionConfig, NotionSyncResult>({ + type: "notion", + label: "Notion", + configSchema: z.object({ + token: z.string().describe("Notion integration token (secret_... or ntn_...)"), + excludePages: z.array(z.string()).optional().describe("Page/database IDs to skip"), + }), + secretFields: ["token"], + sync: (db, provider, config, options) => syncNotion(db, provider, config, options), + summarize: (r) => ({ + added: r.pagesIndexed + r.databasesIndexed, + updated: 0, + deleted: 0, + errors: errorLines(r.errors, (e) => `${e.page}: ${e.error}`), + }), + disconnect: (db) => disconnectNotion(db), + }), + slack: defineConnector<SlackConfig, SlackSyncResult>({ + type: "slack", + label: "Slack", + configSchema: z.object({ + token: z.string().describe("Slack bot or user token (xoxb-... or xoxp-...)"), + channels: z.array(z.string()).describe("Channel names or IDs, or ['all']"), + excludeChannels: z.array(z.string()).optional().describe("Channel names to skip"), + threadMode: z.enum(["aggregate", "separate"]).describe("One document per thread or reply"), + }), + secretFields: ["token"], + sync: (db, provider, config, options) => syncSlack(db, provider, config, options), + summarize: (r) => ({ + added: r.messagesIndexed + r.threadsIndexed, + updated: 0, + deleted: 0, + errors: errorLines(r.errors, (e) => `#${e.channel}: ${e.error}`), + }), + disconnect: (db) => disconnectSlack(db), + }), + confluence: defineConnector<ConfluenceConfig, ConfluenceSyncResult>({ + type: "confluence", + label: "Confluence", + configSchema: z.object({ + baseUrl: z.string().describe("Confluence base URL, e.g. https://acme.atlassian.net"), + type: z.enum(["cloud", "server"]).optional().describe("Cloud or Server/Data Center"), + email: z.string().optional().describe("User email (Cloud)"), + token: z.string().describe("API token (Cloud) or personal access token (Server)"), + spaces: z.array(z.string()).describe("Space keys, or ['all']"), + excludeSpaces: z.array(z.string()).optional().describe("Space keys to skip"), + }), + secretFields: ["token"], + sync: (db, provider, config, options) => syncConfluence(db, provider, config, options), + summarize: (r) => ({ + added: r.pagesIndexed, + updated: r.pagesUpdated, + deleted: 0, + errors: errorLines(r.errors, (e) => `${e.page}: ${e.error}`), + }), + disconnect: (db) => disconnectConfluence(db), + }), + obsidian: defineConnector<ObsidianConfig, ObsidianSyncResult>({ + type: "obsidian", + label: "Obsidian", + configSchema: z.object({ + vaultPath: z.string().describe("Absolute path of the vault"), + topicMapping: z.enum(["folder", "frontmatter"]).describe("Where topics come from"), + excludePatterns: z.array(z.string()).describe("Globs for files to skip"), + }), + secretFields: [], + sync: (db, provider, config, options) => syncObsidianVault(db, provider, config, options), + summarize: (r) => ({ + added: r.added, + updated: r.updated, + deleted: r.deleted, + errors: errorLines(r.errors, (e) => `${e.file}: ${e.error}`), + }), + disconnect: (db, config) => + disconnectVault(db, requireField(config, "vaultPath", "obsidian").vaultPath), + }), + onenote: defineConnector<OneNoteConfig, OneNoteSyncResult>({ + type: "onenote", + label: "OneNote", + configSchema: z.object({ + clientId: z.string().describe("Azure app client ID"), + tenantId: z.string().describe("Azure tenant ID (default: common)"), + accessToken: z.string().optional().describe("Microsoft Graph access token"), + refreshToken: z.string().optional().describe("Refresh token from device sign-in"), + notebooks: z.array(z.string()).describe("Notebook names, or ['all']"), + excludeSections: z.array(z.string()).describe("Section names to skip"), + }), + secretFields: ["accessToken", "refreshToken"], + sync: (db, provider, config, options) => syncOneNote(db, provider, config, options), + summarize: (r) => ({ + added: r.pagesAdded, + updated: r.pagesUpdated, + deleted: r.pagesDeleted, + errors: errorLines(r.errors, (e) => `${e.page}: ${e.error}`), + }), + disconnect: (db) => disconnectOneNote(db), + }), + docs: defineConnector<DocSiteConfig, DocSiteSyncResult>({ + type: "docs", + label: "Documentation site", + configSchema: z.object({ + url: z.string().describe("Root URL of the documentation site"), + type: z + .enum(["auto", "sphinx", "vitepress", "doxygen", "generic"]) + .optional() + .describe("Site generator (default: auto-detect)"), + library: z.string().optional().describe("Library name for indexed pages"), + version: z.string().optional().describe("Library version for indexed pages"), + maxPages: z.number().int().positive().optional().describe("Page limit (default 500)"), + maxDepth: z.number().int().min(0).optional().describe("Link depth limit (default 10)"), + pathPrefix: z.string().optional().describe("Only crawl URLs under this path"), + }), + secretFields: [], + sync: (db, provider, config, options) => syncDocSite(db, provider, config, options), + summarize: (r) => ({ + added: r.pagesIndexed, + updated: r.pagesUpdated, + deleted: 0, + errors: errorLines(r.errors, (e) => `${e.url}: ${e.error}`), + }), + disconnect: (db, config) => disconnectDocSite(db, requireField(config, "url", "docs").url), + }), +}; + +/** The definition for `type`. Throws ValidationError for an unknown type. */ +export function getConnector(type: string): ConnectorDefinition { + if (!isConnectorType(type)) { + throw new ValidationError( + `Unknown connector type "${type}". Known types: ${Object.keys(CONNECTORS).join(", ")}`, + ); + } + return CONNECTORS[type]; +} + +/** + * Re-sync saved connector `name` of type `type` with its saved config (used by the scheduler + * and the sync operation). Writes exactly one connector_syncs row, recorded under `name`, and + * saves the new lastSync. Errors are rethrown. + */ +export async function runSavedConnectorSync( + db: Database.Database, + provider: EmbeddingProvider, + type: ConnectorType, + name: string, + options: ConnectorSyncOptions = {}, +): Promise<ConnectorSyncSummary> { + const connector = CONNECTORS[type]; + const result = await syncSavedConnector<{ lastSync?: string | undefined }, unknown>( + db, + type, + name, + (config) => connector.sync(db, provider, config, { ...options, syncName: name }), + ); + return connector.summarize(result); +} + +/** Connector type of saved connection `name`. Throws NotFoundError when it does not exist. */ +function savedConnectionType(name: string): ConnectorType { + if (!hasNamedConnectorConfig(name)) { + throw new NotFoundError( + `No saved connection named "${name}". Create one with 'libscope connect <type>'.`, + "CONNECTION_NOT_FOUND", + ); + } + const type = resolveConnectorType(name, loadNamedConnectorConfig<Record<string, unknown>>(name)); + return getConnector(type).type; +} + +/** Sync saved connection `name`. */ +export async function syncConnection( + db: Database.Database, + provider: EmbeddingProvider, + name: string, + options: ConnectorSyncOptions = {}, +): Promise<{ name: string; type: ConnectorType; summary: ConnectorSyncSummary }> { + const type = savedConnectionType(name); + const summary = await runSavedConnectorSync(db, provider, type, name, options); + return { name, type, summary }; +} + +export interface ConnectionInfo { + name: string; + type: string; + /** Saved settings with secret fields replaced by "***". */ + settings: Record<string, unknown>; + schedule?: string | undefined; + lastSync?: string | undefined; + /** Status of the most recent recorded run, if any. */ + lastRun?: { status: string; startedAt: string; error: string | null } | undefined; +} + +/** Hide secret values of a saved config (the field stays so users see it is set). */ +export function maskConnectorSecrets( + type: string, + config: Record<string, unknown>, +): Record<string, unknown> { + const secrets = new Set(isConnectorType(type) ? CONNECTORS[type].secretFields : []); + return Object.fromEntries( + Object.entries(config) + .filter(([key]) => key !== "schedule" && key !== "lastSync" && key !== "connectorType") + .map(([key, value]) => [key, secrets.has(key) && value ? "***" : value]), + ); +} + +/** Every saved connection with masked settings, schedule and last run. */ +export function listConnections(db: Database.Database): ConnectionInfo[] { + return listNamedConnectorConfigs().map(({ name, config }) => { + const type = resolveConnectorType(name, config); + const schedule = config["schedule"] as { cronExpression?: string } | undefined; + const lastSync = typeof config["lastSync"] === "string" ? config["lastSync"] : undefined; + const last = getConnectorStatus(db, undefined, name)[0]; + return { + name, + type, + settings: maskConnectorSecrets(type, config), + ...(schedule?.cronExpression ? { schedule: schedule.cronExpression } : {}), + ...(lastSync ? { lastSync } : {}), + ...(last + ? { + lastRun: { + status: last.status, + startedAt: last.started_at, + error: last.error_message, + }, + } + : {}), + }; + }); +} + +/** + * Disconnect saved connection `name`: remove the documents its connector created (unless + * `keepDocuments`) and delete the saved config (it may hold credentials). Without a saved + * config, `type` must be given and only documents are removed. + */ +export async function disconnectConnection( + db: Database.Database, + name: string, + options: { type?: string | undefined; keepDocuments?: boolean | undefined } = {}, +): Promise<{ + name: string; + type: ConnectorType; + documentsRemoved: number; + configRemoved: boolean; +}> { + const saved = hasNamedConnectorConfig(name); + const type = saved ? savedConnectionType(name) : getConnector(options.type ?? name).type; + if (options.type && options.type !== type) { + throw new ValidationError(`Connection "${name}" is a ${type} connection, not ${options.type}`); + } + const config = saved ? loadSavedConnectorConfig<Record<string, unknown>>(type, name) : undefined; + const documentsRemoved = options.keepDocuments + ? 0 + : await CONNECTORS[type].disconnect(db, config); + const configRemoved = saved && deleteNamedConnectorConfig(name); + return { name, type, documentsRemoved, configRemoved }; +} diff --git a/src/connectors/saved-config.ts b/src/connectors/saved-config.ts index 7c0daeea..cd904a27 100644 --- a/src/connectors/saved-config.ts +++ b/src/connectors/saved-config.ts @@ -4,7 +4,6 @@ * scheduler and the MCP sync tools read them. */ import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; import { ConfigError, ValidationError } from "../errors.js"; import { hasNamedConnectorConfig, @@ -12,15 +11,18 @@ import { loadNamedConnectorConfig, saveNamedConnectorConfig, } from "./index.js"; -import { startSync, failSync, type ConnectorSyncOptions } from "./sync-tracker.js"; -import { syncNotion, type NotionConfig } from "./notion.js"; -import { syncSlack, type SlackConfig } from "./slack.js"; -import { syncConfluence, type ConfluenceConfig } from "./confluence.js"; -import { syncObsidianVault, type ObsidianConfig } from "./obsidian.js"; -import { syncOneNote, type OneNoteConfig } from "./onenote.js"; +import { startSync, failSync } from "./sync-tracker.js"; +import type { OneNoteConfig } from "./onenote.js"; /** Connector types that can be saved, re-synced and scheduled. */ -export const CONNECTOR_TYPES = ["notion", "slack", "confluence", "obsidian", "onenote"] as const; +export const CONNECTOR_TYPES = [ + "notion", + "slack", + "confluence", + "obsidian", + "onenote", + "docs", +] as const; export type ConnectorType = (typeof CONNECTOR_TYPES)[number]; export function isConnectorType(value: unknown): value is ConnectorType { @@ -163,50 +165,6 @@ export async function syncSavedConnector<C extends HasLastSync, R>( return runAndSaveConnector(type, name, config, sync); } -type SavedSyncRunner = ( - db: Database.Database, - provider: EmbeddingProvider, - name: string, - options: ConnectorSyncOptions, -) => Promise<unknown>; - -const SAVED_SYNC_RUNNERS: Record<ConnectorType, SavedSyncRunner> = { - notion: (db, provider, name, options) => - syncSavedConnector<NotionConfig, unknown>(db, "notion", name, (c) => - syncNotion(db, provider, c, options), - ), - slack: (db, provider, name, options) => - syncSavedConnector<SlackConfig, unknown>(db, "slack", name, (c) => - syncSlack(db, provider, c, options), - ), - confluence: (db, provider, name, options) => - syncSavedConnector<ConfluenceConfig, unknown>(db, "confluence", name, (c) => - syncConfluence(db, provider, c, options), - ), - obsidian: (db, provider, name, options) => - syncSavedConnector<ObsidianConfig, unknown>(db, "obsidian", name, (c) => - syncObsidianVault(db, provider, c, options), - ), - onenote: (db, provider, name, options) => - syncSavedConnector<OneNoteConfig, unknown>(db, "onenote", name, (c) => - syncOneNote(db, provider, c, options), - ), -}; - -/** - * Re-sync saved connector `name` of type `type` (used by the scheduler). Writes exactly one - * connector_syncs row, recorded under `name`. - */ -export async function runSavedConnectorSync( - db: Database.Database, - provider: EmbeddingProvider, - type: ConnectorType, - name: string, - options: ConnectorSyncOptions = {}, -): Promise<void> { - await SAVED_SYNC_RUNNERS[type](db, provider, name, { ...options, syncName: name }); -} - /** Per-call values for resolveSyncConfig; undefined means "not given". */ export type SyncConfigOverrides<C> = { [K in keyof C]?: C[K] | undefined }; diff --git a/src/core/scheduler.ts b/src/core/scheduler.ts index 1a41a3dc..00a0fcd2 100644 --- a/src/core/scheduler.ts +++ b/src/core/scheduler.ts @@ -1,19 +1,17 @@ import type Database from "better-sqlite3"; -import { readdirSync, readFileSync, existsSync } from "node:fs"; -import { join } from "node:path"; // NOTE: @types/node-cron v3 is used with node-cron v4 — no v4 types are published yet. // The schedule() and ScheduledTask.stop() APIs are compatible across versions. import cron from "node-cron"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; -import { getConnectorsDir } from "../connectors/index.js"; +import { listNamedConnectorConfigs } from "../connectors/index.js"; import { isConnectorType, recordFailedSync, resolveConnectorType, - runSavedConnectorSync, } from "../connectors/saved-config.js"; +import { runSavedConnectorSync } from "../connectors/registry.js"; export interface ScheduleConfig { cronExpression: string; @@ -191,34 +189,16 @@ export class ConnectorScheduler { * Each connector config can have a `schedule` field with a `cronExpression`. */ export function loadScheduleEntries(): ConnectorScheduleEntry[] { - const log = getLogger(); const entries: ConnectorScheduleEntry[] = []; - - const connectorsDir = getConnectorsDir(); - if (!existsSync(connectorsDir)) { - return entries; - } - - const files = readdirSync(connectorsDir).filter((f) => f.endsWith(".json")); - for (const file of files) { - try { - const raw = readFileSync(join(connectorsDir, file), "utf-8"); - const config = JSON.parse(raw) as Record<string, unknown>; - const schedule = config.schedule as { cronExpression?: string } | undefined; - - if (schedule?.cronExpression) { - const connectorName = file.slice(0, -".json".length); - const connectorType = resolveConnectorType(connectorName, config); - entries.push({ - connectorType, - connectorName, - cronExpression: schedule.cronExpression, - }); - } - } catch (err) { - log.warn({ file, err }, "Failed to read connector config for scheduling"); + for (const { name, config } of listNamedConnectorConfigs()) { + const schedule = config["schedule"] as { cronExpression?: string } | undefined; + if (schedule?.cronExpression) { + entries.push({ + connectorType: resolveConnectorType(name, config), + connectorName: name, + cronExpression: schedule.cronExpression, + }); } } - return entries; } diff --git a/src/db/schema.ts b/src/db/schema.ts index a5651429..c4809350 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -188,6 +188,9 @@ const MIGRATIONS: Record<number, string> = { INSERT INTO schema_version (version) VALUES (8); `, + // connector_configs is unused: no CLI, MCP, REST or scheduler code reads or writes it. + // Connector settings live in ~/.libscope/connectors/<name>.json. Only the exported + // save/load/deleteDbConnectorConfig helpers in src/connectors/index.ts touch it. 9: ` CREATE TABLE IF NOT EXISTS connector_configs ( type TEXT PRIMARY KEY, diff --git a/tests/unit/connector-registry.test.ts b/tests/unit/connector-registry.test.ts new file mode 100644 index 00000000..46ab4b2f --- /dev/null +++ b/tests/unit/connector-registry.test.ts @@ -0,0 +1,66 @@ +import { describe, it, expect } from "vitest"; +import { CONNECTORS, getConnector, maskConnectorSecrets } from "../../src/connectors/registry.js"; +import { CONNECTOR_TYPES } from "../../src/connectors/saved-config.js"; +import { createTestDb } from "../fixtures/test-db.js"; +import { ValidationError } from "../../src/errors.js"; + +describe("connector registry", () => { + it("has one definition per connector type", () => { + expect(Object.keys(CONNECTORS).sort()).toEqual([...CONNECTOR_TYPES].sort()); + for (const type of CONNECTOR_TYPES) { + expect(getConnector(type).type).toBe(type); + for (const field of getConnector(type).secretFields) { + expect(getConnector(type).configSchema.shape).toHaveProperty(field); + } + } + }); + + it("rejects unknown connector types", () => { + expect(() => getConnector("myspace")).toThrow(ValidationError); + }); + + it("masks secret fields and drops bookkeeping fields", () => { + expect( + maskConnectorSecrets("confluence", { + baseUrl: "https://acme.atlassian.net", + token: "secret", + connectorType: "confluence", + lastSync: "2026-01-01", + schedule: { cronExpression: "0 * * * *" }, + }), + ).toEqual({ baseUrl: "https://acme.atlassian.net", token: "***" }); + expect(maskConnectorSecrets("unknown", { token: "visible" })).toEqual({ token: "visible" }); + }); + + it("maps each connector's result to the same summary shape", () => { + expect( + CONNECTORS.onenote.summarize({ + notebooks: 1, + sections: 2, + pagesAdded: 3, + pagesUpdated: 4, + pagesDeleted: 5, + errors: [{ page: "p", error: "e" }], + }), + ).toEqual({ added: 3, updated: 4, deleted: 5, errors: ["p: e"] }); + expect( + CONNECTORS.slack.summarize({ + channels: 1, + messagesIndexed: 2, + threadsIndexed: 3, + errors: [{ channel: "c", error: "e" }], + }), + ).toEqual({ added: 5, updated: 0, deleted: 0, errors: ["#c: e"] }); + }); + + it("needs the saved path to disconnect a vault or docs site", () => { + const db = createTestDb(); + try { + expect(() => CONNECTORS.obsidian.disconnect(db, undefined)).toThrow(ValidationError); + expect(() => CONNECTORS.docs.disconnect(db, {})).toThrow(ValidationError); + expect(CONNECTORS.docs.disconnect(db, { url: "https://docs.example.com/" })).toBe(0); + } finally { + db.close(); + } + }); +}); diff --git a/tests/unit/connector-saved-config.test.ts b/tests/unit/connector-saved-config.test.ts index a4cae217..2cecd48d 100644 --- a/tests/unit/connector-saved-config.test.ts +++ b/tests/unit/connector-saved-config.test.ts @@ -32,8 +32,8 @@ const { loadSavedConnectorConfig, findSavedOneNoteConfig, resolveConnectorType, - runSavedConnectorSync, } = await import("../../src/connectors/saved-config.js"); +const { runSavedConnectorSync } = await import("../../src/connectors/registry.js"); const { saveConnectorConfig, getConnectorsDir } = await import("../../src/connectors/index.js"); const { ConnectorScheduler, loadScheduleEntries } = await import("../../src/core/scheduler.js"); const { _resetRateLimiter } = await import("../../src/connectors/onenote.js"); From faeb0ffd309af58833279f75fcbac546792512e6 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:45:14 +0000 Subject: [PATCH 36/83] feat(core): operation layer with one schema per action src/core/operations defines 52 operations (documents, search, links, tags, topics, saved searches, packs, connectors, admin, analytics, webhooks, tasks). Each has a zod input schema with descriptions and defaults, annotations and an optional REST mapping. runOperation validates input (ZodError -> ValidationError); startOperationTask runs one in the shared task registry. --- src/core/operations/admin.ts | 232 +++++++++++++ src/core/operations/analytics.ts | 69 ++++ src/core/operations/connectors.ts | 101 ++++++ src/core/operations/documents.ts | 318 ++++++++++++++++++ src/core/operations/index.ts | 50 +++ src/core/operations/links.ts | 108 ++++++ src/core/operations/packs.ts | 148 ++++++++ src/core/operations/schemas.ts | 39 +++ src/core/operations/search.ts | 133 ++++++++ src/core/operations/searches.ts | 93 +++++ src/core/operations/tags.ts | 78 +++++ src/core/operations/tasks.ts | 53 +++ src/core/operations/topics.ts | 65 ++++ src/core/operations/types.ts | 172 ++++++++++ src/core/operations/webhooks.ts | 81 +++++ tests/unit/operations/connectors.test.ts | 110 ++++++ tests/unit/operations/documents.test.ts | 223 ++++++++++++ tests/unit/operations/helpers.ts | 83 +++++ .../unit/operations/links-tags-topics.test.ts | 147 ++++++++ tests/unit/operations/packs-admin.test.ts | 201 +++++++++++ tests/unit/operations/registry.test.ts | 59 ++++ tests/unit/operations/search.test.ts | 111 ++++++ .../operations/searches-analytics.test.ts | 95 ++++++ tests/unit/operations/webhooks-tasks.test.ts | 134 ++++++++ 24 files changed, 2903 insertions(+) create mode 100644 src/core/operations/admin.ts create mode 100644 src/core/operations/analytics.ts create mode 100644 src/core/operations/connectors.ts create mode 100644 src/core/operations/documents.ts create mode 100644 src/core/operations/index.ts create mode 100644 src/core/operations/links.ts create mode 100644 src/core/operations/packs.ts create mode 100644 src/core/operations/schemas.ts create mode 100644 src/core/operations/search.ts create mode 100644 src/core/operations/searches.ts create mode 100644 src/core/operations/tags.ts create mode 100644 src/core/operations/tasks.ts create mode 100644 src/core/operations/topics.ts create mode 100644 src/core/operations/types.ts create mode 100644 src/core/operations/webhooks.ts create mode 100644 tests/unit/operations/connectors.test.ts create mode 100644 tests/unit/operations/documents.test.ts create mode 100644 tests/unit/operations/helpers.ts create mode 100644 tests/unit/operations/links-tags-topics.test.ts create mode 100644 tests/unit/operations/packs-admin.test.ts create mode 100644 tests/unit/operations/registry.test.ts create mode 100644 tests/unit/operations/search.test.ts create mode 100644 tests/unit/operations/searches-analytics.test.ts create mode 100644 tests/unit/operations/webhooks-tasks.test.ts diff --git a/src/core/operations/admin.ts b/src/core/operations/admin.ts new file mode 100644 index 00000000..76de926b --- /dev/null +++ b/src/core/operations/admin.ts @@ -0,0 +1,232 @@ +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { describeEmbeddingIdentity, rebuildVectorTable } from "../../db/index-meta.js"; +import { createVectorTable } from "../../db/schema.js"; +import { bulkDelete, bulkMove, bulkRetag, type BulkSelector } from "../bulk.js"; +import { findDuplicates } from "../dedup.js"; +import { exportKnowledgeBase, importFromBackup } from "../export.js"; +import { getOverview } from "../overview.js"; +import { reindex } from "../reindex.js"; +import { resolveTopicId } from "../topics.js"; +import { pruneExpiredDocuments } from "../ttl.js"; +import * as s from "./schemas.js"; +import { defineOperation, type OperationContext } from "./types.js"; + +export const overviewOperation = defineOperation({ + name: "overview", + group: "admin", + summary: + "Knowledge base overview: counts, topics, installed packs, embedding model of the index, and health", + input: z.object({}), + annotations: { readOnly: true }, + http: { method: "GET", path: "/overview" }, + handler: (ctx) => getOverview(ctx.db, ctx.provider), +}); + +export const reindexOperation = defineOperation({ + name: "reindex", + group: "admin", + summary: "Re-embed chunks with the configured embedding model (rebuild after changing models)", + input: z.object({ + documentIds: z.array(s.documentId).optional().describe("Only these documents"), + since: z.iso + .date() + .or(z.iso.datetime()) + .optional() + .describe("Only documents created on or after"), + before: z.iso + .date() + .or(z.iso.datetime()) + .optional() + .describe("Only documents created on or before"), + batchSize: z + .number() + .int() + .min(1) + .max(500) + .optional() + .describe("Chunks per embedding call (default 50)"), + rebuild: z + .boolean() + .default(false) + .describe( + "Drop and recreate the vector table for the configured model, then re-embed everything", + ), + }), + annotations: { longRunning: true, idempotent: true }, + http: { method: "POST", path: "/admin/reindex" }, + async handler(ctx, input) { + const filtered = input.documentIds ?? input.since ?? input.before; + if (input.rebuild && filtered !== undefined) { + throw new ValidationError( + "rebuild re-embeds every chunk; it cannot be combined with filters", + ); + } + let index: string | undefined; + if (input.rebuild) { + index = describeEmbeddingIdentity(await rebuildVectorTable(ctx.db, ctx.provider)); + } else { + createVectorTable(ctx.db, ctx.provider); + } + const result = await reindex(ctx.db, ctx.provider, { + documentIds: input.documentIds, + since: input.since, + before: input.before, + batchSize: input.batchSize, + signal: ctx.signal, + onProgress: (p) => ctx.onProgress?.({ done: p.completed + p.failed, total: p.total }), + }); + return { ...result, ...(index === undefined ? {} : { rebuiltIndex: index }) }; + }, +}); + +export const dedupeOperation = defineOperation({ + name: "dedupe", + group: "admin", + summary: "Find groups of duplicate or near-duplicate documents", + input: z.object({ + threshold: z + .number() + .min(0) + .max(1) + .default(0.95) + .describe("Similarity threshold for near-duplicates"), + strategy: z + .enum(["exact", "semantic", "both"]) + .default("both") + .describe("exact: same content hash; semantic: embedding similarity"), + }), + annotations: { readOnly: true, longRunning: true }, + http: { method: "GET", path: "/admin/duplicates" }, + async handler(ctx, input) { + return { items: await findDuplicates(ctx.db, ctx.provider, input) }; + }, +}); + +function assertLocalFilesAllowed(ctx: OperationContext, what: string): void { + if (ctx.surface === "api" || ctx.surface === "mcp") { + throw new ValidationError(`${what} is only available from the CLI and SDK`); + } +} + +export const backupOperation = defineOperation({ + name: "backup", + group: "admin", + summary: "Export the whole knowledge base to a JSON file", + input: z.object({ outputPath: z.string().min(1).describe("File to write") }), + annotations: { readOnly: true }, + handler(ctx, input) { + assertLocalFilesAllowed(ctx, "Backup"); + const data = exportKnowledgeBase(ctx.db, input.outputPath); + return { path: input.outputPath, counts: data.metadata.counts }; + }, +}); + +export const restoreOperation = defineOperation({ + name: "restore", + group: "admin", + summary: "Import a backup file written by backup", + input: z.object({ backupPath: z.string().min(1).describe("Backup file to read") }), + handler(ctx, input) { + assertLocalFilesAllowed(ctx, "Restore"); + const data = importFromBackup(ctx.db, input.backupPath); + return { path: input.backupPath, counts: data.metadata.counts }; + }, +}); + +export const pruneExpiredOperation = defineOperation({ + name: "prune-expired", + group: "admin", + summary: "Delete documents whose expiry time (expiresAt) has passed", + input: z.object({}), + annotations: { destructive: true, idempotent: true }, + http: { method: "POST", path: "/admin/prune-expired" }, + handler: (ctx) => pruneExpiredDocuments(ctx.db), +}); + +const selector = { + topic: s.opt(s.topic), + library: s.opt(s.library), + sourceType: s.opt(s.sourceType), + tags: s.tags.optional().describe("Documents carrying all of these tags"), + dateFrom: z.string().min(1).optional().describe("Created on or after (ISO 8601)"), + dateTo: z.string().min(1).optional().describe("Created on or before (ISO 8601)"), + dryRun: z + .boolean() + .default(false) + .describe("Report the matching documents without changing them"), +}; + +function toSelector( + ctx: OperationContext, + input: { + topic?: string | undefined; + library?: string | undefined; + sourceType?: string | undefined; + tags?: string[] | undefined; + dateFrom?: string | undefined; + dateTo?: string | undefined; + }, +): BulkSelector { + const sel: BulkSelector = {}; + if (input.topic !== undefined) sel.topicId = resolveTopicId(ctx.db, input.topic); + if (input.library !== undefined) sel.library = input.library; + if (input.sourceType !== undefined) sel.sourceType = input.sourceType; + if (input.tags !== undefined) sel.tags = input.tags; + if (input.dateFrom !== undefined) sel.dateFrom = input.dateFrom; + if (input.dateTo !== undefined) sel.dateTo = input.dateTo; + return sel; +} + +export const bulkDeleteOperation = defineOperation({ + name: "bulk-delete", + group: "admin", + summary: "Delete every document matching the filters (at most 1000 per call)", + input: z.object(selector), + annotations: { destructive: true }, + http: { method: "POST", path: "/bulk/delete" }, + handler: (ctx, input) => bulkDelete(ctx.db, toSelector(ctx, input), input.dryRun), +}); + +export const bulkRetagOperation = defineOperation({ + name: "bulk-retag", + group: "admin", + summary: "Add and/or remove tags on every document matching the filters", + input: z.object({ + ...selector, + addTags: s.tags.optional().describe("Tags to add"), + removeTags: s.tags.optional().describe("Tags to remove"), + }), + annotations: { idempotent: true }, + http: { method: "POST", path: "/bulk/retag" }, + handler: (ctx, input) => + bulkRetag(ctx.db, toSelector(ctx, input), input.addTags, input.removeTags, input.dryRun), +}); + +export const bulkMoveOperation = defineOperation({ + name: "bulk-move", + group: "admin", + summary: "Move every document matching the filters to another topic", + input: z.object({ ...selector, targetTopic: s.topic.describe("Destination topic ID or name") }), + annotations: { idempotent: true }, + http: { method: "POST", path: "/bulk/move" }, + handler: (ctx, input) => + bulkMove( + ctx.db, + toSelector(ctx, input), + resolveTopicId(ctx.db, input.targetTopic), + input.dryRun, + ), +}); + +export const adminOperations = [ + overviewOperation, + reindexOperation, + dedupeOperation, + backupOperation, + restoreOperation, + pruneExpiredOperation, + bulkDeleteOperation, + bulkRetagOperation, + bulkMoveOperation, +] as const; diff --git a/src/core/operations/analytics.ts b/src/core/operations/analytics.ts new file mode 100644 index 00000000..c2ee4f2a --- /dev/null +++ b/src/core/operations/analytics.ts @@ -0,0 +1,69 @@ +import { z } from "zod"; +import { + getKnowledgeGaps, + getPopularDocuments, + getSearchAnalytics, + getStaleDocuments, + getTopQueries, +} from "../analytics.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +const days = (defaultValue: number): z.ZodDefault<z.ZodNumber> => + z + .number() + .int() + .min(1) + .max(3650) + .default(defaultValue) + .describe(`Look-back days (default ${defaultValue})`); + +export const popularOperation = defineOperation({ + name: "popular", + group: "analytics", + summary: "Documents returned most often in search results", + input: z.object({ limit: s.limit(10, 100) }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/analytics/popular" }, + handler: (ctx, input) => ({ items: getPopularDocuments(ctx.db, input.limit) }), +}); + +export const staleOperation = defineOperation({ + name: "stale", + group: "analytics", + summary: "Documents that no search returned in the given number of days", + input: z.object({ days: days(90) }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/analytics/stale" }, + handler: (ctx, input) => ({ items: getStaleDocuments(ctx.db, input.days) }), +}); + +export const topQueriesOperation = defineOperation({ + name: "top-queries", + group: "analytics", + summary: "Most frequent search queries", + input: z.object({ limit: s.limit(10, 100) }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/analytics/top-queries" }, + handler: (ctx, input) => ({ items: getTopQueries(ctx.db, input.limit) }), +}); + +export const searchAnalyticsOperation = defineOperation({ + name: "search-analytics", + group: "analytics", + summary: "Search volume, top and zero-result queries, and knowledge gaps", + input: z.object({ days: days(30) }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/analytics/searches" }, + handler: (ctx, input) => ({ + ...getSearchAnalytics(ctx.db, input.days), + knowledgeGaps: getKnowledgeGaps(ctx.db, input.days), + }), +}); + +export const analyticsOperations = [ + popularOperation, + staleOperation, + topQueriesOperation, + searchAnalyticsOperation, +] as const; diff --git a/src/core/operations/connectors.ts b/src/core/operations/connectors.ts new file mode 100644 index 00000000..178e8359 --- /dev/null +++ b/src/core/operations/connectors.ts @@ -0,0 +1,101 @@ +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { listNamedConnectorConfigs } from "../../connectors/index.js"; +import { + disconnectConnection, + listConnections, + syncConnection, + type ConnectorSyncSummary, +} from "../../connectors/registry.js"; +import { CONNECTOR_TYPES } from "../../connectors/saved-config.js"; +import { opt } from "./schemas.js"; +import { defineOperation } from "./types.js"; + +const connectionName = z + .string() + .regex(/^[\w-]+$/, "letters, digits, '_' and '-' only") + .describe("Saved connection name (see list-connections)"); + +export const listConnectionsOperation = defineOperation({ + name: "list-connections", + group: "connectors", + summary: "List saved connector connections with their schedule and last sync", + input: z.object({}), + annotations: { readOnly: true }, + http: { method: "GET", path: "/connections" }, + handler: (ctx) => ({ items: listConnections(ctx.db) }), +}); + +type SyncOutcome = + | { name: string; type: string; status: "completed"; summary: ConnectorSyncSummary } + | { name: string; status: "failed"; error: string }; + +export const syncOperation = defineOperation({ + name: "sync", + group: "connectors", + summary: "Sync one saved connection, or all of them, using the saved settings", + input: z.object({ + name: opt(connectionName), + all: z.boolean().default(false).describe("Sync every saved connection"), + }), + annotations: { longRunning: true }, + http: { method: "POST", path: "/sync" }, + async handler(ctx, input): Promise<{ items: SyncOutcome[] }> { + if ((input.name === undefined) === !input.all) { + throw new ValidationError("Give either a connection name or all: true"); + } + if (input.name !== undefined) { + const result = await syncConnection(ctx.db, ctx.provider, input.name, { signal: ctx.signal }); + return { items: [{ ...result, status: "completed" }] }; + } + // One failing connection does not stop the others; each run is recorded in connector_syncs. + const names = listNamedConnectorConfigs().map((c) => c.name); + const items: SyncOutcome[] = []; + for (const [i, name] of names.entries()) { + ctx.signal?.throwIfAborted(); + try { + const result = await syncConnection(ctx.db, ctx.provider, name, { signal: ctx.signal }); + items.push({ ...result, status: "completed" }); + } catch (err) { + if (ctx.signal?.aborted) throw err; + items.push({ + name, + status: "failed", + error: err instanceof Error ? err.message : String(err), + }); + } + ctx.onProgress?.({ done: i + 1, total: names.length, message: name }); + } + return { items }; + }, +}); + +export const disconnectOperation = defineOperation({ + name: "disconnect", + group: "connectors", + summary: "Remove a connection's documents and its saved settings (including credentials)", + input: z.object({ + name: connectionName, + type: z + .enum(CONNECTOR_TYPES) + .optional() + .describe("Connector type; needed only when no saved connection has this name"), + keepDocuments: z + .boolean() + .default(false) + .describe("Keep the documents; only delete the saved settings"), + }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/connections/:name" }, + handler: (ctx, input) => + disconnectConnection(ctx.db, input.name, { + type: input.type, + keepDocuments: input.keepDocuments, + }), +}); + +export const connectorOperations = [ + listConnectionsOperation, + syncOperation, + disconnectOperation, +] as const; diff --git a/src/core/operations/documents.ts b/src/core/operations/documents.ts new file mode 100644 index 00000000..a060bd66 --- /dev/null +++ b/src/core/operations/documents.ts @@ -0,0 +1,318 @@ +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { + assertDocumentExists, + countDocuments, + deleteDocument, + listDocuments, + updateDocument, + type Document, +} from "../documents.js"; +import { getDocumentView } from "../document-view.js"; +import { detectIngestKind, ingest } from "../ingest.js"; +import { rateDocument } from "../ratings.js"; +import { addTagsToDocument, getDocumentTags, removeTagsFromDocument } from "../tags.js"; +import { findTopicId, resolveTopicId } from "../topics.js"; +import { getVersionHistory, rollbackToVersion } from "../versioning.js"; +import * as s from "./schemas.js"; +import { defineOperation, type ListResult, type OperationContext } from "./types.js"; + +/** A document without its content, for listings. */ +export type DocumentSummary = Omit<Document, "content" | "id"> & { + documentId: string; + contentLength: number; +}; + +export function toDocumentSummary(doc: Document): DocumentSummary { + const { id, content, ...rest } = doc; + return { documentId: id, contentLength: content.length, ...rest }; +} + +/** Topic ID for a write: the topic must exist (by ID or name). */ +function topicForWrite(ctx: OperationContext, topic: string | undefined): string | undefined { + return topic === undefined ? undefined : resolveTopicId(ctx.db, topic); +} + +/** Topic ID for a filter: a name is resolved; an unknown value filters by itself (no match). */ +export function topicForFilter( + ctx: OperationContext, + topic: string | undefined, +): string | undefined { + return topic === undefined ? undefined : (findTopicId(ctx.db, topic) ?? topic); +} + +export const addOperation = defineOperation({ + name: "add", + group: "documents", + summary: + "Add content to the knowledge base: inline content, a URL (optionally crawled), a GitHub/GitLab repository, or (CLI/SDK only) a local file or directory", + input: z.object({ + source: z + .string() + .min(1) + .optional() + .describe("What to add: a URL, a repository URL, or (CLI/SDK) a file or directory path"), + content: z.string().min(1).optional().describe("Inline document content (markdown)"), + title: z + .string() + .min(1) + .optional() + .describe("Title (required with content; detected for files and URLs)"), + url: s.url + .optional() + .describe("Source URL. Without content, the URL is fetched; with content, it is stored"), + kind: z + .enum(["auto", "content", "file", "directory", "url", "repo"]) + .default("auto") + .describe("How to read source (default: detect from the value)"), + topic: s.opt(s.topic), + library: s.opt(s.library), + version: s.opt(s.version), + sourceType: s.sourceType + .optional() + .describe( + "Source type (default: library if library is set, topic if topic is set, else manual)", + ), + tags: s.tags.optional().describe("Tags to add to every new document"), + expiresAt: z.iso + .datetime() + .optional() + .describe("ISO 8601 time after which the document is pruned"), + dedup: z + .enum(["skip", "warn", "force"]) + .optional() + .describe( + "Duplicate handling: skip returns the existing document, warn indexes anyway, force skips the check", + ), + format: z.string().min(1).optional().describe("Force a file format, e.g. .pdf"), + include: z.array(z.string()).optional().describe("Directory: globs of files to include"), + exclude: z.array(z.string()).optional().describe("Directory: globs of files to skip"), + spider: z.boolean().default(false).describe("URL: also crawl linked pages"), + maxPages: z + .number() + .int() + .positive() + .optional() + .describe("Crawl: page limit (default 25, max 200)"), + maxDepth: z.number().int().min(0).optional().describe("Crawl: link depth (default 2, max 5)"), + sameDomain: z.boolean().optional().describe("Crawl: stay on the seed domain (default true)"), + pathPrefix: z.string().optional().describe("Crawl: only follow links under this path"), + excludePatterns: z.array(z.string()).optional().describe("Crawl: globs of URLs to skip"), + branch: z + .string() + .min(1) + .optional() + .describe("Repository: branch (default: from URL, else main)"), + paths: z.array(z.string()).optional().describe("Repository: only these subdirectories"), + extensions: z + .array(z.string()) + .optional() + .describe("Repository: file extensions (default .md, .mdx, .txt, .rst)"), + token: z + .string() + .min(1) + .optional() + .describe("Repository: access token for private repositories"), + dryRun: z.boolean().default(false).describe("List what would be added without adding it"), + }), + annotations: { longRunning: true }, + http: { method: "POST", path: "/documents" }, + async handler(ctx, input) { + const source = input.source ?? (input.content === undefined ? input.url : undefined); + const ingestInput = { + ...input, + source, + topic: topicForWrite(ctx, input.topic), + }; + const kind = detectIngestKind(ingestInput); + // Remote callers must not read files from the server's disk. + if ( + (kind === "file" || kind === "directory") && + (ctx.surface === "mcp" || ctx.surface === "api") + ) { + throw new ValidationError( + "Adding local files or directories is only available from the CLI and SDK", + ); + } + return ingest( + { + db: ctx.db, + provider: ctx.provider, + fetchOptions: { + allowPrivateUrls: ctx.config.indexing.allowPrivateUrls, + allowSelfSignedCerts: ctx.config.indexing.allowSelfSignedCerts, + }, + submittedBy: ctx.surface === "mcp" ? "model" : "manual", + signal: ctx.signal, + onProgress: ctx.onProgress, + }, + { ...ingestInput, kind }, + ); + }, +}); + +export const getDocumentOperation = defineOperation({ + name: "get-document", + group: "documents", + summary: "Get a document with its tags, links and rating summary; long content can be paged", + input: z.object({ + documentId: s.documentId, + offset: z.number().int().min(0).default(0).describe("Character offset into the content"), + maxLength: z + .number() + .int() + .positive() + .optional() + .describe("Maximum characters of content to return (default: all)"), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/documents/:documentId" }, + handler: (ctx, input) => + getDocumentView(ctx.db, input.documentId, { offset: input.offset, maxLength: input.maxLength }), +}); + +export const listDocumentsOperation = defineOperation({ + name: "list-documents", + group: "documents", + summary: "List documents (newest first) with optional filters", + input: z.object({ + ...s.documentFilters, + limit: s.limit(50, 1000), + offset: s.offset, + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/documents" }, + handler(ctx, input): ListResult<DocumentSummary> { + const filters = { + library: input.library, + version: input.version, + topicId: topicForFilter(ctx, input.topic), + sourceType: input.sourceType, + tags: input.tags, + }; + return { + items: listDocuments(ctx.db, { ...filters, limit: input.limit, offset: input.offset }).map( + toDocumentSummary, + ), + total: countDocuments(ctx.db, filters), + limit: input.limit, + offset: input.offset, + }; + }, +}); + +/** Make the document's tags exactly `tags`. */ +function replaceTags(ctx: OperationContext, documentId: string, tags: string[]): void { + const wanted = new Set(tags.map((t) => t.trim().toLowerCase())); + const current = getDocumentTags(ctx.db, documentId).map((t) => t.name); + removeTagsFromDocument( + ctx.db, + documentId, + current.filter((t) => !wanted.has(t)), + ); + addTagsToDocument(ctx.db, documentId, [...wanted]); +} + +export const updateDocumentOperation = defineOperation({ + name: "update-document", + group: "documents", + summary: "Update a document's title, content, metadata or tags (content changes are re-indexed)", + input: z.object({ + documentId: s.documentId, + title: z.string().min(1).optional().describe("New title"), + content: z.string().min(1).optional().describe("New content (re-chunked and re-embedded)"), + library: s.library.nullable().optional().describe("New library (null clears it)"), + version: s.version.nullable().optional().describe("New version (null clears it)"), + url: s.url.nullable().optional().describe("New URL (null clears it)"), + topic: s.topic.nullable().optional().describe("New topic ID or name (null clears it)"), + tags: s.tags.optional().describe("Replace the document's tags with these"), + }), + annotations: { idempotent: true }, + http: { method: "PATCH", path: "/documents/:documentId" }, + async handler(ctx, input) { + const { documentId, title, content, tags, topic, ...meta } = input; + if (Object.values(input).filter((v) => v !== undefined).length === 1) { + throw new ValidationError("Nothing to update: give at least one field to change"); + } + assertDocumentExists(ctx.db, documentId); + const metadata = { + ...meta, + ...(topic === undefined + ? {} + : { topicId: topic === null ? null : resolveTopicId(ctx.db, topic) }), + }; + const hasDocChange = + title !== undefined || content !== undefined || Object.keys(metadata).length > 0; + if (hasDocChange) { + await updateDocument(ctx.db, ctx.provider, documentId, { title, content, metadata }); + } + if (tags) replaceTags(ctx, documentId, tags); + return getDocumentView(ctx.db, documentId, { maxLength: 0 }); + }, +}); + +export const deleteDocumentOperation = defineOperation({ + name: "delete-document", + group: "documents", + summary: "Delete a document with its chunks, vectors, tags, links and ratings", + input: z.object({ documentId: s.documentId }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/documents/:documentId" }, + handler(ctx, input) { + deleteDocument(ctx.db, input.documentId); + return { documentId: input.documentId, deleted: true }; + }, +}); + +export const rateDocumentOperation = defineOperation({ + name: "rate-document", + group: "documents", + summary: "Rate a document (1-5), optionally with feedback or a suggested correction", + input: z.object({ + documentId: s.documentId, + chunkId: s.chunkId.optional().describe("Rate one chunk of the document"), + rating: z.number().int().min(1).max(5).describe("1 (poor) to 5 (excellent)"), + feedback: z.string().min(1).optional().describe("What is good or wrong"), + suggestedCorrection: z + .string() + .min(1) + .optional() + .describe("Replacement text if the content is wrong"), + }), + http: { method: "POST", path: "/documents/:documentId/ratings" }, + handler: (ctx, input) => + rateDocument(ctx.db, { ...input, ratedBy: ctx.surface === "mcp" ? "model" : "user" }), +}); + +export const documentHistoryOperation = defineOperation({ + name: "document-history", + group: "documents", + summary: "List saved versions of a document, newest first", + input: z.object({ documentId: s.documentId }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/documents/:documentId/versions" }, + handler: (ctx, input) => ({ items: getVersionHistory(ctx.db, input.documentId) }), +}); + +export const rollbackDocumentOperation = defineOperation({ + name: "rollback-document", + group: "documents", + summary: "Restore a document to a saved version (the current state is saved first)", + input: z.object({ + documentId: s.documentId, + version: z.number().int().positive().describe("Version number from document-history"), + }), + http: { method: "POST", path: "/documents/:documentId/rollback" }, + handler: (ctx, input) => rollbackToVersion(ctx.db, ctx.provider, input.documentId, input.version), +}); + +export const documentOperations = [ + addOperation, + getDocumentOperation, + listDocumentsOperation, + updateDocumentOperation, + deleteDocumentOperation, + rateDocumentOperation, + documentHistoryOperation, + rollbackDocumentOperation, +] as const; diff --git a/src/core/operations/index.ts b/src/core/operations/index.ts new file mode 100644 index 00000000..93014328 --- /dev/null +++ b/src/core/operations/index.ts @@ -0,0 +1,50 @@ +import { adminOperations } from "./admin.js"; +import { analyticsOperations } from "./analytics.js"; +import { connectorOperations } from "./connectors.js"; +import { documentOperations } from "./documents.js"; +import { linkOperations } from "./links.js"; +import { packOperations } from "./packs.js"; +import { savedSearchOperations } from "./searches.js"; +import { searchOperations } from "./search.js"; +import { tagOperations } from "./tags.js"; +import { taskOperations } from "./tasks.js"; +import { topicOperations } from "./topics.js"; +import type { Operation } from "./types.js"; +import { webhookOperations } from "./webhooks.js"; + +export * from "./types.js"; +export * from "./admin.js"; +export * from "./analytics.js"; +export * from "./connectors.js"; +export * from "./documents.js"; +export * from "./links.js"; +export * from "./packs.js"; +export * from "./searches.js"; +export * from "./search.js"; +export * from "./tags.js"; +export * from "./tasks.js"; +export * from "./topics.js"; +export * from "./webhooks.js"; + +/** Every operation, grouped in the order surfaces list them. */ +export const OPERATIONS: readonly Operation[] = [ + ...documentOperations, + ...searchOperations, + ...linkOperations, + ...tagOperations, + ...topicOperations, + ...savedSearchOperations, + ...packOperations, + ...connectorOperations, + ...adminOperations, + ...analyticsOperations, + ...webhookOperations, + ...taskOperations, +] as unknown as readonly Operation[]; + +const BY_NAME = new Map(OPERATIONS.map((op) => [op.name, op])); + +/** The operation called `name`, or undefined. */ +export function getOperation(name: string): Operation | undefined { + return BY_NAME.get(name); +} diff --git a/src/core/operations/links.ts b/src/core/operations/links.ts new file mode 100644 index 00000000..3afa4026 --- /dev/null +++ b/src/core/operations/links.ts @@ -0,0 +1,108 @@ +import { z } from "zod"; +import { assertDocumentExists } from "../documents.js"; +import { + LINK_TYPES, + createLink, + deleteLink, + getDocumentLinks, + getPrerequisiteChain, + listLinks, + type DocumentLink, + type DocumentLinkWithTitle, +} from "../links.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +const linkType = z.enum(LINK_TYPES).describe(`Relationship type: ${LINK_TYPES.join(", ")}`); + +type LinkOut<L extends DocumentLink> = Omit<L, "id"> & { linkId: string }; + +function toLinkOut<L extends DocumentLink>(link: L): LinkOut<L> { + const { id, ...rest } = link; + return { linkId: id, ...rest }; +} + +export const linkDocumentsOperation = defineOperation({ + name: "link-documents", + group: "links", + summary: "Create a typed link from one document to another", + input: z.object({ + documentId: s.documentId.describe("Source document ID"), + targetDocumentId: s.documentId.describe("Target document ID"), + linkType, + label: z.string().min(1).optional().describe("Short description of the relationship"), + }), + annotations: { idempotent: true }, + http: { method: "POST", path: "/documents/:documentId/links" }, + handler: (ctx, input) => + toLinkOut( + createLink(ctx.db, input.documentId, input.targetDocumentId, input.linkType, input.label), + ), +}); + +export const unlinkDocumentsOperation = defineOperation({ + name: "unlink-documents", + group: "links", + summary: "Delete a link between documents", + input: z.object({ linkId: z.string().min(1).describe("Link ID") }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/links/:linkId" }, + handler(ctx, input) { + deleteLink(ctx.db, input.linkId); + return { linkId: input.linkId, deleted: true }; + }, +}); + +export const listLinksOperation = defineOperation({ + name: "list-links", + group: "links", + summary: "List links of one document (outgoing and incoming) or all links", + input: z.object({ + documentId: s.documentId.optional().describe("Only links from or to this document"), + linkType: s.opt(linkType), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/links" }, + handler(ctx, input) { + if (input.documentId === undefined) { + return { items: listLinks(ctx.db, input.linkType).map(toLinkOut) }; + } + assertDocumentExists(ctx.db, input.documentId); + const { outgoing, incoming } = getDocumentLinks(ctx.db, input.documentId); + const withDirection = ( + links: DocumentLinkWithTitle[], + direction: "outgoing" | "incoming", + ): Array<LinkOut<DocumentLinkWithTitle> & { direction: string }> => + links + .filter((l) => input.linkType === undefined || l.linkType === input.linkType) + .map((l) => ({ ...toLinkOut(l), direction })); + return { + items: [...withDirection(outgoing, "outgoing"), ...withDirection(incoming, "incoming")], + }; + }, +}); + +export const prerequisitesOperation = defineOperation({ + name: "prerequisites", + group: "links", + summary: "List the documents to read before this one (following prerequisite links)", + input: z.object({ documentId: s.documentId }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/documents/:documentId/prerequisites" }, + handler(ctx, input) { + assertDocumentExists(ctx.db, input.documentId); + return { + items: getPrerequisiteChain(ctx.db, input.documentId).map((d) => ({ + documentId: d.id, + title: d.title, + })), + }; + }, +}); + +export const linkOperations = [ + linkDocumentsOperation, + unlinkDocumentsOperation, + listLinksOperation, + prerequisitesOperation, +] as const; diff --git a/src/core/operations/packs.ts b/src/core/operations/packs.ts new file mode 100644 index 00000000..843bd4fe --- /dev/null +++ b/src/core/operations/packs.ts @@ -0,0 +1,148 @@ +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { + createPack, + createPackFromSource, + installPack, + listAvailablePacks, + listInstalledPacks, + removePack, + type KnowledgePack, +} from "../packs.js"; +import { resolveTopicId } from "../topics.js"; +import * as s from "./schemas.js"; +import { defineOperation, type OperationContext } from "./types.js"; + +const pack = z.string().min(1).describe("Pack name"); +const registryUrl = z.url().optional().describe("Registry URL (default: the public pack registry)"); + +function isPackFile(nameOrPath: string): boolean { + return nameOrPath.endsWith(".json") || nameOrPath.endsWith(".json.gz"); +} + +/** REST callers must not read or write files on the server's disk. */ +function assertLocalFilesAllowed(ctx: OperationContext, what: string): void { + if (ctx.surface === "api") { + throw new ValidationError(`${what} is not available over the REST API`); + } +} + +export const installPackOperation = defineOperation({ + name: "install-pack", + group: "packs", + summary: "Install a knowledge pack from the registry or a local .json/.json.gz file", + input: z.object({ + pack: z.string().min(1).describe("Pack name from the registry, or a local .json/.json.gz file"), + registryUrl, + batchSize: z + .number() + .int() + .min(1) + .max(500) + .optional() + .describe("Documents per batch (default 10)"), + concurrency: z + .number() + .int() + .min(1) + .max(16) + .optional() + .describe("Batches embedded in parallel (default 4)"), + }), + annotations: { longRunning: true, idempotent: true }, + http: { method: "POST", path: "/packs" }, + handler(ctx, input) { + if (isPackFile(input.pack)) assertLocalFilesAllowed(ctx, "Installing a pack from a file"); + return installPack(ctx.db, ctx.provider, input.pack, { + registryUrl: input.registryUrl, + batchSize: input.batchSize, + concurrency: input.concurrency, + signal: ctx.signal, + onProgress: (done, total, message) => ctx.onProgress?.({ done, total, message }), + }); + }, +}); + +export const removePackOperation = defineOperation({ + name: "remove-pack", + group: "packs", + summary: "Remove an installed pack and its documents", + input: z.object({ pack }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/packs/:pack" }, + handler(ctx, input) { + removePack(ctx.db, input.pack); + return { pack: input.pack, removed: true }; + }, +}); + +export const listPacksOperation = defineOperation({ + name: "list-packs", + group: "packs", + summary: "List installed packs, or packs available in the registry", + input: z.object({ + available: z.boolean().default(false).describe("List registry packs instead of installed ones"), + registryUrl, + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/packs" }, + async handler(ctx, input) { + if (input.available) return { items: await listAvailablePacks(input.registryUrl) }; + return { items: listInstalledPacks(ctx.db) }; + }, +}); + +export const createPackOperation = defineOperation({ + name: "create-pack", + group: "packs", + summary: "Build a pack from indexed documents, or from local files, folders and URLs", + input: z.object({ + name: z.string().min(1).describe("Pack name"), + from: z + .array(z.string().min(1)) + .optional() + .describe("Files, folders or URLs to build from (default: documents in the database)"), + topic: s.opt(s.topic).describe("Database mode: only this topic's documents"), + version: z.string().min(1).optional().describe("Pack version (default 1.0.0)"), + description: z.string().min(1).optional().describe("Pack description"), + author: z.string().min(1).optional().describe("Pack author"), + license: z.string().min(1).optional().describe("Pack license"), + outputPath: z.string().min(1).optional().describe("Write the pack here (.json or .json.gz)"), + extensions: z.array(z.string()).optional().describe("Source mode: file extensions to include"), + exclude: z.array(z.string()).optional().describe("Source mode: globs to skip"), + recursive: z.boolean().default(true).describe("Source mode: walk subdirectories"), + }), + async handler(ctx, input): Promise<KnowledgePack> { + assertLocalFilesAllowed(ctx, "Creating a pack"); + const common = { + name: input.name, + version: input.version, + description: input.description, + author: input.author, + license: input.license, + outputPath: input.outputPath, + }; + if (input.from) { + return createPackFromSource({ + ...common, + from: input.from, + extensions: input.extensions, + exclude: input.exclude, + recursive: input.recursive, + onProgress: ({ file, index, total }) => + ctx.onProgress?.({ done: index + 1, total, message: file }), + }); + } + return createPack(ctx.db, { + ...common, + topic: input.topic === undefined ? undefined : resolveTopicId(ctx.db, input.topic), + }); + }, +}); + +export const packOperations = [ + installPackOperation, + removePackOperation, + listPacksOperation, + createPackOperation, +] as const; diff --git a/src/core/operations/schemas.ts b/src/core/operations/schemas.ts new file mode 100644 index 00000000..c3548fb0 --- /dev/null +++ b/src/core/operations/schemas.ts @@ -0,0 +1,39 @@ +/** Shared input fields, so every operation uses the same names, limits and descriptions. */ +import { z } from "zod"; +import { SOURCE_TYPES } from "../indexing.js"; + +export const documentId = z.string().min(1).describe("Document ID"); +export const chunkId = z.string().min(1).describe("Chunk ID"); +export const sourceType = z.enum(SOURCE_TYPES).describe("Document source type"); +export const topic = z.string().min(1).describe("Topic ID or name"); +export const library = z.string().min(1).describe("Library name"); +export const version = z.string().min(1).describe("Library version"); +export const tags = z.array(z.string().min(1)).describe("Tag names"); +export const url = z.url({ protocol: /^https?$/ }).describe("http(s) URL"); + +/** `schema.optional()` keeping its description (zod does not copy it to the wrapper). */ +export function opt<T extends z.ZodType>(schema: T): z.ZodOptional<T> { + return schema.optional().describe(schema.description ?? ""); +} + +/** `limit` with a default and maximum. */ +export function limit(defaultValue: number, max: number): z.ZodDefault<z.ZodNumber> { + return z + .number() + .int() + .min(1) + .max(max) + .default(defaultValue) + .describe(`Maximum results (default ${defaultValue}, max ${max})`); +} + +export const offset = z.number().int().min(0).default(0).describe("Results to skip (paging)"); + +/** Filters shared by search, ask and list-documents. */ +export const documentFilters = { + topic: opt(topic), + library: opt(library), + version: opt(version), + sourceType: opt(sourceType), + tags: tags.optional().describe("Only documents carrying all of these tags"), +}; diff --git a/src/core/operations/search.ts b/src/core/operations/search.ts new file mode 100644 index 00000000..f3f7cd4c --- /dev/null +++ b/src/core/operations/search.ts @@ -0,0 +1,133 @@ +import type Database from "better-sqlite3"; +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { answer } from "../rag.js"; +import { firstChunkId, getRelatedChunks, searchDocuments, type SearchResult } from "../search.js"; +import { topicForFilter } from "./documents.js"; +import * as s from "./schemas.js"; +import { defineOperation, type ListResult, type OperationContext } from "./types.js"; + +const minRating = z.number().min(1).max(5).optional().describe("Minimum average rating"); + +/** Chunk to start a "related" search from: `ref` as a chunk ID, else a document's first chunk. */ +function relatedSeedChunk(db: Database.Database, ref: string): string { + if (db.prepare("SELECT 1 FROM chunks WHERE id = ?").get(ref)) return ref; + return firstChunkId(db, ref); +} + +interface SearchFilters { + version?: string | undefined; + sourceType?: string | undefined; + minRating?: number | undefined; +} + +/** Filters getRelatedChunks does not apply in SQL. */ +function matchesFilters(r: SearchResult, f: SearchFilters): boolean { + if (f.version !== undefined && r.version !== f.version) return false; + if (f.sourceType !== undefined && r.sourceType !== f.sourceType) return false; + return f.minRating === undefined || (r.avgRating ?? 0) >= f.minRating; +} + +export const searchOperation = defineOperation({ + name: "search", + group: "search", + summary: + "Search the knowledge base by meaning and keywords, or find content related to a document or chunk", + input: z.object({ + query: z.string().min(1).max(10_000).optional().describe("What to search for"), + relatedTo: z + .string() + .min(1) + .optional() + .describe("Document or chunk ID: return similar content instead of running a query"), + ...s.documentFilters, + minRating, + limit: s.limit(10, 100), + offset: s.offset, + maxChunksPerDocument: z + .number() + .int() + .min(1) + .max(50) + .optional() + .describe("At most this many chunks per document (default: no limit)"), + contextChunks: z + .number() + .int() + .min(0) + .max(2) + .default(0) + .describe("Neighbouring chunks to include before and after each result"), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/search" }, + async handler(ctx, input): Promise<ListResult<SearchResult>> { + if ((input.query === undefined) === (input.relatedTo === undefined)) { + throw new ValidationError("Give either query or relatedTo"); + } + const topic = topicForFilter(ctx, input.topic); + if (input.relatedTo !== undefined) { + const related = getRelatedChunks(ctx.db, { + chunkId: relatedSeedChunk(ctx.db, input.relatedTo), + limit: input.offset + input.limit, + ...(topic === undefined ? {} : { topic }), + ...(input.library === undefined ? {} : { library: input.library }), + ...(input.tags === undefined ? {} : { tags: input.tags }), + }); + const matches = related.chunks.filter((r) => matchesFilters(r, input)); + return { + items: matches.slice(input.offset, input.offset + input.limit), + total: matches.length, + limit: input.limit, + offset: input.offset, + }; + } + const { results, totalCount } = await searchDocuments(ctx.db, ctx.provider, { + query: input.query ?? "", + topic, + library: input.library, + version: input.version, + source: input.sourceType, + tags: input.tags, + minRating: input.minRating, + limit: input.limit, + offset: input.offset, + maxChunksPerDocument: input.maxChunksPerDocument, + contextChunks: input.contextChunks, + }); + return { items: results, total: totalCount, limit: input.limit, offset: input.offset }; + }, +}); + +export const askOperation = defineOperation({ + name: "ask", + group: "search", + summary: + "Answer a question from the knowledge base with an LLM, or (passthrough) return the context to answer from", + input: z.object({ + question: z.string().min(1).max(10_000).describe("The question"), + ...s.documentFilters, + minRating, + topK: z.number().int().min(1).max(20).default(5).describe("Chunks to retrieve as context"), + }), + annotations: { readOnly: true }, + http: { method: "POST", path: "/ask" }, + handler: (ctx: OperationContext, input) => + answer( + ctx.db, + ctx.provider, + { passthrough: ctx.isPassthrough(), llm: ctx.isPassthrough() ? null : ctx.getLlm() }, + { + question: input.question, + topK: input.topK, + topic: topicForFilter(ctx, input.topic), + library: input.library, + version: input.version, + sourceType: input.sourceType, + tags: input.tags, + minRating: input.minRating, + }, + ), +}); + +export const searchOperations = [searchOperation, askOperation] as const; diff --git a/src/core/operations/searches.ts b/src/core/operations/searches.ts new file mode 100644 index 00000000..4084e744 --- /dev/null +++ b/src/core/operations/searches.ts @@ -0,0 +1,93 @@ +import { z } from "zod"; +import { + countSavedSearches, + createSavedSearch, + deleteSavedSearch, + listSavedSearches, + runSavedSearch, +} from "../saved-searches.js"; +import type { SearchOptions } from "../search.js"; +import { topicForFilter } from "./documents.js"; +import * as s from "./schemas.js"; +import { defineOperation, type ListResult } from "./types.js"; + +const savedSearch = z.string().min(1).describe("Saved search name or ID"); + +export const saveSearchOperation = defineOperation({ + name: "save-search", + group: "searches", + summary: "Save a search query and its filters under a name", + input: z.object({ + name: z.string().min(1).describe("Unique name"), + query: z.string().min(1).describe("Search query"), + ...s.documentFilters, + minRating: z.number().min(1).max(5).optional().describe("Minimum average rating"), + limit: z.number().int().min(1).max(100).optional().describe("Maximum results"), + }), + http: { method: "POST", path: "/searches" }, + handler(ctx, input) { + const filters: Omit<SearchOptions, "query"> = {}; + const topic = topicForFilter(ctx, input.topic); + if (topic !== undefined) filters.topic = topic; + if (input.library !== undefined) filters.library = input.library; + if (input.version !== undefined) filters.version = input.version; + if (input.sourceType !== undefined) filters.source = input.sourceType; + if (input.tags !== undefined) filters.tags = input.tags; + if (input.minRating !== undefined) filters.minRating = input.minRating; + if (input.limit !== undefined) filters.limit = input.limit; + return createSavedSearch( + ctx.db, + input.name, + input.query, + Object.keys(filters).length > 0 ? filters : undefined, + ); + }, +}); + +export const listSavedSearchesOperation = defineOperation({ + name: "list-saved-searches", + group: "searches", + summary: "List saved searches", + input: z.object({ limit: s.limit(50, 1000), offset: s.offset }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/searches" }, + handler: (ctx, input): ListResult<ReturnType<typeof listSavedSearches>[number]> => ({ + items: listSavedSearches(ctx.db, input.limit, input.offset), + total: countSavedSearches(ctx.db), + limit: input.limit, + offset: input.offset, + }), +}); + +export const runSavedSearchOperation = defineOperation({ + name: "run-saved-search", + group: "searches", + summary: "Run a saved search", + input: z.object({ search: savedSearch }), + annotations: { readOnly: true }, + http: { method: "POST", path: "/searches/:search/run" }, + async handler(ctx, input) { + const { search, results } = await runSavedSearch(ctx.db, ctx.provider, input.search); + return { search, items: results }; + }, +}); + +export const deleteSavedSearchOperation = defineOperation({ + name: "delete-saved-search", + group: "searches", + summary: "Delete a saved search", + input: z.object({ search: savedSearch }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/searches/:search" }, + handler(ctx, input) { + deleteSavedSearch(ctx.db, input.search); + return { search: input.search, deleted: true }; + }, +}); + +export const savedSearchOperations = [ + saveSearchOperation, + listSavedSearchesOperation, + runSavedSearchOperation, + deleteSavedSearchOperation, +] as const; diff --git a/src/core/operations/tags.ts b/src/core/operations/tags.ts new file mode 100644 index 00000000..fc70fd13 --- /dev/null +++ b/src/core/operations/tags.ts @@ -0,0 +1,78 @@ +import { z } from "zod"; +import { assertDocumentExists } from "../documents.js"; +import { + addTagsToDocument, + getDocumentTags, + listTags, + removeTagsFromDocument, + suggestTags, +} from "../tags.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +const documentTags = z.object({ documentId: s.documentId, tags: s.tags.min(1) }); + +export const addTagsOperation = defineOperation({ + name: "add-tags", + group: "tags", + summary: "Add tags to a document (tags are created as needed)", + input: documentTags, + annotations: { idempotent: true }, + http: { method: "POST", path: "/documents/:documentId/tags" }, + handler(ctx, input) { + assertDocumentExists(ctx.db, input.documentId); + addTagsToDocument(ctx.db, input.documentId, input.tags); + return { + documentId: input.documentId, + tags: getDocumentTags(ctx.db, input.documentId).map((t) => t.name), + }; + }, +}); + +export const removeTagsOperation = defineOperation({ + name: "remove-tags", + group: "tags", + summary: "Remove tags from a document", + input: documentTags, + annotations: { idempotent: true }, + http: { method: "DELETE", path: "/documents/:documentId/tags" }, + handler(ctx, input) { + assertDocumentExists(ctx.db, input.documentId); + const removed = removeTagsFromDocument(ctx.db, input.documentId, input.tags); + return { + documentId: input.documentId, + removed, + tags: getDocumentTags(ctx.db, input.documentId).map((t) => t.name), + }; + }, +}); + +export const listTagsOperation = defineOperation({ + name: "list-tags", + group: "tags", + summary: "List all tags with their document counts", + input: z.object({}), + annotations: { readOnly: true }, + http: { method: "GET", path: "/tags" }, + handler: (ctx) => ({ items: listTags(ctx.db) }), +}); + +export const suggestTagsOperation = defineOperation({ + name: "suggest-tags", + group: "tags", + summary: "Suggest tags for a document from its content", + input: z.object({ documentId: s.documentId, limit: s.limit(5, 20) }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/documents/:documentId/suggested-tags" }, + handler: (ctx, input) => ({ + documentId: input.documentId, + suggestions: suggestTags(ctx.db, input.documentId, input.limit), + }), +}); + +export const tagOperations = [ + addTagsOperation, + removeTagsOperation, + listTagsOperation, + suggestTagsOperation, +] as const; diff --git a/src/core/operations/tasks.ts b/src/core/operations/tasks.ts new file mode 100644 index 00000000..ea78b249 --- /dev/null +++ b/src/core/operations/tasks.ts @@ -0,0 +1,53 @@ +import { z } from "zod"; +import { NotFoundError } from "../../errors.js"; +import { taskRegistry, type Task } from "../tasks.js"; +import { defineOperation } from "./types.js"; + +const taskId = z.string().min(1).describe("Task ID returned when a background task started"); + +function requireTask(id: string): Task { + const task = taskRegistry.get(id); + if (!task) { + throw new NotFoundError( + `Task ${id} not found or expired (finished tasks are kept for 1 hour)`, + "TASK_NOT_FOUND", + ); + } + return task; +} + +export const getTaskOperation = defineOperation({ + name: "get-task", + group: "tasks", + summary: "Status, progress and result of a background task", + input: z.object({ taskId }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/tasks/:taskId" }, + handler: (_ctx, input) => requireTask(input.taskId), +}); + +export const cancelTaskOperation = defineOperation({ + name: "cancel-task", + group: "tasks", + summary: "Request cancellation of a pending or running background task", + input: z.object({ taskId }), + annotations: { idempotent: true }, + http: { method: "POST", path: "/tasks/:taskId/cancel" }, + handler(_ctx, input) { + const outcome = taskRegistry.cancel(input.taskId); + const task = requireTask(input.taskId); + return { taskId: input.taskId, cancelRequested: outcome === "cancelled", status: task.status }; + }, +}); + +export const listTasksOperation = defineOperation({ + name: "list-tasks", + group: "tasks", + summary: "List background tasks from the last hour, newest first", + input: z.object({}), + annotations: { readOnly: true }, + http: { method: "GET", path: "/tasks" }, + handler: () => ({ items: taskRegistry.list() }), +}); + +export const taskOperations = [getTaskOperation, cancelTaskOperation, listTasksOperation] as const; diff --git a/src/core/operations/topics.ts b/src/core/operations/topics.ts new file mode 100644 index 00000000..2f56c64a --- /dev/null +++ b/src/core/operations/topics.ts @@ -0,0 +1,65 @@ +import { z } from "zod"; +import { createTopic, deleteTopic, getTopicStats, resolveTopicId } from "../topics.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +export const listTopicsOperation = defineOperation({ + name: "list-topics", + group: "topics", + summary: "List topics with their document counts", + input: z.object({ parent: s.opt(s.topic).describe("Only direct subtopics of this topic") }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/topics" }, + handler(ctx, input) { + const parentId = input.parent === undefined ? undefined : resolveTopicId(ctx.db, input.parent); + const topics = getTopicStats(ctx.db); + return { + items: + parentId === undefined ? topics : topics.filter((topic) => topic.parentId === parentId), + }; + }, +}); + +export const createTopicOperation = defineOperation({ + name: "create-topic", + group: "topics", + summary: "Create a topic", + input: z.object({ + name: z.string().min(1).describe("Topic name"), + description: z.string().min(1).optional().describe("What the topic covers"), + parent: s.opt(s.topic).describe("Parent topic ID or name"), + }), + http: { method: "POST", path: "/topics" }, + handler: (ctx, input) => + createTopic(ctx.db, { + name: input.name, + description: input.description, + parentId: input.parent === undefined ? undefined : resolveTopicId(ctx.db, input.parent), + }), +}); + +export const deleteTopicOperation = defineOperation({ + name: "delete-topic", + group: "topics", + summary: "Delete a topic, optionally with its documents", + input: z.object({ + topic: s.topic, + deleteDocuments: z + .boolean() + .default(false) + .describe("Also delete the topic's documents (default: keep them, without a topic)"), + }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/topics/:topic" }, + handler(ctx, input) { + const topicId = resolveTopicId(ctx.db, input.topic); + deleteTopic(ctx.db, topicId, { deleteDocuments: input.deleteDocuments }); + return { topicId, deleted: true }; + }, +}); + +export const topicOperations = [ + listTopicsOperation, + createTopicOperation, + deleteTopicOperation, +] as const; diff --git a/src/core/operations/types.ts b/src/core/operations/types.ts new file mode 100644 index 00000000..5927982f --- /dev/null +++ b/src/core/operations/types.ts @@ -0,0 +1,172 @@ +/** + * The operation layer: every user-visible action is one Operation (name + zod input schema + + * handler). The CLI, MCP server, REST API and SDK are thin adapters over these, so parameter + * names, defaults, validation and docs are the same everywhere. + */ +import type Database from "better-sqlite3"; +import { z } from "zod"; +import type { LibScopeConfig } from "../../config.js"; +import { ConfigError, ValidationError } from "../../errors.js"; +import type { EmbeddingProvider } from "../../providers/embedding.js"; +import { createLlmProvider, isPassthroughMode, type LlmProvider } from "../rag.js"; +import { taskRegistry, type Task } from "../tasks.js"; + +export type Surface = "cli" | "mcp" | "api" | "sdk"; + +export type OperationGroup = + | "documents" + | "search" + | "links" + | "tags" + | "topics" + | "searches" + | "packs" + | "connectors" + | "admin" + | "webhooks" + | "tasks" + | "analytics"; + +export interface ProgressEvent { + done: number; + total?: number | undefined; + message?: string | undefined; +} + +export interface OperationContext { + db: Database.Database; + provider: EmbeddingProvider; + config: LibScopeConfig; + surface: Surface; + signal?: AbortSignal | undefined; + onProgress?: ((p: ProgressEvent) => void) | undefined; + /** True when `ask` should return context for the caller to answer (passthrough). */ + isPassthrough(): boolean; + /** The configured LLM provider, created on first use; null when none is configured. */ + getLlm(): LlmProvider | null; +} + +export interface OperationAnnotations { + readOnly?: boolean; + destructive?: boolean; + idempotent?: boolean; + /** May take long enough that surfaces should offer to run it as a background task. */ + longRunning?: boolean; +} + +export type HttpMethod = "GET" | "POST" | "PATCH" | "PUT" | "DELETE"; + +export interface Operation<S extends z.ZodObject = z.ZodObject, O = unknown> { + /** kebab-case, unique, e.g. "get-document". */ + name: string; + group: OperationGroup; + /** One line, used by CLI help, MCP tool description and OpenAPI summary. */ + summary: string; + description?: string; + /** Input schema. Every field has .describe(); defaults live here only. */ + input: S; + annotations?: OperationAnnotations; + /** REST mapping; `:name` path segments are filled from input fields of the same name. */ + http?: { method: HttpMethod; path: string }; + handler(ctx: OperationContext, input: z.output<S>): Promise<O> | O; +} + +/** Identity function that keeps the schema and result types of an operation. */ +export function defineOperation<S extends z.ZodObject, O>(op: Operation<S, O>): Operation<S, O> { + return op; +} + +/** "field: message; field: message" for a failed parse. */ +function formatIssues(error: z.ZodError): string { + return error.issues + .map((issue) => { + const path = issue.path.join("."); + return path ? `${path}: ${issue.message}` : issue.message; + }) + .join("; "); +} + +/** Validate `rawInput` against the operation's schema and run it. ZodError -> ValidationError. */ +export async function runOperation<S extends z.ZodObject, O>( + op: Operation<S, O>, + ctx: OperationContext, + rawInput: unknown, +): Promise<O> { + const parsed = op.input.safeParse(rawInput ?? {}); + if (!parsed.success) { + throw new ValidationError(`Invalid input for ${op.name}: ${formatIssues(parsed.error)}`); + } + ctx.signal?.throwIfAborted(); + return op.handler(ctx, parsed.data); +} + +/** + * Validate input now, then run the operation as a background task in the shared task + * registry. The task's signal and progress replace any in `ctx`. Returns the task; its + * `result` is the JSON-encoded operation result. + */ +export function startOperationTask<S extends z.ZodObject, O>( + op: Operation<S, O>, + ctx: OperationContext, + rawInput: unknown, +): Task { + const parsed = op.input.safeParse(rawInput ?? {}); + if (!parsed.success) { + throw new ValidationError(`Invalid input for ${op.name}: ${formatIssues(parsed.error)}`); + } + const { task } = taskRegistry.run( + "operation", + async (signal, onProgress) => { + const result = await op.handler( + { + ...ctx, + signal, + onProgress: (p) => onProgress(p.done, p.total ?? 0), + }, + parsed.data, + ); + return JSON.stringify(result ?? null); + }, + op.name, + ); + return task; +} + +export interface CreateContextOptions { + db: Database.Database; + provider: EmbeddingProvider; + config: LibScopeConfig; + surface: Surface; + signal?: AbortSignal | undefined; + onProgress?: ((p: ProgressEvent) => void) | undefined; + /** Use this LLM instead of the configured one (null: none). */ + llm?: LlmProvider | null | undefined; +} + +/** Build an OperationContext. The LLM is created from config on first `getLlm()` call. */ +export function createOperationContext(options: CreateContextOptions): OperationContext { + const { llm: injected, ...rest } = options; + let llm: LlmProvider | null | undefined = injected; + return { + ...rest, + isPassthrough: (): boolean => injected === undefined && isPassthroughMode(options.config), + getLlm: (): LlmProvider | null => { + if (llm !== undefined) return llm; + try { + llm = createLlmProvider(options.config); + } catch (err) { + if (!(err instanceof ConfigError)) throw err; + llm = null; + } + return llm; + }, + }; +} + +/** Result shape of every list operation. */ +export interface ListResult<T> { + items: T[]; + total: number; + limit: number; + offset: number; +} diff --git a/src/core/operations/webhooks.ts b/src/core/operations/webhooks.ts new file mode 100644 index 00000000..3be69805 --- /dev/null +++ b/src/core/operations/webhooks.ts @@ -0,0 +1,81 @@ +import { z } from "zod"; +import { + TEST_PING_DATA, + WEBHOOK_EVENTS, + createWebhook, + deleteWebhook, + deliverWebhook, + getWebhook, + listWebhooks, + redactWebhook, +} from "../webhooks.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +const webhookId = z.string().min(1).describe("Webhook ID"); + +export const createWebhookOperation = defineOperation({ + name: "create-webhook", + group: "webhooks", + summary: "Register a URL to receive a signed POST for document and search events", + input: z.object({ + url: s.url.describe("Endpoint that receives the POST requests"), + events: z.array(z.enum(WEBHOOK_EVENTS)).min(1).describe("Events to subscribe to"), + secret: z + .string() + .min(1) + .optional() + .describe("HMAC-SHA256 signing secret (requires LIBSCOPE_SECRET_KEY to store it)"), + }), + http: { method: "POST", path: "/webhooks" }, + handler: async (ctx, input) => + redactWebhook(await createWebhook(ctx.db, input.url, input.events, input.secret)), +}); + +export const listWebhooksOperation = defineOperation({ + name: "list-webhooks", + group: "webhooks", + summary: "List registered webhooks (secrets are never shown)", + input: z.object({ limit: s.limit(50, 1000), offset: s.offset }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/webhooks" }, + handler: (ctx, input) => ({ + items: listWebhooks(ctx.db, input.limit, input.offset).map(redactWebhook), + }), +}); + +export const deleteWebhookOperation = defineOperation({ + name: "delete-webhook", + group: "webhooks", + summary: "Delete a webhook", + input: z.object({ webhookId }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/webhooks/:webhookId" }, + handler(ctx, input) { + deleteWebhook(ctx.db, input.webhookId); + return { webhookId: input.webhookId, deleted: true }; + }, +}); + +export const testWebhookOperation = defineOperation({ + name: "test-webhook", + group: "webhooks", + summary: "Send a test event to a webhook and report the HTTP status", + input: z.object({ webhookId }), + http: { method: "POST", path: "/webhooks/:webhookId/test" }, + async handler(ctx, input) { + const response = await deliverWebhook( + getWebhook(ctx.db, input.webhookId), + "document.created", + TEST_PING_DATA, + ); + return { webhookId: input.webhookId, status: response.status, statusText: response.statusText }; + }, +}); + +export const webhookOperations = [ + createWebhookOperation, + listWebhooksOperation, + deleteWebhookOperation, + testWebhookOperation, +] as const; diff --git a/tests/unit/operations/connectors.test.ts b/tests/unit/operations/connectors.test.ts new file mode 100644 index 00000000..da8ca829 --- /dev/null +++ b/tests/unit/operations/connectors.test.ts @@ -0,0 +1,110 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import { NotFoundError, ValidationError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +// Saved connections live under a temp HOME, never the real ~/.libscope. +let tempHome = join(tmpdir(), `libscope-ops-connectors-${process.pid}`); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const ops = await import("../../../src/core/operations/index.js"); +const { saveConnectorSettings } = await import("../../../src/connectors/saved-config.js"); +const { hasNamedConnectorConfig } = await import("../../../src/connectors/index.js"); + +initLogger("silent"); + +describe("connector operations", () => { + let t: TestContext; + let vault: string; + + beforeEach(() => { + tempHome = join(tmpdir(), `libscope-ops-connectors-${randomUUID()}`); + vault = join(tempHome, "vault"); + mkdirSync(vault, { recursive: true }); + writeFileSync(join(vault, "note.md"), "# Note\n\nA note in the vault."); + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + rmSync(tempHome, { recursive: true, force: true }); + }); + + function saveVault(name: string): void { + saveConnectorSettings("obsidian", name, { + vaultPath: vault, + topicMapping: "folder", + excludePatterns: [], + }); + } + + it("lists, syncs by name, syncs all and disconnects a saved connection", async () => { + saveVault("notes"); + saveConnectorSettings("slack", "team-slack", { + token: "xoxb-secret", + channels: ["all"], + threadMode: "aggregate", + }); + + const listed = await run(ops.listConnectionsOperation, t.ctx, {}); + expect(listed.items.map((c) => c.name)).toEqual(["notes", "team-slack"]); + expect(listed.items[1]?.settings["token"]).toBe("***"); + + const one = await run(ops.syncOperation, t.ctx, { name: "notes" }); + expect(one.items[0]).toMatchObject({ + name: "notes", + type: "obsidian", + status: "completed", + summary: { added: 1 }, + }); + + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("offline"))); + try { + const all = await run(ops.syncOperation, t.ctx, { all: true }); + expect(all.items.map((i) => [i.name, i.status])).toEqual([ + ["notes", "completed"], + ["team-slack", "failed"], + ]); + } finally { + vi.unstubAllGlobals(); + } + + const afterSync = await run(ops.listConnectionsOperation, t.ctx, {}); + expect(afterSync.items[0]?.lastRun?.status).toBe("completed"); + + const removed = await run(ops.disconnectOperation, t.ctx, { name: "notes" }); + expect(removed).toMatchObject({ type: "obsidian", documentsRemoved: 1, configRemoved: true }); + expect(hasNamedConnectorConfig("notes")).toBe(false); + }); + + it("needs exactly one of name and all, and a safe name", async () => { + await expectValidationError(run(ops.syncOperation, t.ctx, {})); + await expectValidationError(run(ops.syncOperation, t.ctx, { name: "a", all: true })); + await expectValidationError(run(ops.disconnectOperation, t.ctx, { name: "../etc" })); + }); + + it("throws NotFoundError for an unknown connection", async () => { + await expect(run(ops.syncOperation, t.ctx, { name: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + }); + + it("disconnects by type when nothing is saved, and checks the type", async () => { + const result = await run(ops.disconnectOperation, t.ctx, { name: "slack" }); + expect(result).toMatchObject({ type: "slack", documentsRemoved: 0, configRemoved: false }); + await expect(run(ops.disconnectOperation, t.ctx, { name: "unknown" })).rejects.toBeInstanceOf( + ValidationError, + ); + saveVault("notes"); + await expect( + run(ops.disconnectOperation, t.ctx, { name: "notes", type: "slack" }), + ).rejects.toBeInstanceOf(ValidationError); + }); +}); diff --git a/tests/unit/operations/documents.test.ts b/tests/unit/operations/documents.test.ts new file mode 100644 index 00000000..e00d7575 --- /dev/null +++ b/tests/unit/operations/documents.test.ts @@ -0,0 +1,223 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + addOperation, + deleteDocumentOperation, + documentHistoryOperation, + getDocumentOperation, + listDocumentsOperation, + rateDocumentOperation, + rollbackDocumentOperation, + updateDocumentOperation, +} from "../../../src/core/operations/index.js"; +import { createTopic } from "../../../src/core/topics.js"; +import { saveVersion } from "../../../src/core/versioning.js"; +import { DocumentNotFoundError, NotFoundError, TopicNotFoundError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +initLogger("silent"); + +describe("document operations", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + }); + + describe("add", () => { + it("adds inline content with tags and the default source type", async () => { + createTopic(t.db, { name: "Guides" }); + const result = await run(addOperation, t.ctx, { + title: "Setup", + content: "How to set up the project", + topic: "guides", + tags: ["Setup"], + }); + expect(result.kind).toBe("content"); + expect(result.documents).toHaveLength(1); + const id = result.documents[0]!.documentId; + const view = await run(getDocumentOperation, t.ctx, { documentId: id }); + expect(view.document.sourceType).toBe("topic"); + expect(view.tags).toEqual(["setup"]); + }); + + it("rejects input without content or source", async () => { + await expectValidationError(run(addOperation, t.ctx, {})); + }); + + it("rejects a malformed url", async () => { + await expectValidationError(run(addOperation, t.ctx, { url: "not a url" })); + }); + + it("fails for an unknown topic", async () => { + await expect( + run(addOperation, t.ctx, { title: "A", content: "B", topic: "nope" }), + ).rejects.toBeInstanceOf(TopicNotFoundError); + }); + + it("adds a local directory from the CLI but refuses it over MCP", async () => { + const dir = mkdtempSync(join(tmpdir(), "libscope-add-op-")); + try { + writeFileSync(join(dir, "a.md"), "# A\n\nAlpha content"); + const result = await run(addOperation, t.ctx, { source: dir }); + expect(result.kind).toBe("directory"); + expect(result.documents).toHaveLength(1); + + const mcp = makeContext({ db: t.db, surface: "mcp" }); + await expectValidationError(run(addOperation, mcp.ctx, { source: dir })); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + }); + + describe("get-document", () => { + it("returns paged content with tags, links and ratings", async () => { + const id = await addDoc(t, "Paged", "0123456789"); + const view = await run(getDocumentOperation, t.ctx, { + documentId: id, + offset: 2, + maxLength: 3, + }); + expect(view.content).toBe("234"); + expect(view.nextOffset).toBe(5); + expect(view.contentLength).toBe(10); + expect(view.document.documentId).toBe(id); + expect(view.ratings.totalRatings).toBe(0); + expect(view.links).toEqual({ outgoing: [], incoming: [] }); + }); + + it("rejects a negative offset", async () => { + await expectValidationError( + run(getDocumentOperation, t.ctx, { documentId: "x", offset: -1 }), + ); + }); + + it("throws NotFoundError for an unknown document", async () => { + await expect( + run(getDocumentOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("list-documents", () => { + it("pages with total and filters by library and tags", async () => { + await addDoc(t, "One", "first doc", { library: "react" }); + await addDoc(t, "Two", "second doc", { library: "react" }); + await addDoc(t, "Three", "third doc"); + const page = await run(listDocumentsOperation, t.ctx, { library: "react", limit: 1 }); + expect(page.total).toBe(2); + expect(page.items).toHaveLength(1); + expect(page.items[0]).not.toHaveProperty("content"); + expect(page.limit).toBe(1); + expect(page.offset).toBe(0); + }); + + it("rejects limit 0", async () => { + await expectValidationError(run(listDocumentsOperation, t.ctx, { limit: 0 })); + }); + }); + + describe("update-document", () => { + it("updates the title and replaces tags", async () => { + const id = await addDoc(t, "Old"); + await run(updateDocumentOperation, t.ctx, { documentId: id, tags: ["a", "b"] }); + const view = await run(updateDocumentOperation, t.ctx, { + documentId: id, + title: "New", + tags: ["b", "c"], + }); + expect(view.document.title).toBe("New"); + expect(view.tags).toEqual(["b", "c"]); + }); + + it("rejects an update with no fields", async () => { + await expectValidationError(run(updateDocumentOperation, t.ctx, { documentId: "x" })); + }); + + it("throws NotFoundError for an unknown document", async () => { + await expect( + run(updateDocumentOperation, t.ctx, { documentId: "missing", title: "x" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("delete-document", () => { + it("deletes a document", async () => { + const id = await addDoc(t, "Gone"); + expect(await run(deleteDocumentOperation, t.ctx, { documentId: id })).toEqual({ + documentId: id, + deleted: true, + }); + await expect(run(getDocumentOperation, t.ctx, { documentId: id })).rejects.toBeInstanceOf( + NotFoundError, + ); + }); + + it("rejects an empty id", async () => { + await expectValidationError(run(deleteDocumentOperation, t.ctx, { documentId: "" })); + }); + + it("throws NotFoundError for an unknown document", async () => { + await expect( + run(deleteDocumentOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("rate-document", () => { + it("records a rating attributed to the surface", async () => { + const id = await addDoc(t, "Rated"); + const mcp = makeContext({ db: t.db, surface: "mcp" }); + const rating = await run(rateDocumentOperation, mcp.ctx, { documentId: id, rating: 4 }); + expect(rating).toMatchObject({ documentId: id, rating: 4, ratedBy: "model" }); + }); + + it("rejects a rating of 6", async () => { + await expectValidationError( + run(rateDocumentOperation, t.ctx, { documentId: "x", rating: 6 }), + ); + }); + + it("throws NotFoundError for an unknown document", async () => { + await expect( + run(rateDocumentOperation, t.ctx, { documentId: "missing", rating: 3 }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("document-history and rollback-document", () => { + it("lists versions and rolls back", async () => { + const id = await addDoc(t, "Original"); + saveVersion(t.db, id); + await run(updateDocumentOperation, t.ctx, { documentId: id, title: "Changed" }); + const history = await run(documentHistoryOperation, t.ctx, { documentId: id }); + expect(history.items.length).toBeGreaterThanOrEqual(1); + const restored = await run(rollbackDocumentOperation, t.ctx, { documentId: id, version: 1 }); + expect(restored.title).toBe("Original"); + }); + + it("rejects version 0", async () => { + await expectValidationError( + run(rollbackDocumentOperation, t.ctx, { documentId: "x", version: 0 }), + ); + }); + + it("throws NotFoundError for an unknown document or version", async () => { + await expect( + run(documentHistoryOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + const id = await addDoc(t, "No versions"); + await expect( + run(rollbackDocumentOperation, t.ctx, { documentId: id, version: 9 }), + ).rejects.toBeInstanceOf(NotFoundError); + }); + }); +}); diff --git a/tests/unit/operations/helpers.ts b/tests/unit/operations/helpers.ts new file mode 100644 index 00000000..c270102d --- /dev/null +++ b/tests/unit/operations/helpers.ts @@ -0,0 +1,83 @@ +import type Database from "better-sqlite3"; +import { expect } from "vitest"; +import type { z } from "zod"; +import type { LibScopeConfig } from "../../../src/config.js"; +import { indexDocument } from "../../../src/core/indexing.js"; +import { + createOperationContext, + runOperation, + type Operation, + type OperationContext, + type Surface, +} from "../../../src/core/operations/index.js"; +import type { LlmProvider } from "../../../src/core/rag.js"; +import { ValidationError } from "../../../src/errors.js"; +import { createTestDb } from "../../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../../fixtures/mock-provider.js"; + +/** Minimal config for operation tests (no files read). */ +export function testConfig(llmProvider?: string): LibScopeConfig { + return { + embedding: { provider: "local" }, + llm: llmProvider ? { provider: llmProvider } : {}, + database: {}, + indexing: { maxDocumentSize: 1_000_000, allowPrivateUrls: false, allowSelfSignedCerts: false }, + logging: { level: "silent" }, + } as LibScopeConfig; +} + +export interface TestContext { + db: Database.Database; + provider: MockEmbeddingProvider; + ctx: OperationContext; +} + +export function makeContext( + options: { + db?: Database.Database; + surface?: Surface; + llm?: LlmProvider | null; + config?: LibScopeConfig; + } = {}, +): TestContext { + const db = options.db ?? createTestDb(); + const provider = new MockEmbeddingProvider(); + const ctx = createOperationContext({ + db, + provider, + config: options.config ?? testConfig(), + surface: options.surface ?? "cli", + // Default: no LLM. Pass `llm: undefined` explicitly to use the configured one. + llm: "llm" in options ? options.llm : null, + }); + return { db, provider, ctx }; +} + +/** Run an operation with raw (unvalidated) input. */ +export function run<S extends z.ZodObject, O>( + op: Operation<S, O>, + ctx: OperationContext, + input: unknown = {}, +): Promise<O> { + return runOperation(op, ctx, input); +} + +export async function expectValidationError(promise: Promise<unknown>): Promise<void> { + await expect(promise).rejects.toBeInstanceOf(ValidationError); +} + +/** Index a small document and return its ID. */ +export async function addDoc( + t: TestContext, + title: string, + content = `${title} body text about testing operations`, + extra: { library?: string; topicId?: string } = {}, +): Promise<string> { + const doc = await indexDocument(t.db, t.provider, { + title, + content, + sourceType: extra.library ? "library" : "manual", + ...extra, + }); + return doc.id; +} diff --git a/tests/unit/operations/links-tags-topics.test.ts b/tests/unit/operations/links-tags-topics.test.ts new file mode 100644 index 00000000..8787d47e --- /dev/null +++ b/tests/unit/operations/links-tags-topics.test.ts @@ -0,0 +1,147 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + addTagsOperation, + createTopicOperation, + deleteTopicOperation, + linkDocumentsOperation, + listLinksOperation, + listTagsOperation, + listTopicsOperation, + prerequisitesOperation, + removeTagsOperation, + suggestTagsOperation, + unlinkDocumentsOperation, +} from "../../../src/core/operations/index.js"; +import { DocumentNotFoundError, NotFoundError, TopicNotFoundError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +initLogger("silent"); + +describe("link, tag and topic operations", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + }); + + describe("links", () => { + it("links, lists with direction, follows prerequisites and unlinks", async () => { + const basics = await addDoc(t, "Basics"); + const advanced = await addDoc(t, "Advanced"); + const link = await run(linkDocumentsOperation, t.ctx, { + documentId: basics, + targetDocumentId: advanced, + linkType: "prerequisite", + }); + expect(link).toMatchObject({ sourceId: basics, targetId: advanced }); + expect(link.linkId).toEqual(expect.any(String)); + + const links = await run(listLinksOperation, t.ctx, { documentId: advanced }); + expect(links.items).toEqual([ + expect.objectContaining({ linkId: link.linkId, direction: "incoming" }), + ]); + const all = await run(listLinksOperation, t.ctx, {}); + expect(all.items).toHaveLength(1); + + const prereqs = await run(prerequisitesOperation, t.ctx, { documentId: advanced }); + expect(prereqs.items).toEqual([{ documentId: basics, title: "Basics" }]); + + await run(unlinkDocumentsOperation, t.ctx, { linkId: link.linkId }); + expect((await run(listLinksOperation, t.ctx, {})).items).toHaveLength(0); + }); + + it("rejects an unknown link type", async () => { + await expectValidationError( + run(linkDocumentsOperation, t.ctx, { + documentId: "a", + targetDocumentId: "b", + linkType: "friend", + }), + ); + }); + + it("throws NotFoundError for unknown documents and links", async () => { + const a = await addDoc(t, "A"); + await expect( + run(linkDocumentsOperation, t.ctx, { + documentId: a, + targetDocumentId: "missing", + linkType: "related", + }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + await expect(run(unlinkDocumentsOperation, t.ctx, { linkId: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + await expect( + run(listLinksOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + await expect( + run(prerequisitesOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("tags", () => { + it("adds, lists, suggests and removes tags", async () => { + const id = await addDoc(t, "Tagged", "database indexing database performance tuning"); + const added = await run(addTagsOperation, t.ctx, { documentId: id, tags: ["DB", "perf"] }); + expect(added.tags).toEqual(["db", "perf"]); + const listed = await run(listTagsOperation, t.ctx, {}); + expect(listed.items.map((tag) => tag.name)).toEqual(["db", "perf"]); + const removed = await run(removeTagsOperation, t.ctx, { documentId: id, tags: ["perf"] }); + expect(removed).toMatchObject({ removed: ["perf"], tags: ["db"] }); + const suggested = await run(suggestTagsOperation, t.ctx, { documentId: id, limit: 3 }); + expect(suggested.suggestions.length).toBeLessThanOrEqual(3); + }); + + it("rejects an empty tag list", async () => { + await expectValidationError(run(addTagsOperation, t.ctx, { documentId: "x", tags: [] })); + await expectValidationError(run(suggestTagsOperation, t.ctx, { documentId: "x", limit: 0 })); + }); + + it("throws NotFoundError for an unknown document", async () => { + for (const op of [addTagsOperation, removeTagsOperation]) { + await expect(run(op, t.ctx, { documentId: "missing", tags: ["x"] })).rejects.toBeInstanceOf( + DocumentNotFoundError, + ); + } + await expect( + run(suggestTagsOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("topics", () => { + it("creates, lists by parent name and deletes topics", async () => { + const parent = await run(createTopicOperation, t.ctx, { name: "Backend" }); + await run(createTopicOperation, t.ctx, { name: "Databases", parent: "backend" }); + const children = await run(listTopicsOperation, t.ctx, { parent: "Backend" }); + expect(children.items.map((topic) => topic.name)).toEqual(["Databases"]); + expect(children.items[0]?.documentCount).toBe(0); + + const deleted = await run(deleteTopicOperation, t.ctx, { topic: "Databases" }); + expect(deleted.deleted).toBe(true); + expect((await run(listTopicsOperation, t.ctx, {})).items.map((x) => x.id)).toEqual([ + parent.id, + ]); + }); + + it("rejects an empty topic name", async () => { + await expectValidationError(run(createTopicOperation, t.ctx, { name: "" })); + }); + + it("throws TopicNotFoundError for unknown topics", async () => { + await expect(run(deleteTopicOperation, t.ctx, { topic: "nope" })).rejects.toBeInstanceOf( + TopicNotFoundError, + ); + await expect(run(listTopicsOperation, t.ctx, { parent: "nope" })).rejects.toBeInstanceOf( + TopicNotFoundError, + ); + }); + }); +}); diff --git a/tests/unit/operations/packs-admin.test.ts b/tests/unit/operations/packs-admin.test.ts new file mode 100644 index 00000000..044cb493 --- /dev/null +++ b/tests/unit/operations/packs-admin.test.ts @@ -0,0 +1,201 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + backupOperation, + bulkDeleteOperation, + bulkMoveOperation, + bulkRetagOperation, + createPackOperation, + dedupeOperation, + installPackOperation, + listDocumentsOperation, + listPacksOperation, + overviewOperation, + pruneExpiredOperation, + reindexOperation, + removePackOperation, + restoreOperation, +} from "../../../src/core/operations/index.js"; +import { indexDocument } from "../../../src/core/indexing.js"; +import { createTopic } from "../../../src/core/topics.js"; +import { NotFoundError, TopicNotFoundError, ValidationError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +initLogger("silent"); + +describe("pack and admin operations", () => { + let t: TestContext; + let dir: string; + + beforeEach(() => { + t = makeContext(); + dir = mkdtempSync(join(tmpdir(), "libscope-ops-")); + }); + + afterEach(() => { + t.db.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + function writePack(name: string): string { + const path = join(dir, `${name}.json`); + writeFileSync( + path, + JSON.stringify({ + name, + version: "1.0.0", + description: "Test pack", + documents: [{ title: "Pack doc", content: "Content from a pack", source: "test" }], + metadata: { author: "tests", license: "MIT", createdAt: new Date().toISOString() }, + }), + ); + return path; + } + + describe("packs", () => { + it("installs from a file, lists, and removes", async () => { + const result = await run(installPackOperation, t.ctx, { pack: writePack("demo") }); + expect(result).toMatchObject({ packName: "demo", documentsInstalled: 1 }); + const listed = await run(listPacksOperation, t.ctx, {}); + expect(listed.items.map((p) => p.name)).toEqual(["demo"]); + await run(removePackOperation, t.ctx, { pack: "demo" }); + expect((await run(listPacksOperation, t.ctx, {})).items).toEqual([]); + }); + + it("refuses pack files over the REST API", async () => { + const api = makeContext({ db: t.db, surface: "api" }); + await expectValidationError(run(installPackOperation, api.ctx, { pack: writePack("x") })); + await expectValidationError(run(createPackOperation, api.ctx, { name: "x" })); + }); + + it("rejects an empty pack name", async () => { + await expectValidationError(run(installPackOperation, t.ctx, { pack: "" })); + await expectValidationError(run(removePackOperation, t.ctx, {})); + }); + + it("throws NotFoundError when removing a pack that is not installed", async () => { + await expect(run(removePackOperation, t.ctx, { pack: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + }); + + it("creates a pack file from the database", async () => { + await addDoc(t, "Packed"); + const outputPath = join(dir, "out.json"); + const pack = await run(createPackOperation, t.ctx, { name: "out", outputPath }); + expect(pack.documents).toHaveLength(1); + expect(existsSync(outputPath)).toBe(true); + }); + }); + + describe("overview", () => { + it("reports counts, topics, packs and health", async () => { + createTopic(t.db, { name: "Guides" }); + await addDoc(t, "Doc"); + const overview = await run(overviewOperation, t.ctx, {}); + expect(overview.stats.totalDocuments).toBe(1); + expect(overview.topics.map((x) => x.name)).toEqual(["Guides"]); + expect(overview.health).toMatchObject({ database: "ok", fts: "ok" }); + expect(overview.index.configured).toMatchObject({ provider: "mock", dimensions: 4 }); + }); + }); + + describe("reindex and dedupe", () => { + it("re-embeds every chunk", async () => { + await addDoc(t, "Reindexed"); + const result = await run(reindexOperation, t.ctx, {}); + expect(result.total).toBeGreaterThan(0); + expect(result.failed).toBe(0); + }); + + it("rejects rebuild combined with filters, and bad dates", async () => { + await expectValidationError( + run(reindexOperation, t.ctx, { rebuild: true, documentIds: ["a"] }), + ); + await expectValidationError(run(reindexOperation, t.ctx, { since: "yesterday" })); + }); + + it("groups exact duplicates", async () => { + await indexDocument(t.db, t.provider, { + title: "Copy one", + content: "identical body", + sourceType: "manual", + }); + await indexDocument(t.db, t.provider, { + title: "Copy two", + content: "identical body", + sourceType: "manual", + dedup: "force", + }); + const result = await run(dedupeOperation, t.ctx, { strategy: "exact" }); + expect(result.items).toHaveLength(1); + await expectValidationError(run(dedupeOperation, t.ctx, { threshold: 2 })); + }); + }); + + describe("backup and restore", () => { + it("writes and reads a backup from the CLI, but not over MCP", async () => { + await addDoc(t, "Saved"); + const outputPath = join(dir, "backup.json"); + const backup = await run(backupOperation, t.ctx, { outputPath }); + expect(backup.counts.documents).toBe(1); + + const target = makeContext(); + try { + const restored = await run(restoreOperation, target.ctx, { backupPath: outputPath }); + expect(restored.counts.documents).toBe(1); + } finally { + target.db.close(); + } + + const mcp = makeContext({ db: t.db, surface: "mcp" }); + await expectValidationError(run(backupOperation, mcp.ctx, { outputPath })); + await expectValidationError(run(restoreOperation, t.ctx, {})); + }); + }); + + describe("prune-expired", () => { + it("deletes expired documents only", async () => { + await indexDocument(t.db, t.provider, { + title: "Old news", + content: "expired content", + sourceType: "manual", + expiresAt: "2000-01-01T00:00:00Z", + }); + await addDoc(t, "Fresh"); + expect(await run(pruneExpiredOperation, t.ctx, {})).toEqual({ pruned: 1 }); + expect((await run(listDocumentsOperation, t.ctx, {})).total).toBe(1); + }); + }); + + describe("bulk operations", () => { + it("deletes (with dry run), retags and moves by filter", async () => { + createTopic(t.db, { name: "Target" }); + await addDoc(t, "R1", "react one", { library: "react" }); + await addDoc(t, "R2", "react two", { library: "react" }); + + const dry = await run(bulkDeleteOperation, t.ctx, { library: "react", dryRun: true }); + expect(dry.affected).toBe(2); + const retag = await run(bulkRetagOperation, t.ctx, { library: "react", addTags: ["ui"] }); + expect(retag.affected).toBe(2); + const moved = await run(bulkMoveOperation, t.ctx, { tags: ["ui"], targetTopic: "Target" }); + expect(moved.affected).toBe(2); + const deleted = await run(bulkDeleteOperation, t.ctx, { topic: "target" }); + expect(deleted.affected).toBe(2); + }); + + it("rejects an empty selector and an invalid source type", async () => { + await expect(run(bulkDeleteOperation, t.ctx, {})).rejects.toBeInstanceOf(ValidationError); + await expectValidationError(run(bulkDeleteOperation, t.ctx, { sourceType: "web" })); + }); + + it("throws TopicNotFoundError for an unknown target topic", async () => { + await expect( + run(bulkMoveOperation, t.ctx, { library: "x", targetTopic: "nowhere" }), + ).rejects.toBeInstanceOf(TopicNotFoundError); + }); + }); +}); diff --git a/tests/unit/operations/registry.test.ts b/tests/unit/operations/registry.test.ts new file mode 100644 index 00000000..4134eb3d --- /dev/null +++ b/tests/unit/operations/registry.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from "vitest"; +import { z } from "zod"; +import { OPERATIONS, getOperation, runOperation } from "../../../src/core/operations/index.js"; +import { ValidationError } from "../../../src/errors.js"; +import { makeContext } from "./helpers.js"; + +describe("OPERATIONS", () => { + it("has unique kebab-case names", () => { + const names = OPERATIONS.map((op) => op.name); + expect(new Set(names).size).toBe(names.length); + for (const name of names) expect(name).toMatch(/^[a-z]+(?:-[a-z]+)*$/); + }); + + it("describes every input field", () => { + const missing = OPERATIONS.flatMap((op) => + Object.entries(op.input.shape) + .filter(([, schema]) => !(schema as z.ZodType).description) + .map(([field]) => `${op.name}.${field}`), + ); + expect(missing).toEqual([]); + }); + + it("maps every :param in an HTTP path to an input field, with unique routes", () => { + const routes = new Set<string>(); + for (const op of OPERATIONS) { + if (!op.http) continue; + const route = `${op.http.method} ${op.http.path}`; + expect(routes.has(route), route).toBe(false); + routes.add(route); + for (const segment of op.http.path.split("/")) { + if (segment.startsWith(":")) expect(op.input.shape, route).toHaveProperty(segment.slice(1)); + } + } + }); + + it("produces JSON Schema for every input (for MCP and OpenAPI)", () => { + for (const op of OPERATIONS) { + expect(() => z.toJSONSchema(op.input, { io: "input" }), op.name).not.toThrow(); + } + }); + + it("looks operations up by name", () => { + expect(getOperation("search")?.group).toBe("search"); + expect(getOperation("nope")).toBeUndefined(); + }); + + it("turns non-object input into a ValidationError naming the operation", async () => { + const { db, ctx } = makeContext(); + try { + const op = getOperation("get-document")!; + await expect(runOperation(op, ctx, "doc-1")).rejects.toBeInstanceOf(ValidationError); + await expect(runOperation(op, ctx, { documentId: 5 })).rejects.toThrow( + "Invalid input for get-document: documentId:", + ); + } finally { + db.close(); + } + }); +}); diff --git a/tests/unit/operations/search.test.ts b/tests/unit/operations/search.test.ts new file mode 100644 index 00000000..451cb637 --- /dev/null +++ b/tests/unit/operations/search.test.ts @@ -0,0 +1,111 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { createDatabase } from "../../../src/db/connection.js"; +import { createVectorTable, runMigrations } from "../../../src/db/schema.js"; +import { askOperation, searchOperation } from "../../../src/core/operations/index.js"; +import type { LlmProvider } from "../../../src/core/rag.js"; +import { ConfigError, DocumentNotFoundError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { MockEmbeddingProvider } from "../../fixtures/mock-provider.js"; +import { + addDoc, + expectValidationError, + makeContext, + run, + testConfig, + type TestContext, +} from "./helpers.js"; + +initLogger("silent"); + +const fakeLlm: LlmProvider = { + model: "fake-model", + complete: (prompt: string) => + Promise.resolve({ text: `answered (${prompt.length} chars)`, tokensUsed: 7 }), +}; + +describe("search operations", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + }); + + describe("search", () => { + it("finds documents by keyword and returns documentId and chunkId", async () => { + const id = await addDoc(t, "Kubernetes", "Kubernetes pods and deployments explained"); + await addDoc(t, "Cooking", "Recipes for pasta"); + const result = await run(searchOperation, t.ctx, { query: "kubernetes" }); + expect(result.total).toBeGreaterThanOrEqual(1); + expect(result.items[0]).toMatchObject({ documentId: id }); + expect(result.items[0]?.chunkId).toEqual(expect.any(String)); + expect(result.limit).toBe(10); + }); + + it("needs exactly one of query and relatedTo", async () => { + await expectValidationError(run(searchOperation, t.ctx, {})); + await expectValidationError(run(searchOperation, t.ctx, { query: "a", relatedTo: "b" })); + }); + + it("rejects limit above the maximum", async () => { + await expectValidationError(run(searchOperation, t.ctx, { query: "a", limit: 500 })); + }); + + it("throws NotFoundError when relatedTo names nothing", async () => { + await expect(run(searchOperation, t.ctx, { relatedTo: "missing" })).rejects.toBeInstanceOf( + DocumentNotFoundError, + ); + }); + + it("finds content related to a document (sqlite-vec)", async () => { + const db = createDatabase(":memory:"); + runMigrations(db); + createVectorTable(db, new MockEmbeddingProvider()); + const v = makeContext({ db }); + try { + const id = await addDoc(v, "Seed", "Vector search seed document"); + await addDoc(v, "Other", "Another document to compare with"); + const result = await run(searchOperation, v.ctx, { relatedTo: id }); + expect(result.items.every((r) => r.documentId !== id)).toBe(true); + expect(result.total).toBe(result.items.length); + } finally { + db.close(); + } + }); + }); + + describe("ask", () => { + it("answers with the LLM and lists sources", async () => { + const llmCtx = makeContext({ db: t.db, llm: fakeLlm }); + await addDoc(t, "Install", "Run npm install to install libscope"); + const result = await run(askOperation, llmCtx.ctx, { question: "How do I install?" }); + expect(result.mode).toBe("answer"); + if (result.mode === "answer") { + expect(result.model).toBe("fake-model"); + expect(result.sources.length).toBeGreaterThan(0); + } + }); + + it("returns context in passthrough mode", async () => { + const p = makeContext({ db: t.db, config: testConfig("passthrough"), llm: undefined }); + await addDoc(t, "Install", "Run npm install to install libscope"); + const result = await run(askOperation, p.ctx, { question: "install" }); + expect(result.mode).toBe("context"); + if (result.mode === "context") expect(result.contextPrompt).toContain("install"); + }); + + it("throws ConfigError when no LLM is configured", async () => { + await expect(run(askOperation, t.ctx, { question: "anything" })).rejects.toBeInstanceOf( + ConfigError, + ); + }); + + it("rejects an empty question and topK above 20", async () => { + await expectValidationError(run(askOperation, t.ctx, { question: "" })); + await expectValidationError(run(askOperation, t.ctx, { question: "q", topK: 21 })); + }); + }); +}); diff --git a/tests/unit/operations/searches-analytics.test.ts b/tests/unit/operations/searches-analytics.test.ts new file mode 100644 index 00000000..42741ef8 --- /dev/null +++ b/tests/unit/operations/searches-analytics.test.ts @@ -0,0 +1,95 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + deleteSavedSearchOperation, + listSavedSearchesOperation, + popularOperation, + runSavedSearchOperation, + saveSearchOperation, + searchAnalyticsOperation, + searchOperation, + staleOperation, + topQueriesOperation, +} from "../../../src/core/operations/index.js"; +import { NotFoundError, ValidationError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +initLogger("silent"); + +describe("saved search and analytics operations", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + }); + + describe("saved searches", () => { + it("saves with filters, lists, runs and deletes", async () => { + await addDoc(t, "Hooks", "React hooks guide", { library: "react" }); + const saved = await run(saveSearchOperation, t.ctx, { + name: "react-hooks", + query: "hooks", + library: "react", + sourceType: "library", + }); + expect(saved.filters).toEqual({ library: "react", source: "library" }); + + const list = await run(listSavedSearchesOperation, t.ctx, {}); + expect(list).toMatchObject({ total: 1, limit: 50, offset: 0 }); + + const ran = await run(runSavedSearchOperation, t.ctx, { search: "react-hooks" }); + expect(ran.items.length).toBe(1); + expect(ran.search.resultCount).toBe(1); + + await run(deleteSavedSearchOperation, t.ctx, { search: saved.id }); + expect((await run(listSavedSearchesOperation, t.ctx, {})).total).toBe(0); + }); + + it("rejects a missing query and a duplicate name", async () => { + await expectValidationError(run(saveSearchOperation, t.ctx, { name: "x" })); + await run(saveSearchOperation, t.ctx, { name: "dup", query: "q" }); + await expect( + run(saveSearchOperation, t.ctx, { name: "dup", query: "q" }), + ).rejects.toBeInstanceOf(ValidationError); + }); + + it("throws NotFoundError for an unknown saved search", async () => { + await expect(run(runSavedSearchOperation, t.ctx, { search: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + await expect( + run(deleteSavedSearchOperation, t.ctx, { search: "nope" }), + ).rejects.toBeInstanceOf(NotFoundError); + }); + }); + + describe("analytics", () => { + it("reports popular documents, stale documents, top queries and search analytics", async () => { + const id = await addDoc(t, "Popular", "frequently searched content"); + await addDoc(t, "Lonely", "nobody searches this"); + await run(searchOperation, t.ctx, { query: "frequently" }); + await run(searchOperation, t.ctx, { query: "frequently" }); + + const popular = await run(popularOperation, t.ctx, {}); + expect(popular.items[0]).toMatchObject({ documentId: id }); + const stale = await run(staleOperation, t.ctx, { days: 30 }); + expect(stale.items.map((d) => d.title)).toEqual(["Lonely"]); + const top = await run(topQueriesOperation, t.ctx, { limit: 5 }); + expect(top.items[0]).toMatchObject({ query: "frequently", count: 2 }); + const analytics = await run(searchAnalyticsOperation, t.ctx, {}); + expect(analytics.totalSearches).toBe(2); + expect(analytics.knowledgeGaps).toEqual(expect.any(Array)); + }); + + it("rejects out-of-range input", async () => { + await expectValidationError(run(popularOperation, t.ctx, { limit: 0 })); + await expectValidationError(run(staleOperation, t.ctx, { days: 0 })); + await expectValidationError(run(topQueriesOperation, t.ctx, { limit: "ten" })); + await expectValidationError(run(searchAnalyticsOperation, t.ctx, { days: 1.5 })); + }); + }); +}); diff --git a/tests/unit/operations/webhooks-tasks.test.ts b/tests/unit/operations/webhooks-tasks.test.ts new file mode 100644 index 00000000..12ebcaa8 --- /dev/null +++ b/tests/unit/operations/webhooks-tasks.test.ts @@ -0,0 +1,134 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { z } from "zod"; +import { NotFoundError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +vi.mock("node:dns", async (importOriginal) => { + const actual: typeof import("node:dns") = await importOriginal(); + return { + ...actual, + promises: { + ...actual.promises, + resolve4: vi.fn().mockResolvedValue(["93.184.216.34"]), + resolve6: vi.fn().mockResolvedValue([]), + }, + }; +}); + +const ops = await import("../../../src/core/operations/index.js"); + +initLogger("silent"); + +describe("webhook and task operations", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + t.db.close(); + }); + + describe("webhooks", () => { + it("creates, lists, tests and deletes a webhook", async () => { + const created = await run(ops.createWebhookOperation, t.ctx, { + url: "https://hooks.example.com/libscope", + events: ["document.created"], + }); + expect(created).not.toHaveProperty("secret"); + const listed = await run(ops.listWebhooksOperation, t.ctx, {}); + expect(listed.items.map((w) => w.id)).toEqual([created.id]); + + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(new Response(null, { status: 204, statusText: "No Content" })), + ); + const tested = await run(ops.testWebhookOperation, t.ctx, { webhookId: created.id }); + expect(tested).toMatchObject({ status: 204 }); + + await run(ops.deleteWebhookOperation, t.ctx, { webhookId: created.id }); + expect((await run(ops.listWebhooksOperation, t.ctx, {})).items).toEqual([]); + }); + + it("rejects unknown events and non-http URLs", async () => { + await expectValidationError( + run(ops.createWebhookOperation, t.ctx, { + url: "https://hooks.example.com", + events: ["document.exploded"], + }), + ); + await expectValidationError( + run(ops.createWebhookOperation, t.ctx, { + url: "ftp://hooks.example.com", + events: ["document.created"], + }), + ); + }); + + it("throws NotFoundError for an unknown webhook", async () => { + await expect( + run(ops.deleteWebhookOperation, t.ctx, { webhookId: "nope" }), + ).rejects.toBeInstanceOf(NotFoundError); + await expect( + run(ops.testWebhookOperation, t.ctx, { webhookId: "nope" }), + ).rejects.toBeInstanceOf(NotFoundError); + }); + }); + + describe("tasks", () => { + it("runs an operation as a background task and reports its result", async () => { + await addDoc(t, "Counted"); + const task = ops.startOperationTask(ops.overviewOperation, t.ctx, {}); + expect(task.operation).toBe("overview"); + await vi.waitFor(() => { + expect(ops.getTaskOperation.handler(t.ctx, { taskId: task.id }).status).toBe("completed"); + }); + const done = await run(ops.getTaskOperation, t.ctx, { taskId: task.id }); + expect(JSON.parse(done.result ?? "{}")).toMatchObject({ stats: { totalDocuments: 1 } }); + const listed = await run(ops.listTasksOperation, t.ctx, {}); + expect(listed.items.some((x) => x.id === task.id)).toBe(true); + const cancel = await run(ops.cancelTaskOperation, t.ctx, { taskId: task.id }); + expect(cancel).toMatchObject({ cancelRequested: false, status: "completed" }); + }); + + it("cancels a running task", async () => { + const slow = ops.defineOperation({ + name: "slow", + group: "admin", + summary: "Waits until cancelled", + input: z.object({}), + handler: (ctx) => + new Promise((_resolve, reject) => { + ctx.signal?.addEventListener("abort", () => + reject(new DOMException("aborted", "AbortError")), + ); + }), + }); + const task = ops.startOperationTask(slow, t.ctx, {}); + const cancel = await run(ops.cancelTaskOperation, t.ctx, { taskId: task.id }); + expect(cancel.cancelRequested).toBe(true); + await vi.waitFor(() => { + expect(ops.getTaskOperation.handler(t.ctx, { taskId: task.id }).status).toBe("cancelled"); + }); + }); + + it("validates input before starting a task", () => { + expect(() => ops.startOperationTask(ops.reindexOperation, t.ctx, { batchSize: 0 })).toThrow( + "Invalid input for reindex", + ); + }); + + it("rejects an empty task id and throws NotFoundError for an unknown one", async () => { + await expectValidationError(run(ops.getTaskOperation, t.ctx, { taskId: "" })); + await expect(run(ops.getTaskOperation, t.ctx, { taskId: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + await expect(run(ops.cancelTaskOperation, t.ctx, { taskId: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + }); + }); +}); From 32b063a0b0abf82cd23cba601f09b7afbafc6b51 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:47:49 +0000 Subject: [PATCH 37/83] fix(core): resolve the LLM per surface in operations and reuse the rag no-LLM hint createOperationContext passes the surface to isPassthroughMode and createLlmProvider, so llm.provider "auto" is passthrough under MCP. --- src/core/operations/types.ts | 9 +++++---- src/core/rag.ts | 5 ----- tests/unit/operations/search.test.ts | 7 +++++++ tests/unit/rag-answer.test.ts | 8 ++++---- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/core/operations/types.ts b/src/core/operations/types.ts index 5927982f..8c111742 100644 --- a/src/core/operations/types.ts +++ b/src/core/operations/types.ts @@ -8,10 +8,10 @@ import { z } from "zod"; import type { LibScopeConfig } from "../../config.js"; import { ConfigError, ValidationError } from "../../errors.js"; import type { EmbeddingProvider } from "../../providers/embedding.js"; -import { createLlmProvider, isPassthroughMode, type LlmProvider } from "../rag.js"; +import { createLlmProvider, isPassthroughMode, type LlmProvider, type LlmSurface } from "../rag.js"; import { taskRegistry, type Task } from "../tasks.js"; -export type Surface = "cli" | "mcp" | "api" | "sdk"; +export type Surface = LlmSurface; export type OperationGroup = | "documents" @@ -149,11 +149,12 @@ export function createOperationContext(options: CreateContextOptions): Operation let llm: LlmProvider | null | undefined = injected; return { ...rest, - isPassthrough: (): boolean => injected === undefined && isPassthroughMode(options.config), + isPassthrough: (): boolean => + injected === undefined && isPassthroughMode(options.config, { surface: options.surface }), getLlm: (): LlmProvider | null => { if (llm !== undefined) return llm; try { - llm = createLlmProvider(options.config); + llm = createLlmProvider(options.config, { surface: options.surface }); } catch (err) { if (!(err instanceof ConfigError)) throw err; llm = null; diff --git a/src/core/rag.ts b/src/core/rag.ts index cac22216..d372ff0f 100644 --- a/src/core/rag.ts +++ b/src/core/rag.ts @@ -433,11 +433,6 @@ export type AnswerResult = | ({ mode: "answer" } & RagResult) | ({ mode: "context" } & PassthroughResult); -/** Message for `ask` when no LLM is available. */ -export const NO_LLM_HINT = - "No LLM provider configured. Set one with `libscope config set llm.provider <openai|anthropic|ollama|passthrough>` " + - "or the LIBSCOPE_LLM_PROVIDER environment variable."; - /** * Answer a question from the knowledge base. Every surface uses this one function: * passthrough mode returns the context prompt and sources; otherwise the LLM writes the answer. diff --git a/tests/unit/operations/search.test.ts b/tests/unit/operations/search.test.ts index 451cb637..70407b3f 100644 --- a/tests/unit/operations/search.test.ts +++ b/tests/unit/operations/search.test.ts @@ -97,6 +97,13 @@ describe("search operations", () => { if (result.mode === "context") expect(result.contextPrompt).toContain("install"); }); + it("resolves llm.provider auto to passthrough under MCP", async () => { + const mcp = makeContext({ db: t.db, surface: "mcp", llm: undefined }); + await addDoc(t, "Install", "Run npm install to install libscope"); + const result = await run(askOperation, mcp.ctx, { question: "install" }); + expect(result.mode).toBe("context"); + }); + it("throws ConfigError when no LLM is configured", async () => { await expect(run(askOperation, t.ctx, { question: "anything" })).rejects.toBeInstanceOf( ConfigError, diff --git a/tests/unit/rag-answer.test.ts b/tests/unit/rag-answer.test.ts index e2cca744..437e677f 100644 --- a/tests/unit/rag-answer.test.ts +++ b/tests/unit/rag-answer.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import type Database from "better-sqlite3"; import { createTestDb } from "../fixtures/test-db.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; -import { answer, NO_LLM_HINT, type LlmProvider } from "../../src/core/rag.js"; +import { answer, type LlmProvider } from "../../src/core/rag.js"; import { indexDocument } from "../../src/core/indexing.js"; import { addTagsToDocument } from "../../src/core/tags.js"; import { ConfigError } from "../../src/errors.js"; @@ -66,8 +66,8 @@ describe("answer", () => { }); it("throws a ConfigError with setup instructions when no LLM is configured", async () => { - await expect( - answer(db, provider, { passthrough: false, llm: null }, { question: "q" }), - ).rejects.toThrow(new ConfigError(NO_LLM_HINT)); + const result = answer(db, provider, { passthrough: false, llm: null }, { question: "q" }); + await expect(result).rejects.toBeInstanceOf(ConfigError); + await expect(result).rejects.toThrow("No LLM provider configured"); }); }); From c20f243c43c4493c293efaf84d2c8ea05e7275e4 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 17:47:54 +0000 Subject: [PATCH 38/83] docs(migration): list Library API renames and not-found error changes --- docs/migration-v2.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 186d566c..1d83e537 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -48,3 +48,24 @@ These variables do not change: `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_LLM_PROV | `embedding.ollamaUrl`, `ollamaModel`, `openaiModel` | `embedding.url`, `embedding.model` | | `llm.ollamaUrl` | `llm.url` | | `createLlmProvider(config)` read `ANTHROPIC_API_KEY` itself | `createLlmProvider(config, { surface })` reads only `config`; `loadConfig()` reads the env vars. | + +## Library API + +### Renamed exports + +| Old (1.x) | New (2.0) | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `removeTagFromDocument(db, documentId, tagId)` | `removeTagsFromDocument(db, documentId, tagNames)`. Takes tag names (case-insensitive) and returns the names that were removed. | + +### Error changes + +Every "not found" error is now a `NotFoundError` (`DocumentNotFoundError`, `ChunkNotFoundError` and `TopicNotFoundError` extend it). The REST API returns HTTP 404 for all of them. Check `err.code` to tell them apart. + +| Function | 1.x error | 2.0 error (`code`) | +| ---------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------ | +| `deleteLink` with an unknown link ID | `ValidationError` | `NotFoundError` (`LINK_NOT_FOUND`) | +| `getWebhook`, `deleteWebhook`, `updateWebhook` with an unknown ID | `ValidationError` | `NotFoundError` (`WEBHOOK_NOT_FOUND`) | +| `getSavedSearch`, `deleteSavedSearch`, `runSavedSearch` with an unknown name or ID | `DocumentNotFoundError` | `NotFoundError` (`SAVED_SEARCH_NOT_FOUND`) | +| `getVersion`, `rollbackToVersion` with an unknown version | `DocumentNotFoundError` | `NotFoundError` (`VERSION_NOT_FOUND`) | +| `removePack` for a pack that is not installed | `ValidationError` | `NotFoundError` (`PACK_NOT_FOUND`) | +| `getRelatedChunks` with an unknown chunk ID | `Error` | `ChunkNotFoundError` (`CHUNK_NOT_FOUND`) | From afb0a654b5cef9940ff16c371bdde04519df6e1f Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:07:03 +0000 Subject: [PATCH 39/83] feat(core): graph operation for the knowledge graph view --- src/core/operations/graph.ts | 40 +++++++++++++++++++++++++++++ src/core/operations/index.ts | 3 +++ tests/unit/operations/graph.test.ts | 36 ++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 src/core/operations/graph.ts create mode 100644 tests/unit/operations/graph.test.ts diff --git a/src/core/operations/graph.ts b/src/core/operations/graph.ts new file mode 100644 index 00000000..5cd92af5 --- /dev/null +++ b/src/core/operations/graph.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; +import { buildKnowledgeGraph } from "../graph.js"; +import { topicForFilter } from "./documents.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +export const graphOperation = defineOperation({ + name: "graph", + group: "links", + summary: + "Knowledge graph: documents, topics and tags as nodes; links, tags and similarity as edges", + input: z.object({ + topic: s.opt(s.topic).describe("Only documents in this topic (ID or name)"), + tag: z.string().min(1).optional().describe("Only documents carrying this tag"), + threshold: z + .number() + .min(0) + .max(1) + .default(0.85) + .describe("Minimum similarity for a similarity edge"), + maxNodes: z + .number() + .int() + .min(1) + .max(5000) + .default(200) + .describe("Maximum document nodes (default 200)"), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/graph" }, + handler: (ctx, input) => + buildKnowledgeGraph(ctx.db, { + similarityThreshold: input.threshold, + maxNodes: input.maxNodes, + topicFilter: topicForFilter(ctx, input.topic), + tagFilter: input.tag, + }), +}); + +export const graphOperations = [graphOperation] as const; diff --git a/src/core/operations/index.ts b/src/core/operations/index.ts index 93014328..8dfe55a9 100644 --- a/src/core/operations/index.ts +++ b/src/core/operations/index.ts @@ -2,6 +2,7 @@ import { adminOperations } from "./admin.js"; import { analyticsOperations } from "./analytics.js"; import { connectorOperations } from "./connectors.js"; import { documentOperations } from "./documents.js"; +import { graphOperations } from "./graph.js"; import { linkOperations } from "./links.js"; import { packOperations } from "./packs.js"; import { savedSearchOperations } from "./searches.js"; @@ -17,6 +18,7 @@ export * from "./admin.js"; export * from "./analytics.js"; export * from "./connectors.js"; export * from "./documents.js"; +export * from "./graph.js"; export * from "./links.js"; export * from "./packs.js"; export * from "./searches.js"; @@ -31,6 +33,7 @@ export const OPERATIONS: readonly Operation[] = [ ...documentOperations, ...searchOperations, ...linkOperations, + ...graphOperations, ...tagOperations, ...topicOperations, ...savedSearchOperations, diff --git a/tests/unit/operations/graph.test.ts b/tests/unit/operations/graph.test.ts new file mode 100644 index 00000000..37006eed --- /dev/null +++ b/tests/unit/operations/graph.test.ts @@ -0,0 +1,36 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { graphOperation } from "../../../src/core/operations/index.js"; +import { createTopic } from "../../../src/core/topics.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +initLogger("silent"); + +describe("graph operation", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + }); + + it("returns document and topic nodes, filtered by topic name", async () => { + const topic = createTopic(t.db, { name: "Guides" }); + await addDoc(t, "In topic", undefined, { topicId: topic.id }); + await addDoc(t, "Elsewhere"); + + const all = await run(graphOperation, t.ctx); + expect(all.nodes.filter((n) => n.type === "document")).toHaveLength(2); + + const filtered = await run(graphOperation, t.ctx, { topic: "Guides" }); + const docs = filtered.nodes.filter((n) => n.type === "document"); + expect(docs.map((n) => n.label)).toEqual(["In topic"]); + }); + + it("rejects an out-of-range threshold", async () => { + await expectValidationError(run(graphOperation, t.ctx, { threshold: 2 })); + }); +}); From 18421aae0969d20840d0c6f234092646510a60f7 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:07:21 +0000 Subject: [PATCH 40/83] feat(api)!: generate REST routes and OpenAPI from operations Every /api/v1 route now runs the operation that declares it (http method + path); input comes from path params plus the query string (GET, DELETE, typed by the zod schema) or the JSON body. Long-running operations answer 202 with a task ID (poll GET /api/v1/tasks/:taskId). ValidationError -> 400, NotFoundError -> 404. The OpenAPI 3.1 document is generated from the same routes. The dashboard server serves its unversioned URLs from operations and no longer sends Access-Control-Allow-Origin: *; cross-origin writes are rejected on both servers. Servers take the bootstrap() result. BREAKING CHANGE: REST paths, parameters and response shapes follow the operations (see docs/migration-v2.md). The REST repo indexer (/api/v1/index/*, LIBSCOPE_REPOS_CONFIG) is removed; add repos with POST /api/v1/documents. startApiServer/startWebServer take { db, provider, config } instead of (db, provider). --- src/api/adapter.ts | 164 +++++ src/api/indexing/repoConfig.ts | 49 -- src/api/indexing/repoIndexer.ts | 225 ------ src/api/middleware.ts | 89 ++- src/api/openapi.ts | 682 ++++++------------ src/api/routes.ts | 1103 ++++------------------------- src/api/server.ts | 119 ++-- src/cli/index.ts | 8 +- src/web/graph-api.ts | 31 - src/web/server.ts | 395 +++++------ tests/unit/api-routes.test.ts | 550 ++++++++++++++ tests/unit/api-server.test.ts | 70 +- tests/unit/api.test.ts | 967 +++---------------------- tests/unit/web-server.test.ts | 67 +- tests/unit/webhook-events.test.ts | 13 +- vitest.config.ts | 3 - 16 files changed, 1566 insertions(+), 2969 deletions(-) create mode 100644 src/api/adapter.ts delete mode 100644 src/api/indexing/repoConfig.ts delete mode 100644 src/api/indexing/repoIndexer.ts delete mode 100644 src/web/graph-api.ts create mode 100644 tests/unit/api-routes.test.ts diff --git a/src/api/adapter.ts b/src/api/adapter.ts new file mode 100644 index 00000000..2e60f151 --- /dev/null +++ b/src/api/adapter.ts @@ -0,0 +1,164 @@ +/** + * HTTP adapter over the operation layer, shared by the REST API and the dashboard server: + * build an operation's input from a request, run it, and map errors to HTTP statuses. + */ +import type { IncomingMessage, ServerResponse } from "node:http"; +import { z } from "zod"; +import { + runOperation, + startOperationTask, + type Operation, + type OperationContext, +} from "../core/operations/index.js"; +import { ConfigError, FetchError, NotFoundError, ValidationError } from "../errors.js"; +import { getLogger } from "../logger.js"; +import { HttpError, parseJsonBody, sendError } from "./middleware.js"; + +/** The subset of JSON Schema the adapter and the OpenAPI generator read. */ +export interface JsonSchema { + type?: string | string[]; + anyOf?: JsonSchema[]; + items?: JsonSchema; + properties?: Record<string, JsonSchema>; + required?: string[]; + [key: string]: unknown; +} + +const schemaCache = new WeakMap<Operation, JsonSchema>(); + +/** JSON Schema of an operation's input (what callers send), without the `$schema` key. */ +export function inputJsonSchema(op: Operation): JsonSchema { + let schema = schemaCache.get(op); + if (!schema) { + schema = { + ...(z.toJSONSchema(op.input, { io: "input", unrepresentable: "any" }) as JsonSchema), + }; + delete schema["$schema"]; + schemaCache.set(op, schema); + } + return schema; +} + +/** The non-null branch of a nullable schema (zod writes `anyOf: [T, {type: "null"}]`). */ +function nonNull(schema: JsonSchema | undefined): JsonSchema | undefined { + if (!schema?.anyOf) return schema; + return schema.anyOf.find((alt) => alt.type !== "null"); +} + +function coerceScalar(raw: string, schema: JsonSchema | undefined): unknown { + const type = nonNull(schema)?.type; + if (type === "integer" || type === "number") { + const n = Number(raw); + return raw.trim() !== "" && Number.isFinite(n) ? n : raw; + } + if (type === "boolean") { + if (raw === "true") return true; + if (raw === "false") return false; + } + return raw; +} + +/** + * Operation input from URL values (query string or path parameters). Each value is converted + * to the type the operation's schema expects: numbers and booleans are parsed, and array + * fields take repeated parameters and/or comma-separated values (`tags=a,b` or `tags=a&tags=b`). + * Values that do not convert are passed through so validation reports them. + */ +export function inputFromParams( + op: Operation, + params: URLSearchParams | Record<string, string>, +): Record<string, unknown> { + const search = params instanceof URLSearchParams ? params : new URLSearchParams(params); + const properties = inputJsonSchema(op).properties ?? {}; + const input: Record<string, unknown> = {}; + for (const name of new Set(search.keys())) { + const schema = nonNull(properties[name]); + if (schema?.type === "array") { + input[name] = search + .getAll(name) + .flatMap((v) => v.split(",")) + .map((v) => v.trim()) + .filter((v) => v !== "") + .map((v) => coerceScalar(v, schema.items)); + } else { + input[name] = coerceScalar(search.get(name) ?? "", schema); + } + } + return input; +} + +/** A decoded URL path segment; a malformed escape is a 400. */ +export function decodeSegment(segment: string): string { + try { + return decodeURIComponent(segment); + } catch { + throw new HttpError(400, "VALIDATION_ERROR", "Malformed URL path"); + } +} + +/** The JSON request body as an object ({} when empty). */ +export async function readJsonObject(req: IncomingMessage): Promise<Record<string, unknown>> { + const body = await parseJsonBody(req); + if (body === null) return {}; + if (typeof body !== "object" || Array.isArray(body)) { + throw new HttpError(400, "VALIDATION_ERROR", "Request body must be a JSON object"); + } + return body as Record<string, unknown>; +} + +export interface OperationResponse { + status: number; + data: unknown; + /** Set for a background task: where to poll it. */ + location?: string | undefined; +} + +/** + * Run an operation for an HTTP request. A long-running operation starts as a background task + * and answers 202 with the task ID; poll `GET {tasksPath}/{taskId}`. + */ +export async function respondWithOperation( + op: Operation, + ctx: OperationContext, + input: Record<string, unknown>, + tasksPath: string, +): Promise<OperationResponse> { + if (op.annotations?.longRunning) { + const task = startOperationTask(op, ctx, input); + return { + status: 202, + data: { taskId: task.id, operation: op.name, status: task.status }, + location: `${tasksPath}/${task.id}`, + }; + } + return { status: 200, data: await runOperation(op, ctx, input) }; +} + +/** HTTP status, code and client-safe message for an error thrown while handling a request. */ +export function describeError(err: unknown): { status: number; code: string; message: string } { + if (err instanceof HttpError) return { status: err.status, code: err.code, message: err.message }; + if (err instanceof ValidationError) return { status: 400, code: err.code, message: err.message }; + if (err instanceof NotFoundError) return { status: 404, code: err.code, message: err.message }; + if (err instanceof FetchError) return { status: 502, code: err.code, message: err.message }; + // Config problems (e.g. no LLM for ask) carry a hint the caller needs; other errors may leak + // internals, so they get a generic message. + if (err instanceof ConfigError) return { status: 500, code: err.code, message: err.message }; + return { status: 500, code: "INTERNAL_ERROR", message: "Internal server error" }; +} + +/** Log an error and send it as `{ error: { code, message } }`. */ +export function sendRequestError( + res: ServerResponse, + err: unknown, + request: { method: string; pathname: string }, +): void { + const { status, code, message } = describeError(err); + const log = getLogger(); + if (status >= 500) log.error({ err, ...request }, "API request failed"); + else log.debug({ err, ...request }, "API request rejected"); + if (res.headersSent) { + res.end(); + return; + } + sendError(res, status, code, message); +} diff --git a/src/api/indexing/repoConfig.ts b/src/api/indexing/repoConfig.ts deleted file mode 100644 index bee2b670..00000000 --- a/src/api/indexing/repoConfig.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { join } from "node:path"; -import { homedir } from "node:os"; -import { readFileSync, mkdirSync } from "node:fs"; -import { LibScopeLite } from "../../lite/core.js"; -import { createDatabase } from "../../db/connection.js"; -import { runMigrations, createVectorTable } from "../../db/schema.js"; -import { LocalEmbeddingProvider } from "../../providers/local.js"; - -export interface RepoEntry { - cloneUrl: string; - branch?: string; - include?: string[]; - exclude?: string[]; -} - -export interface ReposConfig { - repos: Record<string, RepoEntry>; -} - -export function loadReposConfig(): ReposConfig { - const configPath = process.env["LIBSCOPE_REPOS_CONFIG"]; - if (!configPath) { - return { repos: {} }; - } - try { - const raw = readFileSync(configPath, "utf8"); - return JSON.parse(raw) as ReposConfig; - } catch { - return { repos: {} }; - } -} - -export function repoDbPath(repoSlug: string): string { - return join(homedir(), ".libscope", "repos", `${repoSlug}.db`); -} - -export function createRepoLibScope(repoSlug: string): LibScopeLite { - const dbPath = repoDbPath(repoSlug); - mkdirSync(join(homedir(), ".libscope", "repos"), { recursive: true }); - const provider = new LocalEmbeddingProvider(); - const db = createDatabase(dbPath); - runMigrations(db); - try { - createVectorTable(db, provider); - } catch { - /* sqlite-vec not loaded */ - } - return new LibScopeLite({ db, provider }); -} diff --git a/src/api/indexing/repoIndexer.ts b/src/api/indexing/repoIndexer.ts deleted file mode 100644 index 7a097fb2..00000000 --- a/src/api/indexing/repoIndexer.ts +++ /dev/null @@ -1,225 +0,0 @@ -import { execFileSync } from "node:child_process"; -import { readdirSync, readFileSync, rmSync, statSync, existsSync } from "node:fs"; -import { join, relative, extname } from "node:path"; -import { globToRegExp, toPosixPath } from "../../utils/glob.js"; -import { tmpdir } from "node:os"; -import { randomUUID } from "node:crypto"; -import { LibScopeLite } from "../../lite/core.js"; -import { TreeSitterChunker } from "../../lite/chunker-treesitter.js"; -import type { LiteDoc } from "../../lite/types.js"; -import type { RepoEntry } from "./repoConfig.js"; - -const chunker = new TreeSitterChunker(); - -/** - * Absolute path to the git executable, resolved once at module load time. - * Using an absolute path means the OS never consults PATH, eliminating the - * risk of a tampered environment redirecting "git" to a malicious binary. - */ -const GIT_EXECUTABLE = ((): string => { - const candidates = ["/usr/bin/git", "/usr/local/bin/git", "/opt/homebrew/bin/git"]; - for (const candidate of candidates) { - if (existsSync(candidate)) return candidate; - } - throw new Error("git executable not found at any known path"); -})(); - -export interface IndexJobStats { - filesIndexed: number; - chunksCreated: number; - filesSkipped: number; - errors: string[]; - durationMs: number; -} - -/** Map file extension to language string understood by TreeSitterChunker. */ -function detectLanguage(filePath: string): string | undefined { - const ext = extname(filePath).toLowerCase().replace(/^\./, ""); - const EXTENSION_MAP: Record<string, string> = { - ts: "typescript", - tsx: "typescript", - js: "javascript", - jsx: "javascript", - mjs: "javascript", - cjs: "javascript", - py: "python", - cs: "csharp", - cpp: "cpp", - cc: "cpp", - cxx: "cpp", - hpp: "cpp", - h: "cpp", - c: "c", - go: "go", - }; - return EXTENSION_MAP[ext]; -} - -/** Recursively list all files under dir, returning paths relative to dir. */ -function walkFiles(dir: string, include?: string[], exclude?: string[]): string[] { - const includeRegexes = include?.map(globToRegExp); - const excludeRegexes = exclude?.map(globToRegExp); - const results: string[] = []; - - function recurse(current: string): void { - let entries; - try { - entries = readdirSync(current, { withFileTypes: true }); - } catch { - return; - } - for (const entry of entries) { - const full = join(current, entry.name); - const rel = relative(dir, full); - if (entry.isDirectory()) { - // Skip hidden directories - if (entry.name.startsWith(".")) continue; - recurse(full); - } else if (entry.isFile()) { - const posixRel = toPosixPath(rel); - if (excludeRegexes?.some((rx) => rx.test(posixRel))) continue; - if (includeRegexes && !includeRegexes.some((rx) => rx.test(posixRel))) continue; - results.push(rel); - } - } - } - - recurse(dir); - return results; -} - -/** - * Validate a git branch name against the git-check-ref-format allowlist. - * Rejects anything outside [a-zA-Z0-9._\-/] to prevent argument injection. - */ -function validateBranchName(branch: string): void { - if (!/^[a-zA-Z0-9._\-/]+$/.test(branch)) { - throw new Error( - `Invalid branch name "${branch}": only alphanumerics, '.', '_', '-', and '/' are allowed`, - ); - } -} - -/** Clone a repo at branch into a temp directory. Returns the temp dir path. */ -function cloneToTemp(cloneUrl: string, branch?: string): string { - const tempDir = join(tmpdir(), `libscope-repo-${randomUUID()}`); - const args = ["clone", "--depth=1"]; - if (branch) { - validateBranchName(branch); - args.push("--branch", branch); - } - // Use the absolute path resolved at startup — PATH is never consulted. - // execFileSync with an array bypasses the shell entirely. - // The "--" separator prevents git from interpreting the URL as a flag. - args.push("--", cloneUrl, tempDir); - execFileSync(GIT_EXECUTABLE, args, { stdio: "ignore" }); - return tempDir; -} - -/** Chunk a source file with tree-sitter and append resulting LiteDocs. */ -async function chunkAndIndex( - relPath: string, - source: string, - lang: string, - repoSlug: string, - allDocs: LiteDoc[], - stats: IndexJobStats, -): Promise<void> { - try { - const chunks = await chunker.chunk(source, lang); - for (const chunk of chunks) { - allDocs.push({ - title: `${relPath}:L${String(chunk.startLine)}-L${String(chunk.endLine)} [${chunk.nodeType}]`, - content: chunk.content, - library: repoSlug, - url: `repo://${repoSlug}/${relPath}#L${String(chunk.startLine)}`, - }); - stats.chunksCreated++; - } - stats.filesIndexed++; - } catch (err) { - stats.errors.push(`${relPath} (chunking): ${err instanceof Error ? err.message : String(err)}`); - stats.filesSkipped++; - } -} - -/** Read one file from the cloned repo and append its docs to allDocs. */ -async function processFile( - relPath: string, - tempDir: string, - repoSlug: string, - allDocs: LiteDoc[], - stats: IndexJobStats, -): Promise<void> { - const fullPath = join(tempDir, relPath); - let source: string; - try { - const stat = statSync(fullPath); - if (!stat.isFile()) { - stats.filesSkipped++; - return; - } - source = readFileSync(fullPath, "utf8"); - } catch (err) { - stats.filesSkipped++; - stats.errors.push(`${relPath}: ${err instanceof Error ? err.message : String(err)}`); - return; - } - - const lang = detectLanguage(relPath); - if (lang && chunker.supports(lang)) { - await chunkAndIndex(relPath, source, lang, repoSlug, allDocs, stats); - } else { - // Non-code file or unsupported language — index as raw text - allDocs.push({ - title: relPath, - content: source, - library: repoSlug, - url: `repo://${repoSlug}/${relPath}`, - }); - stats.chunksCreated++; - stats.filesIndexed++; - } -} - -export async function indexRepo( - libscope: LibScopeLite, - repoSlug: string, - entry: RepoEntry, - opts: { branch?: string; files?: string[] }, -): Promise<IndexJobStats> { - const start = Date.now(); - const stats: IndexJobStats = { - filesIndexed: 0, - chunksCreated: 0, - filesSkipped: 0, - errors: [], - durationMs: 0, - }; - - const branch = opts.branch ?? entry.branch; - const tempDir = cloneToTemp(entry.cloneUrl, branch); - - try { - const isIncremental = opts.files !== undefined && opts.files.length > 0; - if (!isIncremental) { - libscope.deleteByLibrary(repoSlug); - } - - const filesToIndex = isIncremental - ? opts.files! - : walkFiles(tempDir, entry.include, entry.exclude); - - const allDocs: LiteDoc[] = []; - for (const relPath of filesToIndex) { - await processFile(relPath, tempDir, repoSlug, allDocs, stats); - } - - await libscope.indexBatch(allDocs, { concurrency: 4 }); - } finally { - rmSync(tempDir, { recursive: true, force: true }); - stats.durationMs = Date.now() - start; - } - - return stats; -} diff --git a/src/api/middleware.ts b/src/api/middleware.ts index 7776fed2..d3e44b6e 100644 --- a/src/api/middleware.ts +++ b/src/api/middleware.ts @@ -1,24 +1,52 @@ import type { IncomingMessage, ServerResponse } from "node:http"; import { timingSafeEqual } from "node:crypto"; +import { LibScopeError } from "../errors.js"; + +/** An error with its own HTTP status (bad body, payload too large, forbidden origin). */ +export class HttpError extends LibScopeError { + constructor( + public readonly status: number, + code: string, + message: string, + ) { + super(message, code); + this.name = "HttpError"; + } +} + +const READ_METHODS = new Set(["GET", "HEAD", "OPTIONS"]); -/** Set CORS headers and handle OPTIONS preflight. Returns true if request was handled (preflight). */ +/** True when `origin` is the origin the request was sent to (same scheme-less host and port). */ +function isSameOrigin(req: IncomingMessage, origin: string): boolean { + try { + return new URL(origin).host === req.headers.host; + } catch { + return false; + } +} + +/** + * Set CORS headers and answer OPTIONS preflight. Returns true when the request was handled. + * `origins` lists the allowed browser origins. "*" allows any origin to read (GET) only: + * write methods are offered only to origins listed by name. + */ export function corsMiddleware( req: IncomingMessage, res: ServerResponse, origins: string[], ): boolean { - const origin = req.headers["origin"] ?? "*"; - const allowedOrigin = origins.includes("*") ? "*" : origins.includes(origin) ? origin : ""; - - if (allowedOrigin) { - res.setHeader("Access-Control-Allow-Origin", allowedOrigin); + const origin = req.headers.origin; + if (origin !== undefined && origins.includes(origin)) { + res.setHeader("Access-Control-Allow-Origin", origin); + res.setHeader("Vary", "Origin"); + res.setHeader("Access-Control-Allow-Methods", "GET, POST, PATCH, PUT, DELETE, OPTIONS"); + } else if (origins.includes("*")) { + res.setHeader("Access-Control-Allow-Origin", "*"); + res.setHeader("Access-Control-Allow-Methods", "GET, OPTIONS"); } - res.setHeader("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE, OPTIONS"); res.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization"); res.setHeader("Access-Control-Max-Age", "86400"); - setSecurityHeaders(res); - if (req.method === "OPTIONS") { res.writeHead(204); res.end(); @@ -27,16 +55,39 @@ export function corsMiddleware( return false; } -/** Set standard security response headers. */ -export function setSecurityHeaders(res: ServerResponse): void { +/** + * Reject a write request (POST, PATCH, PUT, DELETE) sent by a browser page from another origin + * that is not listed by name in `origins`. Browsers send such requests without a preflight + * when they have no body or a "simple" content type, so CORS headers alone do not stop them. + * Returns true when the request was rejected (403 sent). + */ +export function rejectForeignWrite( + req: IncomingMessage, + res: ServerResponse, + origins: string[], +): boolean { + const origin = req.headers.origin; + if (READ_METHODS.has(req.method ?? "GET") || origin === undefined) return false; + if (origins.includes(origin) || isSameOrigin(req, origin)) return false; + sendError(res, 403, "FORBIDDEN_ORIGIN", "Write requests from this origin are not allowed"); + return true; +} + +/** + * Set standard security response headers. `contentSecurityPolicy` false leaves CSP unset + * (the dashboard loads d3 from a CDN). + */ +export function setSecurityHeaders(res: ServerResponse, contentSecurityPolicy = true): void { res.setHeader("X-Content-Type-Options", "nosniff"); res.setHeader("X-Frame-Options", "DENY"); res.setHeader("X-XSS-Protection", "1; mode=block"); res.setHeader("Referrer-Policy", "strict-origin-when-cross-origin"); - res.setHeader( - "Content-Security-Policy", - "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:", - ); + if (contentSecurityPolicy) { + res.setHeader( + "Content-Security-Policy", + "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:", + ); + } } /** Maximum request body size in bytes (default 1 MB). */ @@ -138,7 +189,7 @@ export function checkApiKey(req: IncomingMessage, res: ServerResponse): boolean return true; } -/** Parse the request body as JSON. Returns the parsed object or null on failure. */ +/** Parse the request body as JSON. Resolves null for an empty body; rejects with HttpError. */ export async function parseJsonBody( req: IncomingMessage, maxBytes: number = MAX_BODY_SIZE, @@ -150,7 +201,9 @@ export async function parseJsonBody( received += chunk.length; if (received > maxBytes) { req.destroy(); - reject(new Error(`Request body too large (max ${maxBytes} bytes)`)); + reject( + new HttpError(413, "PAYLOAD_TOO_LARGE", `Request body too large (max ${maxBytes} bytes)`), + ); return; } chunks.push(chunk); @@ -164,7 +217,7 @@ export async function parseJsonBody( try { resolve(JSON.parse(raw)); } catch { - reject(new Error("Invalid JSON body")); + reject(new HttpError(400, "INVALID_JSON", "Request body contains invalid JSON")); } }); req.on("error", reject); diff --git a/src/api/openapi.ts b/src/api/openapi.ts index e61b4f22..2aba9dac 100644 --- a/src/api/openapi.ts +++ b/src/api/openapi.ts @@ -1,497 +1,211 @@ -export const OPENAPI_SPEC = { - openapi: "3.0.3", - info: { - title: "LibScope REST API", - version: "1.0.0", - description: "AI-powered knowledge base with semantic search, document indexing, and RAG Q&A.", - }, - servers: [{ url: "http://localhost:3378", description: "Local development" }], - paths: { - "/api/v1/search": { - get: { - summary: "Semantic search", - operationId: "searchDocuments", - parameters: [ - { - name: "q", - in: "query", - required: true, - schema: { type: "string" }, - description: "Search query", - }, - { - name: "topic", - in: "query", - schema: { type: "string" }, - description: "Filter by topic", - }, - { name: "tag", in: "query", schema: { type: "string" }, description: "Filter by tag" }, - { - name: "limit", - in: "query", - schema: { type: "integer", default: 10 }, - description: "Max results", - }, - ], - responses: { - "200": { - description: "Search results", - content: { - "application/json": { - schema: { $ref: "#/components/schemas/SearchResponse" }, - example: { - data: { - results: [ - { - documentId: "abc123", - title: "Getting Started", - content: "...", - score: 0.95, - }, - ], - totalCount: 1, - }, - meta: { took: 42 }, - }, - }, - }, - }, - }, - }, - }, - "/api/v1/documents": { - get: { - summary: "List documents with filters", - operationId: "listDocuments", - parameters: [ - { name: "topic", in: "query", schema: { type: "string" } }, - { name: "tag", in: "query", schema: { type: "string" } }, - { name: "limit", in: "query", schema: { type: "integer", default: 50 } }, - { name: "offset", in: "query", schema: { type: "integer", default: 0 } }, - ], - responses: { - "200": { - description: "List of documents", - content: { - "application/json": { schema: { $ref: "#/components/schemas/DocumentListResponse" } }, - }, - }, - }, - }, - post: { - summary: "Index a new document", - operationId: "indexDocument", - requestBody: { - required: true, - content: { - "application/json": { - schema: { $ref: "#/components/schemas/IndexDocumentRequest" }, - example: { content: "# Guide\nHow to...", title: "My Guide", topic: "tutorials" }, - }, - }, - }, - responses: { - "201": { - description: "Document indexed", - content: { - "application/json": { schema: { $ref: "#/components/schemas/DocumentResponse" } }, - }, - }, +/** + * OpenAPI 3.1 document generated from the REST routes. Operation routes take their summary, + * description and parameter schemas from the operation (zod -> JSON Schema), so the spec + * cannot drift from what the server accepts. + */ +import { createRequire } from "node:module"; +import { inputJsonSchema, type JsonSchema } from "./adapter.js"; +import type { ApiRoute } from "./routes.js"; + +const pkg = createRequire(import.meta.url)("../../package.json") as { version: string }; + +const ref = (name: string): { $ref: string } => ({ $ref: `#/components/responses/${name}` }); + +const ERROR_RESPONSES = { + "400": ref("BadRequest"), + "401": ref("Unauthorized"), + "404": ref("NotFound"), + "429": ref("RateLimited"), + "500": ref("InternalError"), +}; + +const json = (schema: unknown): { "application/json": { schema: unknown } } => ({ + "application/json": { schema }, +}); + +const SUCCESS = { + description: "Success. `data` holds the operation result.", + content: json({ $ref: "#/components/schemas/Success" }), +}; + +const TASK_ACCEPTED = { + description: + "Started as a background task. Poll GET /api/v1/tasks/{taskId}; when it completes, `result` holds the JSON-encoded operation result.", + headers: { Location: { description: "URL of the task", schema: { type: "string" } } }, + content: json({ $ref: "#/components/schemas/TaskAccepted" }), +}; + +function errorResponse(description: string): Record<string, unknown> { + return { description, content: json({ $ref: "#/components/schemas/Error" }) }; +} + +const COMPONENTS = { + schemas: { + Success: { + type: "object", + required: ["data"], + properties: { + data: { description: "Operation result" }, + meta: { + type: "object", + properties: { took: { type: "integer", description: "Handling time in milliseconds" } }, }, }, }, - "/api/v1/documents/{id}": { - get: { - summary: "Get a single document", - operationId: "getDocument", - parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], - responses: { - "200": { - description: "Document found", - content: { - "application/json": { schema: { $ref: "#/components/schemas/DocumentResponse" } }, - }, - }, - "404": { description: "Document not found" }, - }, - }, - delete: { - summary: "Delete a document", - operationId: "deleteDocument", - parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], - responses: { - "200": { description: "Document deleted" }, - "404": { description: "Document not found" }, - }, - }, - }, - "/api/v1/documents/url": { - post: { - summary: "Index document from URL (with optional spidering)", - operationId: "indexFromUrl", - requestBody: { - required: true, - content: { - "application/json": { - schema: { $ref: "#/components/schemas/IndexFromUrlRequest" }, - examples: { - single: { - summary: "Single URL", - value: { url: "https://example.com/page", topic: "guides" }, - }, - spider: { - summary: "Spider mode", - value: { - url: "https://docs.example.com", - spider: true, - maxPages: 50, - maxDepth: 2, - }, - }, - }, - }, - }, - }, - responses: { - "201": { - description: - "Document(s) indexed. Returns DocumentResponse for single-URL mode, SpiderResponse for spider mode.", - content: { - "application/json": { - schema: { - oneOf: [ - { $ref: "#/components/schemas/DocumentResponse" }, - { $ref: "#/components/schemas/SpiderResponse" }, - ], - }, - }, - }, + TaskAccepted: { + type: "object", + required: ["data"], + properties: { + data: { + type: "object", + required: ["taskId", "operation", "status"], + properties: { + taskId: { type: "string" }, + operation: { type: "string" }, + status: { type: "string", enum: ["pending", "running", "completed", "failed"] }, }, }, + meta: { $ref: "#/components/schemas/Success/properties/meta" }, }, }, - "/api/v1/ask": { - post: { - summary: "RAG Q&A — ask a question", - operationId: "askQuestion", - requestBody: { - required: true, - content: { - "application/json": { - schema: { $ref: "#/components/schemas/AskRequest" }, - example: { question: "How do I configure authentication?", topic: "security" }, - }, - }, - }, - responses: { - "200": { - description: "Answer with sources", - content: { - "application/json": { schema: { $ref: "#/components/schemas/AskResponse" } }, - }, - }, - }, - }, - }, - "/api/v1/topics": { - get: { - summary: "List all topics", - operationId: "listTopics", - responses: { - "200": { - description: "List of topics", - content: { - "application/json": { schema: { $ref: "#/components/schemas/TopicListResponse" } }, - }, - }, - }, - }, - post: { - summary: "Create a topic", - operationId: "createTopic", - requestBody: { - required: true, - content: { - "application/json": { - schema: { $ref: "#/components/schemas/CreateTopicRequest" }, - example: { name: "tutorials", parentId: null }, - }, - }, - }, - responses: { - "201": { - description: "Topic created", - content: { - "application/json": { schema: { $ref: "#/components/schemas/TopicResponse" } }, - }, - }, - }, - }, - }, - "/api/v1/tags": { - get: { - summary: "List all tags", - operationId: "listTags", - responses: { - "200": { - description: "List of tags", - content: { - "application/json": { schema: { $ref: "#/components/schemas/TagListResponse" } }, - }, - }, - }, - }, - }, - "/api/v1/documents/{id}/tags": { - post: { - summary: "Add tags to a document", - operationId: "addTagsToDocument", - parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], - requestBody: { - required: true, - content: { - "application/json": { - schema: { $ref: "#/components/schemas/AddTagsRequest" }, - example: { tags: ["typescript", "api"] }, - }, - }, - }, - responses: { - "200": { description: "Tags added" }, - }, - }, - }, - "/api/v1/stats": { - get: { - summary: "Usage statistics", - operationId: "getStats", - responses: { - "200": { - description: "Statistics", - content: { - "application/json": { schema: { $ref: "#/components/schemas/StatsResponse" } }, - }, - }, - }, - }, - }, - "/api/v1/health": { - get: { - summary: "Health check", - operationId: "healthCheck", - responses: { - "200": { - description: "Service health", - content: { - "application/json": { - example: { data: { status: "ok", docCount: 42, dbSize: 1048576 } }, - }, - }, - }, - }, - }, - }, - "/openapi.json": { - get: { - summary: "OpenAPI 3.0 specification", - operationId: "getOpenApiSpec", - responses: { - "200": { description: "OpenAPI spec JSON" }, + Error: { + type: "object", + required: ["error"], + properties: { + error: { + type: "object", + required: ["code", "message"], + properties: { code: { type: "string" }, message: { type: "string" } }, }, }, }, }, - components: { - schemas: { - SearchResponse: { - type: "object", - properties: { - data: { - type: "object", - properties: { - results: { - type: "array", - items: { - type: "object", - properties: { - documentId: { type: "string" }, - title: { type: "string" }, - content: { type: "string" }, - score: { type: "number" }, - }, - }, - }, - totalCount: { type: "integer" }, - }, - }, - meta: { type: "object", properties: { took: { type: "number" } } }, - }, - }, - DocumentListResponse: { - type: "object", - properties: { data: { type: "array", items: { $ref: "#/components/schemas/Document" } } }, - }, - DocumentResponse: { - type: "object", - properties: { data: { $ref: "#/components/schemas/Document" } }, - }, - Document: { - type: "object", - properties: { - id: { type: "string" }, - title: { type: "string" }, - content: { type: "string" }, - sourceType: { type: "string" }, - library: { type: "string", nullable: true }, - topicId: { type: "string", nullable: true }, - createdAt: { type: "string", format: "date-time" }, - updatedAt: { type: "string", format: "date-time" }, - }, - }, - IndexDocumentRequest: { - type: "object", - required: ["content", "title"], - properties: { - content: { type: "string" }, - title: { type: "string" }, - topic: { type: "string" }, - tags: { type: "array", items: { type: "string" } }, - source: { type: "string" }, - }, - }, - IndexFromUrlRequest: { - type: "object", - required: ["url"], - properties: { - url: { type: "string", format: "uri" }, - topic: { type: "string" }, - spider: { - type: "boolean", - description: "When true, crawl linked pages starting from the URL (BFS spider mode).", - }, - maxPages: { - type: "integer", - minimum: 1, - description: - "Maximum total pages to fetch in spider mode (default: 25, hard cap: 200).", - }, - maxDepth: { - type: "integer", - minimum: 0, - description: - "Maximum hop depth from the seed URL in spider mode (default: 2, hard cap: 5).", - }, - sameDomain: { - type: "boolean", - description: "Only follow links sharing the seed hostname (default: true).", - }, - pathPrefix: { - type: "string", - description: "Only follow links whose path starts with this prefix (e.g. '/docs/').", - }, - excludePatterns: { - type: "array", - items: { type: "string" }, - description: "Glob patterns for URLs to skip during spidering (e.g. ['*/changelog*']).", - }, - }, - }, - SpiderResponse: { - type: "object", - properties: { - documents: { - type: "array", - items: { - type: "object", - properties: { - id: { type: "string" }, - title: { type: "string" }, - url: { type: "string" }, - }, - }, - }, - pagesFetched: { type: "integer", description: "Pages successfully fetched and indexed." }, - pagesCrawled: { type: "integer", description: "Total pages attempted." }, - pagesSkipped: { type: "integer", description: "Pages skipped by filters or robots.txt." }, - errors: { - type: "array", - items: { - type: "object", - properties: { - url: { type: "string" }, - error: { type: "string" }, - }, - }, - }, - abortReason: { - type: "string", - nullable: true, - enum: ["maxPages", "timeout", null], - description: "Set if the crawl was aborted early.", - }, - }, - }, - AskRequest: { - type: "object", - required: ["question"], - properties: { - question: { type: "string" }, - topic: { type: "string" }, - }, - }, - AskResponse: { - type: "object", - properties: { - data: { - type: "object", - properties: { - answer: { type: "string" }, - sources: { type: "array", items: { type: "object" } }, - model: { type: "string" }, - }, - }, - }, - }, - CreateTopicRequest: { - type: "object", - required: ["name"], - properties: { - name: { type: "string" }, - parentId: { type: "string", nullable: true }, - }, - }, - TopicResponse: { - type: "object", - properties: { data: { type: "object" } }, - }, - TopicListResponse: { - type: "object", - properties: { data: { type: "array", items: { type: "object" } } }, - }, - TagListResponse: { - type: "object", - properties: { data: { type: "array", items: { type: "object" } } }, - }, - AddTagsRequest: { - type: "object", - required: ["tags"], - properties: { tags: { type: "array", items: { type: "string" } } }, - }, - StatsResponse: { - type: "object", - properties: { data: { $ref: "#/components/schemas/Stats" } }, - }, - Stats: { - type: "object", - properties: { - totalDocuments: { type: "integer" }, - totalChunks: { type: "integer" }, - totalTopics: { type: "integer" }, - databaseSizeBytes: { type: "integer" }, - totalSearches: { type: "integer" }, - avgLatencyMs: { type: "number" }, - }, - }, + responses: { + BadRequest: errorResponse("Invalid input (code VALIDATION_ERROR or INVALID_JSON)"), + Unauthorized: errorResponse("Missing or wrong API key (only when LIBSCOPE_API_KEY is set)"), + NotFound: errorResponse("Route or resource not found"), + RateLimited: errorResponse("More than 120 requests per minute from this address"), + InternalError: errorResponse("Server error"), + }, + securitySchemes: { + bearerAuth: { + type: "http", + scheme: "bearer", + description: "Required only when the server has LIBSCOPE_API_KEY set", }, }, -} as const; +}; + +/** "/api/v1/documents/:documentId" -> "/api/v1/documents/{documentId}" */ +export function toOpenApiPath(path: string): string { + return path + .split("/") + .map((segment) => (segment.startsWith(":") ? `{${segment.slice(1)}}` : segment)) + .join("/"); +} + +/** "get-document" -> "getDocument" */ +function camelCase(name: string): string { + return name + .split("-") + .map((part, i) => (i === 0 ? part : part.charAt(0).toUpperCase() + part.slice(1))) + .join(""); +} + +function pathParamNames(path: string): string[] { + return path + .split("/") + .filter((s) => s.startsWith(":")) + .map((s) => s.slice(1)); +} + +function parameter( + name: string, + location: "path" | "query", + schema: JsonSchema | undefined, + required: boolean, +): Record<string, unknown> { + const isArray = schema?.type === "array"; + return { + name, + in: location, + required, + ...(typeof schema?.description === "string" ? { description: schema.description } : {}), + schema: schema ?? { type: "string" }, + ...(isArray ? { style: "form", explode: true } : {}), + }; +} + +/** The OpenAPI operation object for a route generated from an operation. */ +function operationObject(route: ApiRoute, op: NonNullable<ApiRoute["operation"]>): unknown { + const schema = inputJsonSchema(op); + const properties = schema.properties ?? {}; + const required = new Set(schema.required ?? []); + const inPath = pathParamNames(route.path); + const parameters = inPath.map((name) => parameter(name, "path", properties[name], true)); + const rest = Object.keys(properties).filter((name) => !inPath.includes(name)); + + let requestBody: unknown; + if (route.method === "GET" || route.method === "DELETE") { + for (const name of rest) { + parameters.push(parameter(name, "query", properties[name], required.has(name))); + } + } else if (rest.length > 0) { + const bodyRequired = rest.filter((name) => required.has(name)); + requestBody = { + required: bodyRequired.length > 0, + content: json({ + type: "object", + properties: Object.fromEntries(rest.map((name) => [name, properties[name]])), + ...(bodyRequired.length > 0 ? { required: bodyRequired } : {}), + }), + }; + } + + return { + operationId: camelCase(op.name), + tags: [op.group], + summary: op.summary, + ...(op.description ? { description: op.description } : {}), + ...(parameters.length > 0 ? { parameters } : {}), + ...(requestBody ? { requestBody } : {}), + responses: { + ...(op.annotations?.longRunning ? { "202": TASK_ACCEPTED } : { "200": SUCCESS }), + ...ERROR_RESPONSES, + }, + }; +} + +function fixedRouteObject(route: ApiRoute): unknown { + return { + operationId: route.path.endsWith(".json") ? "getOpenApiSpec" : "health", + tags: ["meta"], + summary: route.summary, + responses: { "200": SUCCESS, "429": ERROR_RESPONSES["429"] }, + }; +} + +/** Build the OpenAPI document for `routes`. */ +export function buildOpenApiSpec(routes: readonly ApiRoute[]): Record<string, unknown> { + const paths: Record<string, Record<string, unknown>> = {}; + for (const route of routes) { + const path = toOpenApiPath(route.path); + paths[path] ??= {}; + paths[path][route.method.toLowerCase()] = route.operation + ? operationObject(route, route.operation) + : fixedRouteObject(route); + } + return { + openapi: "3.1.0", + info: { + title: "LibScope REST API", + version: pkg.version, + description: + "Every /api/v1 route runs one LibScope operation (the same operations as the CLI, MCP server and SDK). Responses are `{ data, meta }`; errors are `{ error: { code, message } }`. Long-running operations answer 202 with a task ID.", + }, + servers: [{ url: "http://localhost:3378", description: "Default `libscope serve --api`" }], + security: [{}, { bearerAuth: [] }], + paths, + components: COMPONENTS, + }; +} diff --git a/src/api/routes.ts b/src/api/routes.ts index 1a6f5690..b1c76bcc 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -1,1022 +1,159 @@ +/** + * REST routes. Every route under /api/v1 is generated from an operation that declares `http`; + * the only hand-written routes are the OpenAPI document and the health check. + */ import type { IncomingMessage, ServerResponse } from "node:http"; -import { URL } from "node:url"; -import { randomUUID } from "node:crypto"; -import type Database from "better-sqlite3"; import { performance } from "node:perf_hooks"; -import type { EmbeddingProvider } from "../providers/embedding.js"; import { - searchDocuments, - listDocuments, - getDocument, - indexDocument, - deleteDocument, - updateDocument, - listTopics, - createTopic, - listTags, - addTagsToDocument, - suggestTags, - getStats, - getSearchAnalytics, - getKnowledgeGaps, - fetchAndConvert, - askQuestion, - askQuestionStream, - createLlmProvider, - createLink, - getDocumentLinks, - deleteLink, - createSavedSearch, - listSavedSearches, - runSavedSearch, - deleteSavedSearch, - bulkDelete, - bulkRetag, - bulkMove, - searchBatch, -} from "../core/index.js"; -import type { LinkType, BulkSelector, BatchSearchRequest } from "../core/index.js"; -import { LINK_TYPES } from "../core/links.js"; -import { loadConfig } from "../config.js"; -import { NotFoundError, FetchError, LibScopeError } from "../errors.js"; -import { getLogger } from "../logger.js"; -import { parseJsonBody, sendJson, sendError } from "./middleware.js"; -import { OPENAPI_SPEC } from "./openapi.js"; -import { getConnectorStatus, getSyncHistory } from "../connectors/sync-tracker.js"; + OPERATIONS, + type HttpMethod, + type Operation, + type OperationContext, +} from "../core/operations/index.js"; +import { HttpError, sendJson } from "./middleware.js"; import { - createWebhook, - listWebhooks, - deleteWebhook, - getWebhook, - deliverWebhook, - TEST_PING_DATA, - redactWebhook, -} from "../core/webhooks.js"; -import type { WebhookEvent } from "../core/webhooks.js"; -import { loadScheduleEntries } from "../core/scheduler.js"; -import { spiderUrl } from "../core/spider.js"; -import type { SpiderOptions } from "../core/spider.js"; -import { loadReposConfig, createRepoLibScope } from "./indexing/repoConfig.js"; -import { indexRepo, type IndexJobStats } from "./indexing/repoIndexer.js"; - -// --------------------------------------------------------------------------- -// Shared types -// --------------------------------------------------------------------------- - -/** Context passed to every route handler. */ -interface RouteContext { - req: IncomingMessage; - res: ServerResponse; - db: Database.Database; - provider: EmbeddingProvider; - url: URL; - start: number; -} - -// --------------------------------------------------------------------------- -// Repo index job registry -// --------------------------------------------------------------------------- - -interface IndexJob { - jobId: string; - repoSlug: string; - status: "queued" | "running" | "completed" | "failed"; - startedAt: string; - completedAt?: string; - stats?: IndexJobStats; - error?: string; -} - -const repoIndexJobs = new Map<string, IndexJob>(); - -// --------------------------------------------------------------------------- -// URL / path helpers -// --------------------------------------------------------------------------- - -function parseUrl(req: IncomingMessage): URL { - return new URL(req.url ?? "/", `http://${req.headers["host"] ?? "localhost"}`); -} - -function extractPathSegments(pathname: string): string[] { - return pathname.split("/").filter(Boolean); -} - -// --------------------------------------------------------------------------- -// Parameter parsing helpers -// --------------------------------------------------------------------------- - -/** Parse an optional integer search param, returning NaN if absent/invalid. */ -function parseOptionalInt(url: URL, name: string): number { - const raw = url.searchParams.get(name); - return raw ? Number.parseInt(raw, 10) : Number.NaN; -} - -/** Parse an optional integer param clamped to [min, max], with a fallback default. */ -function parseClampedInt( - url: URL, - name: string, - min: number, - max: number, - fallback: number, -): number { - const parsed = parseOptionalInt(url, name); - return Number.isNaN(parsed) ? fallback : Math.max(min, Math.min(parsed, max)); -} - -/** Parse an optional integer param clamped to [min, max], returning undefined if absent. */ -function parseClampedIntOrUndefined( - url: URL, - name: string, - min: number, - max: number, -): number | undefined { - const parsed = parseOptionalInt(url, name); - return Number.isNaN(parsed) ? undefined : Math.max(min, Math.min(parsed, max)); + decodeSegment, + inputFromParams, + readJsonObject, + respondWithOperation, + sendRequestError, +} from "./adapter.js"; +import { buildOpenApiSpec } from "./openapi.js"; + +export const API_PREFIX = "/api/v1"; +export const OPENAPI_PATH = "/openapi.json"; +const TASKS_PATH = `${API_PREFIX}/tasks`; + +export interface ApiRoute { + method: HttpMethod; + /** Full path with `:name` parameters, e.g. "/api/v1/documents/:documentId". */ + path: string; + summary: string; + /** The operation this route runs; absent for the fixed routes (spec, health). */ + operation?: Operation | undefined; + /** Response for a fixed route (no envelope for the OpenAPI document). */ + respond?: ((res: ServerResponse, start: number) => void) | undefined; } -/** Compute elapsed milliseconds since start. */ function elapsed(start: number): number { return Math.round(performance.now() - start); } -/** Parse a JSON request body, returning null-guarded record or sending a 400 error. */ -async function requireJsonBody( - req: IncomingMessage, - res: ServerResponse, -): Promise<Record<string, unknown> | null> { - const body = await parseJsonBody(req); - if (!body || typeof body !== "object") { - sendError(res, 400, "VALIDATION_ERROR", "Request body must be a JSON object"); - return null; - } - return body as Record<string, unknown>; -} - -// --------------------------------------------------------------------------- -// Route handlers — each has low cognitive complexity -// --------------------------------------------------------------------------- - -function handleOpenApiSpec(ctx: RouteContext): void { - ctx.res.writeHead(200, { "Content-Type": "application/json" }); - ctx.res.end(JSON.stringify(OPENAPI_SPEC)); -} - -function handleHealthCheck(ctx: RouteContext): void { - const stats = getStats(ctx.db); - sendJson( - ctx.res, - 200, - { status: "ok", docCount: stats.totalDocuments, dbSize: stats.databaseSizeBytes }, - elapsed(ctx.start), - ); -} - -async function handleSearch(ctx: RouteContext): Promise<void> { - const q = ctx.url.searchParams.get("q") ?? ""; - if (!q) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Query parameter 'q' is required"); - return; - } - if (q.length > 10_000) { - sendError( - ctx.res, - 400, - "VALIDATION_ERROR", - "Query parameter 'q' exceeds maximum length (10000)", - ); - return; - } - const topic = ctx.url.searchParams.get("topic") ?? undefined; - const source = ctx.url.searchParams.get("source") ?? undefined; - const tag = ctx.url.searchParams.get("tag") ?? undefined; - const limit = parseClampedIntOrUndefined(ctx.url, "limit", 1, 1000); - const offsetParsed = parseOptionalInt(ctx.url, "offset"); - const offset = Number.isNaN(offsetParsed) ? undefined : Math.max(0, offsetParsed); - const tags = tag ? [tag] : undefined; - const maxChunksPerDocument = parseClampedIntOrUndefined(ctx.url, "maxChunksPerDocument", 1, 100); - - const result = await searchDocuments(ctx.db, ctx.provider, { - query: q, - topic, - tags, - source, - limit, - offset, - maxChunksPerDocument, - }); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); -} - -async function handleBatchSearch(ctx: RouteContext): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (!Array.isArray(b["requests"])) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Field 'requests' must be an array"); - return; - } - const result = await searchBatch(ctx.db, ctx.provider, b["requests"] as BatchSearchRequest[]); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); -} - -function handleListDocuments(ctx: RouteContext): void { - const topicId = ctx.url.searchParams.get("topic") ?? undefined; - const limit = parseClampedInt(ctx.url, "limit", 1, 1000, 100); - const docs = listDocuments(ctx.db, { topicId, limit }); - sendJson(ctx.res, 200, docs, elapsed(ctx.start)); -} - -async function handleCreateDocument(ctx: RouteContext): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (typeof b["content"] !== "string" || typeof b["title"] !== "string") { - sendError( - ctx.res, - 400, - "VALIDATION_ERROR", - "Fields 'content' and 'title' are required strings", - ); - return; - } - - const topicId = typeof b["topic"] === "string" ? b["topic"] : undefined; - const source = typeof b["source"] === "string" ? b["source"] : undefined; - const sourceType = - source === "library" || - source === "topic" || - source === "manual" || - source === "model-generated" - ? source - : "manual"; - - const doc = await indexDocument(ctx.db, ctx.provider, { - content: b["content"], - title: b["title"], - sourceType, - topicId, - }); - - // Add tags if provided - if (Array.isArray(b["tags"])) { - const tagNames = (b["tags"] as unknown[]).filter((t): t is string => typeof t === "string"); - if (tagNames.length > 0) { - addTagsToDocument(ctx.db, doc.id, tagNames); - } - } - - sendJson(ctx.res, 201, doc, elapsed(ctx.start)); -} - -/** Build spider options from the request body fields. */ -function buildSpiderOptions( - b: Record<string, unknown>, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): SpiderOptions { - return { - fetchOptions, - ...(typeof b["maxPages"] === "number" ? { maxPages: b["maxPages"] } : {}), - ...(typeof b["maxDepth"] === "number" ? { maxDepth: b["maxDepth"] } : {}), - ...(typeof b["sameDomain"] === "boolean" ? { sameDomain: b["sameDomain"] } : {}), - ...(typeof b["pathPrefix"] === "string" ? { pathPrefix: b["pathPrefix"] } : {}), - ...(Array.isArray(b["excludePatterns"]) - ? { - excludePatterns: (b["excludePatterns"] as unknown[]).filter( - (p): p is string => typeof p === "string", - ), - } - : {}), - }; -} - -/** Validate that a body field is a positive integer (if present). Returns error message or null. */ -function validatePositiveInt( - b: Record<string, unknown>, - field: string, - label: string, -): string | null { - if (b[field] === undefined) return null; - const v = b[field]; - if (typeof v !== "number" || !Number.isFinite(v) || !Number.isInteger(v) || v < 1) { - return `${label} must be a positive integer`; - } - return null; -} - -/** Validate that a body field is a non-negative integer (if present). Returns error message or null. */ -function validateNonNegativeInt( - b: Record<string, unknown>, - field: string, - label: string, -): string | null { - if (b[field] === undefined) return null; - const v = b[field]; - if (typeof v !== "number" || !Number.isFinite(v) || !Number.isInteger(v) || v < 0) { - return `${label} must be a non-negative integer`; - } - return null; -} - -/** Handle spidering (crawling) a URL and indexing all pages. */ -async function handleSpiderUrl( - ctx: RouteContext, - b: Record<string, unknown>, - urlStr: string, - topicId: string | undefined, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): Promise<void> { - const maxPagesErr = validatePositiveInt(b, "maxPages", "maxPages"); - if (maxPagesErr) { - sendError(ctx.res, 400, "VALIDATION_ERROR", maxPagesErr); - return; - } - const maxDepthErr = validateNonNegativeInt(b, "maxDepth", "maxDepth"); - if (maxDepthErr) { - sendError(ctx.res, 400, "VALIDATION_ERROR", maxDepthErr); - return; - } - - const spiderOptions = buildSpiderOptions(b, fetchOptions); - const indexedDocs: Array<{ id: string; title: string; url: string }> = []; - const errors: Array<{ url: string; error: string }> = []; - - const gen = spiderUrl(urlStr, spiderOptions); - let result = await gen.next(); - while (!result.done) { - const page = result.value; - try { - const doc = await indexDocument(ctx.db, ctx.provider, { - content: page.content, - title: page.title, - sourceType: "manual", - url: page.url, - topicId, - }); - indexedDocs.push({ id: doc.id, title: page.title, url: page.url }); - } catch (indexErr) { - const msg = indexErr instanceof Error ? indexErr.message : String(indexErr); - errors.push({ url: page.url, error: msg }); - } - result = await gen.next(); - } - // result.value is SpiderStats when done (generator is exhausted) - const stats = result.value; +let spec: Record<string, unknown> | undefined; - sendJson( - ctx.res, - 201, - { - documents: indexedDocs, - pagesFetched: indexedDocs.length, - pagesCrawled: stats.pagesCrawled, - pagesSkipped: stats.pagesSkipped, - errors, - abortReason: stats.abortReason ?? null, +const FIXED_ROUTES: ApiRoute[] = [ + { + method: "GET", + path: OPENAPI_PATH, + summary: "This OpenAPI document", + respond: (res): void => { + spec ??= buildOpenApiSpec(API_ROUTES); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify(spec)); }, - elapsed(ctx.start), - ); -} - -async function handleIndexFromUrl(ctx: RouteContext): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (typeof b["url"] !== "string") { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Field 'url' is required"); - return; - } - const urlStr = b["url"]; - const topicId = typeof b["topic"] === "string" ? b["topic"] : undefined; - const config = loadConfig(); - const fetchOptions = { - allowPrivateUrls: config.indexing.allowPrivateUrls, - allowSelfSignedCerts: config.indexing.allowSelfSignedCerts, - }; - - // Spider mode — crawl linked pages - if (b["spider"] === true) { - await handleSpiderUrl(ctx, b, urlStr, topicId, fetchOptions); - return; - } - - // Single-URL mode (default) - const fetched = await fetchAndConvert(urlStr, fetchOptions); - const doc = await indexDocument(ctx.db, ctx.provider, { - content: fetched.content, - title: fetched.title, - sourceType: "manual", - url: urlStr, - topicId, - }); - sendJson(ctx.res, 201, doc, elapsed(ctx.start)); -} - -/** Handle SSE streaming for ask endpoint. */ -async function handleAskStream(ctx: RouteContext, b: Record<string, unknown>): Promise<void> { - const config = loadConfig(); - const llm = createLlmProvider(config); - const topic = typeof b["topic"] === "string" ? b["topic"] : undefined; - - ctx.res.writeHead(200, { - "Content-Type": "text/event-stream", - "Cache-Control": "no-cache", - Connection: "keep-alive", - }); - - let clientDisconnected = false; - const disconnectPromise = new Promise<void>((resolve) => { - ctx.req.on("close", () => { - clientDisconnected = true; - resolve(); - }); - }); - - try { - const stream = askQuestionStream(ctx.db, ctx.provider, llm, { - question: b["question"] as string, - topic, - }); - - for await (const event of stream) { - if (clientDisconnected) break; - const ok = ctx.res.write(`data: ${JSON.stringify(event)}\n\n`); - if (!ok) { - await Promise.race([ - new Promise<void>((resolve) => ctx.res.once("drain", resolve)), - disconnectPromise, - ]); - } - } - } catch (streamErr: unknown) { - const message = streamErr instanceof Error ? streamErr.message : "Internal server error"; - ctx.res.write(`data: ${JSON.stringify({ error: message })}\n\n`); - } - - ctx.res.end(); -} - -async function handleAsk(ctx: RouteContext): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (typeof b["question"] !== "string") { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Field 'question' is required"); - return; - } - - const accept = ctx.req.headers["accept"] ?? ""; - if (accept.includes("text/event-stream")) { - await handleAskStream(ctx, b); - return; - } - - const config = loadConfig(); - const llm = createLlmProvider(config); - const topic = typeof b["topic"] === "string" ? b["topic"] : undefined; - const result = await askQuestion(ctx.db, ctx.provider, llm, { question: b["question"], topic }); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); -} - -function handleListTopics(ctx: RouteContext): void { - const topics = listTopics(ctx.db); - sendJson(ctx.res, 200, topics, elapsed(ctx.start)); -} - -async function handleCreateTopic(ctx: RouteContext): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (typeof b["name"] !== "string") { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Field 'name' is required"); - return; - } - const parentId = typeof b["parentId"] === "string" ? b["parentId"] : undefined; - const topic = createTopic(ctx.db, { name: b["name"], parentId }); - sendJson(ctx.res, 201, topic, elapsed(ctx.start)); -} - -function handleListTags(ctx: RouteContext): void { - const tags = listTags(ctx.db); - sendJson(ctx.res, 200, tags, elapsed(ctx.start)); -} - -async function handleAddTagsToDocument(ctx: RouteContext, docId: string): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (!Array.isArray(b["tags"])) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Field 'tags' must be an array of strings"); - return; - } - const tagNames = (b["tags"] as unknown[]).filter((t): t is string => typeof t === "string"); - const tags = addTagsToDocument(ctx.db, docId, tagNames); - sendJson(ctx.res, 200, tags, elapsed(ctx.start)); -} - -function handleSuggestTags(ctx: RouteContext, docId: string): void { - const limitRaw = ctx.url.searchParams.get("limit"); - const limit = limitRaw ? Number.parseInt(limitRaw, 10) : 5; - const suggestions = suggestTags(ctx.db, docId, limit); - sendJson(ctx.res, 200, { documentId: docId, suggestions }, elapsed(ctx.start)); -} - -function handleSearchAnalytics(ctx: RouteContext): void { - const days = parseClampedInt(ctx.url, "days", 1, 365, 30); - const analytics = getSearchAnalytics(ctx.db, days); - const gaps = getKnowledgeGaps(ctx.db, days); - sendJson(ctx.res, 200, { ...analytics, knowledgeGaps: gaps }, elapsed(ctx.start)); -} - -function handleConnectorStatus(ctx: RouteContext): void { - const connectorType = ctx.url.searchParams.get("type") ?? undefined; - const connectorName = ctx.url.searchParams.get("name") ?? undefined; - const history = ctx.url.searchParams.get("history"); - const limit = parseClampedIntOrUndefined(ctx.url, "limit", 1, 1000); - - let data; - if (history === "true") { - data = getSyncHistory(ctx.db, connectorType, limit); - } else { - data = getConnectorStatus(ctx.db, connectorType, connectorName); - } - sendJson(ctx.res, 200, data, elapsed(ctx.start)); -} - -function handleScheduleStatus(ctx: RouteContext): void { - const entries = loadScheduleEntries(); - sendJson(ctx.res, 200, { schedules: entries }, elapsed(ctx.start)); -} - -function handleStats(ctx: RouteContext): void { - const stats = getStats(ctx.db); - sendJson(ctx.res, 200, stats, elapsed(ctx.start)); -} - -function handleGetDocument(ctx: RouteContext, docId: string): void { - const doc = getDocument(ctx.db, docId); - sendJson(ctx.res, 200, doc, elapsed(ctx.start)); -} - -function handleDeleteDocument(ctx: RouteContext, docId: string): void { - deleteDocument(ctx.db, docId); - ctx.res.writeHead(204); - ctx.res.end(); -} - -/** Parse and validate URL metadata for PATCH. Returns error message or null. */ -function validateUrlMetadata(urlStr: string): string | null { - try { - const parsedUrl = new URL(urlStr); - if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") { - return "URL must use http or https scheme"; - } - } catch { - return "Invalid URL format"; - } - return null; -} - -/** Extract an optional string field from the body, returning null for non-strings. */ -function extractOptionalString( - body: Record<string, unknown>, - key: string, -): string | null | undefined { - if (body[key] === undefined) return undefined; - return typeof body[key] === "string" ? body[key] : null; -} - -/** Parse metadata fields from an update request body. */ -function parseUpdateMetadata(body: Record<string, unknown>): - | { - library?: string | null; - version?: string | null; - url?: string | null; - topicId?: string | null; - } - | undefined { - const metadata: Record<string, string | null | undefined> = {}; - const library = extractOptionalString(body, "library"); - if (library !== undefined) metadata.library = library; - const version = extractOptionalString(body, "version"); - if (version !== undefined) metadata.version = version; - const url = extractOptionalString(body, "url"); - if (url !== undefined) metadata.url = url; - const topicId = extractOptionalString(body, "topicId"); - if (topicId !== undefined) metadata.topicId = topicId; - return Object.keys(metadata).length > 0 - ? (metadata as { - library?: string | null; - version?: string | null; - url?: string | null; - topicId?: string | null; - }) - : undefined; -} - -async function handleUpdateDocument(ctx: RouteContext, docId: string): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - const title = typeof b["title"] === "string" ? b["title"] : undefined; - const content = typeof b["content"] === "string" ? b["content"] : undefined; - const metadata = parseUpdateMetadata(b); - if (metadata?.url) { - const urlErr = validateUrlMetadata(metadata.url); - if (urlErr) { - sendError(ctx.res, 400, "VALIDATION_ERROR", urlErr); - return; - } - } - const doc = await updateDocument(ctx.db, ctx.provider, docId, { title, content, metadata }); - sendJson(ctx.res, 200, doc, elapsed(ctx.start)); -} - -function handleGetDocumentLinks(ctx: RouteContext, docId: string): void { - const links = getDocumentLinks(ctx.db, docId); - sendJson(ctx.res, 200, links, elapsed(ctx.start)); -} - -async function handleCreateDocumentLink(ctx: RouteContext, docId: string): Promise<void> { - const body = (await parseJsonBody(ctx.req)) as { - targetId?: string; - linkType?: string; - label?: string; - }; - if (!body.targetId || !body.linkType) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "targetId and linkType are required"); - return; - } - const validLinkTypes: readonly string[] = LINK_TYPES; - if (!validLinkTypes.includes(body.linkType)) { - sendError( - ctx.res, - 400, - "VALIDATION_ERROR", - `Invalid linkType: ${body.linkType}. Must be one of: ${validLinkTypes.join(", ")}`, - ); - return; - } - const link = createLink(ctx.db, docId, body.targetId, body.linkType as LinkType, body.label); - sendJson(ctx.res, 201, link, elapsed(ctx.start)); -} - -function handleDeleteLink(ctx: RouteContext, linkId: string): void { - deleteLink(ctx.db, linkId); - ctx.res.writeHead(204); - ctx.res.end(); -} - -async function handleRunSavedSearch(ctx: RouteContext, searchId: string): Promise<void> { - const { search, results } = await runSavedSearch(ctx.db, ctx.provider, searchId); - sendJson(ctx.res, 200, { search, resultCount: results.length, results }, elapsed(ctx.start)); -} - -function handleListSavedSearches(ctx: RouteContext): void { - const searchLimit = parseClampedInt(ctx.url, "limit", 1, 1000, 50); - const searchOffset = parseClampedInt(ctx.url, "offset", 0, Number.MAX_SAFE_INTEGER, 0); - const searches = listSavedSearches(ctx.db, searchLimit, searchOffset); - sendJson(ctx.res, 200, searches, elapsed(ctx.start)); -} - -async function handleCreateSavedSearch(ctx: RouteContext): Promise<void> { - const body = (await parseJsonBody(ctx.req)) as { - name?: string; - query?: string; - filters?: Record<string, unknown>; - }; - if (!body.name || !body.query) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "name and query are required"); - return; - } - const saved = createSavedSearch(ctx.db, body.name, body.query, body.filters); - sendJson(ctx.res, 201, saved, elapsed(ctx.start)); -} - -function handleDeleteSavedSearch(ctx: RouteContext, searchId: string): void { - deleteSavedSearch(ctx.db, searchId); - ctx.res.writeHead(204); - ctx.res.end(); -} - -async function handleBulkOperation(ctx: RouteContext, operation: string): Promise<void> { - const body = (await parseJsonBody(ctx.req)) as { - selector?: BulkSelector; - dryRun?: boolean; - addTags?: string[]; - removeTags?: string[]; - targetTopicId?: string; - }; - - if (!body.selector) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "selector is required"); - return; - } - - if (operation === "delete") { - const result = bulkDelete(ctx.db, body.selector, body.dryRun ?? false); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); - return; - } - - if (operation === "retag") { - const result = bulkRetag( - ctx.db, - body.selector, - body.addTags, - body.removeTags, - body.dryRun ?? false, - ); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); - return; - } - - if (operation === "move") { - if (!body.targetTopicId) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "targetTopicId is required"); - return; - } - const result = bulkMove(ctx.db, body.selector, body.targetTopicId, body.dryRun ?? false); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); - return; - } - - sendError( - ctx.res, - 400, - "VALIDATION_ERROR", - `Invalid bulk operation: ${operation}. Valid operations are: delete, retag, move`, - ); -} - -function handleListWebhooks(ctx: RouteContext): void { - const webhookLimit = parseClampedInt(ctx.url, "limit", 1, 1000, 50); - const webhookOffset = parseClampedInt(ctx.url, "offset", 0, Number.MAX_SAFE_INTEGER, 0); - const webhooks = listWebhooks(ctx.db, webhookLimit, webhookOffset); - sendJson(ctx.res, 200, webhooks.map(redactWebhook), elapsed(ctx.start)); -} - -async function handleCreateWebhook(ctx: RouteContext): Promise<void> { - const body = (await parseJsonBody(ctx.req)) as { - url?: string; - events?: string[]; - secret?: string; - }; - if (!body.url || !body.events) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "url and events are required"); - return; - } - const webhook = await createWebhook(ctx.db, body.url, body.events as WebhookEvent[], body.secret); - sendJson(ctx.res, 201, redactWebhook(webhook), elapsed(ctx.start)); -} - -async function handleTestWebhook(ctx: RouteContext, webhookId: string): Promise<void> { - const webhook = getWebhook(ctx.db, webhookId); - const resp = await deliverWebhook(webhook, "document.created", TEST_PING_DATA); - sendJson(ctx.res, 200, { status: resp.status, statusText: resp.statusText }, elapsed(ctx.start)); -} - -function handleDeleteWebhook(ctx: RouteContext, webhookId: string): void { - deleteWebhook(ctx.db, webhookId); - ctx.res.writeHead(204); - ctx.res.end(); -} - -// --------------------------------------------------------------------------- -// Repo indexing webhook handler -// --------------------------------------------------------------------------- - -function handleGetIndexJob(ctx: RouteContext, jobId: string): void { - const job = repoIndexJobs.get(jobId); - if (!job) { - sendError(ctx.res, 404, "NOT_FOUND", `Index job "${jobId}" not found`); - return; - } - sendJson(ctx.res, 200, job, elapsed(ctx.start)); -} - -async function handleIndexRepo(ctx: RouteContext, repoSlug: string): Promise<void> { - const log = getLogger(); - - // Parse optional body — empty body is valid (full reindex) - let body: Record<string, unknown> = {}; - try { - const parsed = await parseJsonBody(ctx.req); - if (parsed && typeof parsed === "object") { - body = parsed as Record<string, unknown>; - } - } catch { - /* empty body or non-JSON — treat as full reindex request */ - } - - const branch = typeof body["branch"] === "string" ? body["branch"] : undefined; - const files = Array.isArray(body["files"]) - ? (body["files"] as unknown[]).filter((f): f is string => typeof f === "string") - : undefined; - - const config = loadReposConfig(); - const entry = config.repos[repoSlug]; - if (!entry) { - sendError(ctx.res, 404, "NOT_FOUND", `Repo "${repoSlug}" not found in repos config`); - return; - } - - const jobId = randomUUID(); - const job: IndexJob = { - jobId, - repoSlug, - status: "queued", - startedAt: new Date().toISOString(), - }; - repoIndexJobs.set(jobId, job); - - sendJson(ctx.res, 202, { jobId, status: "queued" }, elapsed(ctx.start)); - - // Fire-and-forget indexing job - const libscope = createRepoLibScope(repoSlug); - job.status = "running"; - const indexOpts: { branch?: string; files?: string[] } = {}; - if (branch !== undefined) indexOpts.branch = branch; - if (files !== undefined) indexOpts.files = files; - indexRepo(libscope, repoSlug, entry, indexOpts) - .then((stats) => { - job.status = "completed"; - job.completedAt = new Date().toISOString(); - job.stats = stats; - log.info({ jobId, repoSlug, stats }, "Repo indexing completed"); - }) - .catch((err: unknown) => { - job.status = "failed"; - job.completedAt = new Date().toISOString(); - job.error = err instanceof Error ? err.message : String(err); - log.error({ jobId, repoSlug, err }, "Repo indexing failed"); - }) - .finally(() => { - libscope.close(); - }); -} - -// --------------------------------------------------------------------------- -// Error classification helper -// --------------------------------------------------------------------------- - -function handleRouteError( - res: ServerResponse, - err: unknown, - method: string, - pathname: string, -): void { - const log = getLogger(); - log.error({ err, method, pathname }, "API request error"); - - if (err instanceof NotFoundError) { - sendError(res, 404, "NOT_FOUND", err.message); - return; - } - if (err instanceof FetchError) { - sendError(res, 502, "FETCH_ERROR", err.message); - return; - } - if (err instanceof LibScopeError && err.code === "VALIDATION_ERROR") { - sendError(res, 400, "VALIDATION_ERROR", err.message); - return; - } - if (err instanceof Error && err.message === "Invalid JSON body") { - sendError(res, 400, "INVALID_JSON", "Request body contains invalid JSON"); - return; - } - - log.error({ err }, "Unhandled error in request handler"); - sendError(res, 500, "INTERNAL_ERROR", "Internal server error"); -} - -// --------------------------------------------------------------------------- -// Pathname-based route dispatcher (simple /api/v1/... paths) -// --------------------------------------------------------------------------- - -/** Route table: maps "METHOD /path" → handler. */ -const PATHNAME_ROUTES: Record<string, (ctx: RouteContext) => void | Promise<void>> = { - "GET /openapi.json": handleOpenApiSpec, - "GET /api/v1/health": handleHealthCheck, - "GET /api/v1/search": handleSearch, - "POST /api/v1/batch-search": handleBatchSearch, - "GET /api/v1/documents": handleListDocuments, - "POST /api/v1/documents": handleCreateDocument, - "POST /api/v1/documents/url": handleIndexFromUrl, - "POST /api/v1/ask": handleAsk, - "GET /api/v1/topics": handleListTopics, - "POST /api/v1/topics": handleCreateTopic, - "GET /api/v1/tags": handleListTags, - "GET /api/v1/analytics/searches": handleSearchAnalytics, - "GET /api/v1/connectors/status": handleConnectorStatus, - "GET /api/v1/connectors/schedules": handleScheduleStatus, - "GET /api/v1/stats": handleStats, -}; - -async function dispatchPathnameRoutes( - ctx: RouteContext, - pathname: string, - method: string, -): Promise<boolean> { - const handler = PATHNAME_ROUTES[`${method} ${pathname}`]; - if (!handler) return false; - await handler(ctx); - return true; -} + }, + { + method: "GET", + path: `${API_PREFIX}/health`, + summary: "Liveness check (use GET /api/v1/overview for counts and index health)", + respond: (res, start): void => sendJson(res, 200, { status: "ok" }, elapsed(start)), + }, +]; -// --------------------------------------------------------------------------- -// Pattern-based route dispatcher (/documents/:id, links, searches, webhooks, bulk, index) -// --------------------------------------------------------------------------- +/** Every REST route: the fixed routes, then one per operation with an `http` mapping. */ +export const API_ROUTES: readonly ApiRoute[] = [ + ...FIXED_ROUTES, + ...OPERATIONS.flatMap((op): ApiRoute[] => + op.http + ? [ + { + method: op.http.method, + path: `${API_PREFIX}${op.http.path}`, + summary: op.summary, + operation: op, + }, + ] + : [], + ), +]; -interface PatternRoute { - method: string; +interface CompiledRoute { + route: ApiRoute; parts: string[]; - handler: (ctx: RouteContext, param: string) => void | Promise<void>; - /** A param value this route must not match (it belongs to a literal route). */ - exclude?: string | undefined; + paramCount: number; } -/** Build a pattern route from "METHOD /path/:param" (at most one ":param" segment). */ -function route(spec: string, handler: PatternRoute["handler"], exclude?: string): PatternRoute { - const [method = "", path = ""] = spec.split(" "); - return { method, parts: extractPathSegments(path), handler, exclude }; -} +const COMPILED: CompiledRoute[] = API_ROUTES.map((route) => { + const parts = route.path.split("/").filter(Boolean); + return { route, parts, paramCount: parts.filter((p) => p.startsWith(":")).length }; +}); -/** - * Match path segments against pattern parts. Returns the ":param" segment value ("" when the - * pattern has no param), or null when the segments do not match. - */ -function matchPattern(parts: string[], segments: string[]): string | null { +/** Path parameters when `segments` match `parts`, else null. */ +function matchParts(parts: string[], segments: string[]): Record<string, string> | null { if (parts.length !== segments.length) return null; - let param = ""; + const params: Record<string, string> = {}; for (const [i, part] of parts.entries()) { const segment = segments[i] ?? ""; - if (part.startsWith(":")) param = segment; + if (part.startsWith(":")) params[part.slice(1)] = decodeSegment(segment); else if (part !== segment) return null; } - return param; + return params; } -/** Pattern route table, checked in order after PATHNAME_ROUTES. */ -const PATTERN_ROUTES: PatternRoute[] = [ - route("POST /api/v1/documents/:id/tags", handleAddTagsToDocument), - route("GET /api/v1/documents/:id/suggest-tags", handleSuggestTags), - route("GET /api/v1/documents/:id", handleGetDocument, "url"), - route("DELETE /api/v1/documents/:id", handleDeleteDocument, "url"), - route("PATCH /api/v1/documents/:id", handleUpdateDocument, "url"), - route("GET /api/v1/documents/:id/links", handleGetDocumentLinks), - route("POST /api/v1/documents/:id/links", handleCreateDocumentLink), - route("DELETE /api/v1/links/:id", handleDeleteLink), - route("POST /api/v1/searches/:id/run", handleRunSavedSearch), - route("GET /api/v1/searches", handleListSavedSearches), - route("POST /api/v1/searches", handleCreateSavedSearch), - route("POST /api/v1/bulk/:operation", handleBulkOperation), - route("DELETE /api/v1/searches/:id", handleDeleteSavedSearch), - route("GET /api/v1/index/jobs/:jobId", handleGetIndexJob), - route("POST /api/v1/index/repos/:repoSlug", handleIndexRepo), - route("GET /api/v1/webhooks", handleListWebhooks), - route("POST /api/v1/webhooks", handleCreateWebhook), - route("POST /api/v1/webhooks/:id/test", handleTestWebhook), - route("DELETE /api/v1/webhooks/:id", handleDeleteWebhook), -]; - -async function dispatchPatternRoutes( - ctx: RouteContext, - segments: string[], +/** The route for a request; literal segments win over parameters. */ +export function matchRoute( method: string, -): Promise<boolean> { - for (const r of PATTERN_ROUTES) { - if (r.method !== method) continue; - const param = matchPattern(r.parts, segments); - if (param === null || param === r.exclude) continue; - await r.handler(ctx, param); - return true; + pathname: string, +): { route: ApiRoute; params: Record<string, string> } | null { + const segments = pathname.split("/").filter(Boolean); + let best: { route: ApiRoute; params: Record<string, string>; paramCount: number } | null = null; + for (const c of COMPILED) { + if (c.route.method !== method || (best && best.paramCount <= c.paramCount)) continue; + const params = matchParts(c.parts, segments); + if (params) best = { route: c.route, params, paramCount: c.paramCount }; } - return false; + return best; } -// --------------------------------------------------------------------------- -// Main entry point -// --------------------------------------------------------------------------- +/** Input for an operation route: query string (GET, DELETE) or JSON body, plus path params. */ +async function buildInput( + op: Operation, + req: IncomingMessage, + url: URL, + params: Record<string, string>, +): Promise<Record<string, unknown>> { + const fromRequest = + op.http?.method === "GET" || op.http?.method === "DELETE" + ? inputFromParams(op, url.searchParams) + : await readJsonObject(req); + return { ...fromRequest, ...inputFromParams(op, params) }; +} +/** Handle one REST request (after rate limiting, CORS and auth). */ export async function handleRequest( req: IncomingMessage, res: ServerResponse, - db: Database.Database, - provider: EmbeddingProvider, + ctx: OperationContext, ): Promise<void> { const start = performance.now(); - const url = parseUrl(req); - const pathname = url.pathname; + const url = new URL(req.url ?? "/", "http://localhost"); const method = req.method ?? "GET"; - const segments = extractPathSegments(pathname); - - const ctx: RouteContext = { req, res, db, provider, url, start }; - try { - // Try simple pathname-based routes first - const pathnameHandled = await dispatchPathnameRoutes(ctx, pathname, method); - if (pathnameHandled) return; - - // Try segment-pattern routes (document/:id, links, searches, webhooks, bulk, index) - const segmentHandled = await dispatchPatternRoutes(ctx, segments, method); - if (segmentHandled) return; - - // Unknown route — don't leak method/pathname to prevent endpoint enumeration - sendError(res, 404, "NOT_FOUND", "Route not found"); - } catch (err: unknown) { - handleRouteError(res, err, method, pathname); + const match = matchRoute(method, url.pathname); + // Same answer for unknown paths and wrong methods, so endpoints cannot be enumerated. + if (!match) throw new HttpError(404, "NOT_FOUND", "Route not found"); + const { route, params } = match; + if (route.respond) { + route.respond(res, start); + return; + } + const op = route.operation; + if (!op) throw new HttpError(404, "NOT_FOUND", "Route not found"); + const input = await buildInput(op, req, url, params); + const result = await respondWithOperation(op, ctx, input, TASKS_PATH); + if (result.location) res.setHeader("Location", result.location); + sendJson(res, result.status, result.data, elapsed(start)); + } catch (err) { + sendRequestError(res, err, { method, pathname: url.pathname }); } } diff --git a/src/api/server.ts b/src/api/server.ts index 3b9649c5..13f0a8b0 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -1,10 +1,21 @@ import { createServer, type Server } from "node:http"; -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; +import type { AddressInfo } from "node:net"; +import type { Bootstrapped } from "../core/bootstrap.js"; +import { createOperationContext } from "../core/operations/index.js"; +import { ConnectorScheduler, loadScheduleEntries } from "../core/scheduler.js"; import { getLogger } from "../logger.js"; -import { corsMiddleware, checkRateLimit, checkApiKey } from "./middleware.js"; +import { + checkApiKey, + checkRateLimit, + corsMiddleware, + rejectForeignWrite, + sendError, + setSecurityHeaders, +} from "./middleware.js"; import { handleRequest } from "./routes.js"; -import { ConnectorScheduler, loadScheduleEntries } from "../core/scheduler.js"; + +/** What a server needs from bootstrap(). */ +export type ServerApp = Pick<Bootstrapped, "db" | "provider" | "config">; /** Close an HTTP server, returning a promise that resolves when done. */ function closeHttpServer(server: Server): Promise<void> { @@ -13,17 +24,42 @@ function closeHttpServer(server: Server): Promise<void> { }); } +/** Listen on `port`/`host`; resolves with the port actually bound (for port 0). */ +export function listen(server: Server, port: number, host: string): Promise<number> { + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => { + server.off("error", reject); + resolve((server.address() as AddressInfo).port); + }); + }); +} + +/** Answer 429 when `ip` is over the rate limit. Returns true when the request was rejected. */ +export function rejectRateLimited( + ip: string | undefined, + res: Parameters<typeof sendError>[0], +): boolean { + if (checkRateLimit(ip ?? "unknown")) return false; + res.setHeader("Retry-After", "60"); + sendError(res, 429, "RATE_LIMITED", "Too many requests"); + return true; +} + export interface ApiServerOptions { port?: number | undefined; host?: string | undefined; - /** Allowed CORS origins. Defaults to ["http://localhost", "http://localhost:3000"]. */ + /** + * Browser origins allowed to call the API. Default: ["http://localhost", + * "http://localhost:3000"]. "*" lets any origin read (GET) but not write. + */ corsOrigins?: string[] | undefined; enableScheduler?: boolean | undefined; } +/** Start the REST API server (default http://localhost:3378). */ export async function startApiServer( - db: Database.Database, - provider: EmbeddingProvider, + app: ServerApp, options?: ApiServerOptions, ): Promise<{ close: () => Promise<void>; @@ -31,58 +67,39 @@ export async function startApiServer( scheduler?: ConnectorScheduler | undefined; }> { const log = getLogger(); - const port = options?.port ?? 3378; const host = options?.host ?? "localhost"; const corsOrigins = options?.corsOrigins ?? ["http://localhost", "http://localhost:3000"]; + const ctx = createOperationContext({ ...app, surface: "api" }); const server = createServer((req, res) => { - // Rate limiting - const ip = req.socket.remoteAddress ?? "unknown"; - if (!checkRateLimit(ip)) { - res.writeHead(429, { "Content-Type": "application/json", "Retry-After": "60" }); - res.end(JSON.stringify({ error: { code: "RATE_LIMITED", message: "Too many requests" } })); - return; - } - + setSecurityHeaders(res); + if (rejectRateLimited(req.socket.remoteAddress, res)) return; if (corsMiddleware(req, res, corsOrigins)) return; + if (rejectForeignWrite(req, res, corsOrigins)) return; if (!checkApiKey(req, res)) return; - handleRequest(req, res, db, provider).catch((err: unknown) => { - log.error({ err }, "Unhandled error in request handler"); - if (!res.headersSent) { - res.writeHead(500, { "Content-Type": "application/json" }); - res.end( - JSON.stringify({ error: { code: "INTERNAL_ERROR", message: "Internal server error" } }), - ); - } - }); + // handleRequest answers every error itself. + void handleRequest(req, res, ctx); }); - return new Promise((resolve, reject) => { - server.on("error", reject); - server.listen(port, host, () => { - log.info({ port, host }, "API server started"); + const port = await listen(server, options?.port ?? 3378, host); + log.info({ port, host }, "API server started"); - let scheduler: ConnectorScheduler | undefined; - if (options?.enableScheduler !== false) { - const entries = loadScheduleEntries(); - if (entries.length > 0) { - scheduler = new ConnectorScheduler(db, provider); - scheduler.start(entries); - log.info( - { scheduledJobs: entries.length }, - "Connector scheduler started with API server", - ); - } - } + let scheduler: ConnectorScheduler | undefined; + if (options?.enableScheduler !== false) { + const entries = loadScheduleEntries(); + if (entries.length > 0) { + scheduler = new ConnectorScheduler(app.db, app.provider); + scheduler.start(entries); + log.info({ scheduledJobs: entries.length }, "Connector scheduler started with API server"); + } + } - resolve({ - close: async () => { - await scheduler?.stop(); - await closeHttpServer(server); - }, - port, - scheduler, - }); - }); - }); + return { + close: async (): Promise<void> => { + await scheduler?.stop(); + await closeHttpServer(server); + }, + port, + scheduler, + }; } diff --git a/src/cli/index.ts b/src/cli/index.ts index 26a37f14..3362e877 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1536,19 +1536,19 @@ program .option("--host <host>", "Server host", "localhost") .action(async (opts: { api?: boolean; dashboard?: boolean; port?: string; host: string }) => { if (opts.dashboard) { - const { db, provider } = initializeAppWithEmbedding(); + const app = initializeAppWithEmbedding(); const { startWebServer } = await import("../web/server.js"); const defaultPort = 3377; const port = opts.port ? parseIntOption(opts.port, "--port") : defaultPort; - await startWebServer(db, provider, { port, host: opts.host }); + await startWebServer(app, { port, host: opts.host }); console.log(`LibScope dashboard running at http://${opts.host}:${port}`); console.log("Press Ctrl+C to stop"); } else if (opts.api) { - const { db, provider } = initializeAppWithEmbedding(); + const app = initializeAppWithEmbedding(); const { startApiServer } = await import("../api/server.js"); const defaultPort = 3378; const port = opts.port ? parseIntOption(opts.port, "--port") : defaultPort; - const result = await startApiServer(db, provider, { port, host: opts.host }); + const result = await startApiServer(app, { port, host: opts.host }); console.log(`LibScope API server listening on http://${opts.host}:${result.port}`); console.log("Press Ctrl+C to stop"); } else { diff --git a/src/web/graph-api.ts b/src/web/graph-api.ts deleted file mode 100644 index f832773b..00000000 --- a/src/web/graph-api.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { IncomingMessage, ServerResponse } from "node:http"; -import type Database from "better-sqlite3"; -import { buildKnowledgeGraph } from "../core/graph.js"; - -/** HTTP handler for graph data requests. */ -export async function handleGraphRequest( - db: Database.Database, - req: IncomingMessage, - res: ServerResponse, -): Promise<void> { - const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`); - - const rawThreshold = Number.parseFloat(url.searchParams.get("threshold") ?? "0.85"); - const rawMaxNodes = Number.parseInt(url.searchParams.get("maxNodes") ?? "200", 10); - const topic = url.searchParams.get("topic") ?? undefined; - const tag = url.searchParams.get("tag") ?? undefined; - - // Clamp parameters to safe bounds - const threshold = Number.isNaN(rawThreshold) ? 0.85 : Math.max(0, Math.min(1, rawThreshold)); - const maxNodes = Number.isNaN(rawMaxNodes) ? 200 : Math.max(1, Math.min(5000, rawMaxNodes)); - - const graph = await buildKnowledgeGraph(db, { - similarityThreshold: threshold, - maxNodes, - topicFilter: topic ?? undefined, - tagFilter: tag ?? undefined, - }); - - res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify(graph)); -} diff --git a/src/web/server.ts b/src/web/server.ts index 8ecd7669..ffd6a1e4 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -1,255 +1,194 @@ +/** + * Web dashboard server: the dashboard pages plus the small unversioned JSON API their + * scripts call. Each API route runs an operation and reshapes the result into the format + * the dashboard script expects (src/web/dashboard.ts is not changed). + */ import { createServer, type IncomingMessage, type ServerResponse, type Server } from "node:http"; -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import { searchDocuments } from "../core/search.js"; -import { getDocument, deleteDocument, listDocuments } from "../core/documents.js"; -import { getTopicStats } from "../core/topics.js"; +import type { z } from "zod"; +import { decodeSegment, inputFromParams, sendRequestError } from "../api/adapter.js"; +import { + corsMiddleware, + rejectForeignWrite, + sendError, + setSecurityHeaders, +} from "../api/middleware.js"; +import { listen, rejectRateLimited, type ServerApp } from "../api/server.js"; +import { + createOperationContext, + deleteDocumentOperation, + getDocumentOperation, + graphOperation, + listDocumentsOperation, + listTopicsOperation, + overviewOperation, + runOperation, + searchOperation, + type Operation, + type OperationContext, +} from "../core/operations/index.js"; import { getDashboardHtml, getGraphPageHtml } from "./dashboard.js"; -import { handleGraphRequest } from "./graph-api.js"; -import { DocumentNotFoundError } from "../errors.js"; -import { validateCountRow } from "../utils/db-validation.js"; -import { getLogger } from "../logger.js"; -import { checkRateLimit } from "../api/middleware.js"; export interface WebServerOptions { - port?: number; - host?: string; + port?: number | undefined; + host?: string | undefined; + /** Other browser origins allowed to call the dashboard API. Default: none (same origin only). */ + corsOrigins?: string[] | undefined; } -let server: Server | null = null; - -/** Start the web UI server. */ -export function startWebServer( - db: Database.Database, - provider: EmbeddingProvider, - options?: WebServerOptions, -): Promise<Server> { - const port = options?.port ?? 3377; - const host = options?.host ?? "localhost"; - - return new Promise((resolve, reject) => { - server = createServer((req, res) => { - const ip = req.socket.remoteAddress ?? "unknown"; - if (!checkRateLimit(ip)) { - res.writeHead(429, { "Content-Type": "application/json", "Retry-After": "60" }); - res.end(JSON.stringify({ error: "Too many requests" })); - return; - } - handleRequest(db, provider, req, res).catch((err) => { - getLogger().error({ err, url: req.url }, "Unhandled error in web request handler"); - if (res.headersSent) { - req.socket.destroy(); - } else { - sendJson(res, 500, { error: "Internal server error" }); - } - }); - }); +interface DashboardRoute<S extends z.ZodObject = z.ZodObject, O = unknown> { + method: "GET" | "DELETE"; + /** Path; a trailing "/:documentId" segment is passed to the operation. */ + path: string; + operation: Operation<S, O>; + /** Rename query parameters the dashboard sends under another name. */ + rename?: Record<string, string>; + /** Dashboard response shape. */ + shape: (result: O) => unknown; +} - server.on("error", reject); - server.listen(port, host, () => resolve(server!)); - }); +function route<S extends z.ZodObject, O>(r: DashboardRoute<S, O>): DashboardRoute { + return r as unknown as DashboardRoute; } -/** Gracefully shut down the web server. */ -export function stopWebServer(): Promise<void> { - return new Promise((resolve) => { - if (!server) { - resolve(); - return; +/** The JSON routes the dashboard script fetches, each served by one operation. */ +const DASHBOARD_ROUTES: DashboardRoute[] = [ + route({ + method: "GET", + path: "/api/stats", + operation: overviewOperation, + shape: ({ stats }) => ({ + documentCount: stats.totalDocuments, + topicCount: stats.totalTopics, + chunkCount: stats.totalChunks, + }), + }), + route({ + method: "GET", + path: "/api/topics", + operation: listTopicsOperation, + shape: (result) => result.items, + }), + route({ + method: "GET", + path: "/api/documents", + operation: listDocumentsOperation, + shape: (result) => result.items.map(({ documentId, ...rest }) => ({ id: documentId, ...rest })), + }), + route({ + method: "GET", + path: "/api/search", + operation: searchOperation, + rename: { q: "query" }, + shape: (result) => ({ results: result.items, totalCount: result.total }), + }), + route({ + method: "GET", + path: "/api/graph", + operation: graphOperation, + shape: (graph) => graph, + }), + route({ + method: "GET", + path: "/api/documents/:documentId", + operation: getDocumentOperation, + shape: (view) => ({ + id: view.document.documentId, + ...view.document, + content: view.content, + tags: view.tags, + }), + }), + route({ + method: "DELETE", + path: "/api/documents/:documentId", + operation: deleteDocumentOperation, + shape: () => ({ success: true }), + }), +]; + +const PAGES: Record<string, () => string> = { + "/": getDashboardHtml, + "/graph": getGraphPageHtml, +}; + +/** The route for a path, with the trailing `:documentId` value when the route has one. */ +function findRoute( + method: string, + pathname: string, +): { route: DashboardRoute; params: Record<string, string> } | undefined { + for (const r of DASHBOARD_ROUTES) { + if (r.method !== method) continue; + if (r.path === pathname) return { route: r, params: {} }; + const [prefix, param] = r.path.split("/:"); + if (param === undefined || !pathname.startsWith(`${prefix}/`)) continue; + const value = pathname.slice((prefix ?? "").length + 1); + if (value !== "" && !value.includes("/")) { + return { route: r, params: { [param]: decodeSegment(value) } }; } - server.close(() => { - server = null; - resolve(); - }); - }); + } + return undefined; } async function handleRequest( - db: Database.Database, - provider: EmbeddingProvider, + ctx: OperationContext, req: IncomingMessage, res: ServerResponse, ): Promise<void> { - const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`); - const pathname = url.pathname; + const url = new URL(req.url ?? "/", "http://localhost"); const method = req.method ?? "GET"; - - setCorsHeaders(res); - - if (method === "OPTIONS") { - res.writeHead(204); - res.end(); - return; - } - - if (method === "GET") { - const handled = await handleGetRoute(db, provider, pathname, url, res, req); - if (handled) return; - } - - const docMatch = pathname.match(/^\/api\/documents\/([^/]+)$/); - if (docMatch) { - const id = decodeURIComponent(docMatch[1]!); - const handled = handleDocumentByIdRoute(db, method, id, res); - if (handled) return; - } - - sendJson(res, 404, { error: "Not found" }); -} - -/** Handle all GET routes. Returns true if a route matched. */ -async function handleGetRoute( - db: Database.Database, - provider: EmbeddingProvider, - pathname: string, - url: URL, - res: ServerResponse, - req: IncomingMessage, -): Promise<boolean> { - if (pathname === "/") { - res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); - res.end(getDashboardHtml()); - return true; - } - - if (pathname === "/graph") { - res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); - res.end(getGraphPageHtml()); - return true; - } - - if (pathname === "/api/graph") { - await handleGraphRequest(db, req, res); - return true; - } - - if (pathname === "/api/search") { - await handleSearchRoute(db, provider, url, res); - return true; - } - - if (pathname === "/api/documents") { - handleDocumentsListRoute(db, url, res); - return true; - } - - if (pathname === "/api/topics") { - sendJson(res, 200, getTopicStats(db)); - return true; - } - - if (pathname === "/api/stats") { - handleStatsRoute(db, res); - return true; - } - - return false; -} - -/** Handle GET /api/search */ -async function handleSearchRoute( - db: Database.Database, - provider: EmbeddingProvider, - url: URL, - res: ServerResponse, -): Promise<void> { - const query = url.searchParams.get("q") ?? ""; - const rawLimit = Number.parseInt(url.searchParams.get("limit") ?? "10", 10); - const limit = Number.isNaN(rawLimit) ? 10 : Math.max(1, Math.min(100, rawLimit)); - const topic = url.searchParams.get("topic") ?? undefined; - - if (!query) { - sendJson(res, 400, { error: "Missing query parameter 'q'" }); - return; - } - - const results = await searchDocuments(db, provider, { query, limit, topic }); - sendJson(res, 200, results); -} - -/** Handle GET /api/documents */ -function handleDocumentsListRoute(db: Database.Database, url: URL, res: ServerResponse): void { - const rawLimit = Number.parseInt(url.searchParams.get("limit") ?? "50", 10); - const limit = Number.isNaN(rawLimit) ? 50 : Math.max(1, Math.min(500, rawLimit)); - const topic = url.searchParams.get("topic") ?? undefined; - const docs = listDocuments(db, { limit, topicId: topic }); - sendJson(res, 200, docs); -} - -/** Handle GET/DELETE /api/documents/:id. Returns true if method matched. */ -function handleDocumentByIdRoute( - db: Database.Database, - method: string, - id: string, - res: ServerResponse, -): boolean { - if (method === "GET") { - handleGetDocumentById(db, id, res); - return true; - } - if (method === "DELETE") { - handleDeleteDocumentById(db, id, res); - return true; - } - return false; -} - -/** Handle GET /api/documents/:id */ -function handleGetDocumentById(db: Database.Database, id: string, res: ServerResponse): void { try { - const doc = getDocument(db, id); - sendJson(res, 200, doc); - } catch (err) { - if (err instanceof DocumentNotFoundError) { - sendJson(res, 404, { error: "Document not found" }); - } else { - throw err; + const page = method === "GET" ? PAGES[url.pathname] : undefined; + if (page) { + res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + res.end(page()); + return; } - } -} - -/** Handle DELETE /api/documents/:id */ -function handleDeleteDocumentById(db: Database.Database, id: string, res: ServerResponse): void { - try { - deleteDocument(db, id); - sendJson(res, 200, { success: true }); - } catch (err) { - if (err instanceof DocumentNotFoundError) { - sendJson(res, 404, { error: "Document not found" }); - } else { - throw err; + const match = findRoute(method, url.pathname); + if (!match) { + sendError(res, 404, "NOT_FOUND", "Not found"); + return; } + const { route: r, params } = match; + const query = new URLSearchParams(); + for (const [name, value] of url.searchParams) query.append(r.rename?.[name] ?? name, value); + const input = { ...inputFromParams(r.operation, query), ...params }; + const result = await runOperation(r.operation, ctx, input); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify(r.shape(result))); + } catch (err) { + sendRequestError(res, err, { method, pathname: url.pathname }); } } -/** Handle GET /api/stats */ -function handleStatsRoute(db: Database.Database, res: ServerResponse): void { - const docCount = validateCountRow( - db.prepare("SELECT COUNT(*) AS cnt FROM documents").get(), - "document count", - ); - const topicCount = validateCountRow( - db.prepare("SELECT COUNT(*) AS cnt FROM topics").get(), - "topic count", - ); - const chunkCount = validateCountRow( - db.prepare("SELECT COUNT(*) AS cnt FROM chunks").get(), - "chunk count", - ); - sendJson(res, 200, { documentCount: docCount, topicCount, chunkCount }); -} +let server: Server | null = null; -function sendJson(res: ServerResponse, status: number, body: unknown): void { - res.writeHead(status, { "Content-Type": "application/json" }); - res.end(JSON.stringify(body)); +/** Start the web dashboard (default http://localhost:3377). */ +export async function startWebServer(app: ServerApp, options?: WebServerOptions): Promise<Server> { + const corsOrigins = options?.corsOrigins ?? []; + const ctx = createOperationContext({ ...app, surface: "api" }); + const httpServer = createServer((req, res) => { + // No Content-Security-Policy: the graph page loads d3 from a CDN. + setSecurityHeaders(res, false); + if (rejectRateLimited(req.socket.remoteAddress, res)) return; + if (corsMiddleware(req, res, corsOrigins)) return; + if (rejectForeignWrite(req, res, corsOrigins)) return; + void handleRequest(ctx, req, res); + }); + await listen(httpServer, options?.port ?? 3377, options?.host ?? "localhost"); + server = httpServer; + return httpServer; } -function setCorsHeaders(res: ServerResponse): void { - res.setHeader("Access-Control-Allow-Origin", "*"); - res.setHeader("Access-Control-Allow-Methods", "GET, DELETE, OPTIONS"); - res.setHeader("Access-Control-Allow-Headers", "Content-Type"); - res.setHeader("X-Content-Type-Options", "nosniff"); - res.setHeader("X-Frame-Options", "DENY"); - res.setHeader("X-XSS-Protection", "1; mode=block"); - res.setHeader("Referrer-Policy", "strict-origin-when-cross-origin"); +/** Gracefully shut down the web server. */ +export function stopWebServer(): Promise<void> { + return new Promise((resolve) => { + if (!server) { + resolve(); + return; + } + server.close(() => { + server = null; + resolve(); + }); + }); } diff --git a/tests/unit/api-routes.test.ts b/tests/unit/api-routes.test.ts new file mode 100644 index 00000000..e14ee44b --- /dev/null +++ b/tests/unit/api-routes.test.ts @@ -0,0 +1,550 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from "vitest"; +import { createServer, request, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { testConfig } from "./operations/helpers.js"; +import { initLogger } from "../../src/logger.js"; + +// Saved connections live under a temp HOME, never the real ~/.libscope. +let tempHome = join(tmpdir(), `libscope-api-routes-${process.pid}`); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); +// Webhook URLs resolve to a public address without network access. +vi.mock("node:dns", async (importOriginal) => { + const actual: typeof import("node:dns") = await importOriginal(); + return { + ...actual, + promises: { + ...actual.promises, + resolve4: vi.fn().mockResolvedValue(["93.184.216.34"]), + resolve6: vi.fn().mockResolvedValue([]), + }, + }; +}); + +const { API_ROUTES, handleRequest } = await import("../../src/api/routes.js"); +const { createOperationContext } = await import("../../src/core/operations/index.js"); +const { indexDocument } = await import("../../src/core/indexing.js"); +const { updateDocument } = await import("../../src/core/documents.js"); +const { createTopic } = await import("../../src/core/topics.js"); +const { createLink } = await import("../../src/core/links.js"); +const { createSavedSearch } = await import("../../src/core/saved-searches.js"); +const { createWebhook } = await import("../../src/core/webhooks.js"); +const { taskRegistry } = await import("../../src/core/tasks.js"); +const { saveConnectorSettings } = await import("../../src/connectors/saved-config.js"); + +initLogger("silent"); + +interface Fixtures { + docA: string; + docB: string; + linkId: string; + webhookId: string; + taskId: string; +} + +interface Case { + path: string; + query?: Record<string, string>; + body?: unknown; +} + +interface RouteCases { + ok: (f: Fixtures) => Case; + /** Expected success status when not 200 (202 for background tasks). */ + okStatus?: number; + /** A request that fails validation (400). Absent when no input can be invalid. */ + bad?: (f: Fixtures) => Case; + /** A request naming a missing resource (404). */ + missing?: (f: Fixtures) => Case; +} + +const V = "/api/v1"; +const doc = (f: Fixtures, rest = ""): string => `${V}/documents/${f.docA}${rest}`; + +/** One entry per route; the test below fails when a route has no entry. */ +const CASES: Record<string, RouteCases> = { + "GET /openapi.json": { ok: () => ({ path: "/openapi.json" }) }, + "GET /api/v1/health": { ok: () => ({ path: `${V}/health` }) }, + + "POST /api/v1/documents": { + ok: () => ({ path: `${V}/documents`, body: { title: "New", content: "New body text" } }), + okStatus: 202, + bad: () => ({ path: `${V}/documents`, body: { kind: "bogus" } }), + }, + "GET /api/v1/documents/:documentId": { + ok: (f) => ({ path: doc(f), query: { maxLength: "5" } }), + bad: (f) => ({ path: doc(f), query: { offset: "-1" } }), + missing: () => ({ path: `${V}/documents/nope` }), + }, + "GET /api/v1/documents": { + ok: () => ({ path: `${V}/documents`, query: { topic: "Guides", limit: "5" } }), + bad: () => ({ path: `${V}/documents`, query: { limit: "0" } }), + }, + "PATCH /api/v1/documents/:documentId": { + ok: (f) => ({ path: doc(f), body: { title: "Renamed", tags: ["a", "b"] } }), + bad: (f) => ({ path: doc(f), body: {} }), + missing: () => ({ path: `${V}/documents/nope`, body: { title: "x" } }), + }, + "DELETE /api/v1/documents/:documentId": { + ok: (f) => ({ path: doc(f) }), + missing: () => ({ path: `${V}/documents/nope` }), + }, + "POST /api/v1/documents/:documentId/ratings": { + ok: (f) => ({ path: doc(f, "/ratings"), body: { rating: 5 } }), + bad: (f) => ({ path: doc(f, "/ratings"), body: { rating: 9 } }), + missing: () => ({ path: `${V}/documents/nope/ratings`, body: { rating: 3 } }), + }, + "GET /api/v1/documents/:documentId/versions": { + ok: (f) => ({ path: doc(f, "/versions") }), + missing: () => ({ path: `${V}/documents/nope/versions` }), + }, + "POST /api/v1/documents/:documentId/rollback": { + ok: (f) => ({ path: doc(f, "/rollback"), body: { version: 1 } }), + bad: (f) => ({ path: doc(f, "/rollback"), body: { version: 0 } }), + missing: () => ({ path: `${V}/documents/nope/rollback`, body: { version: 1 } }), + }, + + "GET /api/v1/search": { + ok: () => ({ path: `${V}/search`, query: { query: "typescript", tags: "intro,guide" } }), + bad: () => ({ path: `${V}/search` }), + }, + "POST /api/v1/ask": { + ok: () => ({ path: `${V}/ask`, body: { question: "What is TypeScript?" } }), + bad: () => ({ path: `${V}/ask`, body: {} }), + }, + + "POST /api/v1/documents/:documentId/links": { + ok: (f) => ({ + path: doc(f, "/links"), + body: { targetDocumentId: f.docB, linkType: "references" }, + }), + bad: (f) => ({ path: doc(f, "/links"), body: { targetDocumentId: f.docB, linkType: "x" } }), + missing: (f) => ({ + path: `${V}/documents/nope/links`, + body: { targetDocumentId: f.docB, linkType: "related" }, + }), + }, + "DELETE /api/v1/links/:linkId": { + ok: (f) => ({ path: `${V}/links/${f.linkId}` }), + missing: () => ({ path: `${V}/links/nope` }), + }, + "GET /api/v1/links": { + ok: (f) => ({ path: `${V}/links`, query: { documentId: f.docA } }), + bad: () => ({ path: `${V}/links`, query: { linkType: "bogus" } }), + missing: () => ({ path: `${V}/links`, query: { documentId: "nope" } }), + }, + "GET /api/v1/documents/:documentId/prerequisites": { + ok: (f) => ({ path: doc(f, "/prerequisites") }), + missing: () => ({ path: `${V}/documents/nope/prerequisites` }), + }, + "GET /api/v1/graph": { + ok: () => ({ path: `${V}/graph`, query: { threshold: "0.5", topic: "Guides" } }), + bad: () => ({ path: `${V}/graph`, query: { threshold: "high" } }), + }, + + "POST /api/v1/documents/:documentId/tags": { + ok: (f) => ({ path: doc(f, "/tags"), body: { tags: ["new"] } }), + bad: (f) => ({ path: doc(f, "/tags"), body: { tags: [] } }), + missing: () => ({ path: `${V}/documents/nope/tags`, body: { tags: ["x"] } }), + }, + "DELETE /api/v1/documents/:documentId/tags": { + ok: (f) => ({ path: doc(f, "/tags"), query: { tags: "intro" } }), + bad: (f) => ({ path: doc(f, "/tags") }), + missing: () => ({ path: `${V}/documents/nope/tags`, query: { tags: "intro" } }), + }, + "GET /api/v1/tags": { ok: () => ({ path: `${V}/tags` }) }, + "GET /api/v1/documents/:documentId/suggested-tags": { + ok: (f) => ({ path: doc(f, "/suggested-tags") }), + bad: (f) => ({ path: doc(f, "/suggested-tags"), query: { limit: "0" } }), + missing: () => ({ path: `${V}/documents/nope/suggested-tags` }), + }, + + "GET /api/v1/topics": { + ok: () => ({ path: `${V}/topics` }), + missing: () => ({ path: `${V}/topics`, query: { parent: "nope" } }), + }, + "POST /api/v1/topics": { + ok: () => ({ path: `${V}/topics`, body: { name: "Recipes" } }), + bad: () => ({ path: `${V}/topics`, body: {} }), + }, + "DELETE /api/v1/topics/:topic": { + ok: () => ({ path: `${V}/topics/Guides` }), + bad: () => ({ path: `${V}/topics/Guides`, query: { deleteDocuments: "maybe" } }), + missing: () => ({ path: `${V}/topics/nope` }), + }, + + "POST /api/v1/searches": { + ok: () => ({ path: `${V}/searches`, body: { name: "ts", query: "typescript" } }), + bad: () => ({ path: `${V}/searches`, body: { name: "no query" } }), + }, + "GET /api/v1/searches": { + ok: () => ({ path: `${V}/searches` }), + bad: () => ({ path: `${V}/searches`, query: { offset: "-1" } }), + }, + "POST /api/v1/searches/:search/run": { + ok: () => ({ path: `${V}/searches/saved/run` }), + missing: () => ({ path: `${V}/searches/nope/run` }), + }, + "DELETE /api/v1/searches/:search": { + ok: () => ({ path: `${V}/searches/saved` }), + missing: () => ({ path: `${V}/searches/nope` }), + }, + + "POST /api/v1/packs": { + ok: () => ({ path: `${V}/packs`, body: { pack: "some-pack" } }), + okStatus: 202, + bad: () => ({ path: `${V}/packs`, body: { pack: "p", concurrency: 99 } }), + }, + "DELETE /api/v1/packs/:pack": { + ok: () => ({ path: `${V}/packs/demo-pack` }), + missing: () => ({ path: `${V}/packs/nope` }), + }, + "GET /api/v1/packs": { + ok: () => ({ path: `${V}/packs` }), + bad: () => ({ path: `${V}/packs`, query: { available: "maybe" } }), + }, + + "GET /api/v1/connections": { ok: () => ({ path: `${V}/connections` }) }, + "POST /api/v1/sync": { + ok: () => ({ path: `${V}/sync`, body: { all: true } }), + okStatus: 202, + bad: () => ({ path: `${V}/sync`, body: { name: "bad name!" } }), + }, + "DELETE /api/v1/connections/:name": { + ok: () => ({ path: `${V}/connections/notes` }), + // An unknown name with no saved connection is a 400 (unknown connector type), not a 404. + bad: () => ({ path: `${V}/connections/nope` }), + }, + + "GET /api/v1/overview": { ok: () => ({ path: `${V}/overview` }) }, + "POST /api/v1/admin/reindex": { + ok: () => ({ path: `${V}/admin/reindex`, body: {} }), + okStatus: 202, + bad: () => ({ path: `${V}/admin/reindex`, body: { batchSize: 0 } }), + }, + "GET /api/v1/admin/duplicates": { + ok: () => ({ path: `${V}/admin/duplicates`, query: { strategy: "exact" } }), + okStatus: 202, + bad: () => ({ path: `${V}/admin/duplicates`, query: { threshold: "5" } }), + }, + "POST /api/v1/admin/prune-expired": { ok: () => ({ path: `${V}/admin/prune-expired` }) }, + "POST /api/v1/bulk/delete": { + ok: () => ({ path: `${V}/bulk/delete`, body: { topic: "Guides", dryRun: true } }), + bad: () => ({ path: `${V}/bulk/delete`, body: { topic: "Guides", dryRun: "yes" } }), + missing: () => ({ path: `${V}/bulk/delete`, body: { topic: "nope" } }), + }, + "POST /api/v1/bulk/retag": { + ok: () => ({ path: `${V}/bulk/retag`, body: { topic: "Guides", addTags: ["x"] } }), + bad: () => ({ path: `${V}/bulk/retag`, body: { topic: "Guides", addTags: "x" } }), + missing: () => ({ path: `${V}/bulk/retag`, body: { topic: "nope", addTags: ["x"] } }), + }, + "POST /api/v1/bulk/move": { + ok: () => ({ path: `${V}/bulk/move`, body: { library: "ts", targetTopic: "Guides" } }), + bad: () => ({ path: `${V}/bulk/move`, body: { library: "ts" } }), + missing: () => ({ path: `${V}/bulk/move`, body: { library: "ts", targetTopic: "nope" } }), + }, + + "GET /api/v1/analytics/popular": { + ok: () => ({ path: `${V}/analytics/popular` }), + bad: () => ({ path: `${V}/analytics/popular`, query: { limit: "many" } }), + }, + "GET /api/v1/analytics/stale": { + ok: () => ({ path: `${V}/analytics/stale`, query: { days: "7" } }), + bad: () => ({ path: `${V}/analytics/stale`, query: { days: "0" } }), + }, + "GET /api/v1/analytics/top-queries": { + ok: () => ({ path: `${V}/analytics/top-queries` }), + bad: () => ({ path: `${V}/analytics/top-queries`, query: { limit: "1000" } }), + }, + "GET /api/v1/analytics/searches": { + ok: () => ({ path: `${V}/analytics/searches` }), + bad: () => ({ path: `${V}/analytics/searches`, query: { days: "-3" } }), + }, + + "POST /api/v1/webhooks": { + ok: () => ({ + path: `${V}/webhooks`, + body: { url: "https://hooks.example.com/x", events: ["document.created"] }, + }), + bad: () => ({ path: `${V}/webhooks`, body: { url: "https://hooks.example.com/x" } }), + }, + "GET /api/v1/webhooks": { + ok: () => ({ path: `${V}/webhooks` }), + bad: () => ({ path: `${V}/webhooks`, query: { limit: "0" } }), + }, + "DELETE /api/v1/webhooks/:webhookId": { + ok: (f) => ({ path: `${V}/webhooks/${f.webhookId}` }), + missing: () => ({ path: `${V}/webhooks/nope` }), + }, + "POST /api/v1/webhooks/:webhookId/test": { + ok: (f) => ({ path: `${V}/webhooks/${f.webhookId}/test` }), + missing: () => ({ path: `${V}/webhooks/nope/test` }), + }, + + "GET /api/v1/tasks/:taskId": { + ok: (f) => ({ path: `${V}/tasks/${f.taskId}` }), + missing: () => ({ path: `${V}/tasks/nope` }), + }, + "POST /api/v1/tasks/:taskId/cancel": { + ok: (f) => ({ path: `${V}/tasks/${f.taskId}/cancel` }), + missing: () => ({ path: `${V}/tasks/nope/cancel` }), + }, + "GET /api/v1/tasks": { ok: () => ({ path: `${V}/tasks` }) }, +}; + +interface Reply { + status: number; + headers: Record<string, string | string[] | undefined>; + body: { data?: unknown; error?: { code: string; message: string }; [key: string]: unknown }; +} + +let server: Server; +let port: number; +let db: Database.Database; +let fixtures: Fixtures; + +/** Send a request with node:http (global fetch is stubbed by the webhook test). */ +function send(method: string, c: Case, headers: Record<string, string> = {}): Promise<Reply> { + const qs = c.query ? `?${new URLSearchParams(c.query).toString()}` : ""; + const payload = c.body === undefined ? undefined : JSON.stringify(c.body); + return new Promise((resolve, reject) => { + const req = request( + { + host: "127.0.0.1", + port, + method, + path: c.path + qs, + headers: { ...(payload ? { "Content-Type": "application/json" } : {}), ...headers }, + }, + (res) => { + let raw = ""; + res.setEncoding("utf8"); + res.on("data", (chunk: string) => (raw += chunk)); + res.on("end", () => + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: raw ? (JSON.parse(raw) as Reply["body"]) : {}, + }), + ); + }, + ); + req.on("error", reject); + req.end(payload); + }); +} + +const fakeLlm = { + model: "fake", + complete: (): Promise<{ text: string }> => Promise.resolve({ text: "An answer." }), +}; + +async function seed(): Promise<Fixtures> { + const provider = new MockEmbeddingProvider(); + const topic = createTopic(db, { name: "Guides" }); + const a = await indexDocument(db, provider, { + title: "TypeScript intro", + content: "TypeScript is a typed superset of JavaScript.", + sourceType: "manual", + topicId: topic.id, + }); + const b = await indexDocument(db, provider, { + title: "Node guide", + content: "Node.js runs JavaScript on the server.", + sourceType: "library", + library: "ts", + }); + db.prepare("INSERT INTO tags (id, name) VALUES ('t1', 'intro')").run(); + db.prepare("INSERT INTO document_tags (document_id, tag_id) VALUES (?, 't1')").run(a.id); + await updateDocument(db, provider, a.id, { content: "TypeScript adds static types." }); + const link = createLink(db, a.id, b.id, "related"); + createSavedSearch(db, "saved", "typescript"); + const hook = await createWebhook(db, "https://hooks.example.com/libscope", ["document.created"]); + db.prepare("INSERT INTO packs (name, version) VALUES ('demo-pack', '1.0.0')").run(); + saveConnectorSettings("obsidian", "notes", { + vaultPath: join(tempHome, "vault"), + topicMapping: "folder", + excludePatterns: [], + }); + const { task } = taskRegistry.create("operation", "test"); + return { docA: a.id, docB: b.id, linkId: link.id, webhookId: hook.id, taskId: task.id }; +} + +describe("REST routes generated from operations", () => { + let ctxRef: ReturnType<typeof createOperationContext>; + + beforeAll(async () => { + server = createServer((req, res) => void handleRequest(req, res, ctxRef)); + await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve)); + port = (server.address() as AddressInfo).port; + }); + + afterAll(async () => { + await new Promise<void>((resolve) => server.close(() => resolve())); + }); + + beforeEach(async () => { + tempHome = join(tmpdir(), `libscope-api-routes-${randomUUID()}`); + mkdirSync(join(tempHome, "vault"), { recursive: true }); + db = createTestDbWithVec(); + ctxRef = createOperationContext({ + db, + provider: new MockEmbeddingProvider(), + config: testConfig(), + surface: "api", + llm: fakeLlm, + }); + fixtures = await seed(); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(new Response(null, { status: 204, statusText: "No Content" })), + ); + }); + + afterEach(async () => { + vi.unstubAllGlobals(); + // Let background tasks started by 202 routes settle before the database closes. + await new Promise((resolve) => setTimeout(resolve, 20)); + db.close(); + rmSync(tempHome, { recursive: true, force: true }); + }); + + it("has a contract case for every route, and every case names a route", () => { + const routes = API_ROUTES.map((r) => `${r.method} ${r.path}`).sort(); + expect(Object.keys(CASES).sort()).toEqual(routes); + }); + + for (const [key, cases] of Object.entries(CASES)) { + const [method = "GET"] = key.split(" "); + + it(`${key} succeeds`, async () => { + const reply = await send(method, cases.ok(fixtures)); + expect(reply.body.error, JSON.stringify(reply.body.error)).toBeUndefined(); + expect(reply.status).toBe(cases.okStatus ?? 200); + }); + + const { bad, missing } = cases; + if (bad) { + it(`${key} answers 400 for invalid input`, async () => { + const reply = await send(method, bad(fixtures)); + expect(reply.status, JSON.stringify(reply.body)).toBe(400); + expect(reply.body.error?.code).toMatch(/VALIDATION_ERROR|INVALID_JSON/); + }); + } + if (missing) { + it(`${key} answers 404 for a missing resource`, async () => { + const reply = await send(method, missing(fixtures)); + expect(reply.status, JSON.stringify(reply.body)).toBe(404); + expect(reply.body.error?.code).toMatch(/NOT_FOUND$/); + }); + } + } + + it("wraps results as { data, meta } with ids in the result", async () => { + const reply = await send("GET", { path: `${V}/documents/${fixtures.docA}` }); + expect(reply.body).toMatchObject({ + data: { document: { documentId: fixtures.docA, title: "TypeScript intro" } }, + meta: { took: expect.any(Number) as number }, + }); + }); + + it("runs a long-running operation as a task and reports its result", async () => { + const started = await send("POST", { + path: `${V}/documents`, + body: { title: "Queued", content: "Queued document body" }, + }); + expect(started.status).toBe(202); + const data = started.body.data as { taskId: string; operation: string }; + expect(data.operation).toBe("add"); + expect(started.headers["location"]).toBe(`${V}/tasks/${data.taskId}`); + + let task: { status: string; result?: string } = { status: "running" }; + await vi.waitFor(async () => { + const polled = await send("GET", { path: `${V}/tasks/${data.taskId}` }); + task = polled.body.data as typeof task; + expect(task.status).toBe("completed"); + }); + const result = JSON.parse(task.result ?? "null") as { documents: Array<{ title: string }> }; + expect(JSON.stringify(result)).toContain("Queued"); + }); + + it("coerces query values by the operation schema (numbers, booleans, arrays)", async () => { + const reply = await send("GET", { + path: `${V}/documents`, + query: { limit: "1", offset: "1" }, + }); + expect(reply.body.data).toMatchObject({ limit: 1, offset: 1, total: 2 }); + const tagged = await send("GET", { path: `${V}/documents`, query: { tags: "intro" } }); + expect((tagged.body.data as { total: number }).total).toBe(1); + }); + + it("refuses local files over REST (the add task fails)", async () => { + const started = await send("POST", { path: `${V}/documents`, body: { source: "/etc/hosts" } }); + const { taskId } = started.body.data as { taskId: string }; + await vi.waitFor(async () => { + const polled = await send("GET", { path: `${V}/tasks/${taskId}` }); + expect(polled.body.data).toMatchObject({ + status: "failed", + error: expect.stringContaining("only available from the CLI") as string, + }); + }); + }); + + it("answers 400 for a malformed JSON body and a non-object body", async () => { + const bad = await new Promise<Reply>((resolve, reject) => { + const req = request( + { host: "127.0.0.1", port, method: "POST", path: `${V}/topics` }, + (res) => { + let raw = ""; + res.on("data", (c: Buffer) => (raw += c.toString())); + res.on("end", () => + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: JSON.parse(raw) as Reply["body"], + }), + ); + }, + ); + req.on("error", reject); + req.end("{not json"); + }); + expect(bad.status).toBe(400); + expect(bad.body.error?.code).toBe("INVALID_JSON"); + const array = await send("POST", { path: `${V}/topics`, body: ["x"] }); + expect(array.status).toBe(400); + }); + + it("answers 404 for unknown routes and wrong methods", async () => { + expect((await send("GET", { path: `${V}/nope` })).status).toBe(404); + expect((await send("PUT", { path: `${V}/documents` })).status).toBe(404); + expect((await send("GET", { path: `${V}/documents/url/extra/more` })).status).toBe(404); + }); + + it("answers 400 for a malformed path escape", async () => { + const reply = await send("GET", { path: `${V}/documents/%E0%A4%A` }); + expect(reply.status).toBe(400); + }); + + it("answers 500 with the config hint when ask has no LLM", async () => { + ctxRef = createOperationContext({ + db, + provider: new MockEmbeddingProvider(), + config: testConfig(), + surface: "api", + llm: null, + }); + const reply = await send("POST", { path: `${V}/ask`, body: { question: "Why?" } }); + expect(reply.status).toBe(500); + expect(reply.body.error?.code).toBe("CONFIG_ERROR"); + }); +}); diff --git a/tests/unit/api-server.test.ts b/tests/unit/api-server.test.ts index 8ec7905b..e62579ff 100644 --- a/tests/unit/api-server.test.ts +++ b/tests/unit/api-server.test.ts @@ -3,6 +3,7 @@ import { createTestDbWithVec } from "../fixtures/test-db.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; import { initLogger } from "../../src/logger.js"; import type Database from "better-sqlite3"; +import { testConfig } from "./operations/helpers.js"; // Mock the scheduler to avoid real cron vi.mock("../../src/core/scheduler.js", () => ({ @@ -29,24 +30,30 @@ describe("startApiServer", () => { it("starts on port 0 and returns a close function", async () => { const provider = new MockEmbeddingProvider(); - const result = await startApiServer(db, provider, { - port: 0, - host: "127.0.0.1", - enableScheduler: false, - }); + const result = await startApiServer( + { db, provider, config: testConfig() }, + { + port: 0, + host: "127.0.0.1", + enableScheduler: false, + }, + ); - expect(result.port).toBe(0); + expect(result.port).toBeGreaterThan(0); expect(typeof result.close).toBe("function"); await result.close(); }); it("starts with scheduler enabled but no entries", async () => { const provider = new MockEmbeddingProvider(); - const result = await startApiServer(db, provider, { - port: 0, - host: "127.0.0.1", - enableScheduler: true, - }); + const result = await startApiServer( + { db, provider, config: testConfig() }, + { + port: 0, + host: "127.0.0.1", + enableScheduler: true, + }, + ); expect(typeof result.close).toBe("function"); expect(result.scheduler).toBeUndefined(); @@ -55,12 +62,45 @@ describe("startApiServer", () => { it("starts with default options (scheduler enabled by default)", async () => { const provider = new MockEmbeddingProvider(); - const result = await startApiServer(db, provider, { - port: 0, - host: "127.0.0.1", - }); + const result = await startApiServer( + { db, provider, config: testConfig() }, + { + port: 0, + host: "127.0.0.1", + }, + ); expect(typeof result.close).toBe("function"); await result.close(); }); + + it("serves generated routes behind CORS, origin and API key checks", async () => { + const provider = new MockEmbeddingProvider(); + const server = await startApiServer( + { db, provider, config: testConfig() }, + { port: 0, host: "127.0.0.1", enableScheduler: false, corsOrigins: ["http://app.example"] }, + ); + const base = `http://127.0.0.1:${server.port}/api/v1`; + try { + const health = await fetch(`${base}/health`, { headers: { Origin: "http://app.example" } }); + expect(health.status).toBe(200); + expect(health.headers.get("access-control-allow-origin")).toBe("http://app.example"); + expect(health.headers.get("content-security-policy")).toContain("default-src 'self'"); + + const foreign = await fetch(`${base}/topics`, { + method: "POST", + headers: { Origin: "http://evil.example", "Content-Type": "text/plain" }, + body: JSON.stringify({ name: "x" }), + }); + expect(foreign.status).toBe(403); + + process.env["LIBSCOPE_API_KEY"] = "k"; + expect((await fetch(`${base}/overview`)).status).toBe(401); + const authed = await fetch(`${base}/overview`, { headers: { Authorization: "Bearer k" } }); + expect(authed.status).toBe(200); + } finally { + delete process.env["LIBSCOPE_API_KEY"]; + await server.close(); + } + }); }); diff --git a/tests/unit/api.test.ts b/tests/unit/api.test.ts index 28ba6a65..6386c097 100644 --- a/tests/unit/api.test.ts +++ b/tests/unit/api.test.ts @@ -1,12 +1,9 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { describe, it, expect, afterEach } from "vitest"; import { IncomingMessage, ServerResponse } from "node:http"; import { Socket } from "node:net"; -import type Database from "better-sqlite3"; -import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; -import { createTestDbWithVec } from "../fixtures/test-db.js"; -import { handleRequest } from "../../src/api/routes.js"; import { corsMiddleware, + rejectForeignWrite, parseJsonBody, sendJson, sendError, @@ -14,18 +11,10 @@ import { checkApiKey, getRateLimitMapSize, MAX_RATE_LIMIT_ENTRIES, + setSecurityHeaders, } from "../../src/api/middleware.js"; -import { OPENAPI_SPEC } from "../../src/api/openapi.js"; -import { indexDocument } from "../../src/core/indexing.js"; -import { createTopic } from "../../src/core/topics.js"; - -vi.mock("../../src/core/scheduler.js", async (importOriginal) => { - const orig = await importOriginal<typeof import("../../src/core/scheduler.js")>(); - return { - ...orig, - loadScheduleEntries: vi.fn(() => []), - }; -}); +import { API_ROUTES, OPENAPI_PATH } from "../../src/api/routes.js"; +import { buildOpenApiSpec, toOpenApiPath } from "../../src/api/openapi.js"; interface ApiResponse { data?: Record<string, unknown>; @@ -124,38 +113,62 @@ function createMockRes(): MockRes { describe("API middleware", () => { describe("corsMiddleware", () => { - it("should handle OPTIONS preflight", () => { - const req = createMockReq("OPTIONS", "/api/v1/health"); + it("answers OPTIONS preflight for a listed origin with every method", () => { + const req = createMockReq("OPTIONS", "/api/v1/documents"); + req.headers["origin"] = "http://localhost:3000"; const { res, getStatus, getHeaders } = createMockRes(); - const handled = corsMiddleware(req, res, ["*"]); + const handled = corsMiddleware(req, res, ["http://localhost:3000"]); expect(handled).toBe(true); expect(getStatus()).toBe(204); - expect(getHeaders()["Access-Control-Allow-Origin"]).toBe("*"); + expect(getHeaders()["Access-Control-Allow-Origin"]).toBe("http://localhost:3000"); expect(getHeaders()["Access-Control-Allow-Methods"]).toBe( - "GET, POST, PATCH, DELETE, OPTIONS", + "GET, POST, PATCH, PUT, DELETE, OPTIONS", ); + expect(getHeaders()["Vary"]).toBe("Origin"); }); - it("should set CORS headers for non-preflight requests", () => { - const req = createMockReq("GET", "/api/v1/health"); + it("lets a wildcard origin read but not write", () => { + const req = createMockReq("OPTIONS", "/api/v1/documents"); + req.headers["origin"] = "http://evil.example"; const { res, getHeaders } = createMockRes(); - const handled = corsMiddleware(req, res, ["*"]); + corsMiddleware(req, res, ["*"]); - expect(handled).toBe(false); expect(getHeaders()["Access-Control-Allow-Origin"]).toBe("*"); + expect(getHeaders()["Access-Control-Allow-Methods"]).toBe("GET, OPTIONS"); }); - it("should restrict origin when not wildcard", () => { + it("sets no allow-origin for an unlisted origin", () => { const req = createMockReq("GET", "/api/v1/health"); - req.headers["origin"] = "http://example.com"; + req.headers["origin"] = "http://evil.example"; const { res, getHeaders } = createMockRes(); - corsMiddleware(req, res, ["http://example.com"]); + expect(corsMiddleware(req, res, ["http://localhost"])).toBe(false); + expect(getHeaders()["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + }); + + describe("rejectForeignWrite", () => { + function check(method: string, origin: string | undefined, origins: string[]): number | null { + const req = createMockReq(method, "/api/v1/documents/x"); + if (origin !== undefined) req.headers["origin"] = origin; + const { res, getStatus } = createMockRes(); + return rejectForeignWrite(req, res, origins) ? getStatus() : null; + } - expect(getHeaders()["Access-Control-Allow-Origin"]).toBe("http://example.com"); + it('rejects writes from unlisted origins, also when "*" is allowed', () => { + expect(check("DELETE", "http://evil.example", [])).toBe(403); + expect(check("POST", "http://evil.example", ["*"])).toBe(403); + }); + + it("allows reads, same-origin writes, listed origins and non-browser clients", () => { + expect(check("GET", "http://evil.example", [])).toBeNull(); + expect(check("DELETE", "http://localhost:3378", [])).toBeNull(); + expect(check("POST", "http://app.example", ["http://app.example"])).toBeNull(); + expect(check("PATCH", undefined, [])).toBeNull(); + expect(check("POST", "null", [])).toBe(403); }); }); @@ -168,7 +181,7 @@ describe("API middleware", () => { it("should reject invalid JSON", async () => { const req = createMockReq("POST", "/test", "not json{{{"); - await expect(parseJsonBody(req)).rejects.toThrow("Invalid JSON body"); + await expect(parseJsonBody(req)).rejects.toThrow("Request body contains invalid JSON"); }); it("should return null for empty body", async () => { @@ -199,628 +212,56 @@ describe("API middleware", () => { }); }); -describe("API routes", () => { - let db: Database.Database; - let provider: MockEmbeddingProvider; - - beforeEach(() => { - db = createTestDbWithVec(); - provider = new MockEmbeddingProvider(); - }); - - describe("GET /api/v1/health", () => { - it("should return health status", async () => { - const req = createMockReq("GET", "/api/v1/health"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.data.status).toBe("ok"); - expect(typeof parsed.data.docCount).toBe("number"); - expect(typeof parsed.meta.took).toBe("number"); - }); - }); - - describe("GET /openapi.json", () => { - it("should return the OpenAPI spec", async () => { - const req = createMockReq("GET", "/openapi.json"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.openapi).toBe("3.0.3"); - expect(parsed.info.title).toBe("LibScope REST API"); - }); - }); - - describe("GET /api/v1/search", () => { - it("should return 400 without query param", async () => { - const req = createMockReq("GET", "/api/v1/search"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("VALIDATION_ERROR"); - }); - - it("should search with query params", async () => { - await indexDocument(db, provider, { - title: "Test Doc", - content: "Hello world content", - sourceType: "manual", - }); - - const req = createMockReq("GET", "/api/v1/search?q=hello&limit=5"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.data).toBeDefined(); - expect(parsed.meta.took).toBeDefined(); - }); - - it("should accept offset query parameter for pagination", async () => { - await indexDocument(db, provider, { - title: "Doc A", - content: "First document content", - sourceType: "manual", - }); - await indexDocument(db, provider, { - title: "Doc B", - content: "Second document content", - sourceType: "manual", - }); - - const req = createMockReq("GET", "/api/v1/search?q=document&limit=5&offset=1"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.data).toBeDefined(); - expect(parsed.meta.took).toBeDefined(); - }); - }); - - describe("POST /api/v1/documents", () => { - it("should index a new document", async () => { - const req = createMockReq("POST", "/api/v1/documents", { - title: "My Doc", - content: "Some content here", - }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(201); - const parsed = parseResponse(getBody()); - expect(parsed.data.id).toBeDefined(); - expect(parsed.data.chunkCount).toBeGreaterThanOrEqual(1); - }); - - it("should return 400 for missing fields", async () => { - const req = createMockReq("POST", "/api/v1/documents", { title: "No content" }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("VALIDATION_ERROR"); - }); - - it("should index document with tags", async () => { - const req = createMockReq("POST", "/api/v1/documents", { - title: "Tagged Doc", - content: "Content with tags", - tags: ["typescript", "api"], - }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(201); - const parsed = parseResponse(getBody()); - expect(parsed.data.id).toBeDefined(); - }); - - it("should return 400 for invalid JSON body", async () => { - const req = createMockReq("POST", "/api/v1/documents", "not-valid-json{{{"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("INVALID_JSON"); - }); - }); - - describe("GET /api/v1/documents/:id", () => { - it("should return a document by ID", async () => { - const doc = await indexDocument(db, provider, { - title: "Fetch Me", - content: "Document content", - sourceType: "manual", - }); - - const req = createMockReq("GET", `/api/v1/documents/${doc.id}`); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.data.id).toBe(doc.id); - expect(parsed.data.title).toBe("Fetch Me"); - }); - - it("should return 404 for non-existent document", async () => { - const req = createMockReq("GET", "/api/v1/documents/nonexistent-id"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(404); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("NOT_FOUND"); - }); - }); - - describe("DELETE /api/v1/documents/:id", () => { - it("should delete a document", async () => { - const doc = await indexDocument(db, provider, { - title: "Delete Me", - content: "To be deleted", - sourceType: "manual", - }); - - const req = createMockReq("DELETE", `/api/v1/documents/${doc.id}`); - const { res, getStatus } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(204); - }); - - it("should return 404 for deleting non-existent document", async () => { - const req = createMockReq("DELETE", "/api/v1/documents/nonexistent-id"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(404); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("NOT_FOUND"); - }); - }); - - describe("POST /api/v1/ask", () => { - it("should return 400 without question", async () => { - const req = createMockReq("POST", "/api/v1/ask", { notQuestion: "test" }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("VALIDATION_ERROR"); - }); - - it("should return SSE stream when Accept: text/event-stream", async () => { - // Mock the LLM and search modules to avoid real calls - const ragModule = await import("../../src/core/rag.js"); - const searchModule = await import("../../src/core/search.js"); - const configModule = await import("../../src/config.js"); - - vi.spyOn(configModule, "loadConfig").mockReturnValue({ - embedding: { provider: "local" }, - llm: { provider: "openai" }, - openai: { apiKey: "sk-test" }, - database: { path: ":memory:" }, - indexing: { maxDocumentSize: 1024 }, - logging: { level: "silent" }, - }); - - vi.spyOn(searchModule, "searchDocuments").mockResolvedValue({ - results: [], - totalCount: 0, - }); - - vi.spyOn(ragModule, "createLlmProvider").mockReturnValue({ - model: "mock-model", - complete: vi.fn().mockResolvedValue({ text: "SSE answer", tokensUsed: 10 }), - }); - - const req = createMockReq("POST", "/api/v1/ask", { question: "What is SSE?" }); - req.headers["accept"] = "text/event-stream"; - const { res, getStatus, getBody, getHeaders } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - expect(getHeaders()["Content-Type"]).toBe("text/event-stream"); - expect(getHeaders()["Cache-Control"]).toBe("no-cache"); - expect(getHeaders()["Connection"]).toBe("keep-alive"); - - const body = getBody(); - // Should contain at least one token event and a done event - expect(body).toContain("data: "); - expect(body).toContain('"token"'); - expect(body).toContain('"done":true'); - expect(body).toContain('"sources"'); - - // Parse individual SSE events - const events = body - .split("\n\n") - .filter((line: string) => line.startsWith("data: ")) - .map((line: string) => JSON.parse(line.replace("data: ", "")) as Record<string, unknown>); - - expect(events.length).toBeGreaterThanOrEqual(2); - expect(events[0]).toHaveProperty("token"); - expect(events[events.length - 1]).toMatchObject({ done: true }); - - vi.mocked(configModule.loadConfig).mockRestore(); - vi.mocked(searchModule.searchDocuments).mockRestore(); - vi.mocked(ragModule.createLlmProvider).mockRestore(); - }); - - it("should return normal JSON when Accept header is not SSE", async () => { - const req = createMockReq("POST", "/api/v1/ask", { question: "test" }); - req.headers["accept"] = "application/json"; - const { res, getHeaders } = createMockRes(); - - // This will fail because no LLM is configured, but it should NOT return SSE headers - await handleRequest(req, res, db, provider); - - expect(getHeaders()["Content-Type"]).not.toBe("text/event-stream"); - }); - }); - - describe("GET /api/v1/documents", () => { - it("should list documents", async () => { - await indexDocument(db, provider, { - title: "Doc 1", - content: "Content 1", - sourceType: "manual", - }); - - const req = createMockReq("GET", "/api/v1/documents"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(Array.isArray(parsed.data)).toBe(true); - expect(parsed.data.length).toBeGreaterThanOrEqual(1); - }); - }); - - describe("GET /api/v1/topics", () => { - it("should list topics", async () => { - createTopic(db, { name: "test-topic" }); - - const req = createMockReq("GET", "/api/v1/topics"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(Array.isArray(parsed.data)).toBe(true); - }); - }); - - describe("POST /api/v1/topics", () => { - it("should create a topic", async () => { - const req = createMockReq("POST", "/api/v1/topics", { name: "new-topic" }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(201); - const parsed = parseResponse(getBody()); - expect(parsed.data.name).toBe("new-topic"); - }); - - it("should return 400 without name", async () => { - const req = createMockReq("POST", "/api/v1/topics", { description: "no name" }); - const { res, getStatus } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - }); - }); - - describe("GET /api/v1/tags", () => { - it("should list tags", async () => { - const req = createMockReq("GET", "/api/v1/tags"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(Array.isArray(parsed.data)).toBe(true); - }); - }); - - describe("POST /api/v1/documents/:id/tags", () => { - it("should add tags to a document", async () => { - const doc = await indexDocument(db, provider, { - title: "Tag Target", - content: "Content", - sourceType: "manual", - }); - - const req = createMockReq("POST", `/api/v1/documents/${doc.id}/tags`, { - tags: ["alpha", "beta"], - }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(Array.isArray(parsed.data)).toBe(true); - }); - }); - - describe("GET /api/v1/stats", () => { - it("should return stats", async () => { - const req = createMockReq("GET", "/api/v1/stats"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(typeof parsed.data.totalDocuments).toBe("number"); - }); - - it("should return databaseSizeBytes field", async () => { - const req = createMockReq("GET", "/api/v1/stats"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(typeof parsed.data.databaseSizeBytes).toBe("number"); - }); - }); - - describe("Connector status endpoint", () => { - it("GET /api/v1/connectors/status should return empty array when no syncs", async () => { - const req = createMockReq("GET", "/api/v1/connectors/status"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.data).toEqual([]); - }); - - it("GET /api/v1/connectors/status should return latest sync status", async () => { - // Insert a sync record directly - db.prepare( - `INSERT INTO connector_syncs (connector_type, connector_name, started_at, completed_at, status, docs_added) - VALUES ('obsidian', 'vault1', datetime('now'), datetime('now'), 'completed', 5)`, - ).run(); - - const req = createMockReq("GET", "/api/v1/connectors/status"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - const data = parsed.data as unknown as Array<Record<string, unknown>>; - expect(data).toHaveLength(1); - expect(data[0].connector_type).toBe("obsidian"); - expect(data[0].docs_added).toBe(5); - }); - - it("GET /api/v1/connectors/status?history=true should return full history", async () => { - db.prepare( - `INSERT INTO connector_syncs (connector_type, connector_name, started_at, status) - VALUES ('obsidian', 'vault1', datetime('now'), 'running')`, - ).run(); - db.prepare( - `INSERT INTO connector_syncs (connector_type, connector_name, started_at, completed_at, status) - VALUES ('obsidian', 'vault1', datetime('now'), datetime('now'), 'completed')`, - ).run(); - - const req = createMockReq("GET", "/api/v1/connectors/status?history=true"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - const data = parsed.data as unknown as Array<Record<string, unknown>>; - expect(data).toHaveLength(2); - }); - - it("GET /api/v1/connectors/status?type=notion should filter by type", async () => { - db.prepare( - `INSERT INTO connector_syncs (connector_type, connector_name, started_at, status) - VALUES ('obsidian', 'vault1', datetime('now'), 'completed')`, - ).run(); - db.prepare( - `INSERT INTO connector_syncs (connector_type, connector_name, started_at, status) - VALUES ('notion', 'notion', datetime('now'), 'completed')`, - ).run(); - - const req = createMockReq("GET", "/api/v1/connectors/status?type=notion"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - const data = parsed.data as unknown as Array<Record<string, unknown>>; - expect(data).toHaveLength(1); - expect(data[0].connector_type).toBe("notion"); - }); - }); - - describe("Connector schedules", () => { - it("GET /api/v1/connectors/schedules should return schedule entries", async () => { - const { loadScheduleEntries } = await import("../../src/core/scheduler.js"); - const mockLoad = vi.mocked(loadScheduleEntries); - mockLoad.mockReturnValueOnce([ - { connectorType: "notion", connectorName: "notion", cronExpression: "0 */6 * * *" }, - ]); - - const req = createMockReq("GET", "/api/v1/connectors/schedules"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - const schedules = (parsed.data as unknown as { schedules: unknown[] }).schedules; - expect(schedules).toHaveLength(1); - }); - - it("GET /api/v1/connectors/schedules should return empty array when none configured", async () => { - const { loadScheduleEntries } = await import("../../src/core/scheduler.js"); - const mockLoad = vi.mocked(loadScheduleEntries); - mockLoad.mockReturnValueOnce([]); - - const req = createMockReq("GET", "/api/v1/connectors/schedules"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - const schedules = (parsed.data as unknown as { schedules: unknown[] }).schedules; - expect(schedules).toHaveLength(0); - }); - }); - - describe("Unknown route", () => { - it("should return 404", async () => { - const req = createMockReq("GET", "/api/v1/nonexistent"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(404); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("NOT_FOUND"); - }); - }); - - describe("Webhooks API", () => { - it("should create a webhook via POST /api/v1/webhooks", async () => { - process.env.LIBSCOPE_SECRET_KEY = "test-key"; - try { - const req = createMockReq("POST", "/api/v1/webhooks", { - url: "https://example.com/hook", - events: ["document.created"], - secret: "my-secret", - }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(201); - const parsed = parseResponse(getBody()); - expect(parsed.data.url).toBe("https://example.com/hook"); - expect(parsed.data.hasSecret).toBe(true); - expect(parsed.data.secret).toBeUndefined(); - } finally { - delete process.env.LIBSCOPE_SECRET_KEY; - } - }); - - it("should list webhooks via GET /api/v1/webhooks", async () => { - // Create one first - const createReq = createMockReq("POST", "/api/v1/webhooks", { - url: "https://example.com/hook", - events: ["document.created"], - }); - const { res: createRes } = createMockRes(); - await handleRequest(createReq, createRes, db, provider); - - const req = createMockReq("GET", "/api/v1/webhooks"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(Array.isArray(parsed.data)).toBe(true); - expect(parsed.data.length).toBeGreaterThanOrEqual(1); - }); - - it("should return 400 when creating webhook without url", async () => { - const req = createMockReq("POST", "/api/v1/webhooks", { - events: ["document.created"], - }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("VALIDATION_ERROR"); - }); - - it("should delete a webhook via DELETE /api/v1/webhooks/:id", async () => { - const createReq = createMockReq("POST", "/api/v1/webhooks", { - url: "https://example.com/hook", - events: ["document.created"], - }); - const { res: createRes, getBody: getCreateBody } = createMockRes(); - await handleRequest(createReq, createRes, db, provider); - const created = parseResponse(getCreateBody()); - const id = (created.data as Record<string, unknown>).id as string; - - const req = createMockReq("DELETE", `/api/v1/webhooks/${id}`); - const { res, getStatus } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(204); - }); - }); -}); - describe("OpenAPI spec", () => { - it("should have valid structure", () => { - expect(OPENAPI_SPEC.openapi).toBe("3.0.3"); - expect(OPENAPI_SPEC.info.title).toBeDefined(); - expect(OPENAPI_SPEC.paths).toBeDefined(); - expect(OPENAPI_SPEC.components).toBeDefined(); - }); - - it("should define all endpoints", () => { - const paths = Object.keys(OPENAPI_SPEC.paths); - expect(paths).toContain("/api/v1/search"); - expect(paths).toContain("/api/v1/documents"); - expect(paths).toContain("/api/v1/documents/{id}"); - expect(paths).toContain("/api/v1/ask"); - expect(paths).toContain("/api/v1/topics"); - expect(paths).toContain("/api/v1/tags"); - expect(paths).toContain("/api/v1/stats"); - expect(paths).toContain("/api/v1/health"); - expect(paths).toContain("/openapi.json"); + const spec = buildOpenApiSpec(API_ROUTES) as { + openapi: string; + info: { title: string; version: string }; + paths: Record<string, Record<string, Record<string, unknown>>>; + }; + const specRoutes = Object.entries(spec.paths).flatMap(([path, methods]) => + Object.keys(methods).map((method) => `${method.toUpperCase()} ${path}`), + ); + + it("is OpenAPI 3.1 with the package version", () => { + expect(spec.openapi).toBe("3.1.0"); + expect(spec.info.version).toMatch(/^\d+\.\d+\.\d+/); + }); + + it("documents every route in the router, and nothing else", () => { + const routerRoutes = API_ROUTES.map((r) => `${r.method} ${toOpenApiPath(r.path)}`); + expect([...specRoutes].sort()).toEqual([...routerRoutes].sort()); + expect(specRoutes).toContain(`GET ${OPENAPI_PATH}`); + }); + + it("takes parameters and summaries from the operation schemas", () => { + const search = spec.paths["/api/v1/search"]?.["get"] as { + operationId: string; + summary: string; + parameters: Array<{ name: string; in: string; schema: { type?: string } }>; + }; + expect(search.operationId).toBe("search"); + const limit = search.parameters.find((p) => p.name === "limit"); + expect(limit).toMatchObject({ in: "query", schema: { type: "integer", default: 10 } }); + const getDoc = spec.paths["/api/v1/documents/{documentId}"]?.["get"] as { + parameters: Array<{ name: string; in: string; required: boolean }>; + }; + expect(getDoc.parameters).toContainEqual( + expect.objectContaining({ name: "documentId", in: "path", required: true }), + ); + const add = spec.paths["/api/v1/documents"]?.["post"] as { + requestBody: { content: Record<string, { schema: { properties: object } }> }; + responses: Record<string, unknown>; + }; + expect(add.requestBody.content["application/json"]?.schema.properties).toHaveProperty("url"); + expect(add.responses).toHaveProperty("202"); + const link = spec.paths["/api/v1/documents/{documentId}/links"]?.["post"] as { + requestBody: { + content: Record<string, { schema: { properties: object; required: string[] } }>; + }; + }; + const body = link.requestBody.content["application/json"]?.schema; + expect(body?.properties).not.toHaveProperty("documentId"); + expect(body?.required).toEqual(["targetDocumentId", "linkType"]); }); }); @@ -844,17 +285,20 @@ describe("middleware — security", () => { expect(result).toEqual(data); }); - it("should set security headers via CORS middleware", () => { + it("should set security headers", () => { const socket = new Socket(); const req = new IncomingMessage(socket); req.method = "GET"; const res = new ServerResponse(req); - corsMiddleware(req, res, ["*"]); + setSecurityHeaders(res); expect(res.getHeader("X-Content-Type-Options")).toBe("nosniff"); expect(res.getHeader("X-Frame-Options")).toBe("DENY"); expect(res.getHeader("X-XSS-Protection")).toBe("1; mode=block"); expect(res.getHeader("Referrer-Policy")).toBe("strict-origin-when-cross-origin"); expect(res.getHeader("Content-Security-Policy")).toBeDefined(); + const noCsp = new ServerResponse(req); + setSecurityHeaders(noCsp, false); + expect(noCsp.getHeader("Content-Security-Policy")).toBeUndefined(); }); }); @@ -956,220 +400,3 @@ describe("middleware — rate limiting", () => { // Entry exists; size is at least 1 }); }); - -describe("Saved Searches API", () => { - let db: Database.Database; - const provider = new MockEmbeddingProvider(); - - beforeEach(async () => { - db = createTestDbWithVec(); - await indexDocument(db, provider, { - title: "Test Doc", - content: "# Test\n\nSome test content for searching.", - sourceType: "manual", - }); - }); - - afterEach(() => db.close()); - - it("POST /api/v1/searches creates a saved search", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/searches", { name: "my search", query: "test" }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(201); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); - - it("POST /api/v1/searches returns 400 without name", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/searches", { query: "test" }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(400); - const body = parseResponse(getBody()); - expect(body.error?.code).toBe("VALIDATION_ERROR"); - }); - - it("GET /api/v1/searches lists saved searches", async () => { - // First create one - const req1 = createMockReq("POST", "/api/v1/searches", { name: "my search", query: "test" }); - const mock1 = createMockRes(); - await handleRequest(req1, mock1.res, db, provider); - - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("GET", "/api/v1/searches"); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(200); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); - - it("DELETE /api/v1/searches/:id deletes a saved search", async () => { - // Create one first - const req1 = createMockReq("POST", "/api/v1/searches", { name: "del search", query: "test" }); - const mock1 = createMockRes(); - await handleRequest(req1, mock1.res, db, provider); - const created = parseResponse(mock1.getBody()); - const id = (created.data as Record<string, unknown>)?.id as string; - - const { res, getStatus } = createMockRes(); - const req = createMockReq("DELETE", `/api/v1/searches/${id}`); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(204); - }); - - it("POST /api/v1/searches/:id/run runs a saved search", async () => { - // Create one first - const req1 = createMockReq("POST", "/api/v1/searches", { name: "run search", query: "test" }); - const mock1 = createMockRes(); - await handleRequest(req1, mock1.res, db, provider); - const created = parseResponse(mock1.getBody()); - const id = (created.data as Record<string, unknown>)?.id as string; - - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", `/api/v1/searches/${id}/run`); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(200); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); -}); - -describe("Bulk Operations API", () => { - let db: Database.Database; - const provider = new MockEmbeddingProvider(); - - beforeEach(async () => { - db = createTestDbWithVec(); - await indexDocument(db, provider, { - title: "Bulk Doc", - content: "# Bulk\n\nContent for bulk ops.", - sourceType: "manual", - }); - }); - - afterEach(() => db.close()); - - it("POST /api/v1/bulk/delete returns 400 without selector", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/bulk/delete", {}); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(400); - const body = parseResponse(getBody()); - expect(body.error?.code).toBe("VALIDATION_ERROR"); - }); - - it("POST /api/v1/bulk/delete dry run", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/bulk/delete", { - selector: { sourceType: "manual" }, - dryRun: true, - }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(200); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); - - it("POST /api/v1/bulk/retag adds tags", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/bulk/retag", { - selector: { sourceType: "manual" }, - addTags: ["new-tag"], - dryRun: true, - }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(200); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); - - it("POST /api/v1/bulk/move requires targetTopicId", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/bulk/move", { - selector: { sourceType: "manual" }, - }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(400); - const body = parseResponse(getBody()); - expect(body.error?.code).toBe("VALIDATION_ERROR"); - }); - - it("POST /api/v1/bulk/move with targetTopicId", async () => { - const topic = createTopic(db, { name: "Bulk Target" }); - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/bulk/move", { - selector: { sourceType: "manual" }, - targetTopicId: topic.id, - dryRun: true, - }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(200); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); -}); - -describe("Links API", () => { - let db: Database.Database; - const provider = new MockEmbeddingProvider(); - - beforeEach(() => { - db = createTestDbWithVec(); - }); - - afterEach(() => db.close()); - - it("DELETE /api/v1/links/:id deletes a link", async () => { - // Create two docs and link them - const doc1 = await indexDocument(db, provider, { - title: "Doc A", - content: "# A\n\nContent A.", - sourceType: "manual", - }); - const doc2 = await indexDocument(db, provider, { - title: "Doc B", - content: "# B\n\nContent B.", - sourceType: "manual", - }); - - // Create a link via the API: POST /api/v1/documents/:id/links - const { res: createRes, getBody: createBody } = createMockRes(); - const createReq = createMockReq("POST", `/api/v1/documents/${doc1.id}/links`, { - targetId: doc2.id, - linkType: "related", - }); - await handleRequest(createReq, createRes, db, provider); - const created = parseResponse(createBody()); - const linkId = (created.data as Record<string, unknown>)?.id as string; - - // Delete the link - const { res, getStatus } = createMockRes(); - const req = createMockReq("DELETE", `/api/v1/links/${linkId}`); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(204); - }); - - it("POST /api/v1/documents/:id/links accepts the references link type", async () => { - const doc1 = await indexDocument(db, provider, { - title: "Doc C", - content: "# C\n\nContent C.", - sourceType: "manual", - }); - const doc2 = await indexDocument(db, provider, { - title: "Doc D", - content: "# D\n\nContent D.", - sourceType: "manual", - }); - - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", `/api/v1/documents/${doc1.id}/links`, { - targetId: doc2.id, - linkType: "references", - }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(201); - const created = parseResponse(getBody()); - expect((created.data as Record<string, unknown>)?.linkType).toBe("references"); - }); -}); diff --git a/tests/unit/web-server.test.ts b/tests/unit/web-server.test.ts index 4342b6aa..b9d48344 100644 --- a/tests/unit/web-server.test.ts +++ b/tests/unit/web-server.test.ts @@ -5,6 +5,7 @@ import { createTestDb } from "../fixtures/test-db.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; import { insertDoc, insertChunk } from "../fixtures/helpers.js"; import { startWebServer, stopWebServer } from "../../src/web/server.js"; +import { testConfig } from "./operations/helpers.js"; let db: Database.Database; let provider: MockEmbeddingProvider; @@ -35,7 +36,10 @@ describe("web server", () => { insertDoc(db, "doc-2", "Node Guide"); insertChunk(db, "c2", "doc-2", "Node.js runtime for server-side JavaScript"); - server = await startWebServer(db, provider, { port: 0, host: "127.0.0.1" }); + server = await startWebServer( + { db, provider, config: testConfig() }, + { port: 0, host: "127.0.0.1" }, + ); const addr = server.address(); const port = typeof addr === "object" && addr ? addr.port : 3377; baseUrl = `http://127.0.0.1:${port}`; @@ -81,7 +85,7 @@ describe("web server", () => { it("GET /api/documents/:id returns 404 for missing document", async () => { const { status, body } = await fetchJson("/api/documents/nonexistent"); expect(status).toBe(404); - expect(body.error).toBe("Document not found"); + expect(body.error).toMatchObject({ code: "DOCUMENT_NOT_FOUND" }); }); it("GET /api/topics returns topic list with stats", async () => { @@ -105,13 +109,13 @@ describe("web server", () => { it("GET /api/search without q returns 400", async () => { const { status, body } = await fetchJson("/api/search"); expect(status).toBe(400); - expect(body.error).toContain("Missing"); + expect(body.error).toMatchObject({ code: "VALIDATION_ERROR" }); }); it("returns 404 for unknown routes", async () => { const { status, body } = await fetchJson("/api/unknown"); expect(status).toBe(404); - expect(body.error).toBe("Not found"); + expect(body.error).toMatchObject({ code: "NOT_FOUND" }); }); it("DELETE /api/documents/:id deletes a document", async () => { @@ -124,8 +128,57 @@ describe("web server", () => { expect(s2).toBe(404); }); - it("sets CORS headers", async () => { - const res = await fetch(`${baseUrl}/api/stats`); - expect(res.headers.get("access-control-allow-origin")).toBe("*"); + it("GET /graph returns the graph page and /api/graph its data", async () => { + const page = await fetch(`${baseUrl}/graph`); + expect(page.status).toBe(200); + expect(page.headers.get("content-type")).toContain("text/html"); + const { status, body } = await fetchJson("/api/graph?threshold=0.85&maxNodes=200&topic=ts"); + expect(status).toBe(200); + const nodes = (body as { nodes: Array<{ id: string; type: string }> }).nodes; + expect(nodes.filter((n) => n.type === "document").map((n) => n.id)).toEqual(["doc-1"]); + }); + + it("pages and filters the document list the way the dashboard asks", async () => { + const { body } = await fetchJson("/api/documents?limit=20&offset=0&topic=ts"); + const docs = body as unknown as Array<{ id: string; sourceType: string; updatedAt: string }>; + expect(docs.map((d) => d.id)).toEqual(["doc-1"]); + expect(docs[0]!.updatedAt).toBeDefined(); + const { body: page2 } = await fetchJson("/api/documents?limit=1&offset=1"); + expect(page2 as unknown as unknown[]).toHaveLength(1); + }); + + it("sends no CORS allow-origin to other origins and no CSP", async () => { + const res = await fetch(`${baseUrl}/api/stats`, { headers: { Origin: "http://evil.example" } }); + expect(res.status).toBe(200); + expect(res.headers.get("access-control-allow-origin")).toBeNull(); + expect(res.headers.get("content-security-policy")).toBeNull(); + expect(res.headers.get("x-frame-options")).toBe("DENY"); + }); + + it("rejects a cross-origin DELETE and its preflight grants nothing", async () => { + insertDoc(db, "doc-keep", "Keep me"); + const preflight = await fetch(`${baseUrl}/api/documents/doc-keep`, { + method: "OPTIONS", + headers: { Origin: "http://evil.example", "Access-Control-Request-Method": "DELETE" }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get("access-control-allow-origin")).toBeNull(); + + const { status, body } = await fetchJson("/api/documents/doc-keep", { + method: "DELETE", + headers: { Origin: "http://evil.example" }, + }); + expect(status).toBe(403); + expect(body.error).toMatchObject({ code: "FORBIDDEN_ORIGIN" }); + expect((await fetchJson("/api/documents/doc-keep")).status).toBe(200); + }); + + it("allows a same-origin DELETE", async () => { + insertDoc(db, "doc-same", "Same origin"); + const { status } = await fetchJson("/api/documents/doc-same", { + method: "DELETE", + headers: { Origin: baseUrl }, + }); + expect(status).toBe(200); }); }); diff --git a/tests/unit/webhook-events.test.ts b/tests/unit/webhook-events.test.ts index 68a81886..0da1c665 100644 --- a/tests/unit/webhook-events.test.ts +++ b/tests/unit/webhook-events.test.ts @@ -6,6 +6,7 @@ import { createTestDbWithVec } from "../fixtures/test-db.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; import { initLogger } from "../../src/logger.js"; import { ValidationError } from "../../src/errors.js"; +import { testConfig } from "./operations/helpers.js"; const resolve4 = vi.fn(); vi.mock("node:dns", async (importOriginal) => { @@ -27,6 +28,7 @@ const { updateDocument, deleteDocument } = await import("../../src/core/document const { rateDocument } = await import("../../src/core/ratings.js"); const { searchDocuments } = await import("../../src/core/search.js"); const { handleRequest } = await import("../../src/api/routes.js"); +const { createOperationContext } = await import("../../src/core/operations/index.js"); interface SentPayload { event: string; @@ -231,7 +233,16 @@ describe("deliverWebhook", () => { if (typeof chunk === "string") body = chunk; return res; }) as typeof res.end; - await handleRequest(req, res, db, new MockEmbeddingProvider()); + await handleRequest( + req, + res, + createOperationContext({ + db, + provider: new MockEmbeddingProvider(), + config: testConfig(), + surface: "api", + }), + ); return { status: res.statusCode, body }; }; diff --git a/vitest.config.ts b/vitest.config.ts index f48df703..2454a871 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -14,14 +14,11 @@ export default defineConfig({ "src/providers/local.ts", "src/providers/ollama.ts", "src/providers/openai.ts", - "src/api/indexing/repoConfig.ts", - "src/api/indexing/repoIndexer.ts", "src/db/connection.ts", "src/core/index.ts", "src/db/index.ts", "src/providers/index.ts", "src/providers/embedding.ts", - "src/web/graph-api.ts", ], thresholds: { statements: 75, From 172a2b1bc7e84820802a6312bf90c06aadf2bbff Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:07:59 +0000 Subject: [PATCH 41/83] feat(mcp)!: generate MCP tools from operations (11 core + admin toolset) - One generic registerOperationTool() turns an operation into a tool: zod input schema from the operation (minus intentionally omitted fields), annotations from op.annotations, withErrorHandling, and an `async` flag for long-running operations that runs them through startOperationTask. - registerActionTool() builds link-documents (create|delete) and task (status|cancel|list) from several operations. - Core tools: search, ask (only with passthrough or a configured LLM), get-document, list-documents, overview, submit-document, update-document, delete-document, rate-document, link-documents, task. - Admin toolset (LIBSCOPE_MCP_TOOLSETS=admin|all): sync, install-pack, list-packs, reindex-documents. - Removed tools: search-docs, ask-question, get-related, list-topics, health-check, get-document-links, delete-link, get-task, cancel-task, sync-slack/notion/confluence/onenote/obsidian-vault, search-analytics, suggest-tags, saved searches and webhooks. - Compact text formatters in src/mcp/format.ts carry documentId, chunkId and linkId; task status formats a finished task's result with the formatter of the tool that started it. - Server instructions describe the workflow; version from package.json. - src/mcp/server.ts is side-effect free: createMcpServer({ ctx | bootstrap options }) and runStdioServer(); src/mcp/main.ts is the stdio entry (npm run serve). Logs always go to stderr. BREAKING CHANGE: importing libscope/mcp no longer starts a server; run dist/mcp/main.js or call runStdioServer(). MCP tool names and parameters changed; see docs/migration-v2.md. --- package.json | 2 +- src/mcp/format.ts | 383 +++++++- src/mcp/main.ts | 13 + src/mcp/server.ts | 1750 +++++++-------------------------- tests/unit/mcp-entry.test.ts | 87 ++ tests/unit/mcp-format.test.ts | 319 ++++-- tests/unit/mcp-server.test.ts | 692 ++++++++++--- vitest.config.ts | 2 +- 8 files changed, 1584 insertions(+), 1664 deletions(-) create mode 100644 src/mcp/main.ts create mode 100644 tests/unit/mcp-entry.test.ts diff --git a/package.json b/package.json index 55460ec9..331243ac 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "prepare": "husky", - "serve": "node dist/mcp/server.js", + "serve": "node dist/mcp/main.js", "docs:dev": "vitepress dev docs", "docs:build": "vitepress build docs", "docs:preview": "vitepress preview docs" diff --git a/src/mcp/format.ts b/src/mcp/format.ts index 80b1483a..beabda81 100644 --- a/src/mcp/format.ts +++ b/src/mcp/format.ts @@ -1,67 +1,368 @@ +/** + * Compact text output for MCP tools. Every line that names a document, chunk or link carries + * its ID, so the assistant can pass it straight to the next tool. + */ +import type { IngestResult } from "../core/ingest.js"; +import type { DocumentLinks } from "../core/links.js"; +import type { DocumentView } from "../core/document-view.js"; +import type { DocumentSummary, ListResult } from "../core/operations/index.js"; +import type { Overview } from "../core/overview.js"; +import type { InstallResult, InstalledPack, PackInfo } from "../core/packs.js"; +import type { AnswerResult, RagSource } from "../core/rag.js"; +import type { ReindexResult } from "../core/reindex.js"; +import type { Rating } from "../core/ratings.js"; import type { SearchResult } from "../core/search.js"; -import type { DocumentLink, DocumentLinks } from "../core/links.js"; +import type { Task } from "../core/tasks.js"; +import type { ConnectorSyncSummary } from "../connectors/registry.js"; +import { describeEmbeddingIdentity } from "../db/index-meta.js"; + +/** Join the non-empty parts with " · ". */ +function joinParts(parts: Array<string | null | undefined | false>): string { + return parts.filter((p): p is string => typeof p === "string" && p !== "").join(" · "); +} + +/** "library v1.2" / "library" / "". */ +function libraryLabel(library: string | null, version: string | null): string { + if (!library) return ""; + return version ? `${library} v${version}` : library; +} + +/** "Showing 11-20 of 42" style header for a page of a list result. */ +function pageHeader(what: string, page: ListResult<unknown>): string { + if (page.items.length === 0) { + return page.offset > 0 ? `No more ${what} (total ${page.total}).` : `No ${what} found.`; + } + const first = page.offset + 1; + const last = page.offset + page.items.length; + const more = last < page.total ? ` — next page: offset ${last}` : ""; + const title = what.charAt(0).toUpperCase() + what.slice(1); + return `${title} ${first}-${last} of ${page.total}${more}`; +} -/** Join context chunks into a labelled block, or return "" when there are none. */ function formatContext(label: string, chunks: SearchResult["contextBefore"]): string { if (!chunks || chunks.length === 0) return ""; - return `\n**Context (${label}):**\n${chunks.map((c) => c.content).join("\n\n")}\n`; + return `[context ${label}]\n${chunks.map((c) => c.content).join("\n\n")}\n`; } -/** Format one search-docs result, including the IDs that follow-up tools need. */ -function formatSearchResult(r: SearchResult, index: number): string { - const libraryVersion = r.version ? ` v${r.version}` : ""; +function formatSearchResult(r: SearchResult, n: number): string { + const header = joinParts([ + `[${n}] ${r.title}`, + `score ${r.score.toFixed(2)}`, + r.avgRating ? `rating ${r.avgRating.toFixed(1)}/5` : null, + ]); + const meta = joinParts([ + `documentId: ${r.documentId}`, + `chunkId: ${r.chunkId}`, + libraryLabel(r.library, r.version), + r.url, + ]); return ( - `## Result ${index + 1}: ${r.title} (score: ${r.score.toFixed(2)})\n` + - `**Document ID:** ${r.documentId} | **Chunk ID:** ${r.chunkId}\n` + - (r.library ? `**Library:** ${r.library}${libraryVersion}\n` : "") + - (r.url ? `**Source:** ${r.url}\n` : "") + - (r.avgRating ? `**Rating:** ${r.avgRating.toFixed(1)}/5\n` : "") + + `${header}\n${meta}\n` + formatContext("before", r.contextBefore) + - `\n${r.content}\n` + + `${r.content}\n` + formatContext("after", r.contextAfter) ); } -/** Format the search-docs tool output. */ -export function formatSearchResults(results: SearchResult[], totalCount: number): string { - if (results.length === 0) return "No documents found matching your query."; +/** Output of `search`. */ +export function formatSearchResults(page: ListResult<SearchResult>): string { + const header = pageHeader("results", page); + if (page.items.length === 0) return header; + const body = page.items.map((r, i) => formatSearchResult(r, page.offset + i + 1)); + return `${header}\n\n${body.join("\n---\n")}`; +} + +function formatSources(sources: RagSource[]): string { + if (sources.length === 0) return ""; + const lines = sources.map( + (s) => `- ${s.title} (documentId: ${s.documentId}, score ${s.score.toFixed(2)})`, + ); + return `\n\nSources:\n${lines.join("\n")}`; +} + +/** Output of `ask`: the LLM answer, or (passthrough) the context to answer from. */ +export function formatAnswer(result: AnswerResult): string { + if (result.mode === "context") { + return ( + "Passthrough mode: no LLM was called. Answer the question yourself from this context " + + "and cite the documentIds.\n\n" + + result.contextPrompt + + formatSources(result.sources) + ); + } + const tokens = result.tokensUsed == null ? "" : `, ${result.tokensUsed} tokens`; + return `${result.answer}${formatSources(result.sources)}\n\n(model: ${result.model}${tokens})`; +} + +function formatLinks({ outgoing, incoming }: DocumentLinks): string[] { + const label = (l: { label: string | null }): string => (l.label ? ` "${l.label}"` : ""); + return [ + ...outgoing.map( + (l) => + ` -> ${l.linkType} ${l.targetTitle} (documentId: ${l.targetId}, linkId: ${l.id})${label(l)}`, + ), + ...incoming.map( + (l) => + ` <- ${l.linkType} ${l.sourceTitle} (documentId: ${l.sourceId}, linkId: ${l.id})${label(l)}`, + ), + ]; +} + +/** Header lines of a document view: IDs, metadata, tags, rating, links. */ +function formatDocumentHeader(view: DocumentView): string[] { + const d = view.document; + const { averageRating, totalRatings, corrections } = view.ratings; + const lines = [ + `# ${d.title}`, + joinParts([ + `documentId: ${d.documentId}`, + d.sourceType, + libraryLabel(d.library, d.version), + d.topicId ? `topic: ${d.topicId}` : null, + ]), + ]; + if (d.url) lines.push(`url: ${d.url}`); + if (view.tags.length > 0) lines.push(`tags: ${view.tags.join(", ")}`); + if (totalRatings > 0) { + const fixes = corrections > 0 ? `, ${corrections} suggested corrections` : ""; + lines.push(`rating: ${averageRating.toFixed(1)}/5 (${totalRatings} ratings${fixes})`); + } + const links = formatLinks(view.links); + if (links.length > 0) lines.push("links:", ...links); + lines.push(`updated: ${d.updatedAt}`); + return lines; +} + +/** Output of `get-document`: metadata, then the requested slice of the content. */ +export function formatDocumentView(view: DocumentView): string { + const end = view.offset + view.content.length; + const paged = view.offset > 0 || view.nextOffset !== null; + const range = paged + ? `\n\n[characters ${view.offset}-${end} of ${view.contentLength}` + + (view.nextOffset === null ? "]" : `; next page: offset ${view.nextOffset}]`) + : ""; + return `${formatDocumentHeader(view).join("\n")}${range}\n\n${view.content}`; +} + +/** Output of `update-document`: the document's new metadata (no content). */ +export function formatDocumentUpdated(view: DocumentView): string { + return `Updated.\n${formatDocumentHeader(view).join("\n")}`; +} + +/** Output of `list-documents`. */ +export function formatDocumentList(page: ListResult<DocumentSummary>): string { + const header = pageHeader("documents", page); + const lines = page.items.map( + (d) => + `- ${d.title} (documentId: ${d.documentId}) ` + + joinParts([d.sourceType, libraryLabel(d.library, d.version), d.url]), + ); + return [header, ...lines].join("\n"); +} + +/** Output of `overview`. */ +export function formatOverview(o: Overview): string { + const { stats, health, index } = o; + const lines = [ + `Documents: ${stats.totalDocuments} · chunks: ${stats.totalChunks} · topics: ${stats.totalTopics}`, + `Health: database ${health.database} · keyword index ${health.fts} · vector search ${ + health.vectorSearch ? "ok" : "unavailable" + }`, + `Index embeddings: ${describeEmbeddingIdentity(index.stored)}` + + (index.configured ? ` · configured: ${describeEmbeddingIdentity(index.configured)}` : ""), + ]; + if (o.topics.length > 0) { + lines.push( + "Topics:", + ...o.topics.map((t) => `- ${t.name} (topic: ${t.id}, ${t.documentCount} docs)`), + ); + } + if (o.packs.length > 0) { + lines.push("Packs:", ...o.packs.map(formatInstalledPack)); + } + return lines.join("\n"); +} + +/** Output of `submit-document`. */ +export function formatIngestResult(result: IngestResult): string { + const lines: string[] = []; + if (result.planned) { + lines.push(`Dry run: ${result.planned.length} source(s) would be added.`); + lines.push(...result.planned.map((p) => `- ${p}`)); + } else { + lines.push(`Added ${result.documents.length} document(s).`); + } + lines.push( + ...result.documents.map( + (d) => + `- ${d.title} (documentId: ${d.documentId}, ${d.chunkCount} chunks)` + + (d.source ? ` ${d.source}` : ""), + ), + ); + if (result.crawl) { + const { pagesCrawled, pagesSkipped, abortReason } = result.crawl; + lines.push( + `Crawled ${pagesCrawled} page(s), skipped ${pagesSkipped}` + + (abortReason ? `; stopped early: ${abortReason}` : ""), + ); + } + if (result.skipped.length > 0) { + lines.push(`Skipped ${result.skipped.length}:`); + lines.push(...result.skipped.map((s) => `- ${s.source}: ${s.reason}`)); + } + if (result.errors.length > 0) { + lines.push(`Errors ${result.errors.length}:`); + lines.push(...result.errors.map((e) => `- ${e.source}: ${e.error}`)); + } + return lines.join("\n"); +} + +/** Output of `delete-document`. */ +export function formatDocumentDeleted(result: { documentId: string }): string { + return `Deleted document ${result.documentId}.`; +} + +/** Output of `rate-document`. */ +export function formatRating(r: Rating): string { + return joinParts([ + `Rated ${r.rating}/5 (documentId: ${r.documentId}${r.chunkId ? `, chunkId: ${r.chunkId}` : ""})`, + r.feedback ? `feedback saved` : null, + r.suggestedCorrection ? `correction saved` : null, + ]); +} + +/** Output of `link-documents` with action "create". */ +export function formatLinkCreated(link: { + linkId: string; + sourceId: string; + targetId: string; + linkType: string; + label: string | null; +}): string { + const label = link.label ? ` "${link.label}"` : ""; return ( - `**Total results: ${totalCount}**\n\n` + - results.map((r, i) => formatSearchResult(r, i)).join("\n---\n\n") + `Linked ${link.sourceId} -> ${link.targetId} (${link.linkType})${label}. ` + + `linkId: ${link.linkId}` ); } -/** Format the link-documents tool output, including the link ID needed by delete-link. */ -export function formatLinkCreated(link: DocumentLink): string { - const linkLabel = link.label ? ` — ${link.label}` : ""; +/** Output of `link-documents` with action "delete". */ +export function formatLinkDeleted(result: { linkId: string }): string { + return `Deleted link ${result.linkId}.`; +} + +/** Output when a tool started a background task. */ +export function formatTaskStarted(task: Task): string { return ( - `✓ Link created: ${link.sourceId} → ${link.targetId} (${link.linkType})${linkLabel}\n` + - `Link ID: ${link.id}` + `Started background task. taskId: ${task.id}\n` + + `Poll with task {"action": "status", "taskId": "${task.id}"}.` ); } -/** Format the get-document-links tool output, including each link's ID. */ -export function formatDocumentLinks({ outgoing, incoming }: DocumentLinks): string { - if (outgoing.length === 0 && incoming.length === 0) return "No links found for this document."; +/** Formats an operation's JSON result for a task, or returns undefined to show it raw. */ +export type TaskResultFormatter = (operation: string, result: unknown) => string | undefined; - const lines: string[] = []; - if (outgoing.length > 0) { - lines.push("**Outgoing links:**"); - for (const l of outgoing) { - const outLabel = l.label ? ` — ${l.label}` : ""; - lines.push( - ` → [${l.linkType}] ${l.targetTitle} (${l.targetId})${outLabel} [link ID: ${l.id}]`, - ); +function formatTaskResult(task: Task, formatResult?: TaskResultFormatter): string { + if (task.result === undefined) return ""; + let text: string | undefined; + if (task.operation !== undefined && formatResult) { + try { + text = formatResult(task.operation, JSON.parse(task.result) as unknown); + } catch { + text = undefined; } } - if (incoming.length > 0) { - lines.push("**Incoming links:**"); - for (const l of incoming) { - const inLabel = l.label ? ` — ${l.label}` : ""; - lines.push( - ` ← [${l.linkType}] ${l.sourceTitle} (${l.sourceId})${inLabel} [link ID: ${l.id}]`, - ); - } + return `\nresult:\n${text ?? task.result}`; +} + +function taskLine(task: Task): string { + const progress = task.progress ? ` ${task.progress.current}/${task.progress.total || "?"}` : ""; + return joinParts([ + `taskId: ${task.id}`, + task.operation ?? task.type, + `${task.status}${progress}`, + `created ${task.createdAt.toISOString()}`, + ]); +} + +/** Output of `task` with action "status". */ +export function formatTask(task: Task, formatResult?: TaskResultFormatter): string { + const error = task.error ? `\nerror: ${task.error}` : ""; + return `${taskLine(task)}${error}${formatTaskResult(task, formatResult)}`; +} + +/** Output of `task` with action "cancel". */ +export function formatTaskCancel(result: { + taskId: string; + cancelRequested: boolean; + status: string; +}): string { + return result.cancelRequested + ? `Cancellation requested for task ${result.taskId} (status: ${result.status}).` + : `Task ${result.taskId} is already ${result.status}; nothing to cancel.`; +} + +/** Output of `task` with action "list". */ +export function formatTaskList(result: { items: Task[] }): string { + if (result.items.length === 0) return "No tasks in the last hour."; + return result.items.map((t) => `- ${taskLine(t)}`).join("\n"); +} + +/** Output of `sync`. */ +export function formatSyncResult(result: { + items: Array< + | { name: string; type: string; status: "completed"; summary: ConnectorSyncSummary } + | { name: string; status: "failed"; error: string } + >; +}): string { + if (result.items.length === 0) return "No saved connections to sync."; + return result.items + .map((item) => { + if (item.status === "failed") return `- ${item.name}: failed: ${item.error}`; + const { added, updated, deleted, errors } = item.summary; + const errorLines = errors.map((e) => ` ${e}`); + return [ + `- ${item.name} (${item.type}): ${added} added, ${updated} updated, ${deleted} deleted` + + (errors.length > 0 ? `, ${errors.length} errors:` : ""), + ...errorLines, + ].join("\n"); + }) + .join("\n"); +} + +function formatInstalledPack(p: InstalledPack): string { + return `- ${p.name} v${p.version} (${p.docCount} docs, installed ${p.installedAt})`; +} + +/** Output of `install-pack`. */ +export function formatInstallPack(result: InstallResult): string { + if (result.alreadyInstalled) return `Pack ${result.packName} is already installed.`; + const errors = result.errors > 0 ? `, ${result.errors} failed` : ""; + return `Installed pack ${result.packName}: ${result.documentsInstalled} documents${errors}.`; +} + +/** Output of `list-packs`. */ +export function formatPackList(result: { items: Array<InstalledPack | PackInfo> }): string { + if (result.items.length === 0) return "No packs found."; + return result.items + .map((p) => + "installedAt" in p + ? formatInstalledPack(p) + : `- ${p.name} v${p.version} (${p.docCount} docs): ${p.description}`, + ) + .join("\n"); +} + +/** Output of `reindex-documents`. */ +export function formatReindex(result: ReindexResult & { rebuiltIndex?: string }): string { + const lines = [ + `Re-embedded ${result.completed} of ${result.total} chunks` + + (result.failed > 0 ? `; ${result.failed} failed` : "") + + ".", + ]; + if (result.rebuiltIndex) lines.push(`Rebuilt vector index: ${result.rebuiltIndex}`); + if (result.failedChunkIds.length > 0) { + lines.push(`Failed chunkIds: ${result.failedChunkIds.join(", ")}`); } return lines.join("\n"); } diff --git a/src/mcp/main.ts b/src/mcp/main.ts new file mode 100644 index 00000000..ec9dc98a --- /dev/null +++ b/src/mcp/main.ts @@ -0,0 +1,13 @@ +/** + * Executable entry point: `node dist/mcp/main.js` serves libscope over MCP stdio. + * Everything else lives in server.ts, which has no side effects on import. + */ +import { runStdioServer } from "./server.js"; + +runStdioServer().catch((err: unknown) => { + console.error( + "libscope MCP server failed to start:", + err instanceof Error ? err.message : String(err), + ); + process.exit(1); +}); diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 120ce102..430baaae 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -1,1436 +1,412 @@ +/** + * libscope MCP server. Every tool is generated from an operation in src/core/operations: + * the tool's input schema is the operation's zod schema, and the handler runs the operation + * and formats its result as compact text. + * + * Importing this module starts nothing. `createMcpServer()` builds a server; `runStdioServer()` + * connects one to stdin/stdout (see main.ts for the executable entry point). + */ +import { createRequire } from "node:module"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import type { ToolAnnotations } from "@modelcontextprotocol/sdk/types.js"; import { z } from "zod"; import { loadConfig } from "../config.js"; -import { getDatabase, runMigrations, createVectorTable } from "../db/index.js"; -import { resolveDatabasePath } from "../db/connection.js"; -import { createEmbeddingProvider } from "../providers/index.js"; -import { searchDocuments, getRelatedChunks } from "../core/search.js"; -import { - askQuestion, - createLlmProvider, - getContextForQuestion, - isPassthroughMode, - type LlmProvider, -} from "../core/rag.js"; -import { getDocument, listDocuments, deleteDocument, updateDocument } from "../core/documents.js"; -import { rateDocument, getDocumentRatings } from "../core/ratings.js"; -import { indexDocument } from "../core/indexing.js"; -import { listTopics } from "../core/topics.js"; -import { createLink, getDocumentLinks, deleteLink, LINK_TYPES } from "../core/links.js"; +import { bootstrap, type BootstrapOptions, type Bootstrapped } from "../core/bootstrap.js"; import { - createSavedSearch, - listSavedSearches, - runSavedSearch, - deleteSavedSearch, -} from "../core/saved-searches.js"; -import { createWebhook, listWebhooks, deleteWebhook, redactWebhook } from "../core/webhooks.js"; -import type { WebhookEvent } from "../core/webhooks.js"; -import { suggestTags } from "../core/tags.js"; -import { fetchAndConvert } from "../core/url-fetcher.js"; -import { spiderUrl } from "../core/spider.js"; -import type { SpiderOptions } from "../core/spider.js"; -import { initLogger, getLogger } from "../logger.js"; -import { ConfigError, ValidationError } from "../errors.js"; -import { errorResponse, textResult, withErrorHandling, type ToolResult } from "./errors.js"; -export { errorResponse, withErrorHandling, type ToolResult } from "./errors.js"; -import { formatDocumentLinks, formatLinkCreated, formatSearchResults } from "./format.js"; -import { taskRegistry } from "../core/tasks.js"; -import type { TaskType, TaskWork } from "../core/tasks.js"; -import type { NotionConfig } from "../connectors/notion.js"; -import type { SlackConfig } from "../connectors/slack.js"; -import type { ConfluenceConfig } from "../connectors/confluence.js"; -import type { ObsidianConfig } from "../connectors/obsidian.js"; -import type { OneNoteConfig } from "../connectors/onenote.js"; - -/** Build SpiderOptions from submit-document params. */ -function buildSpiderOptions( - params: { - maxPages?: number | undefined; - maxDepth?: number | undefined; - sameDomain?: boolean | undefined; - pathPrefix?: string | undefined; - excludePatterns?: string[] | undefined; - }, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): SpiderOptions { - const opts: SpiderOptions = { fetchOptions }; - if (params.maxPages !== undefined) opts.maxPages = params.maxPages; - if (params.maxDepth !== undefined) opts.maxDepth = params.maxDepth; - if (params.sameDomain !== undefined) opts.sameDomain = params.sameDomain; - if (params.pathPrefix !== undefined) opts.pathPrefix = params.pathPrefix; - if (params.excludePatterns !== undefined) opts.excludePatterns = params.excludePatterns; - return opts; + addOperation, + askOperation, + cancelTaskOperation, + createOperationContext, + deleteDocumentOperation, + getDocumentOperation, + getTaskOperation, + installPackOperation, + linkDocumentsOperation, + listDocumentsOperation, + listPacksOperation, + listTasksOperation, + overviewOperation, + rateDocumentOperation, + reindexOperation, + runOperation, + searchOperation, + startOperationTask, + syncOperation, + unlinkDocumentsOperation, + updateDocumentOperation, + type Operation, + type OperationAnnotations, + type OperationContext, +} from "../core/operations/index.js"; +import { getLogger, initLogger, type LogLevel } from "../logger.js"; +import { textResult, withErrorHandling } from "./errors.js"; +import * as fmt from "./format.js"; + +export { errorResponse, textResult, withErrorHandling, type ToolResult } from "./errors.js"; + +/** Optional toolsets, enabled with LIBSCOPE_MCP_TOOLSETS (comma list, or "all"). */ +export const MCP_TOOLSETS = ["admin"] as const; +export type McpToolset = (typeof MCP_TOOLSETS)[number]; + +/** Environment variable that enables optional toolsets. */ +export const TOOLSETS_ENV = "LIBSCOPE_MCP_TOOLSETS"; + +/** Formats an operation result as tool output text. */ +type Formatter<O> = (result: O) => string; + +export interface OperationToolOptions<O> { + /** Tool name (default: the operation name). */ + name?: string | undefined; + /** Tool description (default: the operation summary). */ + description?: string | undefined; + /** Operation input fields the tool does not offer. */ + omit?: readonly string[] | undefined; + format: Formatter<O>; } -/** Handle spider mode for submit-document. */ -async function handleSpiderSubmit( - db: import("better-sqlite3").Database, - provider: import("../providers/embedding.js").EmbeddingProvider, - params: { - url?: string | undefined; - library?: string | undefined; - version?: string | undefined; - topic?: string | undefined; - sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; - maxPages?: number | undefined; - maxDepth?: number | undefined; - sameDomain?: boolean | undefined; - pathPrefix?: string | undefined; - excludePatterns?: string[] | undefined; - }, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, - signal?: AbortSignal, -): Promise<string> { - const { url, library, version, topic } = params; - if (!url) { - throw new ValidationError("Field 'url' is required when spider is true"); - } - - const spiderOptions = buildSpiderOptions(params, fetchOptions); - if (signal) spiderOptions.signal = signal; - const indexed: Array<{ id: string; title: string }> = []; - const errors: Array<{ url: string; error: string }> = []; - const sourceType = params.sourceType ?? (library ? "library" : "manual"); - - const gen = spiderUrl(url, spiderOptions); - let result = await gen.next(); - while (!result.done) { - const page = result.value; - try { - const doc = await indexDocument(db, provider, { - title: page.title, - content: page.content, - sourceType, - library, - version, - topicId: topic, - url: page.url, - submittedBy: "model", - }); - indexed.push({ id: doc.id, title: page.title }); - } catch (err) { - errors.push({ url: page.url, error: err instanceof Error ? err.message : String(err) }); - } - result = await gen.next(); - } - const stats = result.value; - - return [ - `Spider complete.`, - `Pages indexed: ${indexed.length}`, - `Pages crawled: ${stats.pagesCrawled}`, - `Pages skipped: ${stats.pagesSkipped}`, - errors.length > 0 ? `Errors: ${errors.length}` : null, - stats.abortReason ? `Stopped early: ${stats.abortReason}` : null, - ] - .filter(Boolean) - .join("\n"); -} - -/** Handle single-document submission for submit-document. */ -async function handleSingleDocSubmit( - db: import("better-sqlite3").Database, - provider: import("../providers/embedding.js").EmbeddingProvider, - params: { - title?: string | undefined; - content?: string | undefined; - url?: string | undefined; - library?: string | undefined; - version?: string | undefined; - topic?: string | undefined; - sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; - }, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, - signal?: AbortSignal, -): Promise<string> { - let { title, content } = params; - const { url, library, version, topic } = params; - - if (url && !content) { - const fetched = await fetchAndConvert(url, fetchOptions); - content = fetched.content; - title ??= fetched.title; - } - - if (!title) { - throw new ValidationError("A title is required when not providing a URL"); - } - if (!content) { - throw new ValidationError("Either content or a URL must be provided"); - } - - const sourceType = params.sourceType ?? (library ? "library" : "manual"); - - signal?.throwIfAborted(); - const result = await indexDocument(db, provider, { - title, - content, - sourceType, - library, - version, - topicId: topic, - url, - submittedBy: "model", - }); - - return ( - `Document indexed successfully.\n` + - `Title: ${title}\n` + - `ID: ${result.id}\n` + - `Chunks: ${result.chunkCount}` + - (url ? `\nSource: ${url}` : "") - ); +const asyncField = z + .boolean() + .default(false) + .describe( + 'Run in the background and return a taskId at once; poll with task {"action": "status"}', + ); + +/** MCP tool hints from operation annotations (several for a multi-action tool). */ +function toolAnnotations(list: OperationAnnotations[]): ToolAnnotations { + if (list.every((a) => a.readOnly === true)) return { readOnlyHint: true }; + return { + readOnlyHint: false, + destructiveHint: list.some((a) => a.destructive === true), + idempotentHint: list.every((a) => a.readOnly === true || a.idempotent === true), + }; } -/** Fire-and-forget helper: creates a task, runs `work` in background, returns task ID response. */ -function startAsyncTask(type: TaskType, work: TaskWork): ToolResult { - const { task } = taskRegistry.run(type, work); - return textResult(`Task queued. ID: ${task.id}\nUse get-task to check status.`); +function shapeOf(op: Operation): Record<string, z.ZodType> { + return op.input.shape as Record<string, z.ZodType>; } -const noProgress = (): void => { - // Inline (sync) runs have no task to report progress to. -}; - /** - * Runs `work` as a background task when `isAsync` is set; otherwise runs it inline with a - * never-aborted signal and returns its text as the tool result. + * Register `op` as an MCP tool. The input schema is the operation's, minus `omit`; a + * long-running operation also gets an `async` flag that runs it as a background task. */ -async function runMaybeAsync( - isAsync: boolean | undefined, - type: TaskType, - work: TaskWork, -): Promise<ToolResult> { - if (isAsync) return startAsyncTask(type, work); - return textResult(await work(new AbortController().signal, noProgress)); -} - -// Start the server -async function main(): Promise<void> { - let config; - try { - config = loadConfig(); - } catch (err) { - console.error("Failed to load configuration:", err instanceof Error ? err.message : err); - process.exit(1); - } - - // stdout carries the MCP protocol stream — logs must go to stderr. - initLogger(config.logging.level, { destination: "stderr" }); - - let db; - try { - db = getDatabase(resolveDatabasePath({ explicitPath: config.database.path })); - runMigrations(db); - } catch (err) { - console.error("Failed to initialize database:", err instanceof Error ? err.message : err); - process.exit(1); - } - - let provider; - let llmProvider: LlmProvider | undefined; - try { - provider = createEmbeddingProvider(config); - createVectorTable(db, provider); - } catch (err) { - console.error( - "Failed to initialize embedding provider:", - err instanceof Error ? err.message : err, - ); - db.close(); - process.exit(1); - } - - try { - llmProvider = createLlmProvider(config); - } catch (err) { - getLogger().warn({ err }, "LLM provider unavailable — ask-question tool will not work"); - } - - process.on("SIGINT", () => { - db.close(); - process.exit(0); - }); - process.on("SIGTERM", () => { - db.close(); - process.exit(0); - }); - - const server = new McpServer({ - name: "libscope", - version: "0.1.0", - }); - - // Tool: search-docs - server.tool( - "search-docs", - "Semantic search across all indexed documentation, library docs, and topics", - { - query: z.string().describe("The search query"), - topic: z.string().optional().describe("Filter by topic ID"), - library: z.string().optional().describe("Filter by library name"), - version: z.string().optional().describe("Filter by library version"), - source: z - .string() - .optional() - .describe("Filter by source type (e.g., 'library', 'topic', 'manual', 'model-generated')"), - minRating: z.number().min(1).max(5).optional().describe("Minimum average rating filter"), - offset: z.number().min(0).optional().describe("Offset for pagination (default: 0)"), - limit: z - .number() - .min(1) - .max(50) - .optional() - .describe("Maximum results to return (default: 10)"), - maxChunksPerDocument: z - .number() - .min(0) - .max(50) - .optional() - .describe( - "Maximum chunks per document in results (default: no limit, set to 2 for diversity)", - ), - contextChunks: z - .number() - .min(0) - .max(2) - .optional() - .describe( - "Number of neighboring chunks to include before/after each result for context (0-2, default: 0)", - ), - }, - withErrorHandling(async (params) => { - const { results, totalCount } = await searchDocuments(db, provider, { - query: params.query, - topic: params.topic, - library: params.library, - version: params.version, - source: params.source, - minRating: params.minRating, - limit: params.limit, - offset: params.offset, - maxChunksPerDocument: params.maxChunksPerDocument, - contextChunks: params.contextChunks, - }); - - return textResult(formatSearchResults(results, totalCount)); - }), - ); - - // Tool: get-related - server.tool( - "get-related", - "Find chunks semantically similar to a given chunk (more-like-this). Returns related content seeded from an existing chunk's stored embedding without requiring a text query.", - { - chunkId: z.string().describe("ID of the source chunk to find related content for"), - limit: z - .number() - .min(1) - .max(50) - .optional() - .describe("Number of results to return (default 10)"), - topic: z.string().optional().describe("Filter results to a specific topic"), - library: z.string().optional().describe("Filter results to a specific library"), - tags: z.array(z.string()).optional().describe("Filter results to documents with these tags"), - minScore: z - .number() - .min(0) - .max(1) - .optional() - .describe("Minimum similarity score threshold (0-1)"), - includeLinkedDocuments: z - .boolean() - .optional() - .describe("Also include explicitly linked documents even if below similarity threshold"), - }, - withErrorHandling( - ({ chunkId, limit, topic, library, tags, minScore, includeLinkedDocuments }) => { - const result = getRelatedChunks(db, { - chunkId, - ...(limit !== undefined && { limit }), - ...(topic !== undefined && { topic }), - ...(library !== undefined && { library }), - ...(tags !== undefined && { tags }), - ...(minScore !== undefined && { minScore }), - ...(includeLinkedDocuments !== undefined && { includeLinkedDocuments }), - }); - return textResult(JSON.stringify(result, null, 2)); - }, - ), - ); - - // Tool: get-document - server.tool( - "get-document", - "Retrieve a specific document by its ID", +export function registerOperationTool<S extends z.ZodObject, O>( + server: McpServer, + op: Operation<S, O>, + ctx: OperationContext, + options: OperationToolOptions<O>, +): string { + const omit = new Set(options.omit ?? []); + const shape = Object.fromEntries( + Object.entries(shapeOf(op as unknown as Operation)).filter(([key]) => !omit.has(key)), + ); + const background = op.annotations?.longRunning === true; + if (background) shape["async"] = asyncField; + const name = options.name ?? op.name; + server.registerTool( + name, { - documentId: z.string().describe("The document ID"), + description: options.description ?? op.summary, + inputSchema: z.object(shape), + annotations: toolAnnotations([op.annotations ?? {}]), }, - withErrorHandling((params) => { - const doc = getDocument(db, params.documentId); - const ratings = getDocumentRatings(db, params.documentId); - - const docVersion = doc.version ? ` v${doc.version}` : ""; - const text = - `# ${doc.title}\n\n` + - `**Type:** ${doc.sourceType}\n` + - (doc.library ? `**Library:** ${doc.library}${docVersion}\n` : "") + - (doc.url ? `**Source:** ${doc.url}\n` : "") + - `**Rating:** ${ratings.averageRating.toFixed(1)}/5 (${ratings.totalRatings} ratings)\n\n` + - doc.content; - - return textResult(text); - }), - ); - - // Tool: delete-document - server.tool( - "delete-document", - "Delete a document from the knowledge base by its ID", - { - documentId: z.string().describe("The document ID to delete"), - }, - withErrorHandling((params) => { - deleteDocument(db, params.documentId); - - return textResult(`Document ${params.documentId} has been deleted successfully.`); - }), - ); - - // Tool: update-document - server.tool( - "update-document", - "Update an existing document's title, content, or metadata", - { - documentId: z.string().describe("The document ID to update"), - title: z.string().optional().describe("New title"), - content: z.string().optional().describe("New content (will re-chunk and re-index)"), - library: z.string().nullable().optional().describe("New library name (null to clear)"), - version: z.string().nullable().optional().describe("New version (null to clear)"), - url: z.string().nullable().optional().describe("New URL (null to clear)"), - topicId: z.string().nullable().optional().describe("New topic ID (null to clear)"), - }, - withErrorHandling(async (params) => { - const metadata: Record<string, string | null | undefined> = {}; - if (params.library !== undefined) metadata.library = params.library; - if (params.version !== undefined) metadata.version = params.version; - if (params.url !== undefined) metadata.url = params.url; - if (params.topicId !== undefined) metadata.topicId = params.topicId; - - const doc = await updateDocument(db, provider, params.documentId, { - title: params.title, - content: params.content, - metadata: - Object.keys(metadata).length > 0 - ? (metadata as { - library?: string | null; - version?: string | null; - url?: string | null; - topicId?: string | null; - }) - : undefined, - }); - return textResult(`Document updated: ${doc.title} (${doc.id})`); - }), - ); - - // Tool: rate-document - server.tool( - "rate-document", - "Rate a document or suggest corrections. Use this when documentation appears outdated, incorrect, or particularly helpful.", - { - documentId: z.string().describe("The document ID to rate"), - chunkId: z.string().optional().describe("Optional specific chunk ID to rate"), - rating: z.number().min(1).max(5).describe("Rating from 1 (poor) to 5 (excellent)"), - feedback: z.string().optional().describe("Text feedback about the document"), - suggestedCorrection: z - .string() - .optional() - .describe("Suggested replacement content if the doc is wrong"), - }, - withErrorHandling((params) => { - const result = rateDocument(db, { - documentId: params.documentId, - chunkId: params.chunkId, - rating: params.rating, - feedback: params.feedback, - suggestedCorrection: params.suggestedCorrection, - ratedBy: "model", - }); - - return textResult( - `Rating submitted: ${result.rating}/5 for document ${result.documentId}` + - (result.feedback ? `\nFeedback: ${result.feedback}` : "") + - (result.suggestedCorrection ? `\nCorrection suggested.` : ""), - ); - }), - ); - - // Tool: submit-document - server.tool( - "submit-document", - "Submit a new document for indexing into the knowledge base. You can provide content directly, or provide a URL to fetch and index automatically. Set spider=true to crawl linked pages from the URL.", - { - title: z - .string() - .optional() - .describe("Document title (auto-detected from URL if not provided)"), - content: z - .string() - .optional() - .describe("Document content in markdown (omit if providing a URL to fetch)"), - url: z - .string() - .optional() - .describe( - "URL to fetch and index. When provided, content is fetched automatically. Title is auto-detected if not specified.", - ), - sourceType: z - .enum(["library", "topic", "manual", "model-generated"]) - .optional() - .describe("Type of document (default: 'manual', or 'library' if library name is given)"), - topic: z.string().optional().describe("Topic ID to categorize under"), - library: z.string().optional().describe("Library name (for library docs)"), - version: z.string().optional().describe("Library version"), - spider: z - .boolean() - .optional() - .describe("When true, crawl pages linked from the URL. Requires 'url'. Default: false."), - maxPages: z - .number() - .int() - .positive() - .optional() - .describe("Maximum pages to index during a spider run (default: 25, hard cap: 200)."), - maxDepth: z - .number() - .int() - .min(0) - .optional() - .describe( - "Maximum link-hop depth from the seed URL (default: 2, hard cap: 5). 0 = seed only.", - ), - sameDomain: z - .boolean() - .optional() - .describe("Only follow links on the same domain as the seed URL (default: true)."), - pathPrefix: z - .string() - .optional() - .describe("Only follow links whose path starts with this prefix (e.g. '/docs/')."), - excludePatterns: z - .array(z.string()) - .optional() - .describe("Glob patterns for URLs to skip (e.g. ['*/changelog*', '*/api/v1/*'])."), - async: z - .boolean() - .optional() - .describe( - "When true, start indexing in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const fetchOptions = { - allowPrivateUrls: config.indexing.allowPrivateUrls, - allowSelfSignedCerts: config.indexing.allowSelfSignedCerts, - }; - - return runMaybeAsync(params.async, "index_document", (signal) => - params.spider - ? handleSpiderSubmit(db, provider, params, fetchOptions, signal) - : handleSingleDocSubmit(db, provider, params, fetchOptions, signal), - ); - }), - ); - - // Tool: list-topics - server.tool( - "list-topics", - "List available documentation topics", - { - parentId: z.string().optional().describe("Filter by parent topic ID for subtopics"), - }, - withErrorHandling((params) => { - const topics = listTopics(db, params.parentId); - - if (topics.length === 0) { - return textResult("No topics found."); - } - - const text = topics - .map((t) => { - const topicDesc = t.description ? `: ${t.description}` : ""; - return `- **${t.name}** (\`${t.id}\`)${topicDesc}`; - }) - .join("\n"); - - return textResult(`## Topics\n\n${text}`); - }), - ); - - // Tool: health-check - server.tool( - "health-check", - "Check the health of the LibScope server, including database connectivity, document and chunk counts, and FTS5 index status", - {}, - () => { - try { - const health: Record<string, unknown> = {}; - - // Check database connectivity - try { - db.prepare("SELECT 1").get(); - health.database = "ok"; - } catch (err: unknown) { - health.database = "error"; - getLogger().warn({ err }, "Health check: database connectivity failed"); - } - - // Document count - try { - const row = db.prepare("SELECT COUNT(*) as count FROM documents").get() as { - count: number; - }; - health.documents = row.count; - } catch (err: unknown) { - health.documents = "error"; - getLogger().warn({ err }, "Health check: document count query failed"); - } - - // Chunk count - try { - const row = db.prepare("SELECT COUNT(*) as count FROM chunks").get() as { - count: number; - }; - health.chunks = row.count; - } catch (err: unknown) { - health.chunks = "error"; - getLogger().warn({ err }, "Health check: chunk count query failed"); - } - - // FTS5 index status - try { - db.prepare("SELECT COUNT(*) FROM chunks_fts").get(); - health.fts5 = "ok"; - } catch (err: unknown) { - health.fts5 = "error"; - getLogger().warn({ err }, "Health check: FTS5 index query failed"); - } - - return textResult(JSON.stringify(health, null, 2)); - } catch (err) { - return errorResponse(err); - } - }, - ); - - // Tool: list-documents - server.tool( - "list-documents", - "List all indexed documents with optional filters", - { - library: z.string().optional().describe("Filter by library name"), - topic: z.string().optional().describe("Filter by topic ID"), - sourceType: z - .enum(["library", "topic", "manual", "model-generated"]) - .optional() - .describe("Filter by source type"), - limit: z.number().min(1).max(100).optional().describe("Maximum results (default: 50)"), - }, - withErrorHandling((params) => { - const docs = listDocuments(db, { - library: params.library, - topicId: params.topic, - sourceType: params.sourceType, - limit: params.limit, - }); - - if (docs.length === 0) { - return textResult("No documents found."); + withErrorHandling(async (args: Record<string, unknown>) => { + const { async: runInBackground, ...input } = args; + if (background && runInBackground === true) { + return textResult(fmt.formatTaskStarted(startOperationTask(op, ctx, input))); } - - const text = docs - .map((d) => { - const docLibVersion = d.version ? ` v${d.version}` : ""; - return ( - `- **${d.title}** (\`${d.id}\`)` + - (d.library ? ` — ${d.library}${docLibVersion}` : "") + - (d.url ? ` — [source](${d.url})` : "") + - ` (${d.sourceType})` - ); - }) - .join("\n"); - - return textResult(`## Documents (${docs.length})\n\n${text}`); - }), - ); - - // Tool: ask-question (RAG) - server.tool( - "ask-question", - "Ask a question and get an LLM-synthesized answer based on indexed documentation (RAG)", - { - question: z.string().describe("The question to answer"), - topK: z - .number() - .min(1) - .max(20) - .optional() - .describe("Number of chunks to retrieve for context (default: 5)"), - topic: z.string().optional().describe("Filter by topic ID"), - library: z.string().optional().describe("Filter by library name"), - }, - withErrorHandling(async (params) => { - if (isPassthroughMode(config)) { - const { contextPrompt, sources } = await getContextForQuestion(db, provider, { - question: params.question, - topK: params.topK, - topic: params.topic, - library: params.library, - }); - - const sourcesText = - sources.length > 0 - ? "\n\n**Sources:**\n" + - sources - .map((s) => `- ${s.title} (score: ${s.score.toFixed(2)}) [${s.documentId}]`) - .join("\n") - : ""; - - return textResult(contextPrompt + sourcesText); - } - - if (!llmProvider) { - throw new ConfigError( - "No LLM provider configured. Set llm.provider to 'openai', 'ollama', or 'passthrough' in your config.", - ); - } - - const result = await askQuestion(db, provider, llmProvider, { - question: params.question, - topK: params.topK, - topic: params.topic, - library: params.library, - }); - - const sourcesText = - result.sources.length > 0 - ? "\n\n**Sources:**\n" + - result.sources - .map((s) => `- ${s.title} (score: ${s.score.toFixed(2)}) [${s.documentId}]`) - .join("\n") - : ""; - - const metaText = - result.tokensUsed != null - ? `\n\n_Model: ${result.model} | Tokens: ${result.tokensUsed}_` - : ""; - - return textResult(result.answer + sourcesText + metaText); - }), - ); - - // Tool: reindex-documents - server.tool( - "reindex-documents", - "Re-embed all document chunks with the current embedding model. Use after switching embedding providers to update vectors without re-fetching content.", - { - documentIds: z - .array(z.string()) - .optional() - .describe("Only reindex chunks belonging to these document IDs"), - since: z - .string() - .optional() - .describe("Only reindex documents created on or after this ISO-8601 date"), - before: z - .string() - .optional() - .describe("Only reindex documents created on or before this ISO-8601 date"), - batchSize: z - .number() - .min(1) - .max(500) - .optional() - .describe("Chunks per embedding batch (default: 50)"), - async: z - .boolean() - .optional() - .describe( - "When true, run reindexing in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { reindex } = await import("../core/reindex.js"); - - return runMaybeAsync(params.async, "reindex_documents", async (signal, onProgress) => { - const result = await reindex(db, provider, { - documentIds: params.documentIds, - since: params.since, - before: params.before, - batchSize: params.batchSize, - signal, - onProgress: (p) => onProgress(p.completed, p.total), - }); - return ( - `Reindex complete.\n` + - `Total chunks: ${result.total}\n` + - `Updated: ${result.completed}\n` + - `Failed: ${result.failed}` + - (result.failedChunkIds.length > 0 - ? `\nFailed chunk IDs: ${result.failedChunkIds.join(", ")}` - : "") - ); - }); - }), - ); - - // Tool: sync-slack - server.tool( - "sync-slack", - "Sync Slack channel messages and threads into the knowledge base", - { - token: z - .string() - .optional() - .describe( - "Slack bot token (xoxb-...) or user token (xoxp-...). Optional when a saved Slack connector exists.", - ), - channels: z - .array(z.string()) - .optional() - .describe("Channel names or IDs to sync, or ['all'] (default: saved config, else ['all'])"), - excludeChannels: z - .array(z.string()) - .optional() - .describe("Channel names to exclude from sync"), - threadMode: z - .enum(["aggregate", "separate"]) - .optional() - .describe( - "Thread handling: aggregate (default) combines thread into one doc, separate creates one doc per reply", - ), - name: z - .string() - .optional() - .describe( - 'Saved connector config name (default: "slack"). Values saved by the CLI connect command fill in any parameter you omit.', - ), - async: z - .boolean() - .optional() - .describe( - "When true, run the sync in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { syncSlack: doSyncSlack } = await import("../connectors/slack.js"); - const { resolveSyncConfig } = await import("../connectors/saved-config.js"); - - const name = params.name ?? "slack"; - const { config: slackConfig } = resolveSyncConfig<SlackConfig>( - "slack", - name, - { - token: params.token, - channels: params.channels, - excludeChannels: params.excludeChannels, - threadMode: params.threadMode, - }, - { defaults: { channels: ["all"], threadMode: "aggregate" }, required: ["token"] }, - ); - - return runMaybeAsync(params.async, "sync_connector", async (signal) => { - const result = await doSyncSlack(db, provider, slackConfig, { syncName: name, signal }); - const slackErrorLines = result.errors.map((e) => ` #${e.channel}: ${e.error}`).join("\n"); - const slackErrors = result.errors.length > 0 ? `\nErrors:\n${slackErrorLines}` : ""; - return ( - `Slack sync complete.\n` + - `Channels: ${result.channels}\n` + - `Messages indexed: ${result.messagesIndexed}\n` + - `Threads indexed: ${result.threadsIndexed}` + - slackErrors - ); - }); + return textResult(options.format(await runOperation(op, ctx, input))); }), ); + return name; +} - // Tool: install-pack - server.tool( - "install-pack", - "Install a knowledge pack from the registry or a local file path", - { - nameOrPath: z.string().describe("Pack name (from registry) or local .json file path"), - registryUrl: z.string().optional().describe("Custom registry URL"), - async: z - .boolean() - .optional() - .describe( - "When true, run installation in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { installPack } = await import("../core/packs.js"); - - return runMaybeAsync(params.async, "install_pack", async (signal, onProgress) => { - const result = await installPack(db, provider, params.nameOrPath, { - registryUrl: params.registryUrl, - signal, - onProgress: (current, total) => onProgress(current, total), - }); - return result.alreadyInstalled - ? `Pack "${result.packName}" is already installed.` - : `Pack "${result.packName}" installed successfully (${result.documentsInstalled} documents).`; - }); - }), - ); - - // Tool: list-packs - server.tool( - "list-packs", - "List installed knowledge packs or available packs from the registry", - { - available: z - .boolean() - .optional() - .describe("If true, list available packs from registry instead of installed packs"), - registryUrl: z.string().optional().describe("Custom registry URL"), - }, - withErrorHandling(async (params) => { - if (params.available) { - const { listAvailablePacks } = await import("../core/packs.js"); - const packs = await listAvailablePacks(params.registryUrl); - if (packs.length === 0) { - return textResult("No packs available in the registry."); - } - const text = packs - .map((p) => `- **${p.name}** v${p.version} — ${p.description} (${p.docCount} docs)`) - .join("\n"); - return textResult(`## Available Packs\n\n${text}`); - } - - const { listInstalledPacks } = await import("../core/packs.js"); - const packs = listInstalledPacks(db); - if (packs.length === 0) { - return textResult("No packs installed."); - } - const text = packs - .map( - (p) => - `- **${p.name}** v${p.version} — ${p.description ?? ""} (${p.docCount} docs, installed ${p.installedAt})`, - ) - .join("\n"); - return textResult(`## Installed Packs\n\n${text}`); - }), - ); - - // Tool: sync-onenote - server.tool( - "sync-onenote", - "Sync OneNote notebooks via Microsoft Graph API", - { - accessToken: z - .string() - .optional() - .describe( - "Microsoft Graph API access token. Optional when a saved OneNote connector exists (its refresh token is used).", - ), - notebookName: z.string().optional().describe("Specific notebook name to sync (default: all)"), - name: z - .string() - .optional() - .describe( - 'Saved connector config name (default: "onenote"). Values saved by the CLI connect command fill in any parameter you omit.', - ), - async: z - .boolean() - .optional() - .describe( - "When true, run the sync in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { syncOneNote } = await import("../connectors/onenote.js"); - const { resolveSyncConfig, saveRefreshedOneNoteTokens } = - await import("../connectors/saved-config.js"); - - const name = params.name ?? "onenote"; - const { config: oneNoteConfig, saved } = resolveSyncConfig<OneNoteConfig>( - "onenote", - name, - { - accessToken: params.accessToken, - notebooks: params.notebookName ? [params.notebookName] : undefined, - }, - { defaults: { clientId: "", tenantId: "common", notebooks: ["all"], excludeSections: [] } }, - ); - if (params.accessToken) { - // Use the caller's token as given; never replace it with a refreshed one. - oneNoteConfig.refreshToken = undefined; - oneNoteConfig.tokenExpiry = undefined; - } - if (!oneNoteConfig.accessToken && !oneNoteConfig.refreshToken) { - throw new ValidationError( - `Missing accessToken for onenote sync. Pass it as a parameter, or save a connector named "${name}" with 'libscope connect onenote'.`, - ); - } - - return runMaybeAsync(params.async, "sync_connector", async (signal) => { - const result = await syncOneNote(db, provider, oneNoteConfig, { syncName: name, signal }); - saveRefreshedOneNoteTokens(name, saved, oneNoteConfig); - const oneNoteErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); - const oneNoteErrors = result.errors.length > 0 ? `\nErrors: ${oneNoteErrorLines}` : ""; - return ( - `OneNote sync complete.\n` + - `Notebooks: ${result.notebooks}\n` + - `Sections: ${result.sections}\n` + - `Pages added: ${result.pagesAdded}\n` + - `Pages updated: ${result.pagesUpdated}\n` + - `Pages deleted: ${result.pagesDeleted}` + - oneNoteErrors - ); - }); - }), - ); - - // Tool: sync-notion - server.tool( - "sync-notion", - "Sync pages and databases from a connected Notion workspace into the knowledge base", - { - token: z - .string() - .optional() - .describe( - "Notion integration token (secret_... or ntn_...). Optional when a saved Notion connector exists.", - ), - lastSync: z - .string() - .optional() - .describe("ISO-8601 timestamp — only sync pages edited after this time"), - excludePages: z - .array(z.string()) - .optional() - .describe("List of Notion page/database IDs to exclude from sync"), - name: z - .string() - .optional() - .describe( - 'Saved connector config name (default: "notion"). Values saved by the CLI connect command fill in any parameter you omit.', - ), - async: z - .boolean() - .optional() - .describe( - "When true, run the sync in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { syncNotion } = await import("../connectors/notion.js"); - const { resolveSyncConfig } = await import("../connectors/saved-config.js"); - - const name = params.name ?? "notion"; - const { config: notionConfig } = resolveSyncConfig<NotionConfig>( - "notion", - name, - { token: params.token, lastSync: params.lastSync, excludePages: params.excludePages }, - { required: ["token"] }, - ); +/** One action of a multi-action tool. */ +export interface ToolAction { + op: Operation; + format: Formatter<unknown>; +} - return runMaybeAsync(params.async, "sync_connector", async (signal) => { - const result = await syncNotion(db, provider, notionConfig, { syncName: name, signal }); - const notionErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); - const notionErrors = result.errors.length > 0 ? `\nErrors: ${notionErrorLines}` : ""; - return ( - `Notion sync complete.\n` + - `Pages indexed: ${result.pagesIndexed}\n` + - `Databases indexed: ${result.databasesIndexed}` + - notionErrors - ); - }); - }), - ); +/** Pair an operation with its formatter, for registerActionTool. */ +export function toolAction<S extends z.ZodObject, O>( + op: Operation<S, O>, + format: Formatter<O>, +): ToolAction { + return { op: op as unknown as Operation, format: format as Formatter<unknown> }; +} - // Tool: sync-obsidian-vault - server.tool( - "sync-obsidian-vault", - "Sync an Obsidian vault into the knowledge base. Parses wikilinks, frontmatter, embeds, and tags with incremental sync support.", +/** + * Register one tool that runs one of several operations, chosen by its `action` field. + * Each field is optional in the tool schema and validated by the chosen operation. + */ +export function registerActionTool( + server: McpServer, + ctx: OperationContext, + tool: { name: string; description: string; actions: Record<string, ToolAction> }, +): string { + const names = Object.keys(tool.actions) as [string, ...string[]]; + const fields = new Map<string, { schema: z.ZodType; actions: string[] }>(); + for (const [action, { op }] of Object.entries(tool.actions)) { + for (const [key, schema] of Object.entries(shapeOf(op))) { + const field = fields.get(key) ?? { schema, actions: [] }; + field.actions.push(action); + fields.set(key, field); + } + } + const shape: Record<string, z.ZodType> = { + action: z.enum(names).describe(`What to do: ${names.join(", ")}`), + }; + for (const [key, { schema, actions }] of fields) { + shape[key] = schema + .optional() + .describe(`${schema.description ?? key} (action: ${actions.join(", ")})`); + } + server.registerTool( + tool.name, { - vaultPath: z - .string() - .optional() - .describe( - "Absolute path to the Obsidian vault directory. Optional when a saved Obsidian connector exists.", - ), - name: z - .string() - .optional() - .describe( - 'Saved connector config name (default: "obsidian"). Values saved by the CLI connect command fill in any parameter you omit.', - ), - async: z - .boolean() - .optional() - .describe( - "When true, run the sync in the background and return a task ID immediately. Use get-task to poll for completion.", - ), + description: tool.description, + inputSchema: z.object(shape), + annotations: toolAnnotations(Object.values(tool.actions).map((a) => a.op.annotations ?? {})), }, - withErrorHandling(async (params) => { - const { syncObsidianVault } = await import("../connectors/obsidian.js"); - const { resolveSyncConfig } = await import("../connectors/saved-config.js"); - - const name = params.name ?? "obsidian"; - const { config: obsidianConfig } = resolveSyncConfig<ObsidianConfig>( - "obsidian", - name, - { vaultPath: params.vaultPath }, - { defaults: { topicMapping: "folder", excludePatterns: [] }, required: ["vaultPath"] }, - ); - - return runMaybeAsync(params.async, "sync_connector", async (signal) => { - const result = await syncObsidianVault(db, provider, obsidianConfig, { - syncName: name, - signal, - }); - const obsidianErrorLines = result.errors.map((e) => `${e.file}: ${e.error}`).join(", "); - const obsidianErrors = result.errors.length > 0 ? `\nErrors: ${obsidianErrorLines}` : ""; - return ( - `Obsidian vault sync complete.\n` + - `Added: ${result.added}\n` + - `Updated: ${result.updated}\n` + - `Deleted: ${result.deleted}` + - obsidianErrors - ); - }); + withErrorHandling(async (args: Record<string, unknown>) => { + const { action, ...input } = args; + const chosen = tool.actions[String(action)]; + if (!chosen) throw new Error(`Unknown action: ${String(action)}`); + return textResult(chosen.format(await runOperation(chosen.op, ctx, input))); }), ); + return tool.name; +} - // Tool: sync-confluence - server.tool( - "sync-confluence", - "Sync Confluence spaces and pages into the knowledge base", - { - baseUrl: z - .string() - .optional() - .describe( - "Confluence base URL (e.g. https://acme.atlassian.net). Optional when a saved Confluence connector exists.", - ), - email: z.string().optional().describe("Confluence user email (Cloud). Optional when saved."), - token: z.string().optional().describe("API token or PAT. Optional when saved."), - spaces: z - .array(z.string()) - .optional() - .describe("Space keys to sync, or ['all'] (default: saved config, else ['all'])"), - excludeSpaces: z.array(z.string()).optional().describe("Space keys to exclude"), - name: z - .string() - .optional() - .describe( - 'Saved connector config name (default: "confluence"). Values saved by the CLI connect command fill in any parameter you omit.', - ), - async: z - .boolean() - .optional() - .describe( - "When true, run the sync in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { syncConfluence } = await import("../connectors/confluence.js"); - const { resolveSyncConfig } = await import("../connectors/saved-config.js"); - - const name = params.name ?? "confluence"; - const { config: confluenceConfig } = resolveSyncConfig<ConfluenceConfig>( - "confluence", - name, - { - baseUrl: params.baseUrl, - email: params.email, - token: params.token, - spaces: params.spaces, - excludeSpaces: params.excludeSpaces, - }, - { defaults: { spaces: ["all"] }, required: ["baseUrl", "token"] }, +/** Toolsets named in `value` (comma list; "all" enables every optional toolset). */ +export function parseToolsets(value: string | undefined): Set<McpToolset> { + const enabled = new Set<McpToolset>(); + for (const raw of (value ?? "").split(",")) { + const name = raw.trim().toLowerCase(); + if (name === "" || name === "core") continue; + if (name === "all") { + for (const t of MCP_TOOLSETS) enabled.add(t); + } else if ((MCP_TOOLSETS as readonly string[]).includes(name)) { + enabled.add(name as McpToolset); + } else { + getLogger().warn( + `Unknown MCP toolset "${name}" in ${TOOLSETS_ENV} (known: ${MCP_TOOLSETS.join(", ")}, all)`, ); + } + } + return enabled; +} - return runMaybeAsync(params.async, "sync_connector", async (signal) => { - const result = await syncConfluence(db, provider, confluenceConfig, { - syncName: name, - signal, - }); - const confluenceErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join(", "); - const confluenceErrors = - result.errors.length > 0 ? `\nErrors: ${confluenceErrorLines}` : ""; - return ( - `Confluence sync complete.\n` + - `Spaces: ${result.spaces}\n` + - `Pages indexed: ${result.pagesIndexed}\n` + - `Pages updated: ${result.pagesUpdated}` + - confluenceErrors - ); - }); - }), - ); - - // Tool: search-analytics - server.tool( - "search-analytics", - "View search analytics dashboard and knowledge gap detection", - { - days: z.number().optional().describe("Look-back period in days (default: 30)"), - }, - withErrorHandling(async (params) => { - const { getSearchAnalytics, getKnowledgeGaps } = await import("../core/analytics.js"); - const days = params.days ?? 30; - const analytics = getSearchAnalytics(db, days); - const gaps = getKnowledgeGaps(db, days); - - const lines: string[] = [ - `Search Analytics (last ${days} days)`, - `Total searches: ${analytics.totalSearches}`, - `Avg result count: ${analytics.avgResultCount}`, - "", - "Top queries:", - ...analytics.topQueries.map((q) => ` ${q.count}x ${q.query}`), - "", - "Zero-result queries:", - ...analytics.zeroResultQueries.map((q) => ` ${q.count}x ${q.query}`), - "", - "Knowledge gaps:", - ...gaps.map((g) => ` ${g.count}x ${g.query} (last: ${g.lastSearched})`), - ]; - return textResult(lines.join("\n")); - }), - ); - - // Tool: link-documents - server.tool( - "link-documents", - `Create a relationship between two documents (${LINK_TYPES.join(", ")})`, - { - sourceId: z.string().describe("The source document ID"), - targetId: z.string().describe("The target document ID"), - linkType: z.enum(LINK_TYPES).describe("Type of relationship"), - label: z.string().optional().describe("Optional human-readable description of the link"), - }, - withErrorHandling((params) => { - const link = createLink(db, params.sourceId, params.targetId, params.linkType, params.label); - return textResult(formatLinkCreated(link)); - }), - ); +/** True when `ask` can run: passthrough (the calling assistant answers) or a configured LLM. */ +function askAvailable(ctx: OperationContext): boolean { + return ctx.isPassthrough() || ctx.getLlm() !== null; +} - // Tool: get-document-links - server.tool( - "get-document-links", - "Get all cross-reference links for a document (both outgoing and incoming)", - { - documentId: z.string().describe("The document ID"), - }, - withErrorHandling((params) => { - return textResult(formatDocumentLinks(getDocumentLinks(db, params.documentId))); - }), - ); +/** Server `instructions`: how an assistant should use the tools. */ +export function buildInstructions(options: { + ask: boolean; + passthrough: boolean; + admin: boolean; +}): string { + const lines = [ + "libscope is a local knowledge base of documents (library docs, wikis, notes), split into chunks that are searched by meaning and keywords.", + "", + "Workflow:", + "1. search {query} finds relevant chunks. Every result has a documentId and chunkId. Narrow with topic, library, version, sourceType, tags or minRating; page with offset. search {relatedTo: documentId or chunkId} finds similar content.", + "2. get-document {documentId} reads a document with its tags, links and ratings. Long documents are paged: pass maxLength, then the offset shown as 'next page'.", + "3. rate-document {documentId, rating 1-5} after you use a document; add feedback or suggestedCorrection when it is wrong or out of date.", + ]; + if (options.ask) { + lines.push( + options.passthrough + ? "- ask {question} returns the retrieved context without calling an LLM: write the answer yourself and cite the documentIds." + : "- ask {question} answers from the knowledge base with the configured LLM and lists its sources.", + ); + } + lines.push( + "- submit-document adds inline content (with title), a web page (url), a site crawl (url + spider: true) or a public repository URL.", + '- Slow work (crawls, repositories) accepts async: true and returns a taskId. Poll task {"action": "status", taskId} until the status is completed or failed; task {"action": "cancel"} stops it.', + "- overview shows counts, topics, installed packs and index health. list-documents pages through documents (offset, total).", + '- update-document (including tags), delete-document and link-documents {"action": "create" | "delete"} maintain the knowledge base. Deleting cannot be undone.', + ); + if (options.admin) { + lines.push( + "- Admin tools: sync runs saved connector connections (set up with the libscope CLI), install-pack and list-packs manage knowledge packs, reindex-documents re-embeds chunks after an embedding model change.", + ); + } + return lines.join("\n"); +} - // Tool: delete-link - server.tool( - "delete-link", - "Remove a cross-reference link between documents", - { - linkId: z.string().describe("The link ID to delete"), - }, - withErrorHandling((params) => { - deleteLink(db, params.linkId); - return textResult(`✓ Link ${params.linkId} deleted.`); - }), - ); +function packageVersion(): string { + const require = createRequire(import.meta.url); + return (require("../../package.json") as { version: string }).version; +} - // Tool: save-search - server.tool( - "save-search", - "Save a search query with optional filters for later re-use", - { - name: z.string().describe("A unique name for this saved search"), - query: z.string().describe("The search query"), - topic: z.string().optional().describe("Filter by topic ID"), - library: z.string().optional().describe("Filter by library name"), - version: z.string().optional().describe("Filter by library version"), - source: z.string().optional().describe("Filter by source type"), - minRating: z.number().min(1).max(5).optional().describe("Minimum average rating filter"), - limit: z.number().min(1).max(50).optional().describe("Maximum results to return"), - tags: z.array(z.string()).optional().describe("Filter by tags"), - }, - withErrorHandling((params) => { - const { name, query, ...rest } = params; - const filters: Record<string, unknown> = {}; - if (rest.topic !== undefined) filters.topic = rest.topic; - if (rest.library !== undefined) filters.library = rest.library; - if (rest.version !== undefined) filters.version = rest.version; - if (rest.source !== undefined) filters.source = rest.source; - if (rest.minRating !== undefined) filters.minRating = rest.minRating; - if (rest.limit !== undefined) filters.limit = rest.limit; - if (rest.tags !== undefined) filters.tags = rest.tags; - const saved = createSavedSearch( - db, - name, - query, - Object.keys(filters).length > 0 ? filters : undefined, - ); - return textResult(JSON.stringify(saved, null, 2)); - }), - ); +export interface CreateMcpServerOptions extends BootstrapOptions { + /** Use this operation context instead of bootstrapping one. The caller owns its database. */ + ctx?: OperationContext | undefined; + /** Optional toolsets to enable (default: from LIBSCOPE_MCP_TOOLSETS). */ + toolsets?: readonly string[] | undefined; +} - // Tool: suggest-tags - server.tool( - "suggest-tags", - "Suggest tags for a document based on content analysis", - { - documentId: z.string().describe("Document ID"), - maxSuggestions: z.number().min(1).max(20).optional().describe("Max suggestions (default: 5)"), - }, - withErrorHandling((params) => { - const suggestions = suggestTags(db, params.documentId, params.maxSuggestions); - return textResult(JSON.stringify({ documentId: params.documentId, suggestions }, null, 2)); - }), - ); +export interface LibScopeMcpServer { + server: McpServer; + ctx: OperationContext; + /** Names of the registered tools, in registration order. */ + tools: string[]; + /** Close the MCP server, and the database if createMcpServer opened it. */ + close(): Promise<void>; +} - // Tool: list-saved-searches - server.tool( - "list-saved-searches", - "List all saved searches", - {}, - withErrorHandling(() => { - const searches = listSavedSearches(db); - return textResult( - searches.length === 0 ? "No saved searches." : JSON.stringify(searches, null, 2), - ); - }), - ); +type AddTool = <S extends z.ZodObject, O>( + op: Operation<S, O>, + options: OperationToolOptions<O>, +) => void; + +/** Register the 11 core tools (10 when `ask` is unavailable), except `task`. */ +function registerCoreTools(ctx: OperationContext, withAsk: boolean, add: AddTool): void { + add(searchOperation, { + format: fmt.formatSearchResults, + description: + "Search the knowledge base by meaning and keywords (query), or find content similar to a document or chunk (relatedTo). Results carry documentId and chunkId.", + }); + if (withAsk) { + add(askOperation, { + format: fmt.formatAnswer, + description: ctx.isPassthrough() + ? "Retrieve the knowledge-base context for a question (passthrough: no LLM is called; answer from the returned context yourself)" + : askOperation.summary, + }); + } + add(getDocumentOperation, { format: fmt.formatDocumentView }); + add(listDocumentsOperation, { format: fmt.formatDocumentList }); + add(overviewOperation, { format: fmt.formatOverview }); + add(addOperation, { + name: "submit-document", + description: + "Add to the knowledge base: inline content (with title), a web page (url), a site crawl (url + spider: true) or a public GitHub/GitLab repository URL. Local file paths are not accepted.", + omit: ["source", "kind", "format", "include", "exclude", "token"], + format: fmt.formatIngestResult, + }); + add(updateDocumentOperation, { format: fmt.formatDocumentUpdated }); + add(deleteDocumentOperation, { format: fmt.formatDocumentDeleted }); + add(rateDocumentOperation, { format: fmt.formatRating }); +} - // Tool: run-saved-search - server.tool( - "run-saved-search", - "Execute a saved search by name or ID and return results", - { - nameOrId: z.string().describe("The name or ID of the saved search to run"), - }, - withErrorHandling(async (params) => { - const { search, results } = await runSavedSearch(db, provider, params.nameOrId); - return textResult(JSON.stringify({ search, resultCount: results.length, results }, null, 2)); - }), - ); +/** Register the admin toolset. */ +function registerAdminTools(add: AddTool): void { + add(syncOperation, { + description: + "Sync one saved connector connection (name) or all of them (all: true) with the settings saved by 'libscope connect'", + format: fmt.formatSyncResult, + }); + add(installPackOperation, { format: fmt.formatInstallPack }); + add(listPacksOperation, { format: fmt.formatPackList }); + add(reindexOperation, { name: "reindex-documents", format: fmt.formatReindex }); +} - // Tool: delete-saved-search - server.tool( - "delete-saved-search", - "Delete a saved search by name or ID", - { - nameOrId: z.string().describe("The name or ID of the saved search to delete"), - }, - withErrorHandling((params) => { - deleteSavedSearch(db, params.nameOrId); - return textResult(`✓ Saved search "${params.nameOrId}" deleted.`); - }), - ); +/** + * Build the libscope MCP server. Without `ctx`, opens the database with `bootstrap(options)`. + * Does not connect a transport. + */ +export function createMcpServer(options: CreateMcpServerOptions = {}): LibScopeMcpServer { + const { ctx: givenCtx, toolsets, ...bootOptions } = options; + let boot: Bootstrapped | undefined; + let ctx = givenCtx; + if (!ctx) { + boot = bootstrap(bootOptions); + ctx = createOperationContext({ + db: boot.db, + provider: boot.provider, + config: boot.config, + surface: "mcp", + }); + } + const opCtx = ctx; + const enabled = parseToolsets(toolsets ? toolsets.join(",") : process.env[TOOLSETS_ENV]); + const withAsk = askAvailable(opCtx); + const admin = enabled.has("admin"); - // Tool: create-webhook - server.tool( - "create-webhook", - "Register a webhook to receive notifications for document events", + const server = new McpServer( + { name: "libscope", version: packageVersion() }, { - url: z.string().describe("The URL to send webhook POST requests to (http:// or https://)"), - events: z - .array(z.string()) - .describe( - "Event types to subscribe to: document.created, document.updated, document.deleted, document.rated, search.executed", - ), - secret: z - .string() - .optional() - .describe("Optional secret for HMAC-SHA256 signature verification"), + instructions: buildInstructions({ ask: withAsk, passthrough: opCtx.isPassthrough(), admin }), }, - withErrorHandling(async (params) => { - const webhook = await createWebhook( - db, - params.url, - params.events as WebhookEvent[], - params.secret, - ); - return textResult(JSON.stringify(redactWebhook(webhook), null, 2)); - }), - ); - - // Tool: list-webhooks - server.tool( - "list-webhooks", - "List all registered webhooks", - {}, - withErrorHandling(() => { - const webhooks = listWebhooks(db); - return textResult(JSON.stringify(webhooks.map(redactWebhook), null, 2)); - }), ); - // Tool: delete-webhook - server.tool( - "delete-webhook", - "Remove a registered webhook by ID", - { - id: z.string().describe("The webhook ID to delete"), - }, - withErrorHandling((params) => { - deleteWebhook(db, params.id); - return textResult(`✓ Webhook "${params.id}" deleted.`); + const tools: string[] = []; + // Formatters of tools that can run in the background, so `task` can format their results. + const taskFormatters = new Map<string, Formatter<unknown>>(); + const add: AddTool = (op, toolOptions) => { + tools.push(registerOperationTool(server, op, opCtx, toolOptions)); + if (op.annotations?.longRunning) { + taskFormatters.set(op.name, toolOptions.format as Formatter<unknown>); + } + }; + + registerCoreTools(opCtx, withAsk, add); + tools.push( + registerActionTool(server, opCtx, { + name: "link-documents", + description: + "Create a typed link from one document to another (action: create), or delete a link by linkId (action: delete). get-document lists a document's links.", + actions: { + create: toolAction(linkDocumentsOperation, fmt.formatLinkCreated), + delete: toolAction(unlinkDocumentsOperation, fmt.formatLinkDeleted), + }, }), ); - - // Tool: get-task - server.tool( - "get-task", - "Get the current status, progress, and result of an async background task", - { - taskId: z.string().describe("Task ID returned by an async operation"), - }, - withErrorHandling((params) => { - const task = taskRegistry.get(params.taskId); - if (!task) { - return textResult( - `Task ${params.taskId} not found or has expired (tasks are kept for 1 hour after completion).`, - ); - } - return textResult(JSON.stringify(task, null, 2)); + const formatTaskResult: fmt.TaskResultFormatter = (operation, result) => + taskFormatters.get(operation)?.(result); + tools.push( + registerActionTool(server, opCtx, { + name: "task", + description: + "Background tasks started with async: true. status: progress and result of a task; cancel: stop it; list: tasks from the last hour.", + actions: { + status: toolAction(getTaskOperation, (task) => fmt.formatTask(task, formatTaskResult)), + cancel: toolAction(cancelTaskOperation, fmt.formatTaskCancel), + list: toolAction(listTasksOperation, fmt.formatTaskList), + }, }), ); + if (admin) registerAdminTools(add); - // Tool: cancel-task - server.tool( - "cancel-task", - "Request cancellation of a pending or running async background task", - { - taskId: z.string().describe("Task ID to cancel"), + return { + server, + ctx: opCtx, + tools, + close: async (): Promise<void> => { + await server.close(); + boot?.close(); }, - withErrorHandling((params) => { - const outcome = taskRegistry.cancel(params.taskId); - if (outcome === "not_found") { - return textResult(`Task ${params.taskId} not found or has expired.`); - } - if (outcome === "already_terminal") { - const task = taskRegistry.get(params.taskId); - const status = task?.status ?? "unknown"; - return textResult(`Task ${params.taskId} cannot be cancelled (current status: ${status}).`); - } - return textResult( - `Cancellation requested for task ${params.taskId}. Running operations will stop at the next checkpoint.`, - ); - }), - ); + }; +} - const transport = new StdioServerTransport(); - await server.connect(transport); +export interface RunStdioServerOptions extends CreateMcpServerOptions { + /** Log level (default: config logging.level). Logs always go to stderr. */ + logLevel?: LogLevel | undefined; } -main().catch((err: unknown) => { - console.error("Fatal error starting LibScope MCP server:", err); - process.exit(1); -}); +/** + * Start the MCP server on stdin/stdout. stdout carries only JSON-RPC: logs go to stderr. + * Closes the server and database on SIGINT/SIGTERM. + */ +export async function runStdioServer( + options: RunStdioServerOptions = {}, +): Promise<LibScopeMcpServer> { + const { logLevel, ...serverOptions } = options; + const config = options.ctx?.config ?? options.config ?? loadConfig(); + initLogger(logLevel ?? config.logging.level, { destination: "stderr" }); + const mcp = createMcpServer({ ...serverOptions, config }); + const shutdown = (): void => { + mcp.close().then( + () => process.exit(0), + () => process.exit(1), + ); + }; + process.once("SIGINT", shutdown); + process.once("SIGTERM", shutdown); + await mcp.server.connect(new StdioServerTransport()); + return mcp; +} diff --git a/tests/unit/mcp-entry.test.ts b/tests/unit/mcp-entry.test.ts new file mode 100644 index 00000000..532ea4bb --- /dev/null +++ b/tests/unit/mcp-entry.test.ts @@ -0,0 +1,87 @@ +import { describe, it, expect, vi } from "vitest"; +import { initLogger } from "../../src/logger.js"; +import { makeContext } from "./operations/helpers.js"; + +const mocks = vi.hoisted(() => ({ + transports: [] as unknown[], + bootstrap: vi.fn(), + initLogger: vi.fn(), +})); + +// A stand-in stdio transport, so no test touches the real stdin/stdout. +vi.mock("@modelcontextprotocol/sdk/server/stdio.js", () => ({ + StdioServerTransport: class { + onclose?: () => void; + constructor() { + mocks.transports.push(this); + } + start(): Promise<void> { + return Promise.resolve(); + } + send(): Promise<void> { + return Promise.resolve(); + } + close(): Promise<void> { + this.onclose?.(); + return Promise.resolve(); + } + }, +})); + +vi.mock("../../src/core/bootstrap.js", async (importOriginal) => { + const orig = await importOriginal<typeof import("../../src/core/bootstrap.js")>(); + return { + ...orig, + bootstrap: (...args: Parameters<typeof orig.bootstrap>): ReturnType<typeof orig.bootstrap> => { + mocks.bootstrap(); + return orig.bootstrap(...args); + }, + }; +}); + +vi.mock("../../src/logger.js", async (importOriginal) => { + const orig = await importOriginal<typeof import("../../src/logger.js")>(); + return { + ...orig, + initLogger: (...args: Parameters<typeof orig.initLogger>): void => { + mocks.initLogger(...args); + orig.initLogger(...args); + }, + }; +}); + +initLogger("silent"); + +describe("libscope/mcp entry points", () => { + it("importing the server module starts nothing", async () => { + const sigint = process.listenerCount("SIGINT"); + const mod = await import("../../src/mcp/server.js"); + expect(typeof mod.createMcpServer).toBe("function"); + expect(typeof mod.runStdioServer).toBe("function"); + expect(mocks.transports).toHaveLength(0); + expect(mocks.bootstrap).not.toHaveBeenCalled(); + expect(process.listenerCount("SIGINT")).toBe(sigint); + }); + + it("runStdioServer logs to stderr and connects the stdio transport", async () => { + const { runStdioServer } = await import("../../src/mcp/server.js"); + const t = makeContext({ surface: "mcp", llm: undefined }); + const before = { SIGINT: process.listeners("SIGINT"), SIGTERM: process.listeners("SIGTERM") }; + const mcp = await runStdioServer({ ctx: t.ctx, logLevel: "silent", toolsets: [] }); + try { + expect(mocks.initLogger).toHaveBeenCalledWith("silent", { destination: "stderr" }); + expect(mocks.transports).toHaveLength(1); + expect(mocks.bootstrap).not.toHaveBeenCalled(); + expect(mcp.tools).toHaveLength(11); + expect(process.listenerCount("SIGINT")).toBe(before.SIGINT.length + 1); + } finally { + for (const signal of ["SIGINT", "SIGTERM"] as const) { + for (const listener of process.listeners(signal)) { + if (!before[signal].includes(listener)) process.removeListener(signal, listener); + } + } + await mcp.close(); + t.db.close(); + } + }); +}); diff --git a/tests/unit/mcp-format.test.ts b/tests/unit/mcp-format.test.ts index 92f2426e..77789d4f 100644 --- a/tests/unit/mcp-format.test.ts +++ b/tests/unit/mcp-format.test.ts @@ -1,14 +1,21 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import type Database from "better-sqlite3"; -import { z } from "zod"; -import { createTestDb } from "../fixtures/test-db.js"; -import { insertDoc } from "../fixtures/helpers.js"; -import { createLink, deleteLink, getDocumentLinks, LINK_TYPES } from "../../src/core/links.js"; +import { describe, it, expect } from "vitest"; import type { SearchResult } from "../../src/core/search.js"; +import type { Task } from "../../src/core/tasks.js"; +import type { DocumentView } from "../../src/core/document-view.js"; import { - formatDocumentLinks, - formatLinkCreated, + formatAnswer, + formatDocumentList, + formatDocumentView, + formatIngestResult, + formatOverview, + formatPackList, + formatRating, + formatReindex, formatSearchResults, + formatSyncResult, + formatTask, + formatTaskCancel, + formatTaskList, } from "../../src/mcp/format.js"; function makeResult(overrides: Partial<SearchResult> = {}): SearchResult { @@ -29,23 +36,32 @@ function makeResult(overrides: Partial<SearchResult> = {}): SearchResult { }; } -describe("MCP output formatting", () => { - describe("formatSearchResults", () => { - it("returns the no-results message for an empty list", () => { - expect(formatSearchResults([], 0)).toBe("No documents found matching your query."); - }); +function makeTask(overrides: Partial<Task> = {}): Task { + return { + id: "task-1", + type: "operation", + operation: "add", + status: "running", + createdAt: new Date("2026-01-01T00:00:00Z"), + ...overrides, + }; +} - it("includes the document ID and chunk ID of every result", () => { - const text = formatSearchResults( - [makeResult(), makeResult({ documentId: "doc-2", chunkId: "chunk-2", title: "Doc Two" })], - 2, - ); - expect(text).toContain("**Document ID:** doc-1 | **Chunk ID:** chunk-1"); - expect(text).toContain("**Document ID:** doc-2 | **Chunk ID:** chunk-2"); - }); +const page = <T>( + items: T[], + total = items.length, + offset = 0, +): { items: T[]; total: number; limit: number; offset: number } => ({ + items, + total, + limit: 10, + offset, +}); - it("keeps the existing header, metadata, context, and separator layout", () => { - const text = formatSearchResults( +describe("MCP output formatting", () => { + it("search results carry documentId, chunkId, metadata and context", () => { + const text = formatSearchResults( + page( [ makeResult({ library: "react", @@ -55,81 +71,218 @@ describe("MCP output formatting", () => { contextBefore: [{ chunkId: "c0", content: "before text", chunkIndex: 0 }], contextAfter: [{ chunkId: "c2", content: "after text", chunkIndex: 2 }], }), - makeResult({ title: "Second" }), + makeResult({ documentId: "doc-2", chunkId: "chunk-2", title: "Second" }), ], - 7, - ); - expect(text.startsWith("**Total results: 7**\n\n## Result 1: Doc One (score: 0.50)\n")).toBe( - true, - ); - expect(text).toContain("**Library:** react v18\n"); - expect(text).toContain("**Source:** https://example.com\n"); - expect(text).toContain("**Rating:** 4.0/5\n"); - expect(text).toContain("**Context (before):**\nbefore text\n"); - expect(text).toContain("\nchunk body\n"); - expect(text).toContain("**Context (after):**\nafter text\n"); - expect(text).toContain("\n---\n\n## Result 2: Second"); - }); + 20, + 10, + ), + ); + expect(text).toMatch(/^Results 11-12 of 20 — next page: offset 12\n/); + expect(text).toContain("[11] Doc One · score 0.50 · rating 4.0/5"); + expect(text).toContain( + "documentId: doc-1 · chunkId: chunk-1 · react v18 · https://example.com", + ); + expect(text).toContain( + "[context before]\nbefore text\nchunk body\n[context after]\nafter text", + ); + expect(text).toContain("---\n[12] Second"); + expect(text).toContain("documentId: doc-2 · chunkId: chunk-2"); }); - describe("link formatting", () => { - let db: Database.Database; + it("an empty page past the end says so", () => { + expect(formatSearchResults(page([], 3, 10))).toBe("No more results (total 3)."); + expect(formatSearchResults(page([], 0))).toBe("No results found."); + }); - beforeEach(() => { - db = createTestDb(); - insertDoc(db, "doc-a", "Document A"); - insertDoc(db, "doc-b", "Document B"); - }); + it("search header offers the next page when more results exist", () => { + expect(formatSearchResults(page([makeResult()], 5))).toMatch( + /^Results 1-1 of 5 — next page: offset 1\n/, + ); + }); - afterEach(() => { - db.close(); - }); + it("ask: passthrough context and LLM answers list sources with documentIds", () => { + const sources = [{ documentId: "doc-1", title: "Doc One", chunk: "x", score: 0.9 }]; + const context = formatAnswer({ mode: "context", contextPrompt: "PROMPT", sources }); + expect(context).toMatch(/^Passthrough mode: no LLM was called/); + expect(context).toContain("PROMPT\n\nSources:\n- Doc One (documentId: doc-1, score 0.90)"); - it("formatLinkCreated includes the link ID, which delete-link accepts", () => { - const link = createLink(db, "doc-a", "doc-b", "see_also", "read next"); - const text = formatLinkCreated(link); - expect(text).toContain("doc-a → doc-b (see_also) — read next"); - const id = /Link ID: (\S+)/.exec(text)?.[1]; - expect(id).toBe(link.id); - deleteLink(db, id!); - expect(getDocumentLinks(db, "doc-a").outgoing).toHaveLength(0); + const answer = formatAnswer({ mode: "answer", answer: "Yes.", sources: [], model: "m" }); + expect(answer).toBe("Yes.\n\n(model: m)"); + const withTokens = formatAnswer({ + mode: "answer", + answer: "Yes.", + sources, + model: "m", + tokensUsed: 12, }); + expect(withTokens).toContain("(model: m, 12 tokens)"); + }); - it("formatDocumentLinks includes the link ID on outgoing and incoming lines", () => { - const link = createLink(db, "doc-a", "doc-b", "references"); - const outgoing = formatDocumentLinks(getDocumentLinks(db, "doc-a")); - const incoming = formatDocumentLinks(getDocumentLinks(db, "doc-b")); - expect(outgoing).toContain( - `**Outgoing links:**\n → [references] Document B (doc-b) [link ID: ${link.id}]`, - ); - expect(incoming).toContain( - `**Incoming links:**\n ← [references] Document A (doc-a) [link ID: ${link.id}]`, - ); + it("document view shows metadata, url, suggested corrections and the whole content", () => { + const view: DocumentView = { + document: { + documentId: "doc-1", + title: "Title", + sourceType: "library", + library: "lib", + version: null, + topicId: "topic-1", + url: "https://x.test", + contentHash: null, + submittedBy: "manual", + createdAt: "c", + updatedAt: "u", + }, + content: "body", + contentLength: 4, + offset: 0, + nextOffset: null, + tags: [], + links: { outgoing: [], incoming: [] }, + ratings: { documentId: "doc-1", averageRating: 2, totalRatings: 2, corrections: 1 }, + }; + const text = formatDocumentView(view); + expect(text).toContain("documentId: doc-1 · library · lib · topic: topic-1"); + expect(text).toContain("url: https://x.test"); + expect(text).toContain("rating: 2.0/5 (2 ratings, 1 suggested corrections)"); + expect(text).not.toContain("characters"); + expect(text.endsWith("\n\nbody")).toBe(true); + }); + + it("document list includes IDs and metadata", () => { + const text = formatDocumentList( + page([ + { + documentId: "doc-1", + title: "T", + sourceType: "library", + library: "lib", + version: "2", + topicId: null, + url: null, + contentHash: null, + submittedBy: "manual", + createdAt: "c", + updatedAt: "u", + contentLength: 3, + }, + ]), + ); + expect(text).toBe("Documents 1-1 of 1\n- T (documentId: doc-1) library · lib v2"); + }); + + it("overview lists topics with IDs and the configured model", () => { + const text = formatOverview({ + stats: { + totalDocuments: 2, + totalChunks: 3, + totalTopics: 1, + databaseSizeBytes: 0, + totalSearches: 0, + avgLatencyMs: 0, + }, + topics: [{ id: "t1", name: "Infra", description: null, parentId: null, documentCount: 2 }], + packs: [], + index: { + stored: {}, + configured: { provider: "mock", model: "m", dimensions: 4 }, + }, + health: { database: "ok", fts: "error", vectorSearch: false }, }); + expect(text).toContain("keyword index error · vector search unavailable"); + expect(text).toContain("Index embeddings: unknown · configured: mock/m, 4 dimensions"); + expect(text).toContain("Topics:\n- Infra (topic: t1, 2 docs)"); + }); - it("formatDocumentLinks returns the no-links message when there are none", () => { - expect(formatDocumentLinks(getDocumentLinks(db, "doc-a"))).toBe( - "No links found for this document.", - ); + it("ingest results: dry run, crawl stats, skipped and errors", () => { + const text = formatIngestResult({ + kind: "url", + documents: [{ documentId: "d1", title: "Page", chunkCount: 2, source: "https://a.test" }], + errors: [{ source: "https://b.test", error: "404" }], + skipped: [{ source: "https://c.test", reason: "unchanged" }], + planned: ["https://a.test"], + crawl: { pagesCrawled: 3, pagesSkipped: 1, abortReason: "maxPages" }, }); + expect(text).toContain("Dry run: 1 source(s) would be added.\n- https://a.test"); + expect(text).toContain("- Page (documentId: d1, 2 chunks) https://a.test"); + expect(text).toContain("Crawled 3 page(s), skipped 1; stopped early: maxPages"); + expect(text).toContain("Skipped 1:\n- https://c.test: unchanged"); + expect(text).toContain("Errors 1:\n- https://b.test: 404"); + }); + + it("rating of a chunk includes the chunkId", () => { + expect( + formatRating({ + id: "r1", + documentId: "d1", + chunkId: "c1", + rating: 2, + feedback: null, + suggestedCorrection: "fix", + ratedBy: "model", + createdAt: "now", + }), + ).toBe("Rated 2/5 (documentId: d1, chunkId: c1) · correction saved"); }); - describe("LINK_TYPES", () => { - it("includes 'references' and every value is accepted by createLink", () => { - const db = createTestDb(); - insertDoc(db, "doc-a", "Document A"); - insertDoc(db, "doc-b", "Document B"); - expect(LINK_TYPES).toContain("references"); - for (const type of LINK_TYPES) { - expect(createLink(db, "doc-a", "doc-b", type).linkType).toBe(type); - } - db.close(); + it("task status shows progress, errors, and formats or passes through results", () => { + const failed = formatTask( + makeTask({ status: "failed", error: "boom", progress: { current: 1, total: 0 } }), + ); + expect(failed).toContain("taskId: task-1 · add · failed 1/?"); + expect(failed).toContain("\nerror: boom"); + + const done = makeTask({ status: "completed", result: '{"n":1}' }); + expect(formatTask(done)).toContain('result:\n{"n":1}'); + expect(formatTask(done, () => "formatted")).toContain("result:\nformatted"); + expect(formatTask(done, () => undefined)).toContain('result:\n{"n":1}'); + const broken = makeTask({ status: "completed", result: "not json" }); + expect(formatTask(broken, () => "formatted")).toContain("result:\nnot json"); + const legacy = makeTask({ type: "sync_connector", operation: undefined, status: "pending" }); + expect(formatTask(legacy)).toContain("sync_connector · pending"); + }); + + it("task cancel and list", () => { + expect(formatTaskCancel({ taskId: "t", cancelRequested: false, status: "completed" })).toBe( + "Task t is already completed; nothing to cancel.", + ); + expect(formatTaskList({ items: [] })).toBe("No tasks in the last hour."); + expect(formatTaskList({ items: [makeTask()] })).toMatch(/^- taskId: task-1 · add · running/); + }); + + it("sync results show failures and per-item errors", () => { + const text = formatSyncResult({ + items: [ + { name: "a", status: "failed", error: "offline" }, + { + name: "b", + type: "slack", + status: "completed", + summary: { added: 1, updated: 2, deleted: 0, errors: ["#general: denied"] }, + }, + ], }); + expect(text).toBe( + "- a: failed: offline\n- b (slack): 1 added, 2 updated, 0 deleted, 1 errors:\n #general: denied", + ); + }); + + it("pack list formats registry packs", () => { + expect( + formatPackList({ items: [{ name: "p", version: "1", description: "desc", docCount: 3 }] }), + ).toBe("- p v1 (3 docs): desc"); + }); - it("works as the link-documents zod enum", () => { - const schema = z.enum(LINK_TYPES); - expect(schema.parse("references")).toBe("references"); - expect(() => schema.parse("bogus")).toThrow(); + it("reindex reports failures and a rebuilt index", () => { + const text = formatReindex({ + total: 3, + completed: 2, + failed: 1, + failedChunkIds: ["c9"], + rebuiltIndex: "mock/m, 4 dimensions", }); + expect(text).toBe( + "Re-embedded 2 of 3 chunks; 1 failed.\nRebuilt vector index: mock/m, 4 dimensions\nFailed chunkIds: c9", + ); }); }); diff --git a/tests/unit/mcp-server.test.ts b/tests/unit/mcp-server.test.ts index 1e0f65a8..4b7fe74c 100644 --- a/tests/unit/mcp-server.test.ts +++ b/tests/unit/mcp-server.test.ts @@ -1,209 +1,599 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { createTestDbWithVec } from "../fixtures/test-db.js"; -import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import type { LlmProvider } from "../../src/core/rag.js"; +import { createDatabase } from "../../src/db/connection.js"; +import { createVectorTable, runMigrations } from "../../src/db/schema.js"; +import { LibScopeError, ValidationError, DocumentNotFoundError } from "../../src/errors.js"; import { initLogger } from "../../src/logger.js"; import { errorResponse, withErrorHandling, type ToolResult } from "../../src/mcp/errors.js"; -import { LibScopeError, ValidationError, DocumentNotFoundError } from "../../src/errors.js"; -import type Database from "better-sqlite3"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { addDoc, makeContext, testConfig, type TestContext } from "./operations/helpers.js"; + +// Saved connector settings live under a temp HOME, never the real ~/.libscope. +let tempHome = join(tmpdir(), `libscope-mcp-${process.pid}`); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const { createMcpServer, parseToolsets, buildInstructions } = + await import("../../src/mcp/server.js"); +const { saveConnectorSettings } = await import("../../src/connectors/saved-config.js"); +const { taskRegistry } = await import("../../src/core/tasks.js"); + +initLogger("silent"); + +const CORE_TOOLS = [ + "search", + "ask", + "get-document", + "list-documents", + "overview", + "submit-document", + "update-document", + "delete-document", + "rate-document", + "link-documents", + "task", +]; +const ADMIN_TOOLS = ["sync", "install-pack", "list-packs", "reindex-documents"]; + +const fakeLlm: LlmProvider = { + model: "fake-model", + complete: () => Promise.resolve({ text: "The answer is 42.", tokensUsed: 9 }), +}; + +interface CallResult { + isError?: boolean; + content: Array<{ type: string; text?: string }>; +} + +/** + * MCP context where `ask` is passthrough (llm.provider "auto" on the MCP surface), on a + * database with a real sqlite-vec table so `search {relatedTo}` works. + */ +function passthroughContext(): TestContext { + const db = createDatabase(":memory:"); + runMigrations(db); + createVectorTable(db, new MockEmbeddingProvider()); + return makeContext({ db, surface: "mcp", llm: undefined }); +} + +async function connect( + t: TestContext, + toolsets: string[] = [], +): Promise<{ client: Client; close: () => Promise<void> }> { + const mcp = createMcpServer({ ctx: t.ctx, toolsets }); + const client = new Client({ name: "test-client", version: "1.0.0" }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await Promise.all([mcp.server.connect(serverTransport), client.connect(clientTransport)]); + return { + client, + close: async (): Promise<void> => { + await client.close(); + await mcp.close(); + }, + }; +} + +async function call( + client: Client, + name: string, + args: Record<string, unknown> = {}, +): Promise<{ isError: boolean; text: string }> { + const result = (await client.callTool({ name, arguments: args })) as CallResult; + const text = result.content.map((c) => c.text ?? "").join("\n"); + return { isError: result.isError === true, text }; +} + +async function callOk( + client: Client, + name: string, + args: Record<string, unknown> = {}, +): Promise<string> { + const result = await call(client, name, args); + expect(result.isError, result.text).toBe(false); + return result.text; +} + +async function callError( + client: Client, + name: string, + args: Record<string, unknown> = {}, +): Promise<string> { + const result = await call(client, name, args); + expect(result.isError, result.text).toBe(true); + return result.text; +} + +/** documentId from a submit-document result. */ +function idFrom(text: string, key = "documentId"): string { + const match = new RegExp(`${key}: ([\\w-]+)`).exec(text); + if (!match?.[1]) throw new Error(`no ${key} in: ${text}`); + return match[1]; +} + +async function waitForTask(taskId: string): Promise<void> { + for (let i = 0; i < 200; i++) { + const status = taskRegistry.get(taskId)?.status; + if (status && status !== "pending" && status !== "running") return; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error(`task ${taskId} did not finish`); +} + +describe("MCP error helpers", () => { + it("errorResponse formats LibScopeError, Error and other values", () => { + expect(errorResponse(new ValidationError("invalid input")).content[0]!.text).toBe( + "Error: invalid input", + ); + expect(errorResponse(new TypeError("bad type")).content[0]!.text).toBe( + "Error: TypeError: bad type", + ); + expect(errorResponse("raw").content[0]!.text).toContain("raw"); + expect(errorResponse(new LibScopeError("base")).isError).toBe(true); + expect(() => errorResponse(undefined)).not.toThrow(); + }); + + it("withErrorHandling passes results through and converts throws and rejections", async () => { + const ok: ToolResult = { content: [{ type: "text", text: "ok" }] }; + expect(await withErrorHandling(() => ok)({})).toEqual(ok); + const thrown = await withErrorHandling(() => { + throw new ValidationError("bad input"); + })({}); + expect(thrown.isError).toBe(true); + const rejected = await withErrorHandling(() => + Promise.reject(new DocumentNotFoundError("doc-123")), + )({}); + expect(rejected.content[0]!.text).toContain("doc-123"); + }); +}); + +describe("MCP server: tool list", () => { + let t: TestContext; -describe("MCP server helpers", () => { beforeEach(() => { - initLogger("silent"); + t = passthroughContext(); }); - describe("errorResponse", () => { - it("returns isError: true with text content", () => { - const result = errorResponse(new Error("something went wrong")); - expect(result.isError).toBe(true); - expect(result.content).toHaveLength(1); - expect(result.content[0]!.type).toBe("text"); - }); + afterEach(() => { + t.db.close(); + vi.unstubAllEnvs(); + }); - it("formats LibScopeError using just the message", () => { - const result = errorResponse(new ValidationError("invalid input")); - expect(result.content[0]!.text).toBe("Error: invalid input"); + it("registers the 11 core tools with annotations", async () => { + const { client, close } = await connect(t); + const { tools } = await client.listTools(); + expect(tools.map((tool) => tool.name)).toEqual(CORE_TOOLS); + + const byName = new Map(tools.map((tool) => [tool.name, tool])); + for (const name of ["search", "ask", "get-document", "list-documents", "overview"]) { + expect(byName.get(name)?.annotations?.readOnlyHint, name).toBe(true); + } + expect(byName.get("delete-document")?.annotations).toMatchObject({ + readOnlyHint: false, + destructiveHint: true, }); - - it("formats a generic Error using name: message", () => { - const err = new TypeError("bad type"); - const result = errorResponse(err); - expect(result.content[0]!.text).toBe("Error: TypeError: bad type"); + expect(byName.get("update-document")?.annotations).toMatchObject({ + destructiveHint: false, + idempotentHint: true, }); - - it("formats non-Error values using String()", () => { - const result = errorResponse("raw string error"); - expect(result.content[0]!.text).toContain("raw string error"); + expect(byName.get("submit-document")?.annotations).toMatchObject({ + readOnlyHint: false, + destructiveHint: false, }); - - it("formats null/undefined without throwing", () => { - expect(() => errorResponse(null)).not.toThrow(); - expect(() => errorResponse(undefined)).not.toThrow(); + expect(byName.get("link-documents")?.annotations).toMatchObject({ destructiveHint: true }); + expect(byName.get("task")?.annotations).toMatchObject({ + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, }); + for (const tool of tools) expect(tool.description, tool.name).toBeTruthy(); + await close(); }); - describe("withErrorHandling", () => { - it("returns the handler result when no error is thrown", async () => { - const expected: ToolResult = { content: [{ type: "text", text: "ok" }] }; - const wrapped = withErrorHandling(() => expected); - const result = await wrapped({}); - expect(result).toEqual(expected); - }); + it("generates input schemas from the operations", async () => { + const { client, close } = await connect(t); + const { tools } = await client.listTools(); + const props = (name: string): Record<string, { enum?: string[] }> => + (tools.find((tool) => tool.name === name)?.inputSchema.properties ?? {}) as Record< + string, + { enum?: string[] } + >; + + expect(Object.keys(props("search"))).toEqual( + expect.arrayContaining(["query", "relatedTo", "topic", "library", "version", "sourceType"]), + ); + expect(Object.keys(props("search"))).toEqual( + expect.arrayContaining(["tags", "minRating", "limit", "offset"]), + ); + const submit = Object.keys(props("submit-document")); + expect(submit).toEqual(expect.arrayContaining(["content", "title", "url", "spider", "async"])); + for (const local of ["source", "kind", "format", "include", "exclude", "token"]) { + expect(submit).not.toContain(local); + } + expect(Object.keys(props("update-document"))).toContain("tags"); + expect(Object.keys(props("get-document"))).toEqual( + expect.arrayContaining(["documentId", "offset", "maxLength"]), + ); + expect(props("link-documents")["action"]?.enum).toEqual(["create", "delete"]); + expect(Object.keys(props("link-documents"))).toEqual( + expect.arrayContaining(["documentId", "targetDocumentId", "linkType", "linkId"]), + ); + expect(props("task")["action"]?.enum).toEqual(["status", "cancel", "list"]); + expect(Object.keys(props("search"))).not.toContain("async"); + await close(); + }); - it("catches synchronous throws and returns an error response", async () => { - const wrapped = withErrorHandling(() => { - throw new ValidationError("bad input"); - }); - const result = await wrapped({}); - expect(result.isError).toBe(true); - expect(result.content[0]!.text).toContain("bad input"); - }); + it("adds the admin toolset only when enabled", async () => { + for (const toolsets of [["admin"], ["all"], ["core", "Admin"]]) { + const { client, close } = await connect(t, toolsets); + const { tools } = await client.listTools(); + expect(tools.map((tool) => tool.name)).toEqual([...CORE_TOOLS, ...ADMIN_TOOLS]); + const byName = new Map(tools.map((tool) => [tool.name, tool])); + expect(byName.get("list-packs")?.annotations?.readOnlyHint).toBe(true); + expect(byName.get("sync")?.inputSchema.properties).not.toHaveProperty("token"); + await close(); + } + }); - it("catches rejected promises and returns an error response", async () => { - const wrapped = withErrorHandling(() => { - return Promise.reject(new DocumentNotFoundError("doc-123")); - }); - const result = await wrapped({}); - expect(result.isError).toBe(true); - }); + it("reads LIBSCOPE_MCP_TOOLSETS when no toolsets are passed", () => { + vi.stubEnv("LIBSCOPE_MCP_TOOLSETS", "admin"); + const mcp = createMcpServer({ ctx: t.ctx }); + expect(mcp.tools).toEqual([...CORE_TOOLS, ...ADMIN_TOOLS]); + vi.stubEnv("LIBSCOPE_MCP_TOOLSETS", ""); + expect(createMcpServer({ ctx: t.ctx }).tools).toEqual(CORE_TOOLS); + }); - it("passes params to the inner handler", async () => { - const handler = vi.fn().mockReturnValue({ content: [{ type: "text", text: "done" }] }); - const wrapped = withErrorHandling(handler); - const params = { docId: "abc", query: "test" }; - await wrapped(params); - expect(handler).toHaveBeenCalledWith(params); - }); + it("parseToolsets accepts a comma list, 'all' and ignores unknown names", () => { + expect([...parseToolsets("admin")]).toEqual(["admin"]); + expect([...parseToolsets(" ALL ")]).toEqual(["admin"]); + expect([...parseToolsets("bogus,core")]).toEqual([]); + expect([...parseToolsets(undefined)]).toEqual([]); + }); - it("returns isError: true for LibScopeError subclasses", async () => { - const wrapped = withErrorHandling(() => { - throw new LibScopeError("base lib error"); - }); - const result = await wrapped({}); - expect(result.isError).toBe(true); - expect(result.content[0]!.text).toBe("Error: base lib error"); - }); + it("omits ask when no LLM is configured and passthrough is off", async () => { + const noLlm = makeContext({ db: t.db, surface: "mcp", llm: null }); + const { client, close } = await connect(noLlm); + const names = (await client.listTools()).tools.map((tool) => tool.name); + expect(names).not.toContain("ask"); + expect(names).toHaveLength(10); + expect(client.getInstructions()).not.toContain("ask {question}"); + await close(); }); -}); -// Integration-style tests for MCP tool behaviors using the underlying core functions -// These verify the business logic that MCP tools delegate to. -describe("MCP tool business logic", () => { - let db: Database.Database; - let provider: MockEmbeddingProvider; + it("registers ask with a configured LLM and returns its answer", async () => { + await addDoc(t, "Answers", "The answer to everything is documented here"); + const withLlm = makeContext({ db: t.db, surface: "mcp", llm: fakeLlm }); + const { client, close } = await connect(withLlm); + const text = await callOk(client, "ask", { question: "What is the answer?" }); + expect(text).toContain("The answer is 42."); + expect(text).toContain("model: fake-model"); + expect(client.getInstructions()).toContain("configured LLM"); + await close(); + }); - beforeEach(() => { - initLogger("silent"); - db = createTestDbWithVec(); - provider = new MockEmbeddingProvider(); + it("sends instructions and the package version", async () => { + const { client, close } = await connect(t, ["admin"]); + const pkg = JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8")) as { + version: string; + }; + expect(client.getServerVersion()).toEqual({ name: "libscope", version: pkg.version }); + const instructions = client.getInstructions() ?? ""; + expect(instructions).toContain("search {query}"); + expect(instructions).toContain("get-document {documentId}"); + expect(instructions).toContain("rate-document"); + expect(instructions).toContain("without calling an LLM"); + expect(instructions).toContain("Admin tools"); + await close(); }); - afterEach(() => { - db.close(); + it("buildInstructions leaves out ask and admin when unavailable", () => { + const text = buildInstructions({ ask: false, passthrough: false, admin: false }); + expect(text).not.toContain("ask {question}"); + expect(text).not.toContain("Admin tools"); }); +}); + +describe("MCP server: tool calls", () => { + let t: TestContext; + let client: Client; + let close: () => Promise<void>; - it("search returns empty response when no documents are indexed", async () => { - const { searchDocuments } = await import("../../src/core/search.js"); - const { results, totalCount } = await searchDocuments(db, provider, { query: "anything" }); - expect(results).toHaveLength(0); - expect(totalCount).toBe(0); + beforeEach(async () => { + tempHome = join(tmpdir(), `libscope-mcp-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + t = passthroughContext(); + ({ client, close } = await connect(t, ["admin"])); }); - it("indexDocument then getDocument returns indexed document", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const { getDocument } = await import("../../src/core/documents.js"); + afterEach(async () => { + await close(); + t.db.close(); + rmSync(tempHome, { recursive: true, force: true }); + }); - const indexed = await indexDocument(db, provider, { - title: "Test Doc", - content: "Some content for testing.", - sourceType: "manual", - }); + async function submit(title: string, content: string): Promise<string> { + return idFrom(await callOk(client, "submit-document", { title, content })); + } + + it("search: finds by query and relatedTo, with IDs; rejects bad input", async () => { + const id = await submit("Kubernetes", "Kubernetes pods and deployments explained"); + await submit("Pods", "More about kubernetes pods"); - expect(indexed.id).toBeTruthy(); + const text = await callOk(client, "search", { query: "kubernetes pods" }); + expect(text).toMatch(/^Results 1-2 of 2/); + expect(text).toContain(`documentId: ${id}`); + expect(text).toContain("chunkId: "); - const fetched = getDocument(db, indexed.id); - expect(fetched.title).toBe("Test Doc"); - expect(fetched.content).toBe("Some content for testing."); + const related = await callOk(client, "search", { relatedTo: id }); + expect(related).not.toContain(`documentId: ${id}`); + + expect(await callOk(client, "search", { query: "kubernetes", library: "none" })).toBe( + "No results found.", + ); + expect(await callError(client, "search", {})).toContain("Give either query or relatedTo"); + expect(await callError(client, "search", { query: 5 })).toMatch(/query/); + expect(await callError(client, "search", { query: "x", limit: 0 })).toMatch(/limit/); }); - it("deleteDocument removes a document", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const { deleteDocument, getDocument } = await import("../../src/core/documents.js"); + it("ask: passthrough returns the retrieved context; rejects a missing question", async () => { + const id = await submit("Deploys", "Deploy with kubectl apply"); + const text = await callOk(client, "ask", { question: "How do I deploy?" }); + expect(text).toContain("Passthrough mode"); + expect(text).toContain("How do I deploy?"); + expect(text).toContain(`documentId: ${id}`); + expect(await callError(client, "ask", {})).toMatch(/question/); + }); - const indexed = await indexDocument(db, provider, { - title: "Delete Me", - content: "This will be deleted.", - sourceType: "manual", + it("get-document: returns metadata, tags, links and paged content", async () => { + const id = await submit("Guide", "abcdefghij".repeat(5)); + const other = await submit("Other", "Other content"); + const linkText = await callOk(client, "link-documents", { + action: "create", + documentId: id, + targetDocumentId: other, + linkType: "see_also", }); + const linkId = idFrom(linkText, "linkId"); + await callOk(client, "update-document", { documentId: id, tags: ["k8s"] }); + + const full = await callOk(client, "get-document", { documentId: id }); + expect(full).toContain("# Guide"); + expect(full).toContain(`documentId: ${id}`); + expect(full).toContain("tags: k8s"); + expect(full).toContain(`linkId: ${linkId}`); + + const page = await callOk(client, "get-document", { documentId: id, maxLength: 20 }); + expect(page).toContain("characters 0-20 of 50; next page: offset 20"); + const last = await callOk(client, "get-document", { documentId: id, offset: 40 }); + expect(last).toContain("characters 40-50 of 50]"); + + expect(await callError(client, "get-document", { documentId: "missing" })).toContain("missing"); + expect(await callError(client, "get-document", {})).toMatch(/documentId/); + }); - deleteDocument(db, indexed.id); - - expect(() => getDocument(db, indexed.id)).toThrow(DocumentNotFoundError); + it("list-documents: pages with offset and total", async () => { + const a = await submit("Doc A", "Content A"); + await submit("Doc B", "Content B"); + const first = await callOk(client, "list-documents", { limit: 1 }); + expect(first).toContain("Documents 1-1 of 2 — next page: offset 1"); + const second = await callOk(client, "list-documents", { limit: 1, offset: 1 }); + expect(second).toContain("Documents 2-2 of 2"); + expect(first + second).toContain(`documentId: ${a}`); + expect(await callError(client, "list-documents", { limit: 0 })).toMatch(/limit/); }); - it("listDocuments returns paginated documents", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const { listDocuments } = await import("../../src/core/documents.js"); + it("overview: shows counts, health and index", async () => { + await submit("Doc", "Content"); + const text = await callOk(client, "overview"); + expect(text).toContain("Documents: 1"); + expect(text).toContain("Health: database ok"); + expect(text).toContain("Index embeddings:"); + }); - await indexDocument(db, provider, { - title: "Doc A", - content: "Content A", - sourceType: "library", - library: "react", + it("submit-document: content, async task, and invalid input", async () => { + const text = await callOk(client, "submit-document", { + title: "Inline", + content: "Inline content", + tags: ["a"], }); - await indexDocument(db, provider, { - title: "Doc B", - content: "Content B", - sourceType: "library", - library: "vue", + expect(text).toContain("Added 1 document(s)."); + expect(text).toMatch(/documentId: [\w-]+, 1 chunks/); + + const started = await callOk(client, "submit-document", { + title: "Background", + content: "Background content", + async: true, }); + const taskId = idFrom(started, "taskId"); + await waitForTask(taskId); + const status = await callOk(client, "task", { action: "status", taskId }); + expect(status).toContain("completed"); + expect(status).toContain("Added 1 document(s)."); + expect(status).toContain("Background (documentId: "); + + expect(await callError(client, "submit-document", {})).toContain("Provide content"); + expect(await callError(client, "submit-document", { url: "/etc/passwd" })).toMatch(/url/); + expect(await callError(client, "submit-document", { title: "T", content: "" })).toMatch( + /content/, + ); + }); - const all = listDocuments(db, {}); - expect(all.length).toBeGreaterThanOrEqual(2); + it("update-document: changes fields and tags; rejects empty updates", async () => { + const id = await submit("Old", "Old content"); + const text = await callOk(client, "update-document", { + documentId: id, + title: "New", + tags: ["x", "y"], + }); + expect(text).toContain("# New"); + expect(text).toContain("tags: x, y"); + expect(await callError(client, "update-document", { documentId: id })).toContain( + "Nothing to update", + ); + expect( + await callError(client, "update-document", { documentId: "missing", title: "x" }), + ).toContain("missing"); + }); - const limited = listDocuments(db, { limit: 1 }); - expect(limited).toHaveLength(1); + it("delete-document: deletes; errors for unknown IDs", async () => { + const id = await submit("Gone", "Soon gone"); + expect(await callOk(client, "delete-document", { documentId: id })).toBe( + `Deleted document ${id}.`, + ); + await callError(client, "get-document", { documentId: id }); + await callError(client, "delete-document", { documentId: id }); + await callError(client, "delete-document", {}); }); - it("getDocumentRatings returns zero ratings for new document", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const { getDocumentRatings } = await import("../../src/core/ratings.js"); + it("rate-document: records a rating; rejects out-of-range ratings", async () => { + const id = await submit("Rated", "Rated content"); + const text = await callOk(client, "rate-document", { + documentId: id, + rating: 4, + feedback: "useful", + }); + expect(text).toContain(`Rated 4/5 (documentId: ${id})`); + expect(text).toContain("feedback saved"); + expect(await callOk(client, "get-document", { documentId: id })).toContain( + "rating: 4.0/5 (1 ratings)", + ); + await callError(client, "rate-document", { documentId: id, rating: 6 }); + await callError(client, "rate-document", { documentId: "missing", rating: 3 }); + }); - const indexed = await indexDocument(db, provider, { - title: "Rate Me", - content: "Rateable content.", - sourceType: "manual", + it("link-documents: creates and deletes links; validates per action", async () => { + const a = await submit("A", "Doc A"); + const b = await submit("B", "Doc B"); + const created = await callOk(client, "link-documents", { + action: "create", + documentId: a, + targetDocumentId: b, + linkType: "prerequisite", + label: "read first", }); + expect(created).toContain(`Linked ${a} -> ${b} (prerequisite) "read first"`); + const linkId = idFrom(created, "linkId"); + expect(await callOk(client, "link-documents", { action: "delete", linkId })).toBe( + `Deleted link ${linkId}.`, + ); + + expect(await callError(client, "link-documents", { action: "create", documentId: a })).toMatch( + /targetDocumentId/, + ); + expect(await callError(client, "link-documents", { action: "delete" })).toMatch(/linkId/); + await callError(client, "link-documents", { action: "rename", linkId }); + await callError(client, "link-documents", { action: "delete", linkId: "missing" }); + }); - const ratings = getDocumentRatings(db, indexed.id); - expect(ratings.totalRatings).toBe(0); - expect(ratings.averageRating).toBe(0); + it("task: status, cancel and list; errors for unknown tasks", async () => { + const { task } = taskRegistry.run( + "operation", + (signal) => + new Promise<string>((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason as Error)); + }), + "add", + ); + const running = await callOk(client, "task", { action: "status", taskId: task.id }); + expect(running).toContain(`taskId: ${task.id}`); + expect(running).toContain("running"); + expect(await callOk(client, "task", { action: "list" })).toContain(task.id); + + const cancelled = await callOk(client, "task", { action: "cancel", taskId: task.id }); + expect(cancelled).toContain(`Cancellation requested for task ${task.id}`); + await waitForTask(task.id); + expect(await callOk(client, "task", { action: "cancel", taskId: task.id })).toContain( + "already cancelled", + ); + + expect(await callError(client, "task", { action: "status", taskId: "nope" })).toContain( + "not found", + ); + await callError(client, "task", { action: "cancel", taskId: "nope" }); + await callError(client, "task", { action: "status" }); + await callError(client, "task", {}); }); - it("rateDocument stores a rating and updates average", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const { rateDocument, getDocumentRatings } = await import("../../src/core/ratings.js"); + it("sync (admin): syncs saved connections without secrets as parameters", async () => { + expect(await callOk(client, "sync", { all: true })).toBe("No saved connections to sync."); - const indexed = await indexDocument(db, provider, { - title: "Rate Me", - content: "Rateable content.", - sourceType: "manual", + const vault = join(tempHome, "vault"); + mkdirSync(vault, { recursive: true }); + writeFileSync(join(vault, "note.md"), "# Note\n\nA note in the vault."); + saveConnectorSettings("obsidian", "notes", { + vaultPath: vault, + topicMapping: "folder", + excludePatterns: [], }); + expect(await callOk(client, "sync", { name: "notes" })).toBe( + "- notes (obsidian): 1 added, 0 updated, 0 deleted", + ); + + expect(await callError(client, "sync", {})).toContain("Give either a connection name"); + await callError(client, "sync", { name: "missing" }); + await callError(client, "sync", { name: "bad name!" }); + }); - rateDocument(db, { documentId: indexed.id, rating: 4, feedback: "good doc" }); - const ratings = getDocumentRatings(db, indexed.id); - expect(ratings.totalRatings).toBe(1); - expect(ratings.averageRating).toBe(4); + it("install-pack and list-packs (admin): install from a file and list", async () => { + const path = join(tempHome, "demo.json"); + writeFileSync( + path, + JSON.stringify({ + name: "demo", + version: "1.0.0", + description: "Test pack", + documents: [{ title: "Pack doc", content: "Content from a pack", source: "test" }], + metadata: { author: "tests", license: "MIT", createdAt: new Date().toISOString() }, + }), + ); + expect(await callOk(client, "list-packs")).toBe("No packs found."); + expect(await callOk(client, "install-pack", { pack: path })).toBe( + "Installed pack demo: 1 documents.", + ); + expect(await callOk(client, "install-pack", { pack: path })).toBe( + "Pack demo is already installed.", + ); + expect(await callOk(client, "list-packs")).toMatch(/^- demo v1\.0\.0 \(1 docs, installed /); + expect(await callOk(client, "overview")).toContain("Packs:\n- demo v1.0.0"); + + await callError(client, "install-pack", {}); + await callError(client, "list-packs", { available: "yes" }); }); - it("listTopics returns empty array when no topics exist", async () => { - const { listTopics } = await import("../../src/core/topics.js"); - const topics = listTopics(db); - expect(topics).toEqual([]); + it("reindex-documents (admin): re-embeds chunks, inline or as a task", async () => { + await submit("Doc", "Reindex me"); + expect(await callOk(client, "reindex-documents")).toBe("Re-embedded 1 of 1 chunks."); + const started = await callOk(client, "reindex-documents", { async: true }); + const taskId = idFrom(started, "taskId"); + await waitForTask(taskId); + expect(await callOk(client, "task", { action: "status", taskId })).toContain( + "Re-embedded 1 of 1 chunks.", + ); + await callError(client, "reindex-documents", { batchSize: 0 }); + await callError(client, "reindex-documents", { rebuild: true, documentIds: ["x"] }); }); +}); - it("errorResponse for DocumentNotFoundError returns proper message", () => { - const result = errorResponse(new DocumentNotFoundError("missing-id")); - expect(result.isError).toBe(true); - expect(result.content[0]!.text).toContain("missing-id"); +describe("createMcpServer without a context", () => { + it("bootstraps its own database and closes it", async () => { + const mcp = createMcpServer({ + dbPath: ":memory:", + config: testConfig(), + provider: new MockEmbeddingProvider(), + vectorTable: "skip", + toolsets: [], + }); + expect(mcp.tools).toEqual(CORE_TOOLS); + expect(mcp.ctx.surface).toBe("mcp"); + expect(mcp.ctx.db.open).toBe(true); + await mcp.close(); + expect(mcp.ctx.db.open).toBe(false); }); }); diff --git a/vitest.config.ts b/vitest.config.ts index f48df703..325a61e4 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,7 +10,7 @@ export default defineConfig({ include: ["src/**/*.ts"], exclude: [ "src/cli/**", - "src/mcp/server.ts", + "src/mcp/main.ts", "src/providers/local.ts", "src/providers/ollama.ts", "src/providers/openai.ts", From fc07fbfefada7b9837d8905dffc5a4e33c6b4570 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:08:13 +0000 Subject: [PATCH 42/83] feat(core): custom chunker hook for add and shared ask input parsing - Chunker type in core/indexing; indexFile and ingest (content, file, directory) pass chunks from an optional chunker to indexDocument. - OperationContext.chunker, forwarded by the add operation. - parseOperationInput() shared by runOperation and startOperationTask. - toRagOptions() shared by the ask operation and streaming ask in the SDK. --- src/core/indexing.ts | 15 ++++++++++++ src/core/ingest.ts | 9 ++++++- src/core/operations/documents.ts | 1 + src/core/operations/search.ts | 41 +++++++++++++++++++------------- src/core/operations/types.ts | 30 +++++++++++++++-------- 5 files changed, 68 insertions(+), 28 deletions(-) diff --git a/src/core/indexing.ts b/src/core/indexing.ts index 2d029f79..a4487e8a 100644 --- a/src/core/indexing.ts +++ b/src/core/indexing.ts @@ -572,8 +572,21 @@ export interface IndexFileOptions { sourceType?: IndexDocumentInput["sourceType"] | undefined; /** ISO 8601 expiry timestamp (see IndexDocumentInput.expiresAt). */ expiresAt?: string | undefined; + /** Custom chunker; see {@link Chunker}. */ + chunker?: Chunker | undefined; } +/** + * Splits a document into chunks instead of the built-in markdown chunker. `source` is the + * file path or URL the content came from ("" for inline content). Return undefined to use + * the built-in chunker for this document. + */ +export type Chunker = (doc: { + content: string; + title: string; + source: string; +}) => Promise<string[] | undefined> | string[] | undefined; + /** Document source types. */ export const SOURCE_TYPES = ["library", "topic", "manual", "model-generated"] as const; export type SourceType = (typeof SOURCE_TYPES)[number]; @@ -628,6 +641,7 @@ export async function indexFile( const content = await parser.parse(buffer); const title = options.title ?? basename(filePath).replace(/\.[^.]+$/, ""); + const preChunked = await options.chunker?.({ content, title, source: filePath }); return indexDocument(db, provider, { title, @@ -638,5 +652,6 @@ export async function indexFile( topicId: options.topic, dedup: options.dedup, expiresAt: options.expiresAt, + preChunked, }); } diff --git a/src/core/ingest.ts b/src/core/ingest.ts index 659a7949..9d817534 100644 --- a/src/core/ingest.ts +++ b/src/core/ingest.ts @@ -15,6 +15,7 @@ import { indexDocument, indexFile, resolveSourceType, + type Chunker, type IndexDocumentInput, type IndexedDocument, type SourceType, @@ -79,6 +80,8 @@ export interface IngestContext { /** URL fetch settings (from config.indexing). */ fetchOptions?: Pick<FetchOptions, "allowPrivateUrls" | "allowSelfSignedCerts"> | undefined; submittedBy?: IndexDocumentInput["submittedBy"]; + /** Custom chunker for inline content and local files. */ + chunker?: Chunker | undefined; signal?: AbortSignal | undefined; onProgress?: | ((p: { done: number; total?: number | undefined; message?: string | undefined }) => void) @@ -206,11 +209,13 @@ async function ingestContent(ctx: IngestContext, input: IngestInput): Promise<In throw new ValidationError("A title is required when adding content directly"); } if (input.dryRun) return { ...result, planned: [input.title] }; + const content = input.content ?? ""; const doc = await indexDocument(ctx.db, ctx.provider, { ...documentDefaults(input, ctx), title: input.title, - content: input.content ?? "", + content, url: input.url, + preChunked: await ctx.chunker?.({ content, title: input.title, source: input.url ?? "" }), }); record(ctx, input, result, doc, input.title, input.url ?? ""); return result; @@ -233,6 +238,7 @@ async function ingestFile(ctx: IngestContext, input: IngestInput): Promise<Inges dedup: input.dedup, sourceType: input.sourceType, expiresAt: input.expiresAt, + chunker: ctx.chunker, }); record(ctx, input, result, doc, input.title ?? fileTitle(path), path); return result; @@ -259,6 +265,7 @@ async function ingestDirectory(ctx: IngestContext, input: IngestInput): Promise< dedup: input.dedup, sourceType: input.sourceType, expiresAt: input.expiresAt, + chunker: ctx.chunker, }); record(ctx, input, result, doc, fileTitle(file), file); } catch (err) { diff --git a/src/core/operations/documents.ts b/src/core/operations/documents.ts index a060bd66..0bac8d28 100644 --- a/src/core/operations/documents.ts +++ b/src/core/operations/documents.ts @@ -143,6 +143,7 @@ export const addOperation = defineOperation({ allowSelfSignedCerts: ctx.config.indexing.allowSelfSignedCerts, }, submittedBy: ctx.surface === "mcp" ? "model" : "manual", + chunker: ctx.chunker, signal: ctx.signal, onProgress: ctx.onProgress, }, diff --git a/src/core/operations/search.ts b/src/core/operations/search.ts index f3f7cd4c..50e28e7a 100644 --- a/src/core/operations/search.ts +++ b/src/core/operations/search.ts @@ -1,7 +1,7 @@ import type Database from "better-sqlite3"; import { z } from "zod"; import { ValidationError } from "../../errors.js"; -import { answer } from "../rag.js"; +import { answer, type RagOptions } from "../rag.js"; import { firstChunkId, getRelatedChunks, searchDocuments, type SearchResult } from "../search.js"; import { topicForFilter } from "./documents.js"; import * as s from "./schemas.js"; @@ -99,17 +99,33 @@ export const searchOperation = defineOperation({ }, }); +const askInput = z.object({ + question: z.string().min(1).max(10_000).describe("The question"), + ...s.documentFilters, + minRating, + topK: z.number().int().min(1).max(20).default(5).describe("Chunks to retrieve as context"), +}); + +/** RAG options for a parsed `ask` input (topic names resolved). Shared with streaming ask. */ +export function toRagOptions(ctx: OperationContext, input: z.output<typeof askInput>): RagOptions { + return { + question: input.question, + topK: input.topK, + topic: topicForFilter(ctx, input.topic), + library: input.library, + version: input.version, + sourceType: input.sourceType, + tags: input.tags, + minRating: input.minRating, + }; +} + export const askOperation = defineOperation({ name: "ask", group: "search", summary: "Answer a question from the knowledge base with an LLM, or (passthrough) return the context to answer from", - input: z.object({ - question: z.string().min(1).max(10_000).describe("The question"), - ...s.documentFilters, - minRating, - topK: z.number().int().min(1).max(20).default(5).describe("Chunks to retrieve as context"), - }), + input: askInput, annotations: { readOnly: true }, http: { method: "POST", path: "/ask" }, handler: (ctx: OperationContext, input) => @@ -117,16 +133,7 @@ export const askOperation = defineOperation({ ctx.db, ctx.provider, { passthrough: ctx.isPassthrough(), llm: ctx.isPassthrough() ? null : ctx.getLlm() }, - { - question: input.question, - topK: input.topK, - topic: topicForFilter(ctx, input.topic), - library: input.library, - version: input.version, - sourceType: input.sourceType, - tags: input.tags, - minRating: input.minRating, - }, + toRagOptions(ctx, input), ), }); diff --git a/src/core/operations/types.ts b/src/core/operations/types.ts index 8c111742..8e902050 100644 --- a/src/core/operations/types.ts +++ b/src/core/operations/types.ts @@ -8,6 +8,7 @@ import { z } from "zod"; import type { LibScopeConfig } from "../../config.js"; import { ConfigError, ValidationError } from "../../errors.js"; import type { EmbeddingProvider } from "../../providers/embedding.js"; +import type { Chunker } from "../indexing.js"; import { createLlmProvider, isPassthroughMode, type LlmProvider, type LlmSurface } from "../rag.js"; import { taskRegistry, type Task } from "../tasks.js"; @@ -40,6 +41,8 @@ export interface OperationContext { surface: Surface; signal?: AbortSignal | undefined; onProgress?: ((p: ProgressEvent) => void) | undefined; + /** Custom chunker used by `add` for inline content and local files (SDK only). */ + chunker?: Chunker | undefined; /** True when `ask` should return context for the caller to answer (passthrough). */ isPassthrough(): boolean; /** The configured LLM provider, created on first use; null when none is configured. */ @@ -86,18 +89,27 @@ function formatIssues(error: z.ZodError): string { .join("; "); } +/** Validate `rawInput` against the operation's schema. ZodError -> ValidationError. */ +export function parseOperationInput<S extends z.ZodObject>( + op: Operation<S, unknown>, + rawInput: unknown, +): z.output<S> { + const parsed = op.input.safeParse(rawInput ?? {}); + if (!parsed.success) { + throw new ValidationError(`Invalid input for ${op.name}: ${formatIssues(parsed.error)}`); + } + return parsed.data; +} + /** Validate `rawInput` against the operation's schema and run it. ZodError -> ValidationError. */ export async function runOperation<S extends z.ZodObject, O>( op: Operation<S, O>, ctx: OperationContext, rawInput: unknown, ): Promise<O> { - const parsed = op.input.safeParse(rawInput ?? {}); - if (!parsed.success) { - throw new ValidationError(`Invalid input for ${op.name}: ${formatIssues(parsed.error)}`); - } + const input = parseOperationInput(op, rawInput); ctx.signal?.throwIfAborted(); - return op.handler(ctx, parsed.data); + return op.handler(ctx, input); } /** @@ -110,10 +122,7 @@ export function startOperationTask<S extends z.ZodObject, O>( ctx: OperationContext, rawInput: unknown, ): Task { - const parsed = op.input.safeParse(rawInput ?? {}); - if (!parsed.success) { - throw new ValidationError(`Invalid input for ${op.name}: ${formatIssues(parsed.error)}`); - } + const input = parseOperationInput(op, rawInput); const { task } = taskRegistry.run( "operation", async (signal, onProgress) => { @@ -123,7 +132,7 @@ export function startOperationTask<S extends z.ZodObject, O>( signal, onProgress: (p) => onProgress(p.done, p.total ?? 0), }, - parsed.data, + input, ); return JSON.stringify(result ?? null); }, @@ -139,6 +148,7 @@ export interface CreateContextOptions { surface: Surface; signal?: AbortSignal | undefined; onProgress?: ((p: ProgressEvent) => void) | undefined; + chunker?: Chunker | undefined; /** Use this LLM instead of the configured one (null: none). */ llm?: LlmProvider | null | undefined; } From 2bb894ee270748b605c1e1d3a0588a295e5d0136 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:08:24 +0000 Subject: [PATCH 43/83] chore(api): temporary compile fixes for the SDK export change routes.ts imports from core modules instead of the package root, and the REST repo indexer (src/api/indexing, POST /api/v1/index/repos/:repoSlug, GET /api/v1/index/jobs/:jobId) is removed per decision B6. The REST agent replaces routes.ts; this commit only keeps this branch compiling. --- src/api/indexing/repoConfig.ts | 49 ------- src/api/indexing/repoIndexer.ts | 225 -------------------------------- src/api/routes.ts | 134 ++----------------- 3 files changed, 12 insertions(+), 396 deletions(-) delete mode 100644 src/api/indexing/repoConfig.ts delete mode 100644 src/api/indexing/repoIndexer.ts diff --git a/src/api/indexing/repoConfig.ts b/src/api/indexing/repoConfig.ts deleted file mode 100644 index bee2b670..00000000 --- a/src/api/indexing/repoConfig.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { join } from "node:path"; -import { homedir } from "node:os"; -import { readFileSync, mkdirSync } from "node:fs"; -import { LibScopeLite } from "../../lite/core.js"; -import { createDatabase } from "../../db/connection.js"; -import { runMigrations, createVectorTable } from "../../db/schema.js"; -import { LocalEmbeddingProvider } from "../../providers/local.js"; - -export interface RepoEntry { - cloneUrl: string; - branch?: string; - include?: string[]; - exclude?: string[]; -} - -export interface ReposConfig { - repos: Record<string, RepoEntry>; -} - -export function loadReposConfig(): ReposConfig { - const configPath = process.env["LIBSCOPE_REPOS_CONFIG"]; - if (!configPath) { - return { repos: {} }; - } - try { - const raw = readFileSync(configPath, "utf8"); - return JSON.parse(raw) as ReposConfig; - } catch { - return { repos: {} }; - } -} - -export function repoDbPath(repoSlug: string): string { - return join(homedir(), ".libscope", "repos", `${repoSlug}.db`); -} - -export function createRepoLibScope(repoSlug: string): LibScopeLite { - const dbPath = repoDbPath(repoSlug); - mkdirSync(join(homedir(), ".libscope", "repos"), { recursive: true }); - const provider = new LocalEmbeddingProvider(); - const db = createDatabase(dbPath); - runMigrations(db); - try { - createVectorTable(db, provider); - } catch { - /* sqlite-vec not loaded */ - } - return new LibScopeLite({ db, provider }); -} diff --git a/src/api/indexing/repoIndexer.ts b/src/api/indexing/repoIndexer.ts deleted file mode 100644 index 7a097fb2..00000000 --- a/src/api/indexing/repoIndexer.ts +++ /dev/null @@ -1,225 +0,0 @@ -import { execFileSync } from "node:child_process"; -import { readdirSync, readFileSync, rmSync, statSync, existsSync } from "node:fs"; -import { join, relative, extname } from "node:path"; -import { globToRegExp, toPosixPath } from "../../utils/glob.js"; -import { tmpdir } from "node:os"; -import { randomUUID } from "node:crypto"; -import { LibScopeLite } from "../../lite/core.js"; -import { TreeSitterChunker } from "../../lite/chunker-treesitter.js"; -import type { LiteDoc } from "../../lite/types.js"; -import type { RepoEntry } from "./repoConfig.js"; - -const chunker = new TreeSitterChunker(); - -/** - * Absolute path to the git executable, resolved once at module load time. - * Using an absolute path means the OS never consults PATH, eliminating the - * risk of a tampered environment redirecting "git" to a malicious binary. - */ -const GIT_EXECUTABLE = ((): string => { - const candidates = ["/usr/bin/git", "/usr/local/bin/git", "/opt/homebrew/bin/git"]; - for (const candidate of candidates) { - if (existsSync(candidate)) return candidate; - } - throw new Error("git executable not found at any known path"); -})(); - -export interface IndexJobStats { - filesIndexed: number; - chunksCreated: number; - filesSkipped: number; - errors: string[]; - durationMs: number; -} - -/** Map file extension to language string understood by TreeSitterChunker. */ -function detectLanguage(filePath: string): string | undefined { - const ext = extname(filePath).toLowerCase().replace(/^\./, ""); - const EXTENSION_MAP: Record<string, string> = { - ts: "typescript", - tsx: "typescript", - js: "javascript", - jsx: "javascript", - mjs: "javascript", - cjs: "javascript", - py: "python", - cs: "csharp", - cpp: "cpp", - cc: "cpp", - cxx: "cpp", - hpp: "cpp", - h: "cpp", - c: "c", - go: "go", - }; - return EXTENSION_MAP[ext]; -} - -/** Recursively list all files under dir, returning paths relative to dir. */ -function walkFiles(dir: string, include?: string[], exclude?: string[]): string[] { - const includeRegexes = include?.map(globToRegExp); - const excludeRegexes = exclude?.map(globToRegExp); - const results: string[] = []; - - function recurse(current: string): void { - let entries; - try { - entries = readdirSync(current, { withFileTypes: true }); - } catch { - return; - } - for (const entry of entries) { - const full = join(current, entry.name); - const rel = relative(dir, full); - if (entry.isDirectory()) { - // Skip hidden directories - if (entry.name.startsWith(".")) continue; - recurse(full); - } else if (entry.isFile()) { - const posixRel = toPosixPath(rel); - if (excludeRegexes?.some((rx) => rx.test(posixRel))) continue; - if (includeRegexes && !includeRegexes.some((rx) => rx.test(posixRel))) continue; - results.push(rel); - } - } - } - - recurse(dir); - return results; -} - -/** - * Validate a git branch name against the git-check-ref-format allowlist. - * Rejects anything outside [a-zA-Z0-9._\-/] to prevent argument injection. - */ -function validateBranchName(branch: string): void { - if (!/^[a-zA-Z0-9._\-/]+$/.test(branch)) { - throw new Error( - `Invalid branch name "${branch}": only alphanumerics, '.', '_', '-', and '/' are allowed`, - ); - } -} - -/** Clone a repo at branch into a temp directory. Returns the temp dir path. */ -function cloneToTemp(cloneUrl: string, branch?: string): string { - const tempDir = join(tmpdir(), `libscope-repo-${randomUUID()}`); - const args = ["clone", "--depth=1"]; - if (branch) { - validateBranchName(branch); - args.push("--branch", branch); - } - // Use the absolute path resolved at startup — PATH is never consulted. - // execFileSync with an array bypasses the shell entirely. - // The "--" separator prevents git from interpreting the URL as a flag. - args.push("--", cloneUrl, tempDir); - execFileSync(GIT_EXECUTABLE, args, { stdio: "ignore" }); - return tempDir; -} - -/** Chunk a source file with tree-sitter and append resulting LiteDocs. */ -async function chunkAndIndex( - relPath: string, - source: string, - lang: string, - repoSlug: string, - allDocs: LiteDoc[], - stats: IndexJobStats, -): Promise<void> { - try { - const chunks = await chunker.chunk(source, lang); - for (const chunk of chunks) { - allDocs.push({ - title: `${relPath}:L${String(chunk.startLine)}-L${String(chunk.endLine)} [${chunk.nodeType}]`, - content: chunk.content, - library: repoSlug, - url: `repo://${repoSlug}/${relPath}#L${String(chunk.startLine)}`, - }); - stats.chunksCreated++; - } - stats.filesIndexed++; - } catch (err) { - stats.errors.push(`${relPath} (chunking): ${err instanceof Error ? err.message : String(err)}`); - stats.filesSkipped++; - } -} - -/** Read one file from the cloned repo and append its docs to allDocs. */ -async function processFile( - relPath: string, - tempDir: string, - repoSlug: string, - allDocs: LiteDoc[], - stats: IndexJobStats, -): Promise<void> { - const fullPath = join(tempDir, relPath); - let source: string; - try { - const stat = statSync(fullPath); - if (!stat.isFile()) { - stats.filesSkipped++; - return; - } - source = readFileSync(fullPath, "utf8"); - } catch (err) { - stats.filesSkipped++; - stats.errors.push(`${relPath}: ${err instanceof Error ? err.message : String(err)}`); - return; - } - - const lang = detectLanguage(relPath); - if (lang && chunker.supports(lang)) { - await chunkAndIndex(relPath, source, lang, repoSlug, allDocs, stats); - } else { - // Non-code file or unsupported language — index as raw text - allDocs.push({ - title: relPath, - content: source, - library: repoSlug, - url: `repo://${repoSlug}/${relPath}`, - }); - stats.chunksCreated++; - stats.filesIndexed++; - } -} - -export async function indexRepo( - libscope: LibScopeLite, - repoSlug: string, - entry: RepoEntry, - opts: { branch?: string; files?: string[] }, -): Promise<IndexJobStats> { - const start = Date.now(); - const stats: IndexJobStats = { - filesIndexed: 0, - chunksCreated: 0, - filesSkipped: 0, - errors: [], - durationMs: 0, - }; - - const branch = opts.branch ?? entry.branch; - const tempDir = cloneToTemp(entry.cloneUrl, branch); - - try { - const isIncremental = opts.files !== undefined && opts.files.length > 0; - if (!isIncremental) { - libscope.deleteByLibrary(repoSlug); - } - - const filesToIndex = isIncremental - ? opts.files! - : walkFiles(tempDir, entry.include, entry.exclude); - - const allDocs: LiteDoc[] = []; - for (const relPath of filesToIndex) { - await processFile(relPath, tempDir, repoSlug, allDocs, stats); - } - - await libscope.indexBatch(allDocs, { concurrency: 4 }); - } finally { - rmSync(tempDir, { recursive: true, force: true }); - stats.durationMs = Date.now() - start; - } - - return stats; -} diff --git a/src/api/routes.ts b/src/api/routes.ts index 1a6f5690..0f56b2f8 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -1,41 +1,25 @@ import type { IncomingMessage, ServerResponse } from "node:http"; import { URL } from "node:url"; -import { randomUUID } from "node:crypto"; import type Database from "better-sqlite3"; import { performance } from "node:perf_hooks"; import type { EmbeddingProvider } from "../providers/embedding.js"; +import { searchDocuments } from "../core/search.js"; +import { listDocuments, getDocument, deleteDocument, updateDocument } from "../core/documents.js"; +import { indexDocument } from "../core/indexing.js"; +import { listTopics, createTopic } from "../core/topics.js"; +import { listTags, addTagsToDocument, suggestTags } from "../core/tags.js"; +import { getStats, getSearchAnalytics, getKnowledgeGaps } from "../core/analytics.js"; +import { fetchAndConvert } from "../core/url-fetcher.js"; +import { askQuestion, askQuestionStream, createLlmProvider } from "../core/rag.js"; +import { createLink, getDocumentLinks, deleteLink, type LinkType } from "../core/links.js"; import { - searchDocuments, - listDocuments, - getDocument, - indexDocument, - deleteDocument, - updateDocument, - listTopics, - createTopic, - listTags, - addTagsToDocument, - suggestTags, - getStats, - getSearchAnalytics, - getKnowledgeGaps, - fetchAndConvert, - askQuestion, - askQuestionStream, - createLlmProvider, - createLink, - getDocumentLinks, - deleteLink, createSavedSearch, listSavedSearches, runSavedSearch, deleteSavedSearch, - bulkDelete, - bulkRetag, - bulkMove, - searchBatch, -} from "../core/index.js"; -import type { LinkType, BulkSelector, BatchSearchRequest } from "../core/index.js"; +} from "../core/saved-searches.js"; +import { bulkDelete, bulkRetag, bulkMove, type BulkSelector } from "../core/bulk.js"; +import { searchBatch, type BatchSearchRequest } from "../core/batch-search.js"; import { LINK_TYPES } from "../core/links.js"; import { loadConfig } from "../config.js"; import { NotFoundError, FetchError, LibScopeError } from "../errors.js"; @@ -56,8 +40,6 @@ import type { WebhookEvent } from "../core/webhooks.js"; import { loadScheduleEntries } from "../core/scheduler.js"; import { spiderUrl } from "../core/spider.js"; import type { SpiderOptions } from "../core/spider.js"; -import { loadReposConfig, createRepoLibScope } from "./indexing/repoConfig.js"; -import { indexRepo, type IndexJobStats } from "./indexing/repoIndexer.js"; // --------------------------------------------------------------------------- // Shared types @@ -73,22 +55,6 @@ interface RouteContext { start: number; } -// --------------------------------------------------------------------------- -// Repo index job registry -// --------------------------------------------------------------------------- - -interface IndexJob { - jobId: string; - repoSlug: string; - status: "queued" | "running" | "completed" | "failed"; - startedAt: string; - completedAt?: string; - stats?: IndexJobStats; - error?: string; -} - -const repoIndexJobs = new Map<string, IndexJob>(); - // --------------------------------------------------------------------------- // URL / path helpers // --------------------------------------------------------------------------- @@ -774,80 +740,6 @@ function handleDeleteWebhook(ctx: RouteContext, webhookId: string): void { ctx.res.end(); } -// --------------------------------------------------------------------------- -// Repo indexing webhook handler -// --------------------------------------------------------------------------- - -function handleGetIndexJob(ctx: RouteContext, jobId: string): void { - const job = repoIndexJobs.get(jobId); - if (!job) { - sendError(ctx.res, 404, "NOT_FOUND", `Index job "${jobId}" not found`); - return; - } - sendJson(ctx.res, 200, job, elapsed(ctx.start)); -} - -async function handleIndexRepo(ctx: RouteContext, repoSlug: string): Promise<void> { - const log = getLogger(); - - // Parse optional body — empty body is valid (full reindex) - let body: Record<string, unknown> = {}; - try { - const parsed = await parseJsonBody(ctx.req); - if (parsed && typeof parsed === "object") { - body = parsed as Record<string, unknown>; - } - } catch { - /* empty body or non-JSON — treat as full reindex request */ - } - - const branch = typeof body["branch"] === "string" ? body["branch"] : undefined; - const files = Array.isArray(body["files"]) - ? (body["files"] as unknown[]).filter((f): f is string => typeof f === "string") - : undefined; - - const config = loadReposConfig(); - const entry = config.repos[repoSlug]; - if (!entry) { - sendError(ctx.res, 404, "NOT_FOUND", `Repo "${repoSlug}" not found in repos config`); - return; - } - - const jobId = randomUUID(); - const job: IndexJob = { - jobId, - repoSlug, - status: "queued", - startedAt: new Date().toISOString(), - }; - repoIndexJobs.set(jobId, job); - - sendJson(ctx.res, 202, { jobId, status: "queued" }, elapsed(ctx.start)); - - // Fire-and-forget indexing job - const libscope = createRepoLibScope(repoSlug); - job.status = "running"; - const indexOpts: { branch?: string; files?: string[] } = {}; - if (branch !== undefined) indexOpts.branch = branch; - if (files !== undefined) indexOpts.files = files; - indexRepo(libscope, repoSlug, entry, indexOpts) - .then((stats) => { - job.status = "completed"; - job.completedAt = new Date().toISOString(); - job.stats = stats; - log.info({ jobId, repoSlug, stats }, "Repo indexing completed"); - }) - .catch((err: unknown) => { - job.status = "failed"; - job.completedAt = new Date().toISOString(); - job.error = err instanceof Error ? err.message : String(err); - log.error({ jobId, repoSlug, err }, "Repo indexing failed"); - }) - .finally(() => { - libscope.close(); - }); -} - // --------------------------------------------------------------------------- // Error classification helper // --------------------------------------------------------------------------- @@ -964,8 +856,6 @@ const PATTERN_ROUTES: PatternRoute[] = [ route("POST /api/v1/searches", handleCreateSavedSearch), route("POST /api/v1/bulk/:operation", handleBulkOperation), route("DELETE /api/v1/searches/:id", handleDeleteSavedSearch), - route("GET /api/v1/index/jobs/:jobId", handleGetIndexJob), - route("POST /api/v1/index/repos/:repoSlug", handleIndexRepo), route("GET /api/v1/webhooks", handleListWebhooks), route("POST /api/v1/webhooks", handleCreateWebhook), route("POST /api/v1/webhooks/:id/test", handleTestWebhook), From b8690be9b9af5102d6eb79d0fd44ed555699e335 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:08:35 +0000 Subject: [PATCH 44/83] feat(sdk)!: LibScope on the operation layer; root exports shrink; lite is a preset - LibScope.create() uses bootstrap(); options: workspace, dbPath, db, config overrides, useConfigFile, provider, llmProvider, chunker. - Top-level add/search/ask/askStream/overview/close plus namespaces docs, topics, tags, links, searches, packs, connectors, tasks, admin, analytics, webhooks, generated from a table of operations (input = z.input of the schema, output = the operation result). Connector operations load on first use. - The package root exports LibScope, its option/result types and the error classes only; the ~137 other runtime exports are removed. - libscope/lite: createLite() returns a LibScope (useConfigFile: false); the LibScopeLite class is removed. createCodeChunker(), TreeSitterChunker and normalizeRawInput stay. - package.json: "." and "./lite" export maps list "types" first. - Tests: namespace calls, every operation reachable from the SDK, lite parity, export list, lite import graph. BREAKING CHANGE: see docs/migration-v2.md "Library API". --- package.json | 10 +- src/LibScope.ts | 448 +++++++++++++++++++------- src/core/index.ts | 307 ++++-------------- src/lite/core.ts | 258 +++------------ src/lite/index.ts | 23 +- src/lite/normalize.ts | 65 +--- src/lite/types.ts | 62 ---- tests/integration/lite-embed.test.ts | 168 +++------- tests/unit/db-location.test.ts | 4 +- tests/unit/libscope.test.ts | 315 ++++++++++++++++++ tests/unit/lite-import-graph.test.ts | 88 +++++ tests/unit/lite.test.ts | 464 +++++++++------------------ tests/unit/root-exports.test.ts | 34 ++ 13 files changed, 1110 insertions(+), 1136 deletions(-) delete mode 100644 src/lite/types.ts create mode 100644 tests/unit/libscope.test.ts create mode 100644 tests/unit/lite-import-graph.test.ts create mode 100644 tests/unit/root-exports.test.ts diff --git a/package.json b/package.json index 55460ec9..f5effcca 100644 --- a/package.json +++ b/package.json @@ -8,12 +8,14 @@ "libscope": "./dist/cli/index.js" }, "exports": { - ".": "./dist/core/index.js", + ".": { + "types": "./dist/core/index.d.ts", + "default": "./dist/core/index.js" + }, "./mcp": "./dist/mcp/server.js", "./lite": { - "import": "./dist/lite/index.js", - "require": "./dist/lite/index.js", - "types": "./dist/lite/index.d.ts" + "types": "./dist/lite/index.d.ts", + "default": "./dist/lite/index.js" } }, "files": [ diff --git a/src/LibScope.ts b/src/LibScope.ts index 7014420f..e4e5ac5f 100644 --- a/src/LibScope.ts +++ b/src/LibScope.ts @@ -1,155 +1,375 @@ +/** + * The public Node.js API. Every method is a typed call to one operation from + * src/core/operations, so the SDK has the same parameters, defaults and results as the CLI, + * MCP server and REST API. Namespaces are generated from the NAMESPACES table below. + */ import type Database from "better-sqlite3"; -import type { LibScopeConfig } from "./config.js"; -import type { EmbeddingProvider } from "./providers/embedding.js"; -import { loadConfig } from "./config.js"; -import { createDatabase, resolveDatabasePath } from "./db/connection.js"; -import { runMigrations, createVectorTable } from "./db/schema.js"; -import { createEmbeddingProvider } from "./providers/index.js"; -import { createLlmProvider } from "./core/rag.js"; -import { - indexDocument, - indexFile, - type IndexDocumentInput, - type IndexFileOptions, - type IndexedDocument, -} from "./core/indexing.js"; -import { searchDocuments, type SearchOptions, type SearchResponse } from "./core/search.js"; +import type { z } from "zod"; +import { ConfigSchema, type LibScopeConfig } from "./config.js"; +import { bootstrap, type Bootstrapped, type ConfigOverrides } from "./core/bootstrap.js"; +import type { Chunker } from "./core/indexing.js"; import { - askQuestion, askQuestionStream, - type RagOptions, - type RagResult, + createLlmProvider, + type LlmProvider, type RagStreamEvent, } from "./core/rag.js"; -import { getStats, type OverviewStats } from "./core/analytics.js"; -import { listDocuments, getDocument, deleteDocument, type Document } from "./core/documents.js"; +import type { EmbeddingProvider } from "./providers/embedding.js"; +import { + backupOperation, + bulkDeleteOperation, + bulkMoveOperation, + bulkRetagOperation, + dedupeOperation, + overviewOperation, + pruneExpiredOperation, + reindexOperation, + restoreOperation, +} from "./core/operations/admin.js"; +import { + popularOperation, + searchAnalyticsOperation, + staleOperation, + topQueriesOperation, +} from "./core/operations/analytics.js"; +import { + addOperation, + deleteDocumentOperation, + documentHistoryOperation, + getDocumentOperation, + listDocumentsOperation, + rateDocumentOperation, + rollbackDocumentOperation, + updateDocumentOperation, +} from "./core/operations/documents.js"; +import { + linkDocumentsOperation, + listLinksOperation, + prerequisitesOperation, + unlinkDocumentsOperation, +} from "./core/operations/links.js"; +import { + createPackOperation, + installPackOperation, + listPacksOperation, + removePackOperation, +} from "./core/operations/packs.js"; +import { askOperation, searchOperation, toRagOptions } from "./core/operations/search.js"; +import { + deleteSavedSearchOperation, + listSavedSearchesOperation, + runSavedSearchOperation, + saveSearchOperation, +} from "./core/operations/searches.js"; +import { + addTagsOperation, + listTagsOperation, + removeTagsOperation, + suggestTagsOperation, +} from "./core/operations/tags.js"; +import { + cancelTaskOperation, + getTaskOperation, + listTasksOperation, +} from "./core/operations/tasks.js"; +import { + createTopicOperation, + deleteTopicOperation, + listTopicsOperation, +} from "./core/operations/topics.js"; import { - searchBatch, - type BatchSearchRequest, - type BatchSearchResponse, -} from "./core/batch-search.js"; + createOperationContext, + parseOperationInput, + runOperation, + type Operation, + type OperationContext, + type ProgressEvent, +} from "./core/operations/types.js"; +import { + createWebhookOperation, + deleteWebhookOperation, + listWebhooksOperation, + testWebhookOperation, +} from "./core/operations/webhooks.js"; + +type ConnectorOperations = typeof import("./core/operations/connectors.js"); export interface LibScopeOptions { /** * Workspace whose database to open. Default: the active workspace (LIBSCOPE_WORKSPACE, - * `.libscope.json` `workspace`, or `libscope workspace use`), the same one the CLI and MCP server use. - * Ignored when `dbPath` or `database.path` is set. + * `.libscope.json` `workspace`, or `libscope workspace use`), the same one the CLI and MCP + * server use. Ignored when `dbPath`, `db` or `database.path` is set. + */ + workspace?: string | undefined; + /** Explicit SQLite database file (":memory:" for an in-memory database). */ + dbPath?: string | undefined; + /** Use an already-open database. It is migrated, and `close()` leaves it open. */ + db?: Database.Database | undefined; + /** Per-section config values merged over the loaded config, e.g. `{ llm: { model: "x" } }`. */ + config?: ConfigOverrides | undefined; + /** + * false: ignore config files, secrets.json and LIBSCOPE_* config variables; start from the + * defaults plus `config`. Default true. */ - workspace?: string; - /** Explicit SQLite database file. Wins over `workspace` and over `database.path` in config files. */ - dbPath?: string; - /** Override config values. */ - config?: Partial<LibScopeConfig>; + useConfigFile?: boolean | undefined; + /** Embedding provider instance to use instead of the configured one. */ + provider?: EmbeddingProvider | undefined; + /** LLM used by ask/askStream instead of the configured one. */ + llmProvider?: LlmProvider | undefined; + /** Custom chunker used by `add` for inline content and local files. */ + chunker?: Chunker | undefined; } -export class LibScope { - private readonly db: Database.Database; - private readonly embeddingProvider: EmbeddingProvider; - private readonly config: LibScopeConfig; +/** Per-call options for long-running methods. */ +export interface RunOptions { + signal?: AbortSignal | undefined; + onProgress?: ((progress: ProgressEvent) => void) | undefined; +} - private constructor( - db: Database.Database, - embeddingProvider: EmbeddingProvider, - config: LibScopeConfig, - ) { - this.db = db; - this.embeddingProvider = embeddingProvider; - this.config = config; - } +/** An operation whose module is loaded on first use (keeps connectors out of `libscope/lite`). */ +interface LazyOperation<Op> { + readonly name: string; + load(): Promise<Op>; +} - /** - * Create a new LibScope instance. Initializes DB, runs migrations, and sets up providers. - */ - static create(options?: LibScopeOptions): LibScope { - const baseConfig = loadConfig(); - const config: LibScopeConfig = { - embedding: { ...baseConfig.embedding, ...options?.config?.embedding }, - llm: { ...baseConfig.llm, ...options?.config?.llm }, - openai: { ...baseConfig.openai, ...options?.config?.openai }, - anthropic: { ...baseConfig.anthropic, ...options?.config?.anthropic }, - database: { ...baseConfig.database, ...options?.config?.database }, - indexing: { ...baseConfig.indexing, ...options?.config?.indexing }, - logging: { ...baseConfig.logging, ...options?.config?.logging }, - }; - - const dbPath = resolveDatabasePath({ - explicitPath: options?.dbPath ?? config.database.path, - workspace: options?.workspace, - }); +function lazy<Op>(name: string, load: () => Promise<Op>): LazyOperation<Op> { + return { name, load }; +} - const db = createDatabase(dbPath); - runMigrations(db); +const loadConnectorOperations = (): Promise<ConnectorOperations> => + import("./core/operations/connectors.js"); - const embeddingProvider = createEmbeddingProvider(config); - createVectorTable(db, embeddingProvider); +/** Namespace -> method -> operation. Every operation appears here or as a top-level method. */ +const NAMESPACES = { + docs: { + get: getDocumentOperation, + list: listDocumentsOperation, + update: updateDocumentOperation, + delete: deleteDocumentOperation, + rate: rateDocumentOperation, + history: documentHistoryOperation, + rollback: rollbackDocumentOperation, + }, + topics: { + list: listTopicsOperation, + create: createTopicOperation, + delete: deleteTopicOperation, + }, + tags: { + add: addTagsOperation, + remove: removeTagsOperation, + list: listTagsOperation, + suggest: suggestTagsOperation, + }, + links: { + create: linkDocumentsOperation, + delete: unlinkDocumentsOperation, + list: listLinksOperation, + prerequisites: prerequisitesOperation, + }, + searches: { + save: saveSearchOperation, + list: listSavedSearchesOperation, + run: runSavedSearchOperation, + delete: deleteSavedSearchOperation, + }, + packs: { + install: installPackOperation, + remove: removePackOperation, + list: listPacksOperation, + create: createPackOperation, + }, + connectors: { + list: lazy( + "list-connections", + async () => (await loadConnectorOperations()).listConnectionsOperation, + ), + sync: lazy("sync", async () => (await loadConnectorOperations()).syncOperation), + disconnect: lazy( + "disconnect", + async () => (await loadConnectorOperations()).disconnectOperation, + ), + }, + tasks: { + get: getTaskOperation, + cancel: cancelTaskOperation, + list: listTasksOperation, + }, + admin: { + reindex: reindexOperation, + dedupe: dedupeOperation, + backup: backupOperation, + restore: restoreOperation, + pruneExpired: pruneExpiredOperation, + bulkDelete: bulkDeleteOperation, + bulkRetag: bulkRetagOperation, + bulkMove: bulkMoveOperation, + }, + analytics: { + popular: popularOperation, + stale: staleOperation, + topQueries: topQueriesOperation, + searches: searchAnalyticsOperation, + }, + webhooks: { + create: createWebhookOperation, + list: listWebhooksOperation, + delete: deleteWebhookOperation, + test: testWebhookOperation, + }, +} as const; - return new LibScope(db, embeddingProvider, config); - } +/** Operations called by the top-level methods (add, search, ask, askStream, overview). */ +const TOP_LEVEL = [addOperation, searchOperation, askOperation, overviewOperation] as const; + +type AnyOperation = Operation<z.ZodObject, unknown>; +type OperationSource = AnyOperation | LazyOperation<AnyOperation>; +type Resolved<Src> = Src extends LazyOperation<infer Op> ? Op : Src; + +/** Input accepted by an operation (before defaults are applied). */ +export type OperationInput<Op> = Op extends Operation<infer S, unknown> ? z.input<S> : never; +/** Result of an operation. */ +export type OperationOutput<Op> = Op extends Operation<z.ZodObject, infer O> ? Awaited<O> : never; + +// eslint-disable-next-line @typescript-eslint/no-empty-object-type -- "every field optional" test +type AllOptional<T> = {} extends T ? true : false; + +type Method<Op> = + AllOptional<OperationInput<Op>> extends true + ? (input?: OperationInput<Op>, run?: RunOptions) => Promise<OperationOutput<Op>> + : (input: OperationInput<Op>, run?: RunOptions) => Promise<OperationOutput<Op>>; + +type Namespaces = typeof NAMESPACES; +export type NamespaceName = keyof Namespaces; +/** The methods of one namespace, e.g. `LibScopeNamespace<"docs">`. */ +export type LibScopeNamespace<N extends NamespaceName> = NamespaceMethods<Namespaces[N]>; +type NamespaceMethods<T> = { readonly [M in keyof T]: Method<Resolved<T[M]>> }; +/** Input of a namespace method, e.g. `LibScopeInput<"docs", "update">`. */ +export type LibScopeInput<N extends NamespaceName, M extends keyof Namespaces[N]> = OperationInput< + Resolved<Namespaces[N][M]> +>; +/** Result of a namespace method, e.g. `LibScopeOutput<"docs", "get">`. */ +export type LibScopeOutput< + N extends NamespaceName, + M extends keyof Namespaces[N], +> = OperationOutput<Resolved<Namespaces[N][M]>>; + +export type AddInput = OperationInput<typeof addOperation>; +export type AddResult = OperationOutput<typeof addOperation>; +export type SearchInput = OperationInput<typeof searchOperation>; +export type SearchOutput = OperationOutput<typeof searchOperation>; +export type AskInput = OperationInput<typeof askOperation>; +export type AskOutput = OperationOutput<typeof askOperation>; + +/** Operation names reachable from the SDK (used by tests to keep the SDK complete). */ +export function sdkOperationNames(): string[] { + const nested = Object.values(NAMESPACES).flatMap((ns) => + Object.values(ns as Record<string, OperationSource>).map((src) => src.name), + ); + return [...TOP_LEVEL.map((op) => op.name), ...nested]; +} - /** Index a document from content. */ - async index(input: IndexDocumentInput): Promise<IndexedDocument> { - return indexDocument(this.db, this.embeddingProvider, input); +/** Config built from the schema defaults only (no files, no environment). */ +function defaultConfig(): LibScopeConfig { + const sections = Object.fromEntries(Object.keys(ConfigSchema.shape).map((key) => [key, {}])); + return ConfigSchema.parse(sections) as unknown as LibScopeConfig; +} + +export class LibScope { + readonly docs = this.bind(NAMESPACES.docs); + readonly topics = this.bind(NAMESPACES.topics); + readonly tags = this.bind(NAMESPACES.tags); + readonly links = this.bind(NAMESPACES.links); + readonly searches = this.bind(NAMESPACES.searches); + readonly packs = this.bind(NAMESPACES.packs); + readonly connectors = this.bind(NAMESPACES.connectors); + readonly tasks = this.bind(NAMESPACES.tasks); + readonly admin = this.bind(NAMESPACES.admin); + readonly analytics = this.bind(NAMESPACES.analytics); + readonly webhooks = this.bind(NAMESPACES.webhooks); + + private readonly ctx: OperationContext; + + private constructor( + private readonly boot: Bootstrapped, + private readonly llm: LlmProvider | undefined, + chunker: Chunker | undefined, + ) { + this.ctx = createOperationContext({ + db: boot.db, + provider: boot.provider, + config: boot.config, + surface: "sdk", + llm, + chunker, + }); } - /** Index a file from disk. */ - async indexFile(filePath: string, options?: IndexFileOptions): Promise<IndexedDocument> { - return indexFile(this.db, this.embeddingProvider, filePath, options); + /** Open (and migrate) the database and create the providers. Call `close()` when done. */ + static create(options: LibScopeOptions = {}): LibScope { + const boot = bootstrap({ + workspace: options.workspace, + dbPath: options.dbPath, + db: options.db, + config: options.useConfigFile === false ? defaultConfig() : undefined, + configOverrides: options.config, + provider: options.provider, + }); + return new LibScope(boot, options.llmProvider, options.chunker); } - /** Search documents. */ - async search(query: string, options?: Omit<SearchOptions, "query">): Promise<SearchResponse> { - return searchDocuments(this.db, this.embeddingProvider, { ...options, query }); + /** Add content, a file, a directory, a URL (optionally crawled) or a repository. */ + add(input: string | AddInput, run?: RunOptions): Promise<AddResult> { + const raw = typeof input === "string" ? { source: input } : input; + return runOperation(addOperation, this.context(run), raw); } - /** Run multiple searches concurrently. */ - async searchBatch(requests: BatchSearchRequest[]): Promise<BatchSearchResponse> { - return searchBatch(this.db, this.embeddingProvider, requests); + /** Search by meaning and keywords, or find content related to a document or chunk. */ + search(input: string | SearchInput, run?: RunOptions): Promise<SearchOutput> { + const raw = typeof input === "string" ? { query: input } : input; + return runOperation(searchOperation, this.context(run), raw); } - /** Ask a question using RAG. */ - async ask(question: string, options?: Omit<RagOptions, "question">): Promise<RagResult> { - const llm = createLlmProvider(this.config); - return askQuestion(this.db, this.embeddingProvider, llm, { ...options, question }); + /** Answer a question with the LLM, or (llm.provider "passthrough") return the context. */ + ask(input: string | AskInput, run?: RunOptions): Promise<AskOutput> { + const raw = typeof input === "string" ? { question: input } : input; + return runOperation(askOperation, this.context(run), raw); } - /** Ask a question with streaming response. */ - async *askStream( - question: string, - options?: Omit<RagOptions, "question">, - ): AsyncGenerator<RagStreamEvent> { - const llm = createLlmProvider(this.config); - yield* askQuestionStream(this.db, this.embeddingProvider, llm, { ...options, question }); + /** Like `ask`, but streams the answer tokens. Needs an LLM (no passthrough). */ + async *askStream(input: string | AskInput): AsyncGenerator<RagStreamEvent> { + const raw = typeof input === "string" ? { question: input } : input; + const parsed = parseOperationInput(askOperation, raw); + const llm = this.llm ?? createLlmProvider(this.boot.config, { surface: "sdk" }); + yield* askQuestionStream(this.boot.db, this.boot.provider, llm, toRagOptions(this.ctx, parsed)); } - /** Get overview stats. */ - stats(): OverviewStats { - return getStats(this.db); + /** Counts, topics, installed packs, index identity and health. */ + overview(): Promise<OperationOutput<typeof overviewOperation>> { + return runOperation(overviewOperation, this.ctx, {}); } - /** List documents. */ - list(options?: { - library?: string; - topicId?: string; - sourceType?: string; - dateFrom?: string; - dateTo?: string; - limit?: number; - }): Document[] { - return listDocuments(this.db, options); + /** Close the database (unless it was passed in as `db`). Safe to call twice. */ + close(): void { + this.boot.close(); } - /** Get a document by ID. */ - get(id: string): Document { - return getDocument(this.db, id); + private context(run: RunOptions | undefined): OperationContext { + if (!run) return this.ctx; + return { ...this.ctx, signal: run.signal, onProgress: run.onProgress }; } - /** Delete a document by ID. */ - delete(id: string): void { - deleteDocument(this.db, id); + private async call(src: OperationSource, input: unknown, run?: RunOptions): Promise<unknown> { + const op = "load" in src ? await src.load() : src; + return runOperation(op, this.context(run), input); } - /** Close the database connection. */ - close(): void { - this.db.close(); + private bind<T extends object>(table: T): NamespaceMethods<T> { + const methods: Record<string, (input?: unknown, run?: RunOptions) => Promise<unknown>> = {}; + for (const [key, src] of Object.entries(table)) { + methods[key] = (input, run): Promise<unknown> => + this.call(src as OperationSource, input, run); + } + return Object.freeze(methods) as unknown as NamespaceMethods<T>; } } diff --git a/src/core/index.ts b/src/core/index.ts index e80d2018..d9e97cf6 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -1,255 +1,58 @@ -export { indexDocument, indexFile, chunkContent } from "./indexing.js"; -export type { - IndexDocumentInput, - IndexFileOptions, - IndexedDocument, - ChunkOptions, -} from "./indexing.js"; - -export { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; -export type { DocumentParser } from "./parsers/index.js"; - -export { checkDuplicate, findDuplicates } from "./dedup.js"; -export type { DedupResult, DedupOptions, DuplicateGroup } from "./dedup.js"; - -export { searchDocuments } from "./search.js"; -export type { - SearchOptions, - SearchResult, - SearchMethod, - ScoreExplanation, - ContextChunk, -} from "./search.js"; - -export { - logSearch, - recordSearchQuery, - getStats, - getPopularDocuments, - getStaleDocuments, - getTopQueries, - getSearchTrends, - getSearchAnalytics, - getKnowledgeGaps, -} from "./analytics.js"; +/** + * Package root (`import ... from "libscope"`): the LibScope class, its option and result + * types, and the error classes. Everything else is reached through LibScope methods. + * tests/unit/root-exports.test.ts lists the runtime exports so additions are deliberate. + */ +export { LibScope } from "../LibScope.js"; export type { - SearchLogEntry, - RecordSearchQueryInput, - OverviewStats, - PopularDocument, - StaleDocument, - TopQuery, - SearchTrend, - SearchAnalytics, - KnowledgeGap, -} from "./analytics.js"; - -export { rateDocument, getDocumentRatings, listRatings } from "./ratings.js"; -export type { RateDocumentInput, Rating, RatingSummary } from "./ratings.js"; - -export { getDocument, deleteDocument, listDocuments, updateDocument } from "./documents.js"; -export type { Document, UpdateDocumentInput } from "./documents.js"; - -export { - saveVersion, - getVersionHistory, - getVersion, - rollbackToVersion, - pruneVersions, - MAX_VERSIONS_DEFAULT, -} from "./versioning.js"; -export type { DocumentVersion } from "./versioning.js"; - -export { - createTopic, - listTopics, - getTopic, - deleteTopic, - renameTopic, - getDocumentsByTopic, - getTopicStats, -} from "./topics.js"; -export type { Topic, CreateTopicInput, GetDocumentsByTopicOptions, TopicStats } from "./topics.js"; - -export { fetchAndConvert, DEFAULT_FETCH_OPTIONS } from "./url-fetcher.js"; -export type { FetchedDocument, FetchOptions } from "./url-fetcher.js"; - -export { exportKnowledgeBase, importFromBackup } from "./export.js"; - -export { batchImport } from "./batch.js"; + LibScopeOptions, + RunOptions, + NamespaceName, + LibScopeNamespace, + LibScopeInput, + LibScopeOutput, + OperationInput, + OperationOutput, + AddInput, + AddResult, + SearchInput, + SearchOutput, + AskInput, + AskOutput, +} from "../LibScope.js"; + +export { + LibScopeError, + DatabaseError, + EmbeddingError, + ValidationError, + FetchError, + ConfigError, + NotFoundError, + DocumentNotFoundError, + ChunkNotFoundError, + TopicNotFoundError, +} from "../errors.js"; + +// Providers and hooks callers can pass to LibScope.create. +export type { EmbeddingProvider } from "../providers/embedding.js"; +export type { LlmProvider } from "./rag.js"; +export type { Chunker } from "./indexing.js"; +export type { ConfigOverrides } from "./bootstrap.js"; +export type { LibScopeConfig } from "../config.js"; + +// Result element types. +export type { ListResult, ProgressEvent } from "./operations/types.js"; +export type { DocumentSummary } from "./operations/documents.js"; +export type { SearchResult, ContextChunk, ScoreExplanation } from "./search.js"; export type { - BatchImportOptions, - BatchImportResult, - BatchProgress, - BatchFileResult, -} from "./batch.js"; - -export { - createTag, - deleteTag, - listTags, - addTagsToDocument, - removeTagsFromDocument, - getDocumentTags, - getDocumentsByTag, - suggestTags, -} from "./tags.js"; -export type { Tag, TagWithCount, GetDocumentsByTagOptions } from "./tags.js"; - -export { FileWatcher, DEFAULT_WATCH_EXTENSIONS } from "./watcher.js"; -export type { WatchOptions } from "./watcher.js"; - -export { - askQuestion, - askQuestionStream, - createLlmProvider, - buildContextPrompt, - extractSources, + AnswerResult, + PassthroughResult, + RagResult, + RagSource, + RagStreamEvent, } from "./rag.js"; -export type { RagOptions, RagResult, RagSource, RagStreamEvent, LlmProvider } from "./rag.js"; - -export { reindex } from "./reindex.js"; -export type { ReindexOptions, ReindexResult, ReindexProgress } from "./reindex.js"; - -export { - listAvailablePacks, - installPack, - removePack, - listInstalledPacks, - createPack, -} from "./packs.js"; -export type { - KnowledgePack, - PackDocument, - PackInfo, - InstalledPack, - InstallResult, - CreatePackOptions, -} from "./packs.js"; - -export { indexRepository, parseRepoUrl } from "./repo.js"; -export type { RepoOptions, RepoResult } from "./repo.js"; - -export { - syncOneNote, - disconnectOneNote, - authenticateDeviceCode, - refreshAccessToken, - convertOneNoteHtml, -} from "../connectors/onenote.js"; -export type { OneNoteConfig, OneNoteSyncResult } from "../connectors/onenote.js"; - -export { - syncObsidianVault, - parseObsidianMarkdown, - disconnectVault, -} from "../connectors/obsidian.js"; -export type { ObsidianConfig, SyncResult } from "../connectors/obsidian.js"; - -export { registerProvider, createEmbeddingProvider } from "../providers/index.js"; -export type { EmbeddingProvider } from "../providers/embedding.js"; -export type { ProviderFactory } from "../providers/index.js"; - -export { - createWorkspace, - deleteWorkspace, - listWorkspaces, - getWorkspacePath, - getWorkspacesDir, - getActiveWorkspace, - setActiveWorkspace, - DEFAULT_WORKSPACE, -} from "./workspace.js"; -export type { Workspace } from "./workspace.js"; - -export { buildKnowledgeGraph, detectClusters } from "./graph.js"; -export type { KnowledgeGraph, GraphNode, GraphEdge, GraphOptions } from "./graph.js"; - -export { - createLink, - getDocumentLinks, - deleteLink, - getPrerequisiteChain, - listLinks, -} from "./links.js"; -export type { LinkType, DocumentLink, DocumentLinkWithTitle, DocumentLinks } from "./links.js"; - -export { startApiServer } from "../api/server.js"; -export type { ApiServerOptions } from "../api/server.js"; - -export { syncNotion, convertNotionBlocks, disconnectNotion } from "../connectors/notion.js"; -export type { NotionConfig, NotionSyncResult, NotionBlock } from "../connectors/notion.js"; - -export { syncSlack, convertSlackMrkdwn, disconnectSlack } from "../connectors/slack.js"; -export type { SlackConfig, SlackSyncResult } from "../connectors/slack.js"; - -export { - saveDbConnectorConfig, - loadDbConnectorConfig, - deleteDbConnectorConfig, - loadConnectorConfig, - saveConnectorConfig, - saveNamedConnectorConfig, - loadNamedConnectorConfig, - hasNamedConnectorConfig, - deleteConnectorDocuments, -} from "../connectors/index.js"; -export type { ConnectorConfig } from "../connectors/index.js"; - -export { - createSavedSearch, - listSavedSearches, - getSavedSearch, - deleteSavedSearch, - runSavedSearch, -} from "./saved-searches.js"; -export type { SavedSearch } from "./saved-searches.js"; - -export { - syncConfluence, - convertConfluenceStorage, - disconnectConfluence, -} from "../connectors/confluence.js"; -export type { ConfluenceConfig, ConfluenceSyncResult } from "../connectors/confluence.js"; - -export { - syncDocSite, - disconnectDocSite, - detectDocSiteType, - extractDocLinks, - extractDocTitle, - extractMainContent, - extractElementByPattern, - extractSitemapUrls, - normalizeUrl as normalizeDocUrl, -} from "../connectors/docs.js"; -export type { DocSiteConfig, DocSiteSyncResult, DocSiteType } from "../connectors/docs.js"; - -export { resolveSelector, bulkDelete, bulkRetag, bulkMove } from "./bulk.js"; -export type { BulkSelector, BulkResult } from "./bulk.js"; - -export { - createWebhook, - listWebhooks, - getWebhook, - deleteWebhook, - updateWebhook, - signPayload, - buildPayload, - fireWebhooks, - redactWebhook, - WEBHOOK_EVENTS, -} from "./webhooks.js"; -export type { Webhook, RedactedWebhook, WebhookEvent, WebhookPayload } from "./webhooks.js"; - -export { ConnectorScheduler, loadScheduleEntries } from "./scheduler.js"; -export type { ScheduleConfig, SchedulerStatus } from "./scheduler.js"; - -export { searchBatch, BATCH_SEARCH_MAX_REQUESTS } from "./batch-search.js"; -export type { BatchSearchRequest, BatchSearchResponse } from "./batch-search.js"; - -export { pruneExpiredDocuments } from "./ttl.js"; -export type { PruneResult } from "./ttl.js"; - -export { LibScope } from "../LibScope.js"; -export type { LibScopeOptions } from "../LibScope.js"; +export type { IngestResult, IngestedDocument, IngestKind } from "./ingest.js"; +export type { DocumentView } from "./document-view.js"; +export type { Overview } from "./overview.js"; +export type { Task, TaskStatus } from "./tasks.js"; diff --git a/src/lite/core.ts b/src/lite/core.ts index 43f55e66..6be6ada9 100644 --- a/src/lite/core.ts +++ b/src/lite/core.ts @@ -1,214 +1,62 @@ -import Database from "better-sqlite3"; -import { homedir } from "node:os"; -import { join } from "node:path"; +import { extname } from "node:path"; +import type Database from "better-sqlite3"; +import { ValidationError } from "../errors.js"; +import { LibScope } from "../LibScope.js"; +import type { ConfigOverrides } from "../core/bootstrap.js"; +import type { Chunker } from "../core/indexing.js"; +import type { LlmProvider } from "../core/rag.js"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { LocalEmbeddingProvider } from "../providers/local.js"; -import { createDatabase } from "../db/connection.js"; -import { runMigrations, createVectorTable } from "../db/schema.js"; -import { ConfigError } from "../errors.js"; -import { indexDocument } from "../core/indexing.js"; -import { searchDocuments } from "../core/search.js"; -import { bulkDelete } from "../core/bulk.js"; -import { rateDocument } from "../core/ratings.js"; -import { askQuestion, getContextForQuestion, type LlmProvider } from "../core/rag.js"; -import { normalizeRawInput } from "./normalize.js"; import { TreeSitterChunker } from "./chunker-treesitter.js"; -import type { - LiteOptions, - LiteDoc, - RawInput, - LiteSearchOptions, - LiteSearchResult, - LiteContextOptions, - LiteAskOptions, -} from "./types.js"; -export class LibScopeLite { - private readonly db: Database.Database; - private readonly provider: EmbeddingProvider; - private readonly llmProvider: LlmProvider | null; - private _chunker: TreeSitterChunker | undefined; - private get chunker(): TreeSitterChunker { - this._chunker ??= new TreeSitterChunker(); - return this._chunker; - } - - constructor(opts: LiteOptions = {}) { - this.provider = opts.provider ?? new LocalEmbeddingProvider(); - this.llmProvider = opts.llmProvider ?? null; - - if (opts.db === undefined) { - const dbPath = opts.dbPath ?? join(homedir(), ".libscope", "lite.db"); - // createDatabase handles directory creation, WAL mode, pragmas, and sqlite-vec loading. - this.db = createDatabase(dbPath); - runMigrations(this.db); - // Create vector table best-effort (requires sqlite-vec to be loaded). - try { - createVectorTable(this.db, this.provider); - } catch (err) { - // A model/index mismatch must surface; anything else means sqlite-vec is - // unavailable and FTS5 search still works. - if (err instanceof ConfigError) throw err; - } - } else { - // Caller-provided DB: skip all setup (migrations, extension loading, vector table). - this.db = opts.db; - } - } - - async index(docs: LiteDoc[]): Promise<void> { - for (const doc of docs) { - let preChunked: string[] | undefined; - - if (doc.language && this.chunker.supports(doc.language)) { - try { - const codeChunks = await this.chunker.chunk(doc.content, doc.language); - preChunked = codeChunks.map((c) => c.content); - } catch { - // tree-sitter not installed or parse failed — fall back to text chunker - } - } - - await indexDocument(this.db, this.provider, { - title: doc.title, - content: doc.content, - sourceType: doc.sourceType ?? "manual", - library: doc.library, - version: doc.version, - topicId: doc.topicId, - url: doc.url, - preChunked, - }); - } - } - - async indexRaw(input: RawInput): Promise<string[]> { - const normalized = await normalizeRawInput(input); - if (normalized.chunks !== undefined && normalized.chunks.length > 1) { - const ids: string[] = []; - for (let i = 0; i < normalized.chunks.length; i++) { - const chunk = normalized.chunks[i]!; - const result = await indexDocument(this.db, this.provider, { - title: `${normalized.title} (part ${String(i + 1)})`, - content: chunk, - sourceType: "manual", - }); - ids.push(result.id); - } - return ids; - } - const result = await indexDocument(this.db, this.provider, { - title: normalized.title, - content: normalized.content, - sourceType: "manual", - url: input.type === "url" ? input.url : undefined, - }); - return [result.id]; - } - - async indexBatch(docs: LiteDoc[], opts: { concurrency: number }): Promise<void> { - const concurrency = Math.max(1, opts.concurrency); - let activeCount = 0; - let idx = 0; - - await new Promise<void>((resolve, reject) => { - if (docs.length === 0) { - resolve(); - return; - } - - const runNext = (): void => { - while (activeCount < concurrency && idx < docs.length) { - const doc = docs[idx]; - if (!doc) break; - idx++; - activeCount++; - this.index([doc]) - .then(() => { - activeCount--; - if (idx >= docs.length && activeCount === 0) { - resolve(); - } else { - runNext(); - } - }) - .catch(reject); - } - }; - - runNext(); - }); - } - - async search(query: string, opts?: LiteSearchOptions): Promise<LiteSearchResult[]> { - const { results } = await searchDocuments(this.db, this.provider, { - query, - limit: opts?.limit ?? 10, - topic: opts?.topic, - library: opts?.library, - tags: opts?.tags, - diversity: opts?.diversity, - }); - return results.map((r) => ({ - docId: r.documentId, - chunkId: r.chunkId, - title: r.title, - content: r.content, - score: r.score, - url: r.url, - })); - } +export interface LiteOptions { + /** SQLite database file (":memory:" for an in-memory database). */ + dbPath?: string | undefined; + /** Use an already-open database instead of `dbPath`. `close()` leaves it open. */ + db?: Database.Database | undefined; + /** Embedding provider. Default: the local provider (Xenova/all-MiniLM-L6-v2). */ + provider?: EmbeddingProvider | undefined; + /** LLM for ask/askStream. Without one, ask throws a ConfigError. */ + llmProvider?: LlmProvider | undefined; + /** Custom chunker, e.g. createCodeChunker(). */ + chunker?: Chunker | undefined; + /** Config values over the defaults (config files and environment are not read). */ + config?: ConfigOverrides | undefined; +} - async getContext(question: string, opts?: LiteContextOptions): Promise<string> { - const { contextPrompt } = await getContextForQuestion(this.db, this.provider, { - question, - topK: opts?.topK ?? 5, - topic: opts?.topic, - library: opts?.library, - }); - return contextPrompt; +/** + * LibScope for embedding in an application: the same object and results as + * `LibScope.create`, but config files and LIBSCOPE_* variables are ignored and the database + * is the one you name. + */ +export function createLite(options: LiteOptions): LibScope { + if (options.dbPath === undefined && options.db === undefined) { + throw new ValidationError("createLite needs dbPath or db"); } + return LibScope.create({ ...options, useConfigFile: false }); +} - async ask(question: string, opts?: LiteAskOptions): Promise<string> { - const llm = opts?.llmProvider ?? this.llmProvider; - if (!llm) { - throw new Error("No LlmProvider configured. Pass llmProvider to constructor or ask() opts."); - } - const result = await askQuestion(this.db, this.provider, llm, { - question, - topK: opts?.topK ?? 5, - topic: opts?.topic, - library: opts?.library, - systemPrompt: opts?.systemPrompt, - }); - return result.answer; - } +function languageOf(name: string): string | undefined { + const ext = extname(name).slice(1).toLowerCase(); + return ext === "" ? undefined : ext; +} - async *askStream(question: string, opts?: LiteAskOptions): AsyncGenerator<string> { - const llm = opts?.llmProvider ?? this.llmProvider; - if (!llm) { - throw new Error("No LlmProvider configured."); +/** + * A Chunker that splits source code at function and class boundaries with tree-sitter (an + * optional peer dependency). The language is `language` if given, else the extension of the + * file path, URL or title. Other documents, and code when tree-sitter is not installed, use + * the built-in chunker. + */ +export function createCodeChunker(options: { language?: string | undefined } = {}): Chunker { + const chunker = new TreeSitterChunker(); + return async ({ content, title, source }) => { + const language = options.language ?? languageOf(source) ?? languageOf(title); + if (language === undefined || !chunker.supports(language)) return undefined; + try { + const chunks = await chunker.chunk(content, language); + return chunks.map((c) => c.content); + } catch { + // tree-sitter not installed or the parse failed: use the built-in chunker + return undefined; } - if (!llm.completeStream) { - throw new Error("This LlmProvider does not support streaming."); - } - const context = await this.getContext(question, opts); - yield* llm.completeStream(context, opts?.systemPrompt); - } - - rate(docId: string, score: number): void { - rateDocument(this.db, { documentId: docId, rating: score }); - } - - deleteByLibrary(library: string): void { - // bulkDelete caps at MAX_BATCH_SIZE=1000; loop until all cleared - let result; - do { - result = bulkDelete(this.db, { library }, false); - } while (result.affected > 0); - } - - close(): void { - this.db.close(); - } + }; } diff --git a/src/lite/index.ts b/src/lite/index.ts index 9a353460..5e78ea54 100644 --- a/src/lite/index.ts +++ b/src/lite/index.ts @@ -1,13 +1,10 @@ -export { LibScopeLite } from "./core.js"; -export { TreeSitterChunker } from "./chunker-treesitter.js"; -export type { CodeChunk } from "./chunker-treesitter.js"; -export type { - LiteOptions, - LiteDoc, - RawInput, - LiteSearchOptions, - LiteSearchResult, - LiteContextOptions, - LiteAskOptions, -} from "./types.js"; -export type { LlmProvider } from "../core/rag.js"; +/** + * `libscope/lite`: LibScope for embedding in an application. Everything from the package + * root, plus a preset that ignores config files, a tree-sitter code chunker and input + * normalization. Must not import the CLI, MCP server or connectors + * (tests/unit/lite-import-graph.test.ts). + */ +export * from "../core/index.js"; +export { createLite, createCodeChunker, type LiteOptions } from "./core.js"; +export { TreeSitterChunker, type CodeChunk } from "./chunker-treesitter.js"; +export { normalizeRawInput, type RawInput, type NormalizedInput } from "./normalize.js"; diff --git a/src/lite/normalize.ts b/src/lite/normalize.ts index b20ac5ff..480b5f8c 100644 --- a/src/lite/normalize.ts +++ b/src/lite/normalize.ts @@ -2,73 +2,36 @@ import { readFileSync } from "node:fs"; import { basename, extname } from "node:path"; import { getParserForFile } from "../core/parsers/index.js"; import { fetchAndConvert } from "../core/url-fetcher.js"; -import type { RawInput } from "./types.js"; + +/** Raw input that normalizeRawInput turns into a title and markdown content. */ +export type RawInput = + | { type: "file"; path: string; title?: string | undefined } + | { type: "url"; url: string; title?: string | undefined } + | { type: "text"; content: string; title: string } + | { type: "buffer"; buffer: Buffer; filename: string; title?: string | undefined }; export interface NormalizedInput { title: string; content: string; - chunks?: string[]; -} - -// Code extensions that trigger tree-sitter attempt -const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py"]); - -type TreeSitterChunkerType = import("./chunker-treesitter.js").TreeSitterChunker; -let treeSitterChunker: TreeSitterChunkerType | null = null; -let treeSitterLoaded = false; - -async function getTreeSitterChunker(): Promise<TreeSitterChunkerType | null> { - if (treeSitterLoaded) return treeSitterChunker; - treeSitterLoaded = true; - try { - const { TreeSitterChunker } = await import("./chunker-treesitter.js"); - treeSitterChunker = new TreeSitterChunker(); - } catch { - /* optional dep not installed — graceful fallback */ - } - return treeSitterChunker; } -function extToLang(ext: string): string { - const map: Record<string, string> = { - ts: "typescript", - tsx: "typescript", - js: "javascript", - jsx: "javascript", - mjs: "javascript", - cjs: "javascript", - py: "python", - }; - return map[ext.slice(1)] ?? ext.slice(1); -} - -/** Shared path for file and buffer inputs: tree-sitter for code, else the registered parser. */ +/** Parse a file's bytes with the registered parser for its extension, else as UTF-8 text. */ async function normalizeBuffer( buf: Buffer, filename: string, explicitTitle: string | undefined, ): Promise<NormalizedInput> { - const ext = extname(filename).toLowerCase(); - const title = explicitTitle ?? basename(filename, ext); - - if (CODE_EXTENSIONS.has(ext)) { - const chunker = await getTreeSitterChunker(); - const lang = extToLang(ext); - if (chunker?.supports(lang)) { - const codeChunks = await chunker.chunk(buf.toString("utf-8"), lang); - return { - title, - content: codeChunks[0]?.content ?? "", - chunks: codeChunks.map((c) => c.content), - }; - } - } - + const title = explicitTitle ?? basename(filename, extname(filename)); const parser = getParserForFile(filename); const content = parser ? await parser.parse(buf) : buf.toString("utf-8"); return { title, content }; } +/** + * Turn a file, buffer, URL or text into `{ title, content }` for `scope.add({ title, content })`. + * Files and buffers use the parser for their extension (PDF, DOCX, HTML, ...); other files + * (e.g. source code) are read as UTF-8 text. + */ export async function normalizeRawInput(input: RawInput): Promise<NormalizedInput> { switch (input.type) { case "text": diff --git a/src/lite/types.ts b/src/lite/types.ts deleted file mode 100644 index 600303b5..00000000 --- a/src/lite/types.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import type { LlmProvider } from "../core/rag.js"; - -export interface LiteOptions { - /** Path to SQLite database file. Defaults to ~/.libscope/lite.db. Use ':memory:' for in-memory. */ - dbPath?: string | undefined; - /** Pre-configured database instance. When provided, dbPath is ignored and no migrations or - * vector table setup are performed — caller is responsible for schema initialization. */ - db?: Database.Database | undefined; - provider?: EmbeddingProvider | undefined; - model?: string | undefined; - llmProvider?: LlmProvider | undefined; -} - -export interface LiteDoc { - title: string; - content: string; - url?: string | undefined; - sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; - library?: string | undefined; - version?: string | undefined; - topicId?: string | undefined; - language?: string | undefined; -} - -export type RawInput = - | { type: "file"; path: string; title?: string | undefined } - | { type: "url"; url: string; title?: string | undefined } - | { type: "text"; content: string; title: string } - | { type: "buffer"; buffer: Buffer; filename: string; title?: string | undefined }; - -export interface LiteSearchOptions { - limit?: number | undefined; - topic?: string | undefined; - library?: string | undefined; - tags?: string[] | undefined; - diversity?: number | undefined; -} - -export interface LiteSearchResult { - docId: string; - chunkId: string; - title: string; - content: string; - score: number; - url: string | null; -} - -export interface LiteContextOptions { - topK?: number | undefined; - topic?: string | undefined; - library?: string | undefined; -} - -export interface LiteAskOptions { - topK?: number | undefined; - topic?: string | undefined; - library?: string | undefined; - systemPrompt?: string | undefined; - llmProvider?: LlmProvider | undefined; -} diff --git a/tests/integration/lite-embed.test.ts b/tests/integration/lite-embed.test.ts index 6a48e9c3..3182422c 100644 --- a/tests/integration/lite-embed.test.ts +++ b/tests/integration/lite-embed.test.ts @@ -1,16 +1,14 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { LibScopeLite } from "../../src/lite/index.js"; +import { createLite, ValidationError, type LibScope } from "../../src/lite/index.js"; +import { initLogger } from "../../src/logger.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; /** - * Integration test: full LibScopeLite workflow. - * - * Uses a real in-memory SQLite database with MockEmbeddingProvider - * to exercise the complete pipeline: indexBatch → search → getContext → rate. + * Integration test: the libscope/lite preset end to end on a real in-memory SQLite database + * with MockEmbeddingProvider: add -> search -> ask (passthrough context) -> rate -> delete. */ -describe("LibScopeLite integration", () => { - let lite: LibScopeLite; - let provider: MockEmbeddingProvider; +describe("libscope/lite integration", () => { + let scope: LibScope; const corpus = [ { @@ -19,6 +17,7 @@ describe("LibScopeLite integration", () => { "The useState hook lets you add state to functional components. " + "Call useState with the initial state value and it returns an array with " + "the current state and a setter function. Re-renders happen when state changes.", + library: "react", }, { title: "React useEffect Hook", @@ -26,6 +25,7 @@ describe("LibScopeLite integration", () => { "useEffect runs side effects in functional components. " + "Pass a function and a dependency array. The effect re-runs when dependencies change. " + "Return a cleanup function for subscriptions or timers.", + library: "react", }, { title: "TypeScript Generics", @@ -41,132 +41,58 @@ describe("LibScopeLite integration", () => { "poll, check, and close. setTimeout and setInterval run in the timers phase. " + "setImmediate runs in the check phase, after I/O callbacks.", }, - { - title: "SQL Indexes", - content: - "Database indexes speed up queries by creating sorted data structures. " + - "B-tree indexes are the default in most databases. " + - "Composite indexes cover multiple columns and follow the leftmost prefix rule.", - }, ]; beforeAll(async () => { - provider = new MockEmbeddingProvider(); - lite = new LibScopeLite({ dbPath: ":memory:", provider }); - await lite.indexBatch(corpus, { concurrency: 2 }); + initLogger("silent"); + scope = createLite({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + config: { llm: { provider: "passthrough" } }, + }); + await Promise.all(corpus.map((doc) => scope.add(doc))); }); afterAll(() => { - lite.close(); + scope.close(); }); - describe("indexBatch → search", () => { - it("should find indexed documents via search", async () => { - const results = await lite.search("React hooks"); - expect(results.length).toBeGreaterThan(0); - }); - - it("should return results with all expected fields", async () => { - const results = await lite.search("generics"); - expect(results.length).toBeGreaterThan(0); - - const r = results[0]!; - expect(typeof r.docId).toBe("string"); - expect(typeof r.chunkId).toBe("string"); - expect(typeof r.title).toBe("string"); - expect(typeof r.content).toBe("string"); - expect(typeof r.score).toBe("number"); - expect(r.score).toBeGreaterThan(0); - }); - - it("should respect the limit option", async () => { - const results = await lite.search("Node.js", { limit: 2 }); - expect(results.length).toBeLessThanOrEqual(2); - }); - - it("should return results for different queries", async () => { - const r1 = await lite.search("React useState"); - const r2 = await lite.search("SQL database index"); - - expect(r1.length).toBeGreaterThan(0); - expect(r2.length).toBeGreaterThan(0); - }); + it("search returns chunk results with document IDs", async () => { + const { items, total } = await scope.search({ query: "generics", limit: 2 }); + expect(items.length).toBeGreaterThan(0); + expect(items.length).toBeLessThanOrEqual(2); + expect(total).toBeGreaterThanOrEqual(items.length); + const r = items[0]!; + expect(typeof r.documentId).toBe("string"); + expect(typeof r.chunkId).toBe("string"); + expect(typeof r.score).toBe("number"); }); - describe("getContext", () => { - it("should return a context prompt string containing relevant content", async () => { - const context = await lite.getContext("How does the Node.js event loop work?"); - expect(typeof context).toBe("string"); - expect(context.length).toBeGreaterThan(0); - }); - - it("should include question in context", async () => { - const context = await lite.getContext("What are TypeScript generics?"); - // The context prompt typically includes the question - expect(context).toContain("TypeScript generics"); - }); + it("filters by library", async () => { + const { items } = await scope.search({ query: "functional components", library: "react" }); + expect(items.length).toBeGreaterThan(0); + expect(items.every((r) => r.library === "react")).toBe(true); }); - describe("rate", () => { - it("should rate a document found via search", async () => { - const results = await lite.search("React hooks"); - expect(results.length).toBeGreaterThan(0); - - const docId = results[0]!.docId; - // Should not throw - lite.rate(docId, 5); - lite.rate(docId, 3); - }); - - it("should reject invalid ratings", async () => { - // We need a valid doc ID first - const rateInvalid = async (): Promise<void> => { - const results = await lite.search("React"); - const docId = results[0]!.docId; - lite.rate(docId, 0); // 0 is out of range - }; - await expect(rateInvalid()).rejects.toThrow(); - }); + it("ask in passthrough mode returns the context prompt and sources", async () => { + const result = await scope.ask("How does the Node.js event loop work?"); + expect(result.mode).toBe("context"); + if (result.mode === "context") { + expect(result.contextPrompt).toContain("event loop"); + expect(result.sources.length).toBeGreaterThan(0); + } }); - describe("full pipeline: index → search → getContext → rate", () => { - it("should execute the complete workflow end-to-end", async () => { - // 1. Index additional docs - const extraLite = new LibScopeLite({ - dbPath: ":memory:", - provider: new MockEmbeddingProvider(), - }); - await extraLite.index([ - { - title: "Docker Basics", - content: - "Docker containers package applications with their dependencies. " + - "Images are built from Dockerfiles. Containers run as isolated processes.", - library: "docker", - }, - { - title: "Kubernetes Pods", - content: - "Kubernetes pods are the smallest deployable units. " + - "A pod can contain one or more containers sharing network and storage.", - library: "kubernetes", - }, - ]); - - // 2. Search - const searchResults = await extraLite.search("Docker containers"); - expect(searchResults.length).toBeGreaterThan(0); - expect(searchResults[0]!.title).toBeDefined(); - - // 3. Get context - const context = await extraLite.getContext("How does Docker work?"); - expect(context.length).toBeGreaterThan(0); - - // 4. Rate - const docId = searchResults[0]!.docId; - extraLite.rate(docId, 4); - - extraLite.close(); - }); + it("rates, rejects an invalid rating, and deletes", async () => { + const { items } = await scope.search("React hooks"); + const documentId = items[0]!.documentId; + await scope.docs.rate({ documentId, rating: 5 }); + await expect(scope.docs.rate({ documentId, rating: 0 })).rejects.toBeInstanceOf( + ValidationError, + ); + expect((await scope.docs.get({ documentId })).ratings.totalRatings).toBe(1); + + await scope.docs.delete({ documentId }); + expect((await scope.docs.list()).total).toBe(corpus.length - 1); }); }); diff --git a/tests/unit/db-location.test.ts b/tests/unit/db-location.test.ts index c446b836..aa3fce60 100644 --- a/tests/unit/db-location.test.ts +++ b/tests/unit/db-location.test.ts @@ -163,12 +163,12 @@ describe("database location (temp HOME)", () => { describe("LibScope.create", () => { const local = { embedding: { provider: "local" } } as const; - it("opens the active workspace database (same as CLI/MCP)", () => { + it("opens the active workspace database (same as CLI/MCP)", async () => { const ls = LibScope.create({ config: local }); try { expect(existsSync(wsDb("default"))).toBe(true); expect(existsSync(legacyDb())).toBe(false); - const stats = ls.stats(); + const { stats } = await ls.overview(); expect(stats.databaseSizeBytes).toBe(statSync(wsDb("default")).size); expect(stats.databaseSizeBytes).toBeGreaterThan(0); } finally { diff --git a/tests/unit/libscope.test.ts b/tests/unit/libscope.test.ts new file mode 100644 index 00000000..46c9b6e8 --- /dev/null +++ b/tests/unit/libscope.test.ts @@ -0,0 +1,315 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + expectTypeOf, + it, + vi, +} from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { LibScope, sdkOperationNames, type LibScopeOptions } from "../../src/LibScope.js"; +import { OPERATIONS } from "../../src/core/operations/index.js"; +import type { LlmProvider } from "../../src/core/rag.js"; +import { ConfigError, NotFoundError, ValidationError } from "../../src/errors.js"; +import { initLogger } from "../../src/logger.js"; +import { createTestDb } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; + +function create(options: LibScopeOptions = {}): LibScope { + return LibScope.create({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + useConfigFile: false, + ...options, + }); +} + +function mockLlm(text = "Mocked answer"): { llm: LlmProvider; complete: ReturnType<typeof vi.fn> } { + const complete = vi.fn().mockResolvedValue({ text, tokensUsed: 3 }); + return { llm: { model: "mock-llm", complete }, complete }; +} + +async function addDoc(scope: LibScope, title: string, content?: string): Promise<string> { + const result = await scope.add({ + content: content ?? `${title} explains testing the LibScope SDK end to end`, + title, + }); + return result.documents[0]!.documentId; +} + +let tempHome: string; +let savedHome: string | undefined; + +beforeAll(() => { + initLogger("silent"); + savedHome = process.env["HOME"]; + tempHome = mkdtempSync(join(tmpdir(), "libscope-sdk-home-")); + process.env["HOME"] = tempHome; +}); + +afterAll(() => { + if (savedHome === undefined) delete process.env["HOME"]; + else process.env["HOME"] = savedHome; + rmSync(tempHome, { recursive: true, force: true }); +}); + +describe("LibScope SDK surface", () => { + it("reaches every operation exactly once", () => { + const names = sdkOperationNames(); + expect(new Set(names).size).toBe(names.length); + expect([...names].sort()).toEqual(OPERATIONS.map((op) => op.name).sort()); + }); + + it("types namespace methods from the operation schemas", () => { + const scope = create(); + try { + expectTypeOf(scope.docs.get).parameter(0).toMatchTypeOf<{ documentId: string }>(); + expectTypeOf(scope.tags.list).parameter(0).toEqualTypeOf<Record<string, never> | undefined>(); + expectTypeOf<Awaited<ReturnType<LibScope["search"]>>["items"][number]>().toHaveProperty( + "chunkId", + ); + } finally { + scope.close(); + } + }); +}); + +describe("LibScope", () => { + let scope: LibScope; + + beforeEach(() => { + scope = create(); + }); + + afterEach(() => { + scope.close(); + }); + + it("add + search + overview", async () => { + const added = await scope.add({ content: "Vitest runs fast unit tests", title: "Vitest" }); + expect(added.kind).toBe("content"); + expect(added.documents[0]!.title).toBe("Vitest"); + + const byString = await scope.search("unit tests"); + const byObject = await scope.search({ query: "unit tests", limit: 5 }); + expect(byString.items.length).toBeGreaterThan(0); + expect(byObject).toMatchObject({ limit: 5, offset: 0 }); + expect(byObject.total).toBeGreaterThan(0); + expect(byString.items[0]!.documentId).toBe(added.documents[0]!.documentId); + + const overview = await scope.overview(); + expect(overview.stats.totalDocuments).toBe(1); + expect(overview.health.database).toBe("ok"); + }); + + it("validates input with ValidationError", async () => { + await expect(scope.search({})).rejects.toBeInstanceOf(ValidationError); + await expect(scope.docs.get({ documentId: "" })).rejects.toBeInstanceOf(ValidationError); + }); + + it("docs: list, get, update, rate, history, rollback, delete", async () => { + const id = await addDoc(scope, "Doc One"); + const list = await scope.docs.list(); + expect(list.total).toBe(1); + expect(list.items[0]!.documentId).toBe(id); + + const view = await scope.docs.get({ documentId: id, maxLength: 5 }); + expect(view.content).toHaveLength(5); + expect(view.document.documentId).toBe(id); + + await scope.docs.update({ documentId: id, content: "Doc One rewritten content", tags: ["a"] }); + expect((await scope.docs.get({ documentId: id })).tags).toEqual(["a"]); + + const rating = await scope.docs.rate({ documentId: id, rating: 4 }); + expect(rating).toMatchObject({ documentId: id, rating: 4 }); + + const history = await scope.docs.history({ documentId: id }); + expect(history.items.length).toBeGreaterThan(0); + await scope.docs.rollback({ documentId: id, version: history.items[0]!.version }); + + await scope.docs.delete({ documentId: id }); + await expect(scope.docs.get({ documentId: id })).rejects.toBeInstanceOf(NotFoundError); + }); + + it("topics, tags and links", async () => { + const topic = await scope.topics.create({ name: "testing" }); + expect((await scope.topics.list()).items.map((t) => t.id)).toContain(topic.id); + + const a = await addDoc(scope, "Doc A"); + const b = await addDoc(scope, "Doc B"); + + expect((await scope.tags.add({ documentId: a, tags: ["x", "y"] })).tags).toEqual(["x", "y"]); + expect((await scope.tags.remove({ documentId: a, tags: ["y"] })).tags).toEqual(["x"]); + expect((await scope.tags.list()).items.map((t) => t.name)).toContain("x"); + expect((await scope.tags.suggest({ documentId: a })).documentId).toBe(a); + + const link = await scope.links.create({ + documentId: b, + targetDocumentId: a, + linkType: "prerequisite", + }); + expect((await scope.links.list({ documentId: b })).items).toHaveLength(1); + expect((await scope.links.prerequisites({ documentId: a })).items[0]!.documentId).toBe(b); + await scope.links.delete({ linkId: link.linkId }); + expect((await scope.links.list()).items).toHaveLength(0); + + await scope.topics.delete({ topic: "testing" }); + expect((await scope.topics.list()).items).toHaveLength(0); + }); + + it("searches: save, list, run, delete", async () => { + await addDoc(scope, "Saved Search Doc"); + await scope.searches.save({ name: "mine", query: "SDK" }); + expect((await scope.searches.list()).total).toBe(1); + const run = await scope.searches.run({ search: "mine" }); + expect(run.items.length).toBeGreaterThan(0); + await scope.searches.delete({ search: "mine" }); + expect((await scope.searches.list()).total).toBe(0); + }); + + it("packs, tasks, webhooks, analytics", async () => { + expect((await scope.packs.list()).items).toEqual([]); + expect(Array.isArray((await scope.tasks.list()).items)).toBe(true); + await expect(scope.tasks.get({ taskId: "missing" })).rejects.toBeInstanceOf(NotFoundError); + expect((await scope.webhooks.list()).items).toEqual([]); + expect((await scope.analytics.popular()).items).toEqual([]); + expect(await scope.analytics.searches()).toHaveProperty("knowledgeGaps"); + }); + + it("connectors are loaded on first use", async () => { + expect((await scope.connectors.list()).items).toEqual([]); + expect((await scope.connectors.sync({ all: true })).items).toEqual([]); + await expect(scope.connectors.sync({})).rejects.toBeInstanceOf(ValidationError); + }); + + it("admin: dedupe, pruneExpired, bulk ops, backup and restore", async () => { + await addDoc(scope, "Admin Doc"); + expect(Array.isArray((await scope.admin.dedupe()).items)).toBe(true); + expect(await scope.admin.pruneExpired()).toMatchObject({ pruned: 0 }); + expect((await scope.admin.bulkDelete({ library: "none", dryRun: true })).affected).toBe(0); + expect( + (await scope.admin.bulkRetag({ dateFrom: "2000-01-01", addTags: ["t"], dryRun: true })) + .affected, + ).toBe(1); + await scope.topics.create({ name: "target" }); + expect( + (await scope.admin.bulkMove({ dateFrom: "2000-01-01", targetTopic: "target", dryRun: true })) + .affected, + ).toBe(1); + + const file = join(tempHome, "backup.json"); + const backup = await scope.admin.backup({ outputPath: file }); + expect(backup.counts.documents).toBe(1); + const restored = await scope.admin.restore({ backupPath: file }); + expect(restored.path).toBe(file); + + const reindexed = await scope.admin.reindex({}, { onProgress: () => undefined }); + expect(reindexed.failed).toBe(0); + }); + + it("passes the per-call signal to the operation", async () => { + const controller = new AbortController(); + controller.abort(); + await expect(scope.docs.list({}, { signal: controller.signal })).rejects.toThrow(); + }); +}); + +describe("LibScope ask", () => { + it("throws a ConfigError with a hint when no LLM is configured", async () => { + const scope = create(); + try { + await addDoc(scope, "Ask Doc"); + await expect(scope.ask("What is this?")).rejects.toBeInstanceOf(ConfigError); + await expect(scope.askStream("What is this?").next()).rejects.toBeInstanceOf(ConfigError); + } finally { + scope.close(); + } + }); + + it("answers with the given LLM provider", async () => { + const { llm, complete } = mockLlm(); + const scope = create({ llmProvider: llm }); + try { + await addDoc(scope, "Ask Doc"); + const result = await scope.ask({ question: "What is tested?", topK: 2 }); + expect(result).toMatchObject({ mode: "answer", answer: "Mocked answer", model: "mock-llm" }); + expect(complete).toHaveBeenCalledOnce(); + } finally { + scope.close(); + } + }); + + it("returns context in passthrough mode", async () => { + const scope = create({ config: { llm: { provider: "passthrough" } } }); + try { + await addDoc(scope, "Ask Doc"); + const result = await scope.ask("What is tested?"); + expect(result.mode).toBe("context"); + if (result.mode === "context") expect(result.contextPrompt).toContain("What is tested?"); + } finally { + scope.close(); + } + }); + + it("streams tokens and then sources", async () => { + const llm: LlmProvider = { + model: "stream-llm", + complete: vi.fn(), + async *completeStream() { + await Promise.resolve(); + yield "Hello"; + yield " world"; + }, + }; + const scope = create({ llmProvider: llm }); + try { + await addDoc(scope, "Stream Doc"); + const events = []; + for await (const event of scope.askStream("Question?")) events.push(event); + expect(events.slice(0, 2)).toEqual([{ token: "Hello" }, { token: " world" }]); + expect(events[2]).toMatchObject({ done: true, model: "stream-llm" }); + } finally { + scope.close(); + } + }); +}); + +describe("LibScope options", () => { + it("uses a custom chunker for added content", async () => { + const chunker = vi.fn().mockReturnValue(["part one", "part two", "part three"]); + const scope = create({ chunker }); + try { + const result = await scope.add({ content: "whole text", title: "Chunked" }); + expect(result.documents[0]!.chunkCount).toBe(3); + expect(chunker).toHaveBeenCalledWith({ content: "whole text", title: "Chunked", source: "" }); + + const file = join(tempHome, "notes.md"); + writeFileSync(file, "# Notes\n\nSome notes."); + const fromFile = await scope.add(file); + expect(fromFile.documents[0]!.chunkCount).toBe(3); + expect(chunker).toHaveBeenLastCalledWith( + expect.objectContaining({ title: "notes", source: file }), + ); + } finally { + scope.close(); + } + }); + + it("leaves an injected database open on close", async () => { + const db = createTestDb(); + const scope = LibScope.create({ + db, + provider: new MockEmbeddingProvider(), + useConfigFile: false, + }); + await addDoc(scope, "Injected"); + scope.close(); + expect(db.open).toBe(true); + db.close(); + }); +}); diff --git a/tests/unit/lite-import-graph.test.ts b/tests/unit/lite-import-graph.test.ts new file mode 100644 index 00000000..06ae0020 --- /dev/null +++ b/tests/unit/lite-import-graph.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; + +/** + * `libscope/lite` (and the package root) must stay embeddable: their static import graph may + * not reach the CLI, MCP server, REST API, web UI or connectors. Connector operations are + * loaded with a dynamic import() the first time `scope.connectors.*` is called. + */ +const SRC = resolve(__dirname, "../../src"); +const FORBIDDEN_DIRS = ["cli/", "mcp/", "api/", "web/", "connectors/"]; +const FORBIDDEN_PACKAGES = ["commander", "@modelcontextprotocol/sdk", "node-cron"]; + +const STATEMENT_STARTS = ["import ", "export {", "export *"]; + +/** The module specifier of one import/export statement, or undefined if it has none. */ +function specifierOf(stmt: string): string | undefined { + const FROM = ' from "'; + const at = stmt.lastIndexOf(FROM); + let start: number; + if (at !== -1) start = at + FROM.length; + else if (stmt.startsWith('import "')) start = 'import "'.length; + else return undefined; + return stmt.slice(start, stmt.indexOf('"', start)); +} + +/** Module specifiers of value (not type-only) static imports and re-exports. */ +function staticImports(source: string): string[] { + const specs: string[] = []; + let stmt: string | undefined; + for (const line of source.split("\n")) { + if (stmt === undefined) { + if (!STATEMENT_STARTS.some((s) => line.startsWith(s))) continue; + stmt = line; + } else { + stmt += ` ${line.trim()}`; + } + if (!line.trimEnd().endsWith(";")) continue; + const typeOnly = stmt.startsWith("import type ") || stmt.startsWith("export type "); + const spec = typeOnly ? undefined : specifierOf(stmt); + if (spec !== undefined) specs.push(spec); + stmt = undefined; + } + return specs; +} + +function importGraph(entry: string): { files: Set<string>; packages: Set<string> } { + const files = new Set<string>(); + const packages = new Set<string>(); + const pending = [entry]; + while (pending.length > 0) { + const file = pending.pop()!; + if (files.has(file)) continue; + files.add(file); + for (const spec of staticImports(readFileSync(file, "utf-8"))) { + if (spec.startsWith(".")) { + pending.push(join(dirname(file), spec).replace(/\.js$/, ".ts")); + } else { + packages.add(spec); + } + } + } + return { files, packages }; +} + +describe.each(["lite/index.ts", "core/index.ts"])("import graph of src/%s", (entry) => { + const { files, packages } = importGraph(join(SRC, entry)); + const rel = [...files].map((f) => relative(SRC, f).replaceAll("\\", "/")); + + it("does not reach the CLI, MCP, REST, web or connector modules", () => { + expect(rel.filter((f) => FORBIDDEN_DIRS.some((d) => f.startsWith(d)))).toEqual([]); + }); + + it("does not import commander, the MCP SDK or the scheduler", () => { + expect([...packages].filter((p) => FORBIDDEN_PACKAGES.includes(p))).toEqual([]); + }); + + it("includes the operation layer (sanity check of the walker)", () => { + expect(rel).toContain("core/operations/documents.ts"); + }); +}); + +describe("import graph walker", () => { + it("finds the connectors behind the full operation index", () => { + const { files } = importGraph(join(SRC, "core/operations/index.ts")); + expect(files).toContain(join(SRC, "connectors/registry.ts")); + }); +}); diff --git a/tests/unit/lite.test.ts b/tests/unit/lite.test.ts index 7e64b11b..74e8611b 100644 --- a/tests/unit/lite.test.ts +++ b/tests/unit/lite.test.ts @@ -1,348 +1,188 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { LibScopeLite } from "../../src/lite/index.js"; -import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + ConfigError, + LibScope, + ValidationError, + createCodeChunker, + createLite, + normalizeRawInput, + type LlmProvider, +} from "../../src/lite/index.js"; import { TreeSitterChunker } from "../../src/lite/chunker-treesitter.js"; -import type { LlmProvider } from "../../src/core/rag.js"; +import { initLogger } from "../../src/logger.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; + +const corpus = [ + { title: "React Hooks", content: "useState and useEffect are the most common React hooks." }, + { title: "Vue Composition", content: "Vue 3 composition API uses the setup function." }, + { title: "Angular DI", content: "Angular uses the dependency injection pattern extensively." }, +]; -function* fakeStream(): Generator<string> { - yield "Hello"; - yield " world"; +async function seed(scope: LibScope): Promise<void> { + for (const doc of corpus) await scope.add(doc); } -describe("LibScopeLite", () => { - let lite: LibScopeLite; - let provider: MockEmbeddingProvider; +/** Search results without the per-database IDs and timestamps. */ +function comparable(items: Array<Record<string, unknown>>): Array<Record<string, unknown>> { + return items.map(({ documentId: _d, chunkId: _c, ...rest }) => rest); +} - beforeEach(() => { - provider = new MockEmbeddingProvider(); - lite = new LibScopeLite({ dbPath: ":memory:", provider }); - }); +beforeAll(() => { + initLogger("silent"); +}); + +describe("createLite", () => { + const open: LibScope[] = []; + const lite = (options: Parameters<typeof createLite>[0]): LibScope => { + const scope = createLite(options); + open.push(scope); + return scope; + }; afterEach(() => { - lite.close(); + for (const scope of open.splice(0)) scope.close(); + vi.restoreAllMocks(); }); - describe("constructor", () => { - it("should create an instance with in-memory DB", () => { - expect(lite).toBeInstanceOf(LibScopeLite); - }); - - it("should accept custom embedding provider", () => { - const custom = new MockEmbeddingProvider(); - const instance = new LibScopeLite({ dbPath: ":memory:", provider: custom }); - expect(instance).toBeInstanceOf(LibScopeLite); - instance.close(); - }); + it("returns a LibScope", () => { + expect(lite({ dbPath: ":memory:", provider: new MockEmbeddingProvider() })).toBeInstanceOf( + LibScope, + ); }); - describe("index()", () => { - it("should index a single document", async () => { - await lite.index([{ title: "Test Doc", content: "This is test content for indexing." }]); - - expect(provider.embedBatchCallCount).toBeGreaterThan(0); - }); - - it("should index multiple documents", async () => { - await lite.index([ - { title: "Doc A", content: "Content of document A about TypeScript." }, - { title: "Doc B", content: "Content of document B about Python." }, - ]); - - // Both docs should have been processed - expect(provider.embedBatchCallCount).toBeGreaterThanOrEqual(2); - }); - - it("should index with optional metadata fields", async () => { - await lite.index([ - { - title: "Library Doc", - content: "React documentation content here.", - library: "react", - sourceType: "library", - version: "18.0.0", - url: "https://react.dev", - }, - ]); - - // Should succeed without errors - expect(provider.embedBatchCallCount).toBeGreaterThan(0); - }); + it("needs dbPath or db", () => { + expect(() => createLite({ provider: new MockEmbeddingProvider() })).toThrow(ValidationError); }); - describe("indexBatch()", () => { - it("should index documents with concurrency control", async () => { - const docs = Array.from({ length: 5 }, (_, i) => ({ - title: `Batch Doc ${i}`, - content: `Batch content number ${i} with enough text to be meaningful.`, - })); - - await lite.indexBatch(docs, { concurrency: 2 }); - - expect(provider.embedBatchCallCount).toBe(5); - }); - - it("should handle empty array", async () => { - await lite.indexBatch([], { concurrency: 2 }); - expect(provider.embedBatchCallCount).toBe(0); - }); - - it("should handle concurrency of 1 (sequential)", async () => { - const docs = [ - { title: "A", content: "Content A for sequential test." }, - { title: "B", content: "Content B for sequential test." }, - ]; - - await lite.indexBatch(docs, { concurrency: 1 }); - expect(provider.embedBatchCallCount).toBe(2); - }); + it("gives the same results as LibScope.create with useConfigFile: false", async () => { + const preset = lite({ dbPath: ":memory:", provider: new MockEmbeddingProvider() }); + const full = LibScope.create({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + useConfigFile: false, + }); + open.push(full); + await seed(preset); + await seed(full); + + const a = await preset.search({ query: "React hooks", limit: 2 }); + const b = await full.search({ query: "React hooks", limit: 2 }); + expect(a.total).toBe(b.total); + expect(comparable(a.items as unknown as Array<Record<string, unknown>>)).toEqual( + comparable(b.items as unknown as Array<Record<string, unknown>>), + ); + expect(a.items[0]).toHaveProperty("documentId"); + + const listA = await preset.docs.list(); + const listB = await full.docs.list(); + expect(listA.items.map((d) => d.title)).toEqual(listB.items.map((d) => d.title)); }); - describe("search()", () => { - beforeEach(async () => { - await lite.index([ - { - title: "React Hooks", - content: "useState and useEffect are the most common React hooks.", - }, - { title: "Vue Composition", content: "Vue 3 composition API uses setup function." }, - { title: "Angular DI", content: "Angular uses dependency injection pattern extensively." }, - ]); - }); - - it("should return search results", async () => { - const results = await lite.search("React hooks"); - expect(results.length).toBeGreaterThan(0); - }); - - it("should return results with expected shape", async () => { - const results = await lite.search("React"); - const first = results[0]; - expect(first).toBeDefined(); - expect(first).toHaveProperty("docId"); - expect(first).toHaveProperty("chunkId"); - expect(first).toHaveProperty("title"); - expect(first).toHaveProperty("content"); - expect(first).toHaveProperty("score"); - expect(first).toHaveProperty("url"); - expect(typeof first?.score).toBe("number"); - }); - - it("should respect limit option", async () => { - const results = await lite.search("API", { limit: 1 }); - expect(results.length).toBeLessThanOrEqual(1); - }); + it("ask without an LLM throws ConfigError; with one returns a RagResult", async () => { + const noLlm = lite({ dbPath: ":memory:", provider: new MockEmbeddingProvider() }); + await seed(noLlm); + await expect(noLlm.ask("What are hooks?")).rejects.toBeInstanceOf(ConfigError); + + const complete = vi.fn().mockResolvedValue({ text: "Hooks are functions." }); + const llm: LlmProvider = { model: "test-model", complete }; + const withLlm = lite({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + llmProvider: llm, + }); + await seed(withLlm); + const result = await withLlm.ask("What are hooks?"); + expect(result).toMatchObject({ + mode: "answer", + answer: "Hooks are functions.", + model: "test-model", + }); + if (result.mode === "answer") expect(result.sources.length).toBeGreaterThan(0); + expect(String(complete.mock.calls[0]?.[0])).toContain("What are hooks?"); }); - describe("getContext()", () => { - beforeEach(async () => { - await lite.index([ - { - title: "Node.js Streams", - content: "Readable streams in Node.js are a fundamental pattern.", - }, - ]); - }); - - it("should return a context string", async () => { - const context = await lite.getContext("How do Node.js streams work?"); - expect(typeof context).toBe("string"); - expect(context.length).toBeGreaterThan(0); + it("ignores config files: llm.provider can be set through config", async () => { + const scope = lite({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + config: { llm: { provider: "passthrough" } }, }); + await seed(scope); + expect((await scope.ask("What are hooks?")).mode).toBe("context"); }); +}); - describe("ask()", () => { - it("should throw when no LlmProvider is configured", async () => { - await lite.index([{ title: "Test", content: "Some content for testing ask." }]); - await expect(lite.ask("What is this about?")).rejects.toThrow("No LlmProvider configured"); - }); - - it("should call LlmProvider.complete with context", async () => { - // Declare the spy separately so we never reference it as an object method - const completeSpy = vi.fn().mockResolvedValue({ text: "Mocked LLM response" }); - const mockLlm: LlmProvider = { model: "test-model", complete: completeSpy }; - - const liteWithLlm = new LibScopeLite({ - dbPath: ":memory:", - provider, - llmProvider: mockLlm, - }); - - await liteWithLlm.index([ - { title: "Test Doc", content: "Information about testing patterns." }, - ]); - - const answer = await liteWithLlm.ask("What are testing patterns?"); - expect(answer).toBe("Mocked LLM response"); - expect(completeSpy).toHaveBeenCalledOnce(); - expect(completeSpy.mock.calls[0]?.[0]).toContain("testing patterns"); - - liteWithLlm.close(); - }); - - it("should allow llmProvider override in ask() opts", async () => { - const mockLlm: LlmProvider = { - model: "override-model", - complete: vi.fn().mockResolvedValue({ text: "Override response" }), - }; - - await lite.index([{ title: "Test", content: "Some content for LLM." }]); - - const answer = await lite.ask("Question?", { llmProvider: mockLlm }); - expect(answer).toBe("Override response"); - }); +describe("createCodeChunker", () => { + afterEach(() => { + vi.restoreAllMocks(); }); - describe("askStream()", () => { - it("should throw when no LlmProvider is configured", async () => { - await lite.index([{ title: "Test", content: "Content here." }]); - const gen = lite.askStream("Question?"); - await expect(gen.next()).rejects.toThrow("No LlmProvider configured"); - }); - - it("should throw when LlmProvider does not support streaming", async () => { - const mockLlm: LlmProvider = { - model: "no-stream", - complete: vi.fn().mockResolvedValue({ text: "done" }), - // No completeStream method - }; - - const liteWithLlm = new LibScopeLite({ - dbPath: ":memory:", - provider, - llmProvider: mockLlm, - }); - - await liteWithLlm.index([{ title: "Test", content: "Content." }]); - - const gen = liteWithLlm.askStream("Question?"); - await expect(gen.next()).rejects.toThrow("does not support streaming"); - - liteWithLlm.close(); - }); - - it("should stream tokens from LlmProvider", async () => { - const mockLlm: LlmProvider = { - model: "stream-model", - complete: vi.fn().mockResolvedValue({ text: "done" }), - completeStream: vi.fn().mockReturnValue(fakeStream()), - }; - - const liteWithLlm = new LibScopeLite({ - dbPath: ":memory:", - provider, - llmProvider: mockLlm, - }); - - await liteWithLlm.index([{ title: "Test", content: "Test content." }]); - - const tokens: string[] = []; - for await (const token of liteWithLlm.askStream("Question?")) { - tokens.push(token); - } - - expect(tokens).toEqual(["Hello", " world"]); - liteWithLlm.close(); - }); + const chunks = [ + { content: "function foo() {}", startLine: 1, endLine: 1, nodeType: "function_declaration" }, + { content: "function bar() {}", startLine: 2, endLine: 2, nodeType: "function_declaration" }, + ]; + + it("chunks code by the extension of the title", async () => { + const chunkSpy = vi.spyOn(TreeSitterChunker.prototype, "chunk").mockResolvedValue(chunks); + const chunker = createCodeChunker(); + const result = await chunker({ content: "src", title: "main.ts", source: "" }); + expect(result).toEqual(["function foo() {}", "function bar() {}"]); + expect(chunkSpy).toHaveBeenCalledWith("src", "ts"); }); - describe("rate()", () => { - it("should rate an indexed document", async () => { - await lite.index([{ title: "Rate Me", content: "Content to rate." }]); - - // Find the doc ID via search - const results = await lite.search("rate"); - expect(results.length).toBeGreaterThan(0); - const docId = results[0]?.docId; - expect(docId).toBeDefined(); - - // Should not throw - lite.rate(docId!, 5); - }); - - it("should throw for nonexistent document", () => { - expect(() => lite.rate("nonexistent-doc", 3)).toThrow(); - }); + it("uses a fixed language when given", async () => { + const chunkSpy = vi.spyOn(TreeSitterChunker.prototype, "chunk").mockResolvedValue(chunks); + await createCodeChunker({ language: "python" })({ content: "x", title: "Doc", source: "" }); + expect(chunkSpy).toHaveBeenCalledWith("x", "python"); }); - describe("close()", () => { - it("should close the database without error", () => { - const instance = new LibScopeLite({ dbPath: ":memory:", provider }); - expect(() => instance.close()).not.toThrow(); - }); + it("returns undefined for prose, unsupported languages and parse failures", async () => { + vi.spyOn(TreeSitterChunker.prototype, "chunk").mockRejectedValue(new Error("not installed")); + const chunker = createCodeChunker(); + expect(await chunker({ content: "x", title: "Guide", source: "" })).toBeUndefined(); + expect(await chunker({ content: "x", title: "main.rb", source: "" })).toBeUndefined(); + expect(await chunker({ content: "x", title: "main.go", source: "" })).toBeUndefined(); }); - describe("index() with language/tree-sitter chunking", () => { - afterEach(() => { - vi.restoreAllMocks(); - }); - - it("calls TreeSitterChunker.chunk() when language is set and supported", async () => { - vi.spyOn(TreeSitterChunker.prototype, "supports").mockReturnValue(true); - const chunkSpy = vi.spyOn(TreeSitterChunker.prototype, "chunk").mockResolvedValue([ - { - content: "function foo() {}", - startLine: 1, - endLine: 3, - nodeType: "function_declaration", - }, - { - content: "function bar() {}", - startLine: 5, - endLine: 7, - nodeType: "function_declaration", - }, - ]); - - await lite.index([ - { - title: "src/main.ts", - content: "function foo() {}\nfunction bar() {}", - language: "typescript", - }, - ]); - - expect(chunkSpy).toHaveBeenCalledWith("function foo() {}\nfunction bar() {}", "typescript"); - }); - - it("does not call chunk() when language is not set", async () => { - const chunkSpy = vi.spyOn(TreeSitterChunker.prototype, "chunk"); - - await lite.index([{ title: "Doc", content: "Some content here." }]); + it("is used by add when passed to createLite", async () => { + vi.spyOn(TreeSitterChunker.prototype, "chunk").mockResolvedValue(chunks); + const scope = createLite({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + chunker: createCodeChunker(), + }); + try { + const result = await scope.add({ content: "function foo() {}", title: "main.ts" }); + expect(result.documents[0]!.chunkCount).toBe(2); + } finally { + scope.close(); + } + }); +}); - expect(chunkSpy).not.toHaveBeenCalled(); +describe("normalizeRawInput", () => { + it("passes text through", async () => { + expect(await normalizeRawInput({ type: "text", title: "T", content: "C" })).toEqual({ + title: "T", + content: "C", }); + }); - it("falls back silently when tree-sitter throws", async () => { - vi.spyOn(TreeSitterChunker.prototype, "supports").mockReturnValue(true); - vi.spyOn(TreeSitterChunker.prototype, "chunk").mockRejectedValue( - new Error("tree-sitter not installed"), - ); - - // Should not throw — fallback to text chunker - await expect( - lite.index([ - { - title: "src/main.go", - content: "package main\nfunc main() {}", - language: "go", - }, - ]), - ).resolves.toBeUndefined(); + it("reads a buffer with the parser for its extension, else as text", async () => { + const md = await normalizeRawInput({ + type: "buffer", + buffer: Buffer.from("# Heading\n\nBody"), + filename: "notes.md", }); + expect(md.title).toBe("notes"); + expect(md.content).toContain("Body"); - it("does not call chunk() when language is set but not supported", async () => { - vi.spyOn(TreeSitterChunker.prototype, "supports").mockReturnValue(false); - const chunkSpy = vi.spyOn(TreeSitterChunker.prototype, "chunk"); - - await lite.index([ - { - title: "src/main.rb", - content: "def hello; end", - language: "ruby", - }, - ]); - - expect(chunkSpy).not.toHaveBeenCalled(); + const code = await normalizeRawInput({ + type: "buffer", + buffer: Buffer.from("const x = 1;"), + filename: "x.ts", + title: "x.ts", }); + expect(code).toEqual({ title: "x.ts", content: "const x = 1;" }); }); }); diff --git a/tests/unit/root-exports.test.ts b/tests/unit/root-exports.test.ts new file mode 100644 index 00000000..8730a4e1 --- /dev/null +++ b/tests/unit/root-exports.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; + +/** + * The runtime exports of the package entry points. Adding a public export is an API + * decision: update this list deliberately (and docs/reference/lite-api.md or + * docs/guide/programmatic-usage.md). + */ +const ROOT_EXPORTS = [ + "ChunkNotFoundError", + "ConfigError", + "DatabaseError", + "DocumentNotFoundError", + "EmbeddingError", + "FetchError", + "LibScope", + "LibScopeError", + "NotFoundError", + "TopicNotFoundError", + "ValidationError", +]; + +const LITE_EXTRAS = ["TreeSitterChunker", "createCodeChunker", "createLite", "normalizeRawInput"]; + +describe("package exports", () => { + it("libscope exports only LibScope and the error classes", async () => { + const root = await import("../../src/core/index.js"); + expect(Object.keys(root).sort()).toEqual(ROOT_EXPORTS); + }); + + it("libscope/lite exports the root plus the lite helpers", async () => { + const lite = await import("../../src/lite/index.js"); + expect(Object.keys(lite).sort()).toEqual([...ROOT_EXPORTS, ...LITE_EXTRAS].sort()); + }); +}); From 0d72be29dc5e2d7ec620deed04173749d5bf5426 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:10:48 +0000 Subject: [PATCH 45/83] docs(mcp): document the 11 core tools, admin toolset and MCP migration Tool parameter tables in docs/reference/mcp-tools.md were generated from the built server's tools/list output. --- docs/guide/mcp-setup.md | 98 ++++---- docs/migration-v2.md | 51 ++++ docs/reference/mcp-tools.md | 455 ++++++++++++++++-------------------- 3 files changed, 311 insertions(+), 293 deletions(-) diff --git a/docs/guide/mcp-setup.md b/docs/guide/mcp-setup.md index 6aec26d3..0853966e 100644 --- a/docs/guide/mcp-setup.md +++ b/docs/guide/mcp-setup.md @@ -78,44 +78,64 @@ If you're using [workspaces](/guide/configuration#workspaces), pass the workspac } ``` +## Admin Tools + +The server starts with 11 core tools. To also give the assistant `sync`, `install-pack`, `list-packs` and `reindex-documents`, set `LIBSCOPE_MCP_TOOLSETS` in the server's environment: + +```json +{ + "mcpServers": { + "libscope": { + "command": "npx", + "args": ["-y", "libscope", "serve"], + "env": { "LIBSCOPE_MCP_TOOLSETS": "admin" } + } + } +} +``` + ## Available Tools -Once connected, your AI assistant gets access to all 31 of LibScope's MCP tools. See the [MCP Tools Reference](/reference/mcp-tools) for full parameter details. - -**Search & Q&A** -- **`search-docs`** — semantic search with topic/library/version/rating filters -- **`ask-question`** — RAG Q&A with synthesized answers and source citations - -**Document Management** -- **`submit-document`** — index new content by text or URL -- **`update-document`** — update title, content, or metadata -- **`get-document`** — retrieve a document by ID -- **`list-documents`** — list docs with filters -- **`delete-document`** — remove a document -- **`rate-document`** — rate 1–5 with feedback -- **`suggest-tags`** — auto-suggest tags based on content - -**Organization** -- **`list-topics`** — browse the topic hierarchy -- **`link-documents`** — create cross-references between docs -- **`get-document-links`** — list a document's incoming and outgoing links -- **`delete-link`** — remove a cross-reference - -**Saved Searches** -- **`save-search`** — save a named query with filters -- **`list-saved-searches`** — list saved searches -- **`run-saved-search`** — execute a saved search -- **`delete-saved-search`** — delete a saved search - -**Similar Content** -- **`get-related`** — find chunks semantically similar to a given chunk - -**Connectors** — trigger syncs directly from your AI assistant: -- **`sync-obsidian-vault`**, **`sync-notion`**, **`sync-confluence`**, **`sync-slack`**, **`sync-onenote`** - -**Packs & Maintenance** -- **`install-pack`**, **`list-packs`** — manage knowledge packs -- **`reindex-documents`** — re-embed after switching providers -- **`health-check`** — DB status and doc/chunk counts - -Your AI assistant will call these tools automatically when it needs information from your docs. +See the [MCP Tools Reference](/reference/mcp-tools) for all parameters. + +**Core tools** + +- **`search`**: search by meaning and keywords, or find content related to a document or chunk (`relatedTo`). Filters: topic, library, version, source type, tags, minimum rating. +- **`ask`**: answer a question from the knowledge base. By default (`llm.provider: auto`) the server returns the retrieved context and your assistant writes the answer. Registered only when an answer is possible. +- **`get-document`**: read a document with its tags, links and ratings. Long content is paged. +- **`list-documents`**: list documents with filters, `offset` and a total count. +- **`overview`**: document, chunk and topic counts, topics, installed packs, the embedding model of the index, and health. +- **`submit-document`**: add inline content, a web page, a site crawl (`spider: true`) or a public repository URL. +- **`update-document`**: change the title, content, metadata or tags. +- **`delete-document`**: delete a document. +- **`rate-document`**: rate a document 1-5, with optional feedback or a correction. +- **`link-documents`**: create (`action: "create"`) or delete (`action: "delete"`) a link between documents. +- **`task`**: status, cancel or list of background tasks started with `async: true`. + +**Admin tools** (with `LIBSCOPE_MCP_TOOLSETS=admin`) + +- **`sync`**: sync one saved connector connection, or all of them. Set up connections with `libscope connect`; the tool does not accept credentials. +- **`install-pack`**, **`list-packs`**: manage knowledge packs. +- **`reindex-documents`**: re-embed chunks after you change the embedding model. + +Your AI assistant calls these tools when it needs information from your docs. The server also sends the assistant short instructions that describe the search, read and rate workflow. + +## Logs + +The MCP server writes JSON-RPC messages to stdout and its logs to stderr. Set the log level with `logging.level` in the config file or `LIBSCOPE_LOGGING_LEVEL`. + +## Embedding the Server + +`libscope/mcp` exports `createMcpServer()`. Importing the module does not start a server. + +```ts +import { createMcpServer, runStdioServer } from "libscope/mcp"; + +// Build a server and connect your own transport. +const { server, tools, close } = createMcpServer({ workspace: "my-project", toolsets: ["admin"] }); + +// Or serve over stdio, the same as `libscope serve`. +await runStdioServer({ workspace: "my-project" }); +``` + +`createMcpServer` accepts the same options as LibScope's startup (`workspace`, `dbPath`, `config`, `provider`) or an existing operation context (`ctx`). `close()` closes the server and the database that it opened. diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 1d83e537..55c46322 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -69,3 +69,54 @@ Every "not found" error is now a `NotFoundError` (`DocumentNotFoundError`, `Chun | `getVersion`, `rollbackToVersion` with an unknown version | `DocumentNotFoundError` | `NotFoundError` (`VERSION_NOT_FOUND`) | | `removePack` for a pack that is not installed | `ValidationError` | `NotFoundError` (`PACK_NOT_FOUND`) | | `getRelatedChunks` with an unknown chunk ID | `Error` | `ChunkNotFoundError` (`CHUNK_NOT_FOUND`) | + +## MCP + +The MCP server now has 11 core tools and an optional admin toolset. Old tool names do not work in 2.0. Update prompts, agent configs and allow-lists that name MCP tools. + +### Server + +| 1.x | 2.0 | +| ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| `import "libscope/mcp"` started a stdio server. | Importing `libscope/mcp` starts nothing. Call `runStdioServer()`, or `createMcpServer()` and connect your own transport. | +| `npm run serve` ran `dist/mcp/server.js`. | `npm run serve` runs `dist/mcp/main.js`. `node dist/mcp/server.js` exits without serving. | +| The server reported version `0.1.0`. | The server reports the package version. | +| Every tool was always registered. | `ask` is registered only with passthrough or a configured LLM. Admin tools are registered only with `LIBSCOPE_MCP_TOOLSETS=admin` (or `all`). | +| Most tools returned markdown or pretty-printed JSON. | Tools return compact text. Every document, chunk, link and task line includes its ID. | +| Long-running tools accepted `async` and polled `get-task`. | Same `async: true`, then `task {"action": "status", "taskId"}`. A finished task shows the same output as the tool run without `async`. | + +### Renamed and merged tools + +| 1.x tool | 2.0 tool | Parameter changes | +| ------------------------------------------------------------------------------------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `search-docs` | `search` | `source` -> `sourceType` (enum). Added `tags`, `relatedTo`. `limit` max 100. `maxChunksPerDocument` minimum is 1. | +| `get-related` | `search` with `relatedTo` | `chunkId` -> `relatedTo` (a chunk ID or a document ID). `minScore` and `includeLinkedDocuments` are removed. Use `version`, `sourceType`, `minRating` as filters. | +| `ask-question` | `ask` | Added `version`, `sourceType`, `tags`, `minRating`. | +| `health-check`, `list-topics` | `overview` | `list-topics` `parentId` is removed; `overview` lists every topic with its document count. | +| `list-packs` | `overview` (installed packs), or admin `list-packs` | Admin `list-packs` keeps `available` and `registryUrl`. | +| `get-document` | `get-document` | Added `offset` and `maxLength` for paging. The output includes tags, links (with `linkId`) and ratings. | +| `get-document-links` | `get-document` | Links are part of the document output. | +| `list-documents` | `list-documents` | Added `version`, `tags`, `offset`. `limit` max 1000. The output shows the total. | +| `submit-document` | `submit-document` | Added `tags`, `expiresAt`, `dedup`, `dryRun` and repository fields (`branch`, `paths`, `extensions`). A GitHub/GitLab repository URL indexes the repository. `topic` must be an existing topic ID or name. | +| `update-document` | `update-document` | `topicId` -> `topic` (ID or name). Added `tags` (replaces the document's tags). | +| `link-documents` | `link-documents` with `action: "create"` | `sourceId` -> `documentId`, `targetId` -> `targetDocumentId`. | +| `delete-link` | `link-documents` with `action: "delete"` | `linkId` unchanged. | +| `get-task` | `task` with `action: "status"` | `taskId` unchanged. An unknown task is an error. | +| `cancel-task` | `task` with `action: "cancel"` | `taskId` unchanged. An unknown task is an error. | +| — | `task` with `action: "list"` | New. | +| `install-pack` (always on) | `install-pack` (admin toolset) | `nameOrPath` -> `pack`. Added `batchSize`, `concurrency`. | +| `reindex-documents` (always on) | `reindex-documents` (admin toolset) | Added `rebuild`. | +| `sync-slack`, `sync-notion`, `sync-confluence`, `sync-onenote`, `sync-obsidian-vault` | `sync` (admin toolset) | `{ "name": "<saved connection>" }` or `{ "all": true }`. Credentials and connector settings are no longer tool parameters: save them with `libscope connect <type>`. | + +`delete-document` and `rate-document` keep their names and parameters. + +### Removed tools + +These operations are available from the CLI, the REST API and the SDK, not over MCP: + +| 1.x tool | Operation (REST path under `/api/v1`) | +| ------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| `save-search`, `list-saved-searches`, `run-saved-search`, `delete-saved-search` | `save-search`, `list-saved-searches`, `run-saved-search`, `delete-saved-search` (`/searches`) | +| `create-webhook`, `list-webhooks`, `delete-webhook` | `create-webhook`, `list-webhooks`, `delete-webhook` (`/webhooks`) | +| `search-analytics` | `search-analytics` (`/analytics/searches`) | +| `suggest-tags` | `suggest-tags` (`/documents/:documentId/suggested-tags`) | diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 4b263e64..235a8fbb 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -1,337 +1,284 @@ # MCP Tools -LibScope exposes the following tools over the Model Context Protocol. Any MCP-compatible client (Claude, Cursor, VS Code, etc.) can call these directly. +LibScope exposes its knowledge base over the Model Context Protocol. Any MCP client (Claude, Cursor, VS Code, and others) can call these tools. See [MCP Setup](/guide/mcp-setup) to connect a client. -## search-docs +The server has 11 core tools. Four admin tools are available when you enable the admin toolset. Each tool is generated from a LibScope operation, so its parameters, defaults and validation are the same as in the CLI, the REST API and the SDK. -Semantic search across your knowledge base. +## Workflow -| Parameter | Type | Required | Description | -| ----------- | ------ | -------- | ---------------------------- | -| `query` | string | ✅ | Search query | -| `library` | string | | Filter by library name | -| `topic` | string | | Filter by topic | -| `version` | string | | Filter by library version | -| `minRating` | number | | Minimum average rating (1–5) | -| `limit` | number | | Max results (default: 10) | -| `offset` | number | | Pagination offset | +1. `search` finds chunks. Each result shows a `documentId` and a `chunkId`. +2. `get-document` reads a document with its tags, links and ratings. For a long document, set `maxLength`, then pass the `offset` that the output shows as "next page". +3. `rate-document` records whether a document was useful, wrong or out of date. -Each result shows a `**Document ID:** … | **Chunk ID:** …` line. Pass these IDs to `get-document`, `rate-document`, `get-related`, and `link-documents`. +The server sends these steps to the client as its `instructions`, so the assistant knows the workflow without extra prompting. -**Search results** include a `scoreExplanation` object on each result: +## Output -```typescript -{ - method: "hybrid" | "vector" | "fts5" | "keyword", - rawScore: number, // raw score before boosts - boostFactors: string[], // e.g. ["title_match:x1.5"] - details: string // human-readable scoring breakdown -} -``` - -## get-document - -Retrieve a document by its ID, including ratings and metadata. - -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | --------------- | -| `documentId` | string | ✅ | The document ID | - -## delete-document - -Delete a document from the knowledge base. - -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | ------------------------- | -| `documentId` | string | ✅ | The document ID to delete | - -## submit-document - -Index a new document. You can provide content directly, or a URL to fetch automatically. - -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | ---------------------------------------------------- | -| `title` | string | | Document title (auto-detected from URL if omitted) | -| `content` | string | | Document content in markdown (omit if providing URL) | -| `url` | string | | URL to fetch and index | -| `library` | string | | Library name | -| `version` | string | | Library version | -| `topic` | string | | Topic to categorize under | -| `sourceType` | string | | `library`, `topic`, `manual`, or `model-generated` | -| `dedup` | string | | Duplicate detection behaviour (see below) | -| `dedupOptions` | object | | Fine-tune duplicate detection (see below) | +Tools return compact text. Each line that names a document, chunk, link or task includes its ID (`documentId`, `chunkId`, `linkId`, `taskId`), so the assistant can pass the ID to the next tool. Lists show their position, for example `Documents 1-50 of 120 — next page: offset 50`. -**`dedup`** *(optional)*: Controls duplicate detection behaviour. -- `"skip"` — If a duplicate is detected, return the existing document without re-indexing -- `"warn"` — Log a warning about the duplicate but index anyway -- `"force"` — Skip duplicate checking entirely and always index -- *(omitted)* — Default behaviour: reject exact duplicates by title+content-length, allow similar content +Errors return `isError: true` with a message. Invalid parameters fail with an input validation error that names the parameter. -**`dedupOptions`** *(optional)*: Fine-tune duplicate detection. -- `threshold` *(number, 0–1)*: Similarity threshold for semantic dedup (default 0.95) -- `strategy` *(string)*: `"exact"` (hash-based) or `"semantic"` (embedding-based) +## Background tasks -## update-document +`submit-document`, `sync`, `install-pack` and `reindex-documents` accept `async: true`. With it, the tool starts a background task and returns a `taskId` immediately. Then: -Update an existing document's title, content, or metadata. Changing content triggers re-chunking and re-embedding. Changing the title, library, or version re-embeds the existing chunks, because these fields are part of the embedded text. +- `task {"action": "status", "taskId": "..."}` shows the status (`pending`, `running`, `completed`, `failed` or `cancelled`), the progress and, when the task is done, the same output that the tool returns without `async`. +- `task {"action": "cancel", "taskId": "..."}` asks the task to stop. +- `task {"action": "list"}` lists the tasks from the last hour. -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | --------------------------------------- | -| `documentId` | string | ✅ | The document ID to update | -| `title` | string | | New title | -| `content` | string | | New content (triggers re-chunking) | -| `library` | string | | New library name (pass `null` to clear) | -| `version` | string | | New version (pass `null` to clear) | -| `url` | string | | New source URL (pass `null` to clear) | -| `topicId` | string | | New topic ID (pass `null` to clear) | +Tasks are kept in memory for one hour after they finish. They are lost when the server stops. -## get-related +## ask and passthrough -Find document chunks that are semantically similar to a given chunk (more-like-this). +`ask` is registered only when it can answer: -| Parameter | Type | Required | Description | -| ----------- | ------ | -------- | ---------------------------------- | -| `chunkId` | string | ✅ | The source chunk ID | -| `limit` | number | | Max results (default: 10) | -| `library` | string | | Filter results by library | -| `topic` | string | | Filter results by topic | +- With `llm.provider` set to `auto` (the default) or `passthrough`, the MCP server does not call an LLM. `ask` returns the retrieved context and its sources. The calling assistant writes the answer. +- With `llm.provider` set to `openai`, `anthropic` or `ollama` and the provider configured, `ask` returns the LLM's answer and its sources. +- When an LLM provider is selected but cannot be created (for example, an API key is missing), the server does not register `ask`. -## rate-document +## Admin toolset -Rate a document and optionally suggest corrections. +Set `LIBSCOPE_MCP_TOOLSETS` in the server's environment to enable optional toolsets. The value is a comma-separated list. `admin` enables `sync`, `install-pack`, `list-packs` and `reindex-documents`. `all` enables every optional toolset. -| Parameter | Type | Required | Description | -| --------------------- | ------ | -------- | ------------------------------------- | -| `documentId` | string | ✅ | The document ID | -| `rating` | number | ✅ | Rating from 1 (poor) to 5 (excellent) | -| `chunkId` | string | | Rate a specific chunk | -| `feedback` | string | | Text feedback | -| `suggestedCorrection` | string | | Suggested replacement content | - -## list-documents - -List documents with optional filters. - -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | ------------------------- | -| `library` | string | | Filter by library | -| `topic` | string | | Filter by topic | -| `sourceType` | string | | Filter by source type | -| `limit` | number | | Max results (default: 50) | - -## list-topics - -List available topics. - -| Parameter | Type | Required | Description | -| ---------- | ------ | -------- | -------------------------------------- | -| `parentId` | string | | Filter by parent topic (for subtopics) | - -## ask-question - -RAG question-answering. Retrieves relevant chunks and synthesizes an answer using your configured LLM. - -| Parameter | Type | Required | Description | -| ---------- | ------ | -------- | ----------------------------------------- | -| `question` | string | ✅ | The question to answer | -| `library` | string | | Filter source docs by library | -| `topic` | string | | Filter source docs by topic | -| `topK` | number | | Number of chunks to retrieve (default: 5) | +```json +{ + "mcpServers": { + "libscope": { + "command": "npx", + "args": ["-y", "libscope", "serve"], + "env": { "LIBSCOPE_MCP_TOOLSETS": "admin" } + } + } +} +``` -## health-check +`sync` uses connections saved with `libscope connect`. It does not accept tokens or other credentials as parameters. -Check database connectivity, document/chunk counts, and FTS5 index status. Takes no parameters. +## Core tools -## reindex-documents +### search -Re-embed all document chunks with the current embedding provider. Use after switching providers. +Search the knowledge base by meaning and keywords (query), or find content similar to a document or chunk (relatedTo). Results carry documentId and chunkId. -| Parameter | Type | Required | Description | -| ------------- | -------- | -------- | --------------------------------------------------- | -| `batchSize` | number | | Chunks per batch (default: 50) | -| `since` | string | | Only reindex docs created after this ISO-8601 date | -| `before` | string | | Only reindex docs created before this ISO-8601 date | -| `documentIds` | string[] | | Only reindex specific documents | +Annotations: read-only. -## Connector sync tools +| Parameter | Type | Required | Description | +| ---------------------- | ----------------------------------------------------- | -------- | ------------------------------------------------------------------------- | +| `query` | string | | What to search for | +| `relatedTo` | string | | Document or chunk ID: return similar content instead of running a query | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | +| `tags` | string[] | | Only documents carrying all of these tags | +| `minRating` | number | | Minimum average rating | +| `limit` | integer | | Maximum results (default 10, max 100) | +| `offset` | integer | | Results to skip (paging) Default: `0`. | +| `maxChunksPerDocument` | integer | | At most this many chunks per document (default: no limit) | +| `contextChunks` | integer | | Neighbouring chunks to include before and after each result Default: `0`. | -The five `sync-*` tools share these behaviors: +### ask -- `name` (string, optional) selects a saved connector config (default: the connector type). Configs are saved by `libscope connect <type>`. Any parameter you omit, including tokens, comes from that config, so an agent does not need secrets in its context. Parameters you pass apply to that call only and are not saved. -- `async` (boolean, optional) runs the sync in the background and returns a task ID. `cancel-task` stops the sync between pages or items, and the run is recorded as failed. -- Each call writes one entry to the connector sync history under `name`. +Retrieve the knowledge-base context for a question (passthrough: no LLM is called; answer from the returned context yourself) -## sync-obsidian-vault +Annotations: read-only. -Sync an Obsidian vault into the knowledge base. Parses wikilinks, frontmatter, embeds, and tags. +| Parameter | Type | Required | Description | +| ------------ | ----------------------------------------------------- | -------- | ------------------------------------------- | +| `question` | string | yes | The question | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | +| `tags` | string[] | | Only documents carrying all of these tags | +| `minRating` | number | | Minimum average rating | +| `topK` | integer | | Chunks to retrieve as context Default: `5`. | -| Parameter | Type | Required | Description | -| ----------- | ------ | --------------- | ------------------------------------ | -| `vaultPath` | string | unless saved | Absolute path to the vault directory | -| `name` | string | | Saved config name (default: `obsidian`) | +### get-document -## sync-notion +Get a document with its tags, links and rating summary; long content can be paged -Sync Notion pages and databases. +Annotations: read-only. -| Parameter | Type | Required | Description | -| -------------- | -------- | ------------ | --------------------------------------- | -| `token` | string | unless saved | Notion integration token | -| `excludePages` | string[] | | Page/database IDs to exclude | -| `lastSync` | string | | ISO-8601 timestamp for incremental sync | -| `name` | string | | Saved config name (default: `notion`) | +| Parameter | Type | Required | Description | +| ------------ | ------- | -------- | ------------------------------------------------------ | +| `documentId` | string | yes | Document ID | +| `offset` | integer | | Character offset into the content Default: `0`. | +| `maxLength` | integer | | Maximum characters of content to return (default: all) | -## sync-confluence +### list-documents -Sync Confluence spaces and pages. +List documents (newest first) with optional filters -| Parameter | Type | Required | Description | -| --------------- | -------- | ------------ | --------------------------------------- | -| `baseUrl` | string | unless saved | Confluence base URL | -| `email` | string | Cloud only | User email | -| `token` | string | unless saved | API token or PAT | -| `spaces` | string[] | | Space keys to sync (default: all) | -| `excludeSpaces` | string[] | | Space keys to exclude | -| `name` | string | | Saved config name (default: `confluence`) | +Annotations: read-only. -## sync-slack +| Parameter | Type | Required | Description | +| ------------ | ----------------------------------------------------- | -------- | ----------------------------------------- | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | +| `tags` | string[] | | Only documents carrying all of these tags | +| `limit` | integer | | Maximum results (default 50, max 1000) | +| `offset` | integer | | Results to skip (paging) Default: `0`. | -Sync Slack channel messages and threads. +### overview -| Parameter | Type | Required | Description | -| ----------------- | -------- | ------------ | -------------------------------------------- | -| `token` | string | unless saved | Slack bot token | -| `channels` | string[] | | Channel names/IDs, or `["all"]` (default: all) | -| `excludeChannels` | string[] | | Channels to exclude | -| `threadMode` | string | | `aggregate` (default) or `separate` | -| `name` | string | | Saved config name (default: `slack`) | +Knowledge base overview: counts, topics, installed packs, embedding model of the index, and health -## sync-onenote +Annotations: read-only. -Sync OneNote notebooks via Microsoft Graph API. +No parameters. -| Parameter | Type | Required | Description | -| -------------- | ------ | ------------ | ------------------------------------------------------------------------ | -| `accessToken` | string | unless saved | Microsoft Graph API access token, used as given | -| `notebookName` | string | | Specific notebook (default: all) | -| `name` | string | | Saved config name (default: `onenote`) | +### submit-document -Without `accessToken`, the saved config's refresh token is used to get a new access token when needed, and the new tokens are saved. +Add to the knowledge base: inline content (with title), a web page (url), a site crawl (url + spider: true) or a public GitHub/GitLab repository URL. Local file paths are not accepted. -## install-pack +Annotations: not destructive. -Install a knowledge pack from the registry or a local file. +| Parameter | Type | Required | Description | +| ----------------- | ----------------------------------------------------- | -------- | -------------------------------------------------------------------------------------------------------- | +| `content` | string | | Inline document content (markdown) | +| `title` | string | | Title (required with content; detected for files and URLs) | +| `url` | string | | Source URL. Without content, the URL is fetched; with content, it is stored | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Source type (default: library if library is set, topic if topic is set, else manual) | +| `tags` | string[] | | Tags to add to every new document | +| `expiresAt` | string | | ISO 8601 time after which the document is pruned | +| `dedup` | `skip` \| `warn` \| `force` | | Duplicate handling: skip returns the existing document, warn indexes anyway, force skips the check | +| `spider` | boolean | | URL: also crawl linked pages Default: `false`. | +| `maxPages` | integer | | Crawl: page limit (default 25, max 200) | +| `maxDepth` | integer | | Crawl: link depth (default 2, max 5) | +| `sameDomain` | boolean | | Crawl: stay on the seed domain (default true) | +| `pathPrefix` | string | | Crawl: only follow links under this path | +| `excludePatterns` | string[] | | Crawl: globs of URLs to skip | +| `branch` | string | | Repository: branch (default: from URL, else main) | +| `paths` | string[] | | Repository: only these subdirectories | +| `extensions` | string[] | | Repository: file extensions (default .md, .mdx, .txt, .rst) | +| `dryRun` | boolean | | List what would be added without adding it Default: `false`. | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"} Default: `false`. | -| Parameter | Type | Required | Description | -| ------------- | ------ | -------- | ---------------------------- | -| `nameOrPath` | string | ✅ | Pack name or local file path | -| `registryUrl` | string | | Custom registry URL | +### update-document -## list-packs +Update a document's title, content, metadata or tags (content changes are re-indexed) -List installed or available knowledge packs. +Annotations: not destructive, idempotent. -| Parameter | Type | Required | Description | -| ------------- | ------- | -------- | ------------------------------------------------ | -| `available` | boolean | | If true, list from registry instead of installed | -| `registryUrl` | string | | Custom registry URL | +| Parameter | Type | Required | Description | +| ------------ | -------------- | -------- | ---------------------------------------- | +| `documentId` | string | yes | Document ID | +| `title` | string | | New title | +| `content` | string | | New content (re-chunked and re-embedded) | +| `library` | string \| null | | New library (null clears it) | +| `version` | string \| null | | New version (null clears it) | +| `url` | string \| null | | New URL (null clears it) | +| `topic` | string \| null | | New topic ID or name (null clears it) | +| `tags` | string[] | | Replace the document's tags with these | -## suggest-tags +### delete-document -Suggest tags for a document based on content analysis (compares against existing tags in the knowledge base). +Delete a document with its chunks, vectors, tags, links and ratings -| Parameter | Type | Required | Description | -| ---------------- | ------ | -------- | ------------------------------------ | -| `documentId` | string | ✅ | The document ID | -| `maxSuggestions` | number | | Maximum suggestions to return (1–20, default: 5) | +Annotations: destructive. -## link-documents +| Parameter | Type | Required | Description | +| ------------ | ------ | -------- | ----------- | +| `documentId` | string | yes | Document ID | -Create a typed cross-reference relationship between two documents. +### rate-document -| Parameter | Type | Required | Description | -| ---------- | ------ | -------- | -------------------------------------------------------------------- | -| `sourceId` | string | ✅ | The source document ID | -| `targetId` | string | ✅ | The target document ID | -| `linkType` | string | ✅ | Relationship type: `see_also`, `prerequisite`, `supersedes`, `related`, `references` | -| `label` | string | | Optional human-readable description of the relationship | +Rate a document (1-5), optionally with feedback or a suggested correction -The output includes a `Link ID:` line. Pass this ID to `delete-link`. +Annotations: not destructive. -## get-document-links +| Parameter | Type | Required | Description | +| --------------------- | ------- | -------- | ---------------------------------------- | +| `documentId` | string | yes | Document ID | +| `chunkId` | string | | Rate one chunk of the document | +| `rating` | integer | yes | 1 (poor) to 5 (excellent) | +| `feedback` | string | | What is good or wrong | +| `suggestedCorrection` | string | | Replacement text if the content is wrong | -Get all cross-reference links for a document, both outgoing and incoming. Each line ends with `[link ID: …]`, which you can pass to `delete-link`. +### link-documents -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | --------------- | -| `documentId` | string | ✅ | The document ID | +Create a typed link from one document to another (action: create), or delete a link by linkId (action: delete). get-document lists a document's links. -## delete-link +Annotations: destructive. -Remove a cross-reference link between documents. +| Parameter | Type | Required | Description | +| ------------------ | ------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------------------------------- | +| `action` | `create` \| `delete` | yes | What to do: create, delete | +| `documentId` | string | | Source document ID (action: create) | +| `targetDocumentId` | string | | Target document ID (action: create) | +| `linkType` | `see_also` \| `prerequisite` \| `supersedes` \| `related` \| `references` | | Relationship type: see_also, prerequisite, supersedes, related, references (action: create) | +| `label` | string | | Short description of the relationship (action: create) | +| `linkId` | string | | Link ID (action: delete) | -| Parameter | Type | Required | Description | -| --------- | ------ | -------- | ------------------------ | -| `linkId` | string | ✅ | The link ID to delete | +### task -## save-search +Background tasks started with async: true. status: progress and result of a task; cancel: stop it; list: tasks from the last hour. -Save a search query with optional filters for later re-use. +Annotations: not destructive, idempotent. -| Parameter | Type | Required | Description | -| ----------- | -------- | -------- | --------------------------------- | -| `name` | string | ✅ | Unique name for this saved search | -| `query` | string | ✅ | The search query | -| `topic` | string | | Filter by topic ID | -| `library` | string | | Filter by library name | -| `version` | string | | Filter by library version | -| `source` | string | | Filter by source type | -| `minRating` | number | | Minimum average rating filter | -| `limit` | number | | Maximum results to return | -| `tags` | string[] | | Filter by tags | +| Parameter | Type | Required | Description | +| --------- | ------------------------------ | -------- | ------------------------------------------------------------------------ | +| `action` | `status` \| `cancel` \| `list` | yes | What to do: status, cancel, list | +| `taskId` | string | | Task ID returned when a background task started (action: status, cancel) | -## list-saved-searches +## Admin tools -List all saved searches. +These tools are registered only when `LIBSCOPE_MCP_TOOLSETS` contains `admin` or `all`. -_No parameters._ +### sync -## run-saved-search +Sync one saved connector connection (name) or all of them (all: true) with the settings saved by 'libscope connect' -Execute a saved search by name or ID and return results. +Annotations: not destructive. -| Parameter | Type | Required | Description | -| ---------- | ------ | -------- | ----------------------------------------- | -| `nameOrId` | string | ✅ | The name or ID of the saved search to run | +| Parameter | Type | Required | Description | +| --------- | ------- | -------- | -------------------------------------------------------------------------------------------------------- | +| `name` | string | | Saved connection name (see `libscope connections`) | +| `all` | boolean | | Sync every saved connection Default: `false`. | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"} Default: `false`. | -## delete-saved-search +### install-pack -Delete a saved search by name or ID. +Install a knowledge pack from the registry or a local .json/.json.gz file -| Parameter | Type | Required | Description | -| ---------- | ------ | -------- | -------------------------------------------- | -| `nameOrId` | string | ✅ | The name or ID of the saved search to delete | +Annotations: not destructive, idempotent. -## get-task +| Parameter | Type | Required | Description | +| ------------- | ------- | -------- | -------------------------------------------------------------------------------------------------------- | +| `pack` | string | yes | Pack name from the registry, or a local .json/.json.gz file | +| `registryUrl` | string | | Registry URL (default: the public pack registry) | +| `batchSize` | integer | | Documents per batch (default 10) | +| `concurrency` | integer | | Batches embedded in parallel (default 4) | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"} Default: `false`. | -Get the status, progress, and result of a background task. Tools called with `async: true` return a task ID. +### list-packs -| Parameter | Type | Required | Description | -| --------- | ------ | -------- | ---------------------------------------- | -| `taskId` | string | ✅ | Task ID returned by an async tool call | +List installed packs, or packs available in the registry -The task `type` is one of `index_document`, `reindex_documents`, `sync_connector`, or `install_pack`. +Annotations: read-only. -## cancel-task +| Parameter | Type | Required | Description | +| ------------- | ------- | -------- | --------------------------------------------------------------- | +| `available` | boolean | | List registry packs instead of installed ones Default: `false`. | +| `registryUrl` | string | | Registry URL (default: the public pack registry) | -Request cancellation of a pending or running background task. +### reindex-documents -| Parameter | Type | Required | Description | -| --------- | ------ | -------- | ------------------ | -| `taskId` | string | ✅ | Task ID to cancel | +Re-embed chunks with the configured embedding model (rebuild after changing models) -`submit-document` (including spider mode), `reindex-documents`, and `install-pack` stop at the next page or batch. A cancelled `install-pack` removes the documents it already inserted. The final status is: +Annotations: not destructive, idempotent. -- `cancelled` — the task stopped because of the cancellation. -- `completed` — the task finished before it could stop. The result is kept. -- `failed` — the task stopped because of a different error. +| Parameter | Type | Required | Description | +| ------------- | ----------------------------------- | -------- | -------------------------------------------------------------------------------------------------------- | +| `documentIds` | string[] | | Only these documents | +| `since` | string (ISO 8601 date or date-time) | | Only documents created on or after | +| `before` | string (ISO 8601 date or date-time) | | Only documents created on or before | +| `batchSize` | integer | | Chunks per embedding call (default 50) | +| `rebuild` | boolean | | Drop and recreate the vector table for the configured model, then re-embed everything Default: `false`. | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"} Default: `false`. | From 04a5cc1c11b0ae0079921e3e483fb3a50260a9cc Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:10:58 +0000 Subject: [PATCH 46/83] docs(api): REST reference, dashboard guide and REST migration notes for 2.0 --- docs/guide/dashboard.md | 43 +++-- docs/migration-v2.md | 96 ++++++++++ docs/reference/rest-api.md | 361 ++++++++++++++++++------------------- 3 files changed, 303 insertions(+), 197 deletions(-) diff --git a/docs/guide/dashboard.md b/docs/guide/dashboard.md index 7ffa6fe5..06a07dea 100644 --- a/docs/guide/dashboard.md +++ b/docs/guide/dashboard.md @@ -18,7 +18,7 @@ libscope serve --dashboard --port 8080 libscope serve --dashboard --host 0.0.0.0 --port 3377 ``` -The dashboard is fully self-contained — no external CDN dependencies. +The dashboard page needs no external files. The knowledge graph page (`/graph`) loads d3 from `d3js.org`. ## Features @@ -61,33 +61,48 @@ The graph is useful for discovering how your documents relate to each other and Click the sun/moon icon in the top-right corner to toggle between light and dark mode. The preference is saved in `localStorage`. +## Data API + +The dashboard pages read their data from a small JSON API on the same server. Each URL runs one LibScope operation, the same operation as the CLI, MCP server and REST API: + +| URL | Operation | +| --------------------------------------------- | --------------------------------------------- | +| `GET /api/stats` | `overview` (document, topic and chunk counts) | +| `GET /api/topics` | `list-topics` | +| `GET /api/documents?limit&offset&topic` | `list-documents` | +| `GET /api/documents/:id` | `get-document` | +| `DELETE /api/documents/:id` | `delete-document` | +| `GET /api/search?q&limit&topic` | `search` | +| `GET /api/graph?threshold&maxNodes&topic&tag` | `graph` | + +These URLs are for the dashboard pages. For your own scripts, use the [REST API](../reference/rest-api.md) (`libscope serve --api`), which has every operation, API-key authentication and an OpenAPI document. + ## Rate Limiting -The dashboard server applies per-IP rate limiting to prevent abuse when exposed on a network. The defaults are generous for local use; if you hit limits, restart with `--host localhost` to restrict access to your machine. +The dashboard server allows 120 requests per minute from each IP address. A request over the limit gets `429`. ## Running Alongside the MCP Server -The dashboard and MCP server run on different ports and can be started independently: +The dashboard, the REST API and the MCP server are separate servers. You can start them independently: ```bash # Dashboard in one terminal libscope serve --dashboard --port 3377 -# MCP server in another (or configured in your AI client) +# REST API in another +libscope serve --api --port 3378 + +# MCP server (or configure it in your AI client) libscope serve ``` -Or run the REST API and use it as the dashboard's data source: - -```bash -libscope serve --api --port 3378 -libscope serve --dashboard --port 3377 -``` +All of them use the same workspace database. ## Security Considerations -The dashboard is designed for **local use only**. If you expose it on a network: +The dashboard is for **local use**. It has no authentication. Anyone who can connect to its port can read and delete documents. -- Use `--host` carefully — binding to `0.0.0.0` exposes it to all network interfaces -- Consider putting it behind a reverse proxy (nginx, Caddy) with authentication -- The REST API (`--api`) does require an `X-API-Key` header for write operations; the dashboard uses the same key automatically when running on the same host +- The server listens on `localhost` by default. If you use `--host 0.0.0.0`, every network interface can reach it. +- The server sends no `Access-Control-Allow-Origin` header, so pages from other origins cannot read its responses. +- The server rejects write requests (`DELETE`) from pages on other origins with `403`. +- To expose the dashboard on a network, put it behind a reverse proxy (nginx, Caddy) that adds authentication. diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 1d83e537..48175419 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -69,3 +69,99 @@ Every "not found" error is now a `NotFoundError` (`DocumentNotFoundError`, `Chun | `getVersion`, `rollbackToVersion` with an unknown version | `DocumentNotFoundError` | `NotFoundError` (`VERSION_NOT_FOUND`) | | `removePack` for a pack that is not installed | `ValidationError` | `NotFoundError` (`PACK_NOT_FOUND`) | | `getRelatedChunks` with an unknown chunk ID | `Error` | `ChunkNotFoundError` (`CHUNK_NOT_FOUND`) | + +## REST API + +Every `/api/v1` route now runs one operation and is generated from it. The [REST API reference](reference/rest-api.md) lists all routes. The OpenAPI document at `GET /openapi.json` is generated from the same routes (OpenAPI 3.1.0, was a hand-written 3.0.3 document that listed 14 routes). + +### Routes + +| Old route (1.x) | New route (2.0) | +| --------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| `GET /api/v1/search?q=&source=&tag=` | `GET /api/v1/search?query=&sourceType=&tags=` (also `library`, `version`, `minRating`, `relatedTo`) | +| `POST /api/v1/batch-search` | Removed. Send one `GET /api/v1/search` per query. | +| `POST /api/v1/documents/url` | `POST /api/v1/documents` with `url` (and `spider: true` to crawl) | +| `POST /api/v1/index/repos/:repoSlug` | Removed with the REST repo indexer. `POST /api/v1/documents` with `source: "<repository URL>"`. | +| `GET /api/v1/index/jobs/:jobId` | `GET /api/v1/tasks/:taskId` | +| `GET /api/v1/stats` | `GET /api/v1/overview` (`data.stats`) | +| `GET /api/v1/connectors/status` | `GET /api/v1/connections` (`lastRun` of each connection). The `history=true` sync history is removed. | +| `GET /api/v1/connectors/schedules` | `GET /api/v1/connections` (`schedule` of each connection) | +| `GET /api/v1/documents/:id/links` | `GET /api/v1/links?documentId=:id` | +| `GET /api/v1/documents/:id/suggest-tags` | `GET /api/v1/documents/:id/suggested-tags` | +| `POST /api/v1/bulk/:operation` with any other name | Only `delete`, `retag` and `move` exist; other names are `404`. | +| `POST /api/v1/ask` with `Accept: text/event-stream` | Removed (no streaming). `POST /api/v1/ask` returns the whole answer. | + +These routes did not change their path: `GET /openapi.json`, `GET /api/v1/health`, `GET` and `POST /api/v1/documents`, `GET`, `PATCH` and `DELETE /api/v1/documents/:id`, `POST /api/v1/documents/:id/tags`, `POST /api/v1/documents/:id/links`, `DELETE /api/v1/links/:id`, `POST /api/v1/ask`, `GET` and `POST /api/v1/topics`, `GET /api/v1/tags`, `GET /api/v1/analytics/searches`, the saved-search routes, `POST /api/v1/bulk/{delete,retag,move}` and the webhook routes. + +New routes: `DELETE /api/v1/documents/:id/tags`, `POST /api/v1/documents/:id/ratings`, `GET /api/v1/documents/:id/versions`, `POST /api/v1/documents/:id/rollback`, `GET /api/v1/documents/:id/prerequisites`, `GET /api/v1/links`, `GET /api/v1/graph`, `DELETE /api/v1/topics/:topic`, `GET`, `POST` and `DELETE /api/v1/packs`, `GET /api/v1/connections`, `DELETE /api/v1/connections/:name`, `POST /api/v1/sync`, `GET /api/v1/overview`, `POST /api/v1/admin/reindex`, `GET /api/v1/admin/duplicates`, `POST /api/v1/admin/prune-expired`, `GET /api/v1/analytics/{popular,stale,top-queries}`, `GET /api/v1/tasks`, `GET /api/v1/tasks/:taskId` and `POST /api/v1/tasks/:taskId/cancel`. + +### Request fields + +| Route | Old field (1.x) | New field (2.0) | +| ------------------------------------- | ------------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `POST /api/v1/documents` | `source` (the source type) | `sourceType`. `source` now names what to add (URL or repository). | +| `POST /api/v1/documents` | `topic` (topic ID) | `topic` (topic ID or name; the topic must exist) | +| `PATCH /api/v1/documents/:id` | `topicId` | `topic` (ID or name; `null` clears it). New: `tags` replaces the tags. | +| `POST /api/v1/documents/:id/links` | `targetId` | `targetDocumentId` | +| `POST /api/v1/searches` | `filters: { topic, library, ... }` | The filter fields at the top level: `topic`, `library`, `version`, `sourceType`, `tags`, `minRating`, `limit` | +| `POST /api/v1/bulk/*` | `selector: { topicId, library, ... }` | The selector fields at the top level: `topic`, `library`, `sourceType`, `tags`, `dateFrom`, `dateTo` | +| `POST /api/v1/bulk/move` | `targetTopicId` | `targetTopic` (ID or name) | +| `GET` routes with `limit` or `offset` | Out-of-range values were clamped | Out-of-range values are rejected with `400` | + +### Responses + +| 1.x | 2.0 | +| -------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `POST /api/v1/documents` indexed the document and answered `201` with it. | It starts a task and answers `202` with `taskId`. Poll `GET /api/v1/tasks/:taskId`; `result` (a JSON string) lists the added documents with their `documentId`. | +| Create routes answered `201`; `DELETE` routes answered `204` with no body. | Every synchronous route answers `200` with `{ data, meta }`. `DELETE` returns, for example, `{ "documentId": "...", "deleted": true }`. | +| `GET /api/v1/search` returned `{ results, totalCount }`. | `{ items, total, limit, offset }` | +| `GET /api/v1/documents` returned an array of documents with their content. | `{ items, total, limit, offset }`. Items have `documentId` and `contentLength`, not `id` and `content`. | +| `GET /api/v1/documents/:id` returned the document row. | `{ document, content, contentLength, offset, nextOffset, tags, links, ratings }`. Use `offset` and `maxLength` to page the content. | +| `GET /api/v1/topics`, `/tags`, `/webhooks`, `/searches` returned arrays. | `{ items, ... }` | +| `GET /api/v1/health` returned `{ status, docCount, dbSize }`. | `{ status: "ok" }`. Counts and index health are in `GET /api/v1/overview`. | +| Links had `id`. | Links have `linkId`. | +| Not-found errors had the code `NOT_FOUND`. | The code names the resource: `DOCUMENT_NOT_FOUND`, `TOPIC_NOT_FOUND`, `LINK_NOT_FOUND`, `TASK_NOT_FOUND`, ... Unknown routes keep `NOT_FOUND`. | +| A body over 1 MB answered `500`. | `413` with the code `PAYLOAD_TOO_LARGE` | +| `ask` without an LLM answered `500` with a generic message. | `500` with the code `CONFIG_ERROR` and the setting to change | + +### CORS and browser requests + +| 1.x | 2.0 | +| ----------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `corsOrigins: ["*"]` allowed every method from every origin. | `"*"` allows `GET` only. Name an origin in `corsOrigins` to let it write. | +| A page on any origin could send a write request without a preflight (for example, a form POST). | Write requests from an origin that is not listed, and is not the server's own origin, get `403` (`FORBIDDEN_ORIGIN`). | + +### REST repo indexer (removed) + +The REST repo indexer is removed: the routes `POST /api/v1/index/repos/:repoSlug` and `GET /api/v1/index/jobs/:jobId`, the `LIBSCOPE_REPOS_CONFIG` environment variable and its repos JSON file. It indexed each repository into its own database (`~/.libscope/repos/<slug>.db`) that no other part of LibScope could search, and it always used the local embedding model. + +Add a repository to the workspace database instead: + +```bash +curl -X POST http://localhost:3378/api/v1/documents \ + -H "Content-Type: application/json" \ + -d '{ "source": "https://github.com/org/repo", "branch": "main", "paths": ["docs"] }' +``` + +LibScope 2.0 does not read or delete the files in `~/.libscope/repos/`. Delete them yourself when you no longer need them. + +### Web dashboard server + +The dashboard URLs (`/`, `/graph`, `/api/stats`, `/api/topics`, `/api/documents`, `/api/documents/:id`, `/api/search`, `/api/graph`) keep their paths and response shapes, with these changes: + +| 1.x | 2.0 | +| ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| Every response had `Access-Control-Allow-Origin: *`, also for `DELETE`. | No CORS header by default (same origin only). Cross-origin `DELETE` gets `403`. | +| Errors were `{ "error": "<message>" }`. | `{ "error": { "code", "message" } }` | +| `GET /api/documents` items had `content`. | Items have `contentLength` instead. | +| Out-of-range `limit` values were clamped. | Out-of-range values are rejected with `400` (`limit` 1-1000 for documents, 1-100 for search). | + +### Library code + +| Old (1.x) | New (2.0) | +| ------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `startApiServer(db, provider, options)` | `startApiServer({ db, provider, config }, options)`. Pass the result of `bootstrap()`. The returned `port` is the bound port. | +| `startWebServer(db, provider, options)` | `startWebServer({ db, provider, config }, options)` | +| `handleRequest(req, res, db, provider)` (`src/api/routes.ts`) | `handleRequest(req, res, ctx)` with an `OperationContext` from `createOperationContext` | +| `OPENAPI_SPEC` (`src/api/openapi.ts`) | `buildOpenApiSpec(API_ROUTES)` | +| `handleGraphRequest` (`src/web/graph-api.ts`) | The `graph` operation (`graphOperation`) | diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index a03bbe74..b7ed864a 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -6,139 +6,159 @@ Start the REST API server: libscope serve --api --port 3378 ``` -The OpenAPI 3.0 spec is available at `GET /openapi.json`. +Every route under `/api/v1` runs one LibScope operation: the same operation, with the same parameter names, defaults and validation, as the CLI, the MCP server and the SDK. The OpenAPI 3.1 document at `GET /openapi.json` is generated from the operations, so it always matches the server. -## Endpoints +## Conventions -### Search & Q&A +**Input.** Path parameters (`:documentId`) fill the input field of the same name. `GET` and `DELETE` routes read the other fields from the query string. Numbers and booleans are parsed (`limit=5`, `deleteDocuments=true`). Array fields take repeated or comma-separated values (`tags=a&tags=b` or `tags=a,b`). `POST`, `PATCH` and `PUT` routes read a JSON object body (maximum 1 MB). Unknown fields are ignored. -| Method | Endpoint | Description | -| ------ | ------------------------- | ------------------------------------- | -| `GET` | `/api/v1/search?q=...` | Semantic search | -| `POST` | `/api/v1/batch-search` | Run up to 20 search queries at once | -| `POST` | `/api/v1/ask` | RAG question-answering | +**Responses.** A success is `{ "data": <operation result>, "meta": { "took": <ms> } }` with status `200`. Lists are `{ "items": [...] }`, with `total`, `limit` and `offset` when the list is paged. Results that refer to a document carry `documentId`. Chunk results carry `chunkId`. -### Documents +**Background tasks.** Long-running operations (`add`, `sync`, `reindex`, `dedupe`, `install-pack`) start a task and answer `202` with `{ "data": { "taskId", "operation", "status" } }` and a `Location` header. Poll `GET /api/v1/tasks/:taskId`. When `status` is `completed`, `result` holds the operation result as a JSON string. When `status` is `failed`, `error` holds the message. The server keeps finished tasks in memory for one hour. -| Method | Endpoint | Description | -| -------- | ------------------------------------- | ------------------------------------- | -| `GET` | `/api/v1/documents` | List documents (with filters) | -| `POST` | `/api/v1/documents` | Index a new document | -| `GET` | `/api/v1/documents/:id` | Get a single document | -| `PATCH` | `/api/v1/documents/:id` | Update a document | -| `DELETE` | `/api/v1/documents/:id` | Delete a document | -| `POST` | `/api/v1/documents/url` | Index from a URL | -| `POST` | `/api/v1/documents/:id/tags` | Add tags to a document | -| `GET` | `/api/v1/documents/:id/suggest-tags` | Auto-suggest tags based on content | -| `GET` | `/api/v1/documents/:id/links` | List cross-reference links | -| `POST` | `/api/v1/documents/:id/links` | Create a cross-reference link | - -### Document Links - -| Method | Endpoint | Description | -| -------- | --------------------- | ---------------------- | -| `DELETE` | `/api/v1/links/:id` | Delete a link | - -### Topics & Tags - -| Method | Endpoint | Description | -| ------ | ----------------- | -------------------- | -| `GET` | `/api/v1/topics` | List all topics | -| `POST` | `/api/v1/topics` | Create a topic | -| `GET` | `/api/v1/tags` | List all tags | - -### Saved Searches - -| Method | Endpoint | Description | -| -------- | --------------------------- | ---------------------------- | -| `GET` | `/api/v1/searches` | List saved searches | -| `POST` | `/api/v1/searches` | Create a saved search | -| `POST` | `/api/v1/searches/:id/run` | Run a saved search | -| `DELETE` | `/api/v1/searches/:id` | Delete a saved search | - -### Bulk Operations - -| Method | Endpoint | Description | -| ------ | ----------------------- | ------------------------------ | -| `POST` | `/api/v1/bulk/delete` | Bulk delete documents | -| `POST` | `/api/v1/bulk/retag` | Bulk add/remove tags | -| `POST` | `/api/v1/bulk/move` | Bulk move documents to a topic | - -### Repository Indexing - -| Method | Endpoint | Description | -| ------ | ----------------------------------- | ------------------------------------------- | -| `POST` | `/api/v1/index/repos/:repoSlug` | Trigger a repo index job (async, 202) | -| `GET` | `/api/v1/index/jobs/:jobId` | Poll index job status | - -The `POST` endpoint is designed to be used as a **Bitbucket post-receive webhook**. It clones the repo, walks files, runs tree-sitter chunking for supported languages, and indexes everything into the main libscope database under `library: repoSlug`. The response is immediate (202) with a `jobId`. - -**Configure via `LIBSCOPE_REPOS_CONFIG`** — path to a JSON file: - -```json -{ - "repos": { - "my-service": { - "cloneUrl": "https://git.example.com/org/my-service.git", - "branch": "main", - "include": ["src/**/*.go", "**/*.cs"], - "exclude": ["vendor/**", "**/*_test.go"] - } - } -} -``` +**Errors.** An error is `{ "error": { "code": "...", "message": "..." } }`. -**Trigger a full reindex:** +| Status | When | +| ------ | ------------------------------------------------------------------------------------------------------ | +| `400` | Invalid input (`VALIDATION_ERROR`), invalid JSON (`INVALID_JSON`), or a malformed path | +| `401` | `LIBSCOPE_API_KEY` is set and the request has no `Authorization: Bearer <key>` header, or a wrong key | +| `403` | A browser page from another origin sent a write request (`FORBIDDEN_ORIGIN`) | +| `404` | Unknown route or method, or the resource does not exist (`DOCUMENT_NOT_FOUND`, `TOPIC_NOT_FOUND`, ...) | +| `413` | The body is larger than 1 MB (`PAYLOAD_TOO_LARGE`) | +| `429` | More than 120 requests per minute from one address (`RATE_LIMITED`) | +| `500` | Server error. A `CONFIG_ERROR` message tells you what to configure (for example, an LLM for `ask`) | +| `502` | A fetched URL failed (`FETCH_ERROR`) | -```bash -curl -X POST http://localhost:3378/api/v1/index/repos/my-service \ - -H "Authorization: Bearer $API_KEY" \ - -H "Content-Type: application/json" \ - -d '{}' -# → { "data": { "jobId": "abc-123", "status": "queued" } } -``` +**Authentication.** When the `LIBSCOPE_API_KEY` environment variable is set, every request must send `Authorization: Bearer <key>`. -**Incremental reindex** (specific files only, e.g. from a push webhook): +**CORS.** By default, only `http://localhost` and `http://localhost:3000` can call the API from a browser. To change the list, pass `corsOrigins` to `startApiServer`. The origin `"*"` lets every origin read (`GET`), but not write. A write request from an origin that is not in the list, and is not the server's own origin, gets `403`. -```bash -curl -X POST http://localhost:3378/api/v1/index/repos/my-service \ - -H "Authorization: Bearer $API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ "files": ["src/auth/handler.go", "src/models/user.go"] }' -``` +**Local files.** Over REST, `add` cannot read files or directories on the server, and `install-pack` cannot install a pack file. `backup`, `restore` and `create-pack` have no route. Use the CLI or the SDK for these. -**Poll job status:** +## Endpoints -```bash -curl http://localhost:3378/api/v1/index/jobs/abc-123 -# → { "data": { "jobId": "abc-123", "status": "completed", "stats": { ... } } } -``` +The parameters of each route are in the OpenAPI document (`GET /openapi.json`). -Job status values: `queued` → `running` → `completed` | `failed`. +### Meta + +| Method | Path | Operation | Description | +| ------ | ---------------- | --------- | ---------------------------------------------------------------------- | +| `GET` | `/openapi.json` | — | This OpenAPI document. | +| `GET` | `/api/v1/health` | — | Liveness check (use GET /api/v1/overview for counts and index health). | + +### Documents + +| Method | Path | Operation | Description | +| -------- | ---------------------------------------- | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `POST` | `/api/v1/documents` | `add` | Add content to the knowledge base: inline content, a URL (optionally crawled), a GitHub/GitLab repository, or (CLI/SDK only) a local file or directory. Runs as a task (202). | +| `GET` | `/api/v1/documents/:documentId` | `get-document` | Get a document with its tags, links and rating summary; long content can be paged. | +| `GET` | `/api/v1/documents` | `list-documents` | List documents (newest first) with optional filters. | +| `PATCH` | `/api/v1/documents/:documentId` | `update-document` | Update a document's title, content, metadata or tags (content changes are re-indexed). | +| `DELETE` | `/api/v1/documents/:documentId` | `delete-document` | Delete a document with its chunks, vectors, tags, links and ratings. | +| `POST` | `/api/v1/documents/:documentId/ratings` | `rate-document` | Rate a document (1-5), optionally with feedback or a suggested correction. | +| `GET` | `/api/v1/documents/:documentId/versions` | `document-history` | List saved versions of a document, newest first. | +| `POST` | `/api/v1/documents/:documentId/rollback` | `rollback-document` | Restore a document to a saved version (the current state is saved first). | + +### Search and Q&A + +| Method | Path | Operation | Description | +| ------ | ---------------- | --------- | ---------------------------------------------------------------------------------------------------------- | +| `GET` | `/api/v1/search` | `search` | Search the knowledge base by meaning and keywords, or find content related to a document or chunk. | +| `POST` | `/api/v1/ask` | `ask` | Answer a question from the knowledge base with an LLM, or (passthrough) return the context to answer from. | + +### Links and graph + +| Method | Path | Operation | Description | +| -------- | --------------------------------------------- | ------------------ | ------------------------------------------------------------------------------------------ | +| `POST` | `/api/v1/documents/:documentId/links` | `link-documents` | Create a typed link from one document to another. | +| `DELETE` | `/api/v1/links/:linkId` | `unlink-documents` | Delete a link between documents. | +| `GET` | `/api/v1/links` | `list-links` | List links of one document (outgoing and incoming) or all links. | +| `GET` | `/api/v1/documents/:documentId/prerequisites` | `prerequisites` | List the documents to read before this one (following prerequisite links). | +| `GET` | `/api/v1/graph` | `graph` | Knowledge graph: documents, topics and tags as nodes; links, tags and similarity as edges. | + +### Tags + +| Method | Path | Operation | Description | +| -------- | ---------------------------------------------- | -------------- | ---------------------------------------------------- | +| `POST` | `/api/v1/documents/:documentId/tags` | `add-tags` | Add tags to a document (tags are created as needed). | +| `DELETE` | `/api/v1/documents/:documentId/tags` | `remove-tags` | Remove tags from a document. | +| `GET` | `/api/v1/tags` | `list-tags` | List all tags with their document counts. | +| `GET` | `/api/v1/documents/:documentId/suggested-tags` | `suggest-tags` | Suggest tags for a document from its content. | + +### Topics + +| Method | Path | Operation | Description | +| -------- | ----------------------- | -------------- | ---------------------------------------------- | +| `GET` | `/api/v1/topics` | `list-topics` | List topics with their document counts. | +| `POST` | `/api/v1/topics` | `create-topic` | Create a topic. | +| `DELETE` | `/api/v1/topics/:topic` | `delete-topic` | Delete a topic, optionally with its documents. | + +### Saved searches + +| Method | Path | Operation | Description | +| -------- | ------------------------------ | --------------------- | ------------------------------------------------- | +| `POST` | `/api/v1/searches` | `save-search` | Save a search query and its filters under a name. | +| `GET` | `/api/v1/searches` | `list-saved-searches` | List saved searches. | +| `POST` | `/api/v1/searches/:search/run` | `run-saved-search` | Run a saved search. | +| `DELETE` | `/api/v1/searches/:search` | `delete-saved-search` | Delete a saved search. | + +### Packs + +| Method | Path | Operation | Description | +| -------- | --------------------- | -------------- | ------------------------------------------------------------------------------------------------ | +| `POST` | `/api/v1/packs` | `install-pack` | Install a knowledge pack from the registry or a local .json/.json.gz file. Runs as a task (202). | +| `DELETE` | `/api/v1/packs/:pack` | `remove-pack` | Remove an installed pack and its documents. | +| `GET` | `/api/v1/packs` | `list-packs` | List installed packs, or packs available in the registry. | + +### Connectors + +| Method | Path | Operation | Description | +| -------- | --------------------------- | ------------------ | ------------------------------------------------------------------------------------------ | +| `GET` | `/api/v1/connections` | `list-connections` | List saved connector connections with their schedule and last sync. | +| `POST` | `/api/v1/sync` | `sync` | Sync one saved connection, or all of them, using the saved settings. Runs as a task (202). | +| `DELETE` | `/api/v1/connections/:name` | `disconnect` | Remove a connection's documents and its saved settings (including credentials). | + +### Admin and bulk + +| Method | Path | Operation | Description | +| ------ | ----------------------------- | --------------- | ---------------------------------------------------------------------------------------------------------- | +| `GET` | `/api/v1/overview` | `overview` | Knowledge base overview: counts, topics, installed packs, embedding model of the index, and health. | +| `POST` | `/api/v1/admin/reindex` | `reindex` | Re-embed chunks with the configured embedding model (rebuild after changing models). Runs as a task (202). | +| `GET` | `/api/v1/admin/duplicates` | `dedupe` | Find groups of duplicate or near-duplicate documents. Runs as a task (202). | +| `POST` | `/api/v1/admin/prune-expired` | `prune-expired` | Delete documents whose expiry time (expiresAt) has passed. | +| `POST` | `/api/v1/bulk/delete` | `bulk-delete` | Delete every document matching the filters (at most 1000 per call). | +| `POST` | `/api/v1/bulk/retag` | `bulk-retag` | Add and/or remove tags on every document matching the filters. | +| `POST` | `/api/v1/bulk/move` | `bulk-move` | Move every document matching the filters to another topic. | + +### Analytics + +| Method | Path | Operation | Description | +| ------ | ------------------------------- | ------------------ | --------------------------------------------------------------- | +| `GET` | `/api/v1/analytics/popular` | `popular` | Documents returned most often in search results. | +| `GET` | `/api/v1/analytics/stale` | `stale` | Documents that no search returned in the given number of days. | +| `GET` | `/api/v1/analytics/top-queries` | `top-queries` | Most frequent search queries. | +| `GET` | `/api/v1/analytics/searches` | `search-analytics` | Search volume, top and zero-result queries, and knowledge gaps. | ### Webhooks -| Method | Endpoint | Description | -| -------- | ------------------------------- | ----------------------------- | -| `GET` | `/api/v1/webhooks` | List webhooks | -| `POST` | `/api/v1/webhooks` | Create a webhook | -| `DELETE` | `/api/v1/webhooks/:id` | Delete a webhook | -| `POST` | `/api/v1/webhooks/:id/test` | Send a test ping | +| Method | Path | Operation | Description | +| -------- | ---------------------------------- | ---------------- | ----------------------------------------------------------------------- | +| `POST` | `/api/v1/webhooks` | `create-webhook` | Register a URL to receive a signed POST for document and search events. | +| `GET` | `/api/v1/webhooks` | `list-webhooks` | List registered webhooks (secrets are never shown). | +| `DELETE` | `/api/v1/webhooks/:webhookId` | `delete-webhook` | Delete a webhook. | +| `POST` | `/api/v1/webhooks/:webhookId/test` | `test-webhook` | Send a test event to a webhook and report the HTTP status. | -### System +### Tasks -| Method | Endpoint | Description | -| ------ | -------------------------------- | ---------------------------------- | -| `GET` | `/api/v1/health` | Health check with document count | -| `GET` | `/api/v1/stats` | Usage statistics | -| `GET` | `/api/v1/analytics/searches` | Search analytics and knowledge gaps| -| `GET` | `/api/v1/connectors/status` | Connector sync status and history | -| `GET` | `/api/v1/connectors/schedules` | Scheduled connector entries | -| `GET` | `/openapi.json` | OpenAPI 3.0 specification | +| Method | Path | Operation | Description | +| ------ | ------------------------------ | ------------- | ------------------------------------------------------------- | +| `GET` | `/api/v1/tasks/:taskId` | `get-task` | Status, progress and result of a background task. | +| `POST` | `/api/v1/tasks/:taskId/cancel` | `cancel-task` | Request cancellation of a pending or running background task. | +| `GET` | `/api/v1/tasks` | `list-tasks` | List background tasks from the last hour, newest first. | ## Examples -### Index a document +### Add a document ```bash curl -X POST http://localhost:3378/api/v1/documents \ @@ -148,33 +168,37 @@ curl -X POST http://localhost:3378/api/v1/documents \ "content": "# Authentication\n\nUse OAuth2...", "tags": ["auth", "security"] }' +# -> 202 { "data": { "taskId": "6f1c...", "operation": "add", "status": "running" } } + +curl http://localhost:3378/api/v1/tasks/6f1c... +# -> { "data": { "status": "completed", "result": "{\"documents\":[{\"documentId\":\"...\"}],...}", ... } } ``` -### Search +### Add a web page, a crawled site or a repository ```bash -curl "http://localhost:3378/api/v1/search?q=authentication&limit=5" -``` +# One page +curl -X POST http://localhost:3378/api/v1/documents \ + -H "Content-Type: application/json" \ + -d '{ "url": "https://docs.example.com/guide", "library": "my-lib" }' -### Search with filters +# The page and the pages it links to +curl -X POST http://localhost:3378/api/v1/documents \ + -H "Content-Type: application/json" \ + -d '{ "url": "https://docs.example.com/", "spider": true, "maxPages": 50 }' -```bash -curl "http://localhost:3378/api/v1/search?q=deploy&library=my-lib&topic=backend&limit=10" +# A GitHub or GitLab repository (markdown and text files by default) +curl -X POST http://localhost:3378/api/v1/documents \ + -H "Content-Type: application/json" \ + -d '{ "source": "https://github.com/org/repo", "branch": "main", "paths": ["docs"] }' ``` -### Batch search - -Run multiple search queries concurrently (up to 20). Results are keyed by query string. +### Search ```bash -curl -X POST http://localhost:3378/api/v1/batch-search \ - -H "Content-Type: application/json" \ - -d '{ - "requests": [ - { "query": "authentication" }, - { "query": "deployment", "options": { "library": "my-lib", "limit": 5 } } - ] - }' +curl "http://localhost:3378/api/v1/search?query=authentication&limit=5" +curl "http://localhost:3378/api/v1/search?query=deploy&library=my-lib&topic=backend&tags=ops,prod" +curl "http://localhost:3378/api/v1/search?relatedTo=<documentId>" ``` ### Ask a question @@ -182,33 +206,26 @@ curl -X POST http://localhost:3378/api/v1/batch-search \ ```bash curl -X POST http://localhost:3378/api/v1/ask \ -H "Content-Type: application/json" \ - -d '{ - "question": "How does authentication work?", - "topic": "security" - }' + -d '{ "question": "How does authentication work?", "topic": "security" }' ``` -### Index from a URL +`ask` needs an LLM (see `llm.provider` in the [configuration reference](configuration.md)). Without one, `ask` answers `500` with the code `CONFIG_ERROR` and the setting to change. + +### Update a document ```bash -curl -X POST http://localhost:3378/api/v1/documents/url \ +curl -X PATCH http://localhost:3378/api/v1/documents/<documentId> \ -H "Content-Type: application/json" \ - -d '{ - "url": "https://docs.example.com/guide", - "library": "my-lib" - }' + -d '{ "title": "Auth Guide v2", "tags": ["auth"] }' ``` -### Update a document +### Tags ```bash -curl -X PATCH http://localhost:3378/api/v1/documents/<id> \ +curl -X POST http://localhost:3378/api/v1/documents/<documentId>/tags \ -H "Content-Type: application/json" \ - -d '{ - "title": "Updated Title", - "library": "my-lib", - "version": "2.0.0" - }' + -d '{ "tags": ["security"] }' +curl -X DELETE "http://localhost:3378/api/v1/documents/<documentId>/tags?tags=security" ``` ### Bulk retag @@ -216,55 +233,33 @@ curl -X PATCH http://localhost:3378/api/v1/documents/<id> \ ```bash curl -X POST http://localhost:3378/api/v1/bulk/retag \ -H "Content-Type: application/json" \ - -d '{ - "selector": {"library": "react"}, - "addTags": ["v18"], - "removeTags": ["v17"], - "dryRun": false - }' + -d '{ "library": "my-lib", "addTags": ["reviewed"], "dryRun": true }' ``` -### Create a webhook +### Link documents ```bash -curl -X POST http://localhost:3378/api/v1/webhooks \ +curl -X POST http://localhost:3378/api/v1/documents/<documentId>/links \ -H "Content-Type: application/json" \ - -d '{ - "url": "https://hooks.example.com/libscope", - "events": ["document.created", "document.updated"], - "secret": "my-hmac-secret" - }' + -d '{ "targetDocumentId": "<otherId>", "linkType": "prerequisite" }' +curl "http://localhost:3378/api/v1/links?documentId=<documentId>" ``` -Webhook payloads are signed with HMAC-SHA256 when a secret is provided. The signature is sent in the `X-LibScope-Signature` header. - -Supported events: `document.created`, `document.updated`, `document.deleted`. - -### Create a cross-reference link +### Saved searches ```bash -curl -X POST http://localhost:3378/api/v1/documents/<source-id>/links \ +curl -X POST http://localhost:3378/api/v1/searches \ -H "Content-Type: application/json" \ - -d '{ - "targetId": "<target-id>", - "linkType": "prerequisite", - "label": "Read this first" - }' + -d '{ "name": "auth-docs", "query": "authentication", "library": "my-lib" }' +curl -X POST http://localhost:3378/api/v1/searches/auth-docs/run ``` -Valid `linkType` values: `see_also`, `prerequisite`, `supersedes`, `related`. - -### Create a saved search +### Webhooks ```bash -curl -X POST http://localhost:3378/api/v1/searches \ +curl -X POST http://localhost:3378/api/v1/webhooks \ -H "Content-Type: application/json" \ - -d '{ - "name": "Auth Docs", - "query": "authentication best practices", - "filters": {"library": "my-lib"} - }' - -# Run it later -curl -X POST http://localhost:3378/api/v1/searches/<id>/run + -d '{ "url": "https://hooks.example.com/libscope", "events": ["document.created"] }' ``` + +See [Webhooks](../guide/webhooks.md) for the payload and the signature. From cbacaf879b4d39ba577a26b46a3604c684482a32 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:14:19 +0000 Subject: [PATCH 47/83] feat(core): diversity on search and systemPrompt on ask The SDK and lite accepted MMR diversity and a system prompt in 1.x; the search and ask operations now take them, so every surface can. --- src/core/operations/search.ts | 14 ++++++++++++++ tests/unit/libscope.test.ts | 9 +++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/core/operations/search.ts b/src/core/operations/search.ts index 50e28e7a..24a8550e 100644 --- a/src/core/operations/search.ts +++ b/src/core/operations/search.ts @@ -58,6 +58,12 @@ export const searchOperation = defineOperation({ .max(2) .default(0) .describe("Neighbouring chunks to include before and after each result"), + diversity: z + .number() + .min(0) + .max(1) + .optional() + .describe("Query search: MMR reranking, 0 = relevance only (default), 1 = most diverse"), }), annotations: { readOnly: true }, http: { method: "GET", path: "/search" }, @@ -94,6 +100,7 @@ export const searchOperation = defineOperation({ offset: input.offset, maxChunksPerDocument: input.maxChunksPerDocument, contextChunks: input.contextChunks, + diversity: input.diversity, }); return { items: results, total: totalCount, limit: input.limit, offset: input.offset }; }, @@ -104,6 +111,12 @@ const askInput = z.object({ ...s.documentFilters, minRating, topK: z.number().int().min(1).max(20).default(5).describe("Chunks to retrieve as context"), + systemPrompt: z + .string() + .min(1) + .max(10_000) + .optional() + .describe("System prompt for the LLM (default: answer from the context and cite titles)"), }); /** RAG options for a parsed `ask` input (topic names resolved). Shared with streaming ask. */ @@ -117,6 +130,7 @@ export function toRagOptions(ctx: OperationContext, input: z.output<typeof askIn sourceType: input.sourceType, tags: input.tags, minRating: input.minRating, + systemPrompt: input.systemPrompt, }; } diff --git a/tests/unit/libscope.test.ts b/tests/unit/libscope.test.ts index 46c9b6e8..2a8321ee 100644 --- a/tests/unit/libscope.test.ts +++ b/tests/unit/libscope.test.ts @@ -96,7 +96,7 @@ describe("LibScope", () => { expect(added.documents[0]!.title).toBe("Vitest"); const byString = await scope.search("unit tests"); - const byObject = await scope.search({ query: "unit tests", limit: 5 }); + const byObject = await scope.search({ query: "unit tests", limit: 5, diversity: 0.5 }); expect(byString.items.length).toBeGreaterThan(0); expect(byObject).toMatchObject({ limit: 5, offset: 0 }); expect(byObject.total).toBeGreaterThan(0); @@ -236,9 +236,14 @@ describe("LibScope ask", () => { const scope = create({ llmProvider: llm }); try { await addDoc(scope, "Ask Doc"); - const result = await scope.ask({ question: "What is tested?", topK: 2 }); + const result = await scope.ask({ + question: "What is tested?", + topK: 2, + systemPrompt: "Be brief.", + }); expect(result).toMatchObject({ mode: "answer", answer: "Mocked answer", model: "mock-llm" }); expect(complete).toHaveBeenCalledOnce(); + expect(complete.mock.calls[0]?.[1]).toBe("Be brief."); } finally { scope.close(); } From 3dabd6a6a8f1d846e01c6aac45c097136a23cdf5 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:14:23 +0000 Subject: [PATCH 48/83] docs: LibScope SDK and libscope/lite for 2.0; Library API migration - programmatic-usage: create options, add/search/ask, namespace table, types, errors. - lite guide and API reference: createLite, createCodeChunker, Chunker, normalizeRawInput. - migration-v2 Library API: removed root exports with replacements, LibScope method changes, LibScopeLite -> createLite, package exports. --- docs/guide/lite.md | 450 ++++++--------------- docs/guide/programmatic-usage.md | 173 +++++--- docs/migration-v2.md | 90 +++++ docs/reference/lite-api.md | 655 +++++-------------------------- 4 files changed, 413 insertions(+), 955 deletions(-) diff --git a/docs/guide/lite.md b/docs/guide/lite.md index 1fc75476..6fcd92a8 100644 --- a/docs/guide/lite.md +++ b/docs/guide/lite.md @@ -1,33 +1,18 @@ # LibScope Lite — Embedded Semantic Search -`libscope/lite` is a lightweight, embeddable version of LibScope designed to be imported directly into any Node.js application. Instead of running a standalone CLI process or an MCP server, you call `index()` and `search()` programmatically from your own code. +`libscope/lite` puts LibScope inside your own Node.js application. You call `add()`, `search()` and `ask()` from your code. There is no CLI process and no server. -## When to Use LibScope Lite +`createLite()` returns a normal [`LibScope`](/guide/programmatic-usage) object. The methods and results are the same as `LibScope.create()`. The differences: -Use `libscope/lite` when you need to: +- It does not read config files, `~/.libscope/secrets.json` or `LIBSCOPE_*` config variables. Settings come only from the options that you pass. +- You must name the database (`dbPath` or `db`). +- `libscope/lite` does not load the CLI, the MCP server or the connectors when you import it. -- **Embed semantic search into another application** — e.g., a custom MCP server, a VS Code extension, a CI/CD tool -- **Avoid spawning subprocesses** — no CLI execution, no HTTP server required -- **Control the database lifecycle** — pass `:memory:` for ephemeral sessions or a file path for persistent cross-session reuse -- **Search across code** — tree-sitter powered chunking splits source files at function and class boundaries +## When to use Lite -The primary use case driving this feature: a Bitbucket MCP server that wants semantic search over repository files and Jira/Confluence pages. On repository connect, it calls `indexBatch(repoFiles)` to build a local index. On PR review, it calls `getContext(question)` to retrieve the top-K relevant chunks and inject them into its LLM prompt — replacing 50 raw files of context with 5 highly-relevant chunks. - -## What Lite Does NOT Include - -`libscope/lite` intentionally omits the full-LibScope surface area: - -| Feature | Full `libscope` | `libscope/lite` | -|---|---|---| -| Semantic search | ✅ | ✅ | -| RAG (ask/stream) | ✅ | ✅ | -| Code-aware chunking | ❌ | ✅ | -| CLI commands | ✅ | ❌ | -| MCP server | ✅ | ❌ | -| Connectors (Notion, Slack…) | ✅ | ❌ | -| Topics & packs | ✅ | ❌ | -| Webhooks & registry | ✅ | ❌ | -| Web dashboard | ✅ | ❌ | +- To add semantic search to another application, for example a custom MCP server, an editor extension or a CI tool. +- To control the database: `":memory:"` for a temporary index, or a file for an index that you keep. +- To index source code with tree-sitter chunks at function and class boundaries. ## Installation @@ -35,392 +20,193 @@ The primary use case driving this feature: a Bitbucket MCP server that wants sem npm install libscope ``` -For code indexing, also install the optional peer dependencies for the languages you need: +For code chunking, also install tree-sitter and the grammars that you need (optional peer dependencies): ```bash -# Core parser (required for any language) npm install tree-sitter - -# Install grammars for the languages you need npm install tree-sitter-typescript tree-sitter-javascript tree-sitter-python npm install tree-sitter-c-sharp tree-sitter-cpp tree-sitter-c tree-sitter-go ``` -These are optional — if not installed, code chunking is unavailable but all other features work. +If they are not installed, code is chunked with the built-in text chunker. -## Quick Start +## Quick start ```ts -import { LibScopeLite } from "libscope/lite"; +import { createLite } from "libscope/lite"; -const lite = new LibScopeLite({ dbPath: ":memory:" }); +const scope = createLite({ + dbPath: ":memory:", + config: { llm: { provider: "passthrough" } }, // ask() returns context for your own LLM +}); -// Index some documents -await lite.indexBatch([ - { title: "Auth Guide", content: "Use OAuth2 for all API access. Tokens expire after 1 hour." }, - { title: "Deploy Guide", content: "Deploy to Kubernetes using Helm charts. Set replicas: 3." }, -], { concurrency: 4 }); +await scope.add({ + title: "Auth Guide", + content: "Use OAuth2 for all API access. Tokens expire after 1 hour.", +}); +await scope.add({ + title: "Deploy Guide", + content: "Deploy to Kubernetes using Helm charts. Set replicas: 3.", +}); -// Hybrid vector + FTS5 search -const results = await lite.search("how to authenticate"); -console.log(results[0]?.title); // "Auth Guide" +const { items } = await scope.search("how to authenticate"); +console.log(items[0]?.title, items[0]?.documentId); // "Auth Guide", "<id>" -// RAG context retrieval (for external LLMs) -const context = await lite.getContext("How do I authenticate API requests?"); -// Returns a formatted context string ready to inject into an LLM prompt +const context = await scope.ask("How do I authenticate API requests?"); +if (context.mode === "context") { + // Put context.contextPrompt into your LLM prompt. context.sources lists the chunks. +} -lite.close(); +scope.close(); ``` -## Constructor Options +## Options ```ts -new LibScopeLite(opts?: LiteOptions) +createLite(options: LiteOptions): LibScope ``` -| Option | Type | Default | Description | -|---|---|---|---| -| `dbPath` | `string` | `~/.libscope/lite.db` | SQLite database path. Use `":memory:"` for in-memory. | -| `db` | `Database` | — | Inject an existing `better-sqlite3` instance. When provided, `dbPath` is ignored and no migrations or schema setup are run. | -| `provider` | `EmbeddingProvider` | Local (all-MiniLM-L6-v2) | Embedding provider to use for indexing and search. | -| `llmProvider` | `LlmProvider` | — | LLM provider for `ask()` and `askStream()`. Required to use those methods. | - -### Persistent vs In-Memory Database - -```ts -// In-memory — data is lost when the process exits (good for one-off tasks) -const lite = new LibScopeLite({ dbPath: ":memory:" }); - -// File-backed — persists across sessions (good for long-lived indexes) -const lite = new LibScopeLite({ dbPath: "/data/my-project.db" }); -``` - -### Embedding Providers - -By default, LibScope Lite uses the local `all-MiniLM-L6-v2` model (downloads ~80 MB on first use). To use OpenAI or Ollama: - -```ts -import { LibScopeLite } from "libscope/lite"; -import { createEmbeddingProvider } from "libscope"; - -const provider = createEmbeddingProvider({ - embedding: { provider: "openai", model: "text-embedding-3-small" }, -}); - -const lite = new LibScopeLite({ provider }); -``` +| Option | Type | Default | Description | +| ------------- | ------------------- | ------------------------ | ------------------------------------------------------------------------------------- | +| `dbPath` | `string` | — | SQLite file, or `":memory:"`. Give `dbPath` or `db`. | +| `db` | `Database` | — | An open `better-sqlite3` database. It is migrated. `close()` leaves it open. | +| `provider` | `EmbeddingProvider` | Local (all-MiniLM-L6-v2) | Embedding provider instance. | +| `llmProvider` | `LlmProvider` | — | LLM for `ask()` and `askStream()`. | +| `chunker` | `Chunker` | — | Custom chunker, for example `createCodeChunker()`. | +| `config` | `ConfigOverrides` | — | Config values over the defaults, for example `{ embedding: { provider: "openai" } }`. | -## Indexing +`createLite(options)` is the same as `LibScope.create({ ...options, useConfigFile: false })`. -### `index(docs)` +### Embedding providers -Index an array of pre-parsed documents: +The default is the local `all-MiniLM-L6-v2` model (about 80 MB, downloaded on first use). To use OpenAI or Ollama, set it in `config`: ```ts -await lite.index([ - { - title: "Getting Started", - content: "# Introduction\n\nThis guide covers the basics...", - library: "my-api", - version: "2.0", - url: "https://docs.example.com/getting-started", +const scope = createLite({ + dbPath: "/data/index.db", + config: { + embedding: { provider: "openai", model: "text-embedding-3-small" }, + openai: { apiKey: process.env.OPENAI_API_KEY }, }, -]); -``` - -**`LiteDoc` fields:** - -| Field | Type | Description | -|---|---|---| -| `title` | `string` | Document title (required) | -| `content` | `string` | Document text content (required) | -| `url` | `string?` | Source URL for deduplication and attribution | -| `library` | `string?` | Library namespace for scoped search | -| `version` | `string?` | Library version | -| `sourceType` | `string?` | `"manual"` (default), `"library"`, `"topic"`, or `"model-generated"` | -| `topicId` | `string?` | Topic ID to associate the document with | -| `language` | `string?` | Language alias for code-aware tree-sitter chunking (e.g. `"typescript"`, `"cpp"`, `"go"`). When set and tree-sitter is available, chunks at function/class boundaries instead of text boundaries. | - -### `indexRaw(input)` - -Index from a raw source — a file path, URL, buffer, or plain text. LibScope Lite normalizes the input using the same parser pipeline as full LibScope: - -```ts -// Index a local file (auto-detects format from extension) -const docId = await lite.indexRaw({ type: "file", path: "./docs/guide.pdf" }); - -// Fetch and index a URL -const docId = await lite.indexRaw({ type: "url", url: "https://docs.example.com/guide" }); - -// Index raw text -const docId = await lite.indexRaw({ type: "text", title: "Notes", content: "..." }); - -// Index from a buffer (e.g., uploaded file) -const docId = await lite.indexRaw({ - type: "buffer", - buffer: fileBuffer, - filename: "report.docx", - title: "Q4 Report", }); ``` -**Supported formats:** Markdown, plain text, HTML, PDF (requires `pdf-parse`), DOCX (requires `mammoth`), EPUB (requires `epub2`), PPTX (requires `pizzip`), CSV, JSON, YAML. - -### `indexBatch(docs, opts)` +Or pass your own object that implements `EmbeddingProvider` as `provider`. -Index multiple documents with concurrency control: +## Add content ```ts -await lite.indexBatch(repoFiles, { concurrency: 4 }); +await scope.add({ title: "Notes", content: "...", library: "my-repo", tags: ["notes"] }); +await scope.add("./docs/guide.md"); // a local file (PDF, DOCX, HTML, Markdown, ...) +await scope.add({ source: "https://example.com/page" }); ``` -`concurrency` controls how many documents are embedded in parallel. A value of 4–8 is recommended for most systems. Each document's embeddings are computed concurrently but each database write is atomic. +To add many documents, run the calls in parallel or in a loop. Each result lists the new `documentId` values. -**Pattern for large repos:** +`normalizeRawInput()` turns a file, an in-memory buffer, a URL or text into `{ title, content }`: ```ts -const files = await glob("src/**/*.ts"); -const docs = await Promise.all( - files.map(async (path) => ({ - title: path, - content: await fs.readFile(path, "utf8"), - sourceType: "library" as const, - })) -); - -await lite.indexBatch(docs, { concurrency: 8 }); -``` - -## Searching - -### `search(query, opts?)` - -Hybrid vector + FTS5 search — the same engine used by full LibScope: - -```ts -const results = await lite.search("OAuth2 token refresh", { - limit: 5, // max results (default: 10) - library: "api", // scope to a library - tags: ["auth"], // filter by tags - diversity: 0.3, // MMR reranking (0 = pure relevance, 1 = max diversity) -}); +import { normalizeRawInput } from "libscope/lite"; -for (const result of results) { - console.log(result.title, result.score); - console.log(result.content); // the matching chunk text -} +const doc = await normalizeRawInput({ type: "buffer", buffer: upload, filename: "report.pdf" }); +await scope.add({ title: doc.title, content: doc.content }); ``` -**`LiteSearchResult` fields:** - -| Field | Type | Description | -|---|---|---| -| `docId` | `string` | Document ID | -| `chunkId` | `string` | Chunk ID within the document | -| `title` | `string` | Document title | -| `content` | `string` | Chunk text | -| `score` | `number` | Relevance score (higher is better) | -| `url` | `string \| null` | Source URL if set at index time | - -## RAG - -### `getContext(question, opts?)` - -Retrieve context without running an LLM — useful when you want to inject the context into your own LLM prompt: +## Search and ask ```ts -const context = await lite.getContext("How does the event loop work?", { - topK: 5, // number of chunks to retrieve (default: 5) - library: "node", // optional scope -}); - -// context is a formatted string you can inject into any LLM prompt -const prompt = `Answer based only on this context:\n\n${context}\n\nQuestion: ...`; +const { items, total } = await scope.search({ query: "deploy", library: "my-repo", limit: 5 }); ``` -This is the primary method for agent-to-agent integration patterns — your orchestrating LLM calls `getContext()` and injects the result into its prompt rather than managing a separate RAG system. - -### `ask(question, opts?)` - -Full RAG with an LLM completing the response (requires `llmProvider` in constructor or opts): +`ask()` returns `{ mode: "answer", answer, sources, model }` when there is an LLM (`llmProvider`, or `config.llm`). With `config: { llm: { provider: "passthrough" } }` it returns `{ mode: "context", contextPrompt, sources }`. With neither, it throws a `ConfigError`. ```ts -import { LibScopeLite } from "libscope/lite"; -import { createLlmProvider } from "libscope"; - -const lite = new LibScopeLite({ - llmProvider: createLlmProvider({ llm: { provider: "openai", model: "gpt-4o-mini" } }), +const scope = createLite({ dbPath: ":memory:", llmProvider: myLlm }); +const result = await scope.ask({ + question: "How do I deploy?", + topK: 5, + systemPrompt: "Be brief.", }); -const answer = await lite.ask("How do I configure rate limiting?", { topK: 5 }); -console.log(answer); // string answer from LLM -``` - -### `askStream(question, opts?)` - -Streaming version of `ask()` — returns an `AsyncGenerator<string>` of token chunks: - -```ts -for await (const token of lite.askStream("Explain the deployment process")) { - process.stdout.write(token); +for await (const event of scope.askStream("How do I deploy?")) { + if ("token" in event) process.stdout.write(event.token); } ``` -The LLM provider must support streaming. Providers that don't expose a `completeStream()` method will throw a clear error. +## Code indexing -## Code Indexing +`createCodeChunker()` splits code at function and class boundaries with tree-sitter. It finds the language from the file extension of the source path, URL or title, or uses the `language` that you give. -LibScope Lite can split source code files at function and class boundaries using tree-sitter rather than plain text chunking. The preferred way to enable this is to set the `language` field on a `LiteDoc` — no extra imports or chunking steps required. +```ts +import { createLite, createCodeChunker } from "libscope/lite"; -### Preferred: set `language` on `LiteDoc` +const scope = createLite({ dbPath: "/data/repo.db", chunker: createCodeChunker() }); -```ts -// Preferred: just set language on LiteDoc — chunking is automatic -await lite.index([ - { - title: "src/auth.cpp", - content: fileContent, - library: "my-repo", - language: "cpp", // enables tree-sitter chunking at function boundaries - }, -]); +await scope.add({ title: "src/auth.ts", content: source, library: "my-repo" }); +// or one language for everything: +createCodeChunker({ language: "python" }); ``` -Setting `language` on a `LiteDoc` automatically triggers code-aware tree-sitter chunking. This is the preferred approach over using `TreeSitterChunker` directly. If tree-sitter is not installed or parsing fails, indexing falls back silently to the standard text chunker. +Documents in other languages, and code when tree-sitter is not installed, use the built-in chunker. -Supported languages and their extension aliases: +Supported languages and aliases: -| Language | Aliases | -|---|---| -| `typescript` | `ts`, `tsx` | +| Language | Aliases | +| ------------ | ------------------------- | +| `typescript` | `ts`, `tsx` | | `javascript` | `js`, `jsx`, `mjs`, `cjs` | -| `python` | `py` | -| `csharp` | `cs` | -| `cpp` | `cc`, `cxx`, `hpp`, `h` | -| `c` | — | -| `go` | — | +| `python` | `py` | +| `csharp` | `cs` | +| `cpp` | `cc`, `cxx`, `hpp`, `h` | +| `c` | — | +| `go` | — | -### Advanced: using `TreeSitterChunker` directly +To get the raw chunks with line numbers, use `TreeSitterChunker` directly. A `chunker` can also be your own function: it receives `{ content, title, source }` and returns an array of chunk strings, or `undefined` to use the built-in chunker. -Direct use of `TreeSitterChunker` is rarely needed when using `LibScopeLite` — setting `language` on `LiteDoc` covers most cases. Use `TreeSitterChunker` directly only when you need access to the raw `CodeChunk` objects (e.g., to extract line numbers for display, filter by node type, or build custom chunk titles): +## Delete, rate, close ```ts -import { LibScopeLite } from "libscope/lite"; -import { TreeSitterChunker } from "libscope/lite"; - -const chunker = new TreeSitterChunker(); -const lite = new LibScopeLite({ dbPath: ":memory:" }); - -// Check if a language is supported before chunking -if (chunker.supports("typescript")) { - const source = await fs.readFile("src/auth.ts", "utf8"); - const chunks = await chunker.chunk(source, "typescript"); - - // Each chunk is a function or class with 1-based line numbers - for (const chunk of chunks) { - await lite.index([{ - title: `auth.ts:${chunk.startLine}-${chunk.endLine} (${chunk.nodeType})`, - content: chunk.content, - library: "src", - }]); - } -} -``` - -See [Code Indexing](/guide/code-indexing) for the full guide including supported languages, chunk shape, and large-file strategies. +// Delete all documents of a library (at most 1000 per call; repeat until affected is 0). +let deleted; +do { + deleted = await scope.admin.bulkDelete({ library: "my-repo" }); +} while (deleted.affected > 0); -### `deleteByLibrary(library)` +await scope.docs.rate({ documentId, rating: 5 }); // ratings boost later search results -Delete all indexed documents belonging to a library namespace. Useful before a full reindex to avoid stale chunks accumulating: - -```ts -// Clear all previously indexed content for this repo, then reindex -lite.deleteByLibrary("my-repo"); -await lite.indexBatch(freshDocs, { concurrency: 4 }); +scope.close(); ``` -The library name matches the `library` field set at index time. Deletion is batched internally and loops until all matching documents are removed. +Create one instance for a long-running service. In a script, close it in a `finally` block. -## Feedback - -### `rate(docId, score)` - -Record a quality signal for a document (score 1–5): - -```ts -const results = await lite.search("deployment process"); -const docId = results[0]?.docId; -if (docId) { - lite.rate(docId, 5); // this result was highly relevant -} -``` - -Ratings feed into subsequent searches — highly-rated documents get boosted in results over time. - -## Lifecycle - -### `close()` - -Always close the database when done: +## Example: an MCP server with a repository index ```ts -lite.close(); -``` - -For long-running services, create one `LibScopeLite` instance and reuse it for the lifetime of the service. For one-off scripts, close in a `finally` block: - -```ts -const lite = new LibScopeLite({ dbPath: "/data/repo.db" }); -try { - await lite.indexBatch(docs, { concurrency: 4 }); - const results = await lite.search(query); - // ... -} finally { - lite.close(); -} -``` - -## Integration Pattern: External MCP Server +import { createLite, createCodeChunker } from "libscope/lite"; -The canonical use case — an MCP server that builds a semantic index over repository files: +const scope = createLite({ + dbPath: path.join(os.homedir(), ".bitbucket-mcp", "index.db"), + chunker: createCodeChunker(), + config: { llm: { provider: "passthrough" } }, +}); -```ts -import { LibScopeLite } from "libscope/lite"; -import { TreeSitterChunker } from "libscope/lite"; - -const DB_PATH = path.join(os.homedir(), ".bitbucket-mcp", "index.db"); -const chunker = new TreeSitterChunker(); -let lite: LibScopeLite; - -// Called when a repo is connected -async function onRepoConnect(repoFiles: { path: string; content: string }[]) { - lite = new LibScopeLite({ dbPath: DB_PATH }); - - const docs = await Promise.all(repoFiles.map(async ({ path, content }) => { - if (chunker.supports(path.split(".").pop() ?? "")) { - // Code-aware chunking for supported languages - const chunks = await chunker.chunk(content, path.split(".").pop()!); - return chunks.map((c) => ({ - title: `${path}:${c.startLine}-${c.endLine}`, - content: c.content, - url: path, - })); - } - return [{ title: path, content, url: path }]; - })); - - await lite.indexBatch(docs.flat(), { concurrency: 4 }); +// When a repository is connected: +async function onRepoConnect(repo: string, files: { path: string; content: string }[]) { + for (const { path, content } of files) { + await scope.add({ title: path, content, library: repo }); + } } -// Called during PR review +// During a pull request review: async function onPrReview(question: string): Promise<string> { - return lite.getContext(question, { topK: 5 }); + const result = await scope.ask({ question, topK: 5 }); + return result.mode === "context" ? result.contextPrompt : result.answer; } ``` -## See Also +## See also -- [Code Indexing Guide](/guide/code-indexing) — tree-sitter chunking in depth -- [LibScope Lite API Reference](/reference/lite-api) — full TypeScript API -- [Programmatic Usage](/guide/programmatic-usage) — full `LibScope` SDK (with connectors, packs, topics) -- [How Search Works](/guide/how-search-works) — hybrid vector + FTS5 explained +- [LibScope Lite API Reference](/reference/lite-api) +- [Programmatic Usage](/guide/programmatic-usage): every `LibScope` method +- [Code Indexing](/guide/code-indexing) +- [How Search Works](/guide/how-search-works) diff --git a/docs/guide/programmatic-usage.md b/docs/guide/programmatic-usage.md index 01be9029..ca0e9df8 100644 --- a/docs/guide/programmatic-usage.md +++ b/docs/guide/programmatic-usage.md @@ -1,9 +1,9 @@ # Programmatic Usage -LibScope can be used as a Node.js library via the `LibScope` SDK class. +Use the `LibScope` class to call LibScope from Node.js. Each method calls the same operation as the CLI command, MCP tool and REST route. The parameter names, defaults, validation and results are the same on all of them. -::: tip Looking for embedded / lightweight usage? -[LibScope Lite](/guide/lite) (`libscope/lite`) is a zero-dependency-on-connectors embeddable class with `index()`, `search()`, `getContext()`, and tree-sitter code chunking — designed to be imported directly into external applications without the full CLI/MCP/connector surface area. +::: tip Embedding LibScope in another application? +Use [`libscope/lite`](/guide/lite). `createLite()` returns the same `LibScope` object, but it does not read config files and it uses the database that you name. ::: ## Setup @@ -12,9 +12,24 @@ LibScope can be used as a Node.js library via the `LibScope` SDK class. import { LibScope } from "libscope"; const scope = LibScope.create(); +// ... use scope ... +scope.close(); ``` -You can pass options to `create()`: +`create()` loads the config, opens and migrates the database, and creates the embedding provider. Without options, it opens the database of the active workspace (the same one that the CLI and MCP server use). + +### Options + +| Option | Type | Description | +| --------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `workspace` | `string` | Workspace whose database to open. Default: the active workspace. | +| `dbPath` | `string` | SQLite file. Use `":memory:"` for an in-memory database. Wins over `workspace` and `database.path`. | +| `db` | `Database` | An open `better-sqlite3` database. It is migrated. `close()` does not close it. | +| `config` | `ConfigOverrides` | Values per config section, merged over the loaded config. Example: `{ llm: { provider: "anthropic" } }`. | +| `useConfigFile` | `boolean` | `false`: do not read config files, `secrets.json` or `LIBSCOPE_*` config variables. Start from the defaults and `config`. | +| `provider` | `EmbeddingProvider` | Embedding provider instance to use instead of the configured one. | +| `llmProvider` | `LlmProvider` | LLM for `ask()` and `askStream()` instead of the configured one. | +| `chunker` | `Chunker` | Splits inline content and local files into chunks instead of the built-in markdown chunker. | ```ts const scope = LibScope.create({ @@ -26,111 +41,141 @@ const scope = LibScope.create({ }); ``` -This initializes the database, runs migrations, and sets up embedding/LLM providers automatically. +## Add content -## Indexing Documents +`add()` takes inline content, a file, a directory, a URL or a GitHub/GitLab repository URL. A string is a source path or URL. ```ts -const doc = await scope.index({ +const result = await scope.add({ title: "Auth Guide", content: "# Authentication\n\nUse OAuth2 for all API access...", library: "my-lib", version: "2.0.0", + tags: ["auth"], }); +console.log(result.documents[0]?.documentId); -console.log(doc.id); // document ID +await scope.add("./docs"); // every supported file in the directory +await scope.add({ source: "https://example.com/docs", spider: true, maxPages: 50 }); +await scope.add("https://github.com/org/repo"); ``` -### Document TTL / Auto-Expiry +The result is `{ kind, documents, errors, skipped }`. Each item in `documents` has `documentId`, `title`, `chunkCount` and `source`. + +To make a document expire, set `expiresAt` (ISO 8601). `scope.admin.pruneExpired()` deletes the expired documents. -Set `expiresAt` to an ISO 8601 timestamp to mark a document for automatic expiry: +## Search ```ts -await scope.index({ - title: "Sprint 42 Notes", - content: "...", - expiresAt: "2026-04-01T00:00:00Z", +const { items, total } = await scope.search({ + query: "how to authenticate", + library: "my-lib", + limit: 10, + diversity: 0.3, // MMR reranking: 0 = relevance only, 1 = most diverse }); -``` -Expired documents are not removed automatically — call `pruneExpiredDocuments()` to clean them up: +for (const r of items) console.log(r.documentId, r.chunkId, r.title, r.score); -```ts -import { pruneExpiredDocuments } from "libscope"; - -// Using the low-level function (requires a db handle) -const { pruned } = pruneExpiredDocuments(db); -console.log(`Removed ${pruned} expired documents`); +// A string is the query. +await scope.search("deployment"); +// Content related to a document or chunk: +await scope.search({ relatedTo: items[0]!.documentId }); ``` -## Searching +All list results have the shape `{ items, total, limit, offset }`. -```ts -const { results } = await scope.search("how to authenticate", { - library: "my-lib", - limit: 10, - diversity: 0.3, // MMR diversity reranking (0 = pure relevance, 1 = max diversity) -}); +## Ask -for (const result of results) { - console.log(result.title, result.score); +```ts +const result = await scope.ask({ question: "How does OAuth2 work?", library: "my-lib", topK: 5 }); +if (result.mode === "answer") { + console.log(result.answer, result.sources); +} else { + // llm.provider is "passthrough": answer from result.contextPrompt yourself. + console.log(result.contextPrompt); } ``` -## Batch Search +`ask()` uses `llmProvider`, or the LLM from `llm.provider` (default `auto`: OpenAI if an OpenAI key is set, else Anthropic if an Anthropic key is set, else Ollama if `llm.url` is set or the embedding provider is `ollama`). If there is no LLM, `ask()` throws a `ConfigError` that tells you what to set. -Run up to 20 search queries concurrently: +Streaming: ```ts -const { results } = await scope.searchBatch([ - { query: "authentication" }, - { query: "deployment", options: { library: "my-lib", limit: 5 } }, -]); - -// Results are keyed by query string -console.log(results["authentication"].results.length); -console.log(results["deployment"].results.length); +for await (const event of scope.askStream("How does OAuth2 work?")) { + if ("token" in event) process.stdout.write(event.token); + else console.log("\nSources:", event.sources); +} ``` -## RAG (Ask Questions) +## Namespaces + +The other operations are grouped in namespaces. Each method takes one input object and returns a promise. The input type is the operation's schema input, so your editor shows each field and its description. + +| Method | Operation | +| ----------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | +| `scope.overview()` | Counts, topics, packs, index, health | +| `docs.get({ documentId, offset?, maxLength? })` | Document with tags, links, ratings | +| `docs.list({ topic?, library?, version?, sourceType?, tags?, limit?, offset? })` | List documents | +| `docs.update({ documentId, title?, content?, library?, version?, url?, topic?, tags? })` | Update a document | +| `docs.delete({ documentId })` | Delete a document | +| `docs.rate({ documentId, rating, chunkId?, feedback?, suggestedCorrection? })` | Rate a document | +| `docs.history({ documentId })`, `docs.rollback({ documentId, version })` | Saved versions | +| `topics.list()`, `topics.create({ name })`, `topics.delete({ topic })` | Topics | +| `tags.add`, `tags.remove` (`{ documentId, tags }`), `tags.list()`, `tags.suggest({ documentId })` | Tags | +| `links.create({ documentId, targetDocumentId, linkType })`, `links.delete({ linkId })`, `links.list()`, `links.prerequisites({ documentId })` | Document links | +| `searches.save`, `searches.list`, `searches.run`, `searches.delete` | Saved searches | +| `packs.install`, `packs.remove`, `packs.list`, `packs.create` | Knowledge packs | +| `connectors.list()`, `connectors.sync({ name } \| { all: true })`, `connectors.disconnect({ name })` | Saved connections | +| `tasks.get`, `tasks.cancel`, `tasks.list` | Background tasks | +| `admin.reindex`, `admin.dedupe`, `admin.backup`, `admin.restore`, `admin.pruneExpired`, `admin.bulkDelete`, `admin.bulkRetag`, `admin.bulkMove` | Maintenance | +| `analytics.popular`, `analytics.stale`, `analytics.topQueries`, `analytics.searches` | Search analytics | +| `webhooks.create`, `webhooks.list`, `webhooks.delete`, `webhooks.test` | Webhooks | ```ts -const answer = await scope.ask("How does OAuth2 work?", { - library: "my-lib", - topK: 5, -}); - -console.log(answer.text); -console.log(answer.sources); // cited chunks +const doc = await scope.docs.get({ documentId: "doc-id" }); +await scope.tags.add({ documentId: "doc-id", tags: ["reviewed"] }); +await scope.docs.update({ documentId: "doc-id", topic: "security" }); +const { stats } = await scope.overview(); ``` -For streaming responses: +Long-running methods (`add`, `admin.reindex`, `packs.install`, `connectors.sync`) take a second argument with an `AbortSignal` and a progress callback: ```ts -for await (const event of scope.askStream("How does OAuth2 work?")) { - if (event.type === "text") process.stdout.write(event.text); -} +const controller = new AbortController(); +await scope.admin.reindex( + { rebuild: true }, + { signal: controller.signal, onProgress: (p) => console.log(p.done, p.total) }, +); ``` -## Other Operations +### Types + +The package root exports the types for inputs and results: ```ts -// Get stats -const stats = scope.stats(); +import type { LibScopeInput, LibScopeOutput, SearchOutput, AskOutput } from "libscope"; + +type UpdateInput = LibScopeInput<"docs", "update">; +type DocumentView = LibScopeOutput<"docs", "get">; +``` -// List documents -const docs = scope.list({ library: "my-lib" }); +## Errors -// Get a single document -const doc = scope.get("doc-id"); +Invalid input throws `ValidationError`. An unknown document, chunk, topic, link or other resource throws a `NotFoundError` subclass. A missing LLM or API key throws `ConfigError`. All of them extend `LibScopeError` and have a `code`. -// Delete a document -scope.delete("doc-id"); +```ts +import { NotFoundError, ValidationError } from "libscope"; + +try { + await scope.docs.get({ documentId: "missing" }); +} catch (err) { + if (err instanceof NotFoundError) console.log(err.code); // "DOCUMENT_NOT_FOUND" +} ``` -## Cleanup +## Close -Always close the database connection when done: +Close the database when you are done: ```ts scope.close(); diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 1d83e537..0307021e 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -69,3 +69,93 @@ Every "not found" error is now a `NotFoundError` (`DocumentNotFoundError`, `Chun | `getVersion`, `rollbackToVersion` with an unknown version | `DocumentNotFoundError` | `NotFoundError` (`VERSION_NOT_FOUND`) | | `removePack` for a pack that is not installed | `ValidationError` | `NotFoundError` (`PACK_NOT_FOUND`) | | `getRelatedChunks` with an unknown chunk ID | `Error` | `ChunkNotFoundError` (`CHUNK_NOT_FOUND`) | + +### Package root exports + +`import ... from "libscope"` now gives only the `LibScope` class, its option and result types (`LibScopeOptions`, `RunOptions`, `LibScopeInput`, `LibScopeOutput`, `AddInput`, `SearchOutput`, `AskOutput`, `SearchResult`, `RagResult`, `DocumentView`, `Overview`, `Task`, ...), the provider interfaces (`EmbeddingProvider`, `LlmProvider`, `Chunker`) and the error classes. The other 142 runtime exports are removed. The functions in "Renamed exports" and "Error changes" above are no longer exported from the root; the `LibScope` methods that call them throw the same errors. + +Each method below is on a `LibScope` instance (`const scope = LibScope.create()`) and takes one input object. "—" means there is no replacement in the library API. + +| Removed exports (1.x) | Replacement (2.0) | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `indexDocument`, `indexFile`, `batchImport`, `indexRepository`, `parseRepoUrl`, `fetchAndConvert`, `DEFAULT_FETCH_OPTIONS` | `scope.add()` (content, file, directory, URL with optional `spider`, or repository URL) | +| `chunkContent` | `chunker` option of `LibScope.create()` to replace the chunker; otherwise — | +| `getParserForFile`, `getSupportedExtensions` | `scope.add()`; `normalizeRawInput()` from `libscope/lite` | +| `checkDuplicate`, `findDuplicates` | `scope.add({ dedup })`, `scope.admin.dedupe()` | +| `searchDocuments` | `scope.search()` | +| `searchBatch`, `BATCH_SEARCH_MAX_REQUESTS` | `Promise.all` of `scope.search()` calls | +| `askQuestion`, `askQuestionStream`, `buildContextPrompt`, `extractSources` | `scope.ask()`, `scope.askStream()`; `llm.provider: "passthrough"` returns the context prompt | +| `createLlmProvider` | `llmProvider` option, or `config: { llm: { ... } }` | +| `registerProvider`, `createEmbeddingProvider` | `provider` option (an `EmbeddingProvider` instance), or `config: { embedding: { ... } }` | +| `getDocument`, `listDocuments`, `updateDocument`, `deleteDocument` | `scope.docs.get()`, `.list()`, `.update()`, `.delete()` | +| `rateDocument`, `getDocumentRatings`, `listRatings` | `scope.docs.rate()`; the rating summary is in `scope.docs.get()`; `listRatings` — | +| `getVersionHistory`, `getVersion`, `rollbackToVersion`, `saveVersion`, `pruneVersions`, `MAX_VERSIONS_DEFAULT` | `scope.docs.history()` (includes each version's content), `scope.docs.rollback()`; versions are saved automatically | +| `createTopic`, `listTopics`, `getTopic`, `getTopicStats`, `deleteTopic`, `getDocumentsByTopic`, `renameTopic` | `scope.topics.create()`, `.list()`, `.delete()`; `scope.docs.list({ topic })`; `renameTopic` — | +| `createTag`, `listTags`, `addTagsToDocument`, `removeTagsFromDocument`, `getDocumentTags`, `getDocumentsByTag`, `suggestTags`, `deleteTag` | `scope.tags.add()` (creates tags), `.list()`, `.remove()`, `.suggest()`; `scope.docs.get()` (tags); `scope.docs.list({ tags })`; `deleteTag` — | +| `createLink`, `listLinks`, `getDocumentLinks`, `deleteLink`, `getPrerequisiteChain` | `scope.links.create()`, `.list()`, `.delete()`, `.prerequisites()` | +| `createSavedSearch`, `listSavedSearches`, `getSavedSearch`, `runSavedSearch`, `deleteSavedSearch` | `scope.searches.save()`, `.list()`, `.run()`, `.delete()` | +| `getStats` | `scope.overview()` (`stats`, `topics`, `packs`, `index`, `health`) | +| `getPopularDocuments`, `getStaleDocuments`, `getTopQueries`, `getSearchAnalytics`, `getKnowledgeGaps`, `getSearchTrends` | `scope.analytics.popular()`, `.stale()`, `.topQueries()`, `.searches()` (includes knowledge gaps and queries per day) | +| `logSearch`, `recordSearchQuery` | — (searches are recorded automatically) | +| `listAvailablePacks`, `listInstalledPacks`, `installPack`, `removePack`, `createPack` | `scope.packs.list({ available })`, `.install()`, `.remove()`, `.create()` | +| `reindex` | `scope.admin.reindex()` | +| `exportKnowledgeBase`, `importFromBackup` | `scope.admin.backup()`, `scope.admin.restore()` | +| `pruneExpiredDocuments` | `scope.admin.pruneExpired()` | +| `bulkDelete`, `bulkRetag`, `bulkMove`, `resolveSelector` | `scope.admin.bulkDelete()`, `.bulkRetag()`, `.bulkMove()` (`dryRun: true` lists the matches) | +| `createWebhook`, `listWebhooks`, `getWebhook`, `deleteWebhook`, `updateWebhook`, `signPayload`, `buildPayload`, `fireWebhooks`, `redactWebhook`, `WEBHOOK_EVENTS` | `scope.webhooks.create()`, `.list()`, `.delete()`, `.test()`; `updateWebhook`: delete and create; the others are internal — | +| `syncNotion`, `syncSlack`, `syncConfluence`, `syncOneNote`, `syncObsidianVault`, `syncDocSite` | Save the connection with the CLI (`libscope connect <type>`), then `scope.connectors.sync({ name })` or `sync({ all: true })` | +| `disconnectNotion`, `disconnectSlack`, `disconnectConfluence`, `disconnectOneNote`, `disconnectVault`, `disconnectDocSite`, `deleteConnectorDocuments` | `scope.connectors.disconnect({ name })` | +| `saveConnectorConfig`, `loadConnectorConfig`, `saveNamedConnectorConfig`, `loadNamedConnectorConfig`, `hasNamedConnectorConfig`, `saveDbConnectorConfig`, `loadDbConnectorConfig`, `deleteDbConnectorConfig` | CLI `libscope connect <type>`; `scope.connectors.list()` shows saved connections (secrets masked) | +| `authenticateDeviceCode`, `refreshAccessToken` | CLI `libscope connect onenote` | +| `convertNotionBlocks`, `convertSlackMrkdwn`, `convertConfluenceStorage`, `convertOneNoteHtml`, `parseObsidianMarkdown`, `detectDocSiteType`, `extractDocLinks`, `extractDocTitle`, `extractMainContent`, `extractElementByPattern`, `extractSitemapUrls`, `normalizeDocUrl` | — (connector internals) | +| `ConnectorScheduler`, `loadScheduleEntries` | CLI `libscope connect <type> --schedule <cron>` | +| `createWorkspace`, `deleteWorkspace`, `listWorkspaces`, `getWorkspacePath`, `getWorkspacesDir`, `getActiveWorkspace`, `setActiveWorkspace`, `DEFAULT_WORKSPACE` | `workspace` option of `LibScope.create()`; CLI `libscope workspace` | +| `FileWatcher`, `DEFAULT_WATCH_EXTENSIONS` | CLI `libscope add <dir> --watch` | +| `buildKnowledgeGraph`, `detectClusters` | — (the web dashboard shows the graph) | +| `startApiServer` | CLI `libscope serve api` | + +The removed type exports follow their functions. Use `LibScopeInput<"group", "method">` and `LibScopeOutput<"group", "method">` for the input and result of any namespace method. + +### `LibScope` class + +`LibScope.create()` now uses the same startup as the CLI and MCP server. Its methods call the operations, so inputs and results match the CLI, MCP and REST API. + +| 1.x | 2.0 | +| ------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `LibScope.create({ config: Partial<LibScopeConfig> })` | `config` is per-section overrides (`ConfigOverrides`). New options: `db`, `useConfigFile`, `provider`, `llmProvider`, `chunker`. | +| `scope.index({ title, content, ... })` → `{ id, chunkCount }` | `scope.add({ title, content, ... })` → `{ kind, documents: [{ documentId, title, chunkCount, source }], errors, skipped }`. `topicId` is now `topic` (ID or name). | +| `scope.indexFile(path, options)` | `scope.add(path)` or `scope.add({ source: path, ... })` | +| `scope.search(query, options)` → `{ results, totalCount }` | `scope.search(query)` or `scope.search({ query, ...filters })` → `{ items, total, limit, offset }`. The `source` filter is now `sourceType`. | +| `scope.searchBatch(requests)` | `Promise.all(requests.map((r) => scope.search(r)))` | +| `scope.ask(question, options)` → `RagResult` | `scope.ask(question)` or `scope.ask({ question, ...filters, topK, systemPrompt })` → `{ mode: "answer", ...RagResult }` or, with `llm.provider: "passthrough"`, `{ mode: "context", contextPrompt, sources }` | +| `scope.askStream(question, options)` | `scope.askStream(question)` or `scope.askStream({ question, ... })`. Same events. | +| `scope.stats()` | `(await scope.overview()).stats` | +| `scope.list(options)` → `Document[]` | `scope.docs.list({ topic, library, version, sourceType, tags, limit, offset })` → `{ items, total, limit, offset }`. Items have `documentId` and `contentLength`, not `id` and `content`. `dateFrom`/`dateTo` are removed. | +| `scope.get(id)` → `Document` | `scope.docs.get({ documentId })` → `{ document, content, contentLength, offset, nextOffset, tags, links, ratings }` | +| `scope.delete(id)` | `scope.docs.delete({ documentId })` | +| Synchronous `stats()`, `list()`, `get()`, `delete()` | Every method returns a promise. | + +### `libscope/lite` + +The `LibScopeLite` class is removed. `createLite(options)` returns a `LibScope` with the same methods and results as `LibScope.create()`. It does not read config files or `LIBSCOPE_*` config variables. + +| 1.x | 2.0 | +| --------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `new LibScopeLite({ dbPath, provider, llmProvider })` | `createLite({ dbPath, provider, llmProvider })`. `dbPath` (or `db`) is required; the old default `~/.libscope/lite.db` is removed. | +| `new LibScopeLite({ db })` skipped migrations | `createLite({ db })` migrates the database and creates the vector table. | +| `model` option (unused) | Removed. Use `config: { embedding: { model } }` or `provider`. | +| `lite.index(docs)`, `lite.indexBatch(docs, { concurrency })` | `scope.add({ title, content, ... })` per document (run them in parallel if you want). | +| `LiteDoc.language` (tree-sitter chunking) | `createLite({ chunker: createCodeChunker() })`. The language comes from the title or source extension, or `createCodeChunker({ language })`. | +| `lite.indexRaw(input)` → `string[]` | `const { title, content } = await normalizeRawInput(input)`, then `scope.add({ title, content })`. Code is no longer split into "part N" documents; use a chunker. | +| `lite.search(query, opts)` → `LiteSearchResult[]` with `docId` | `scope.search({ query, ...opts })` → `{ items, total, limit, offset }`; items have `documentId`. | +| `lite.getContext(question, opts)` → `string` | `createLite({ ..., config: { llm: { provider: "passthrough" } } })`, then `(await scope.ask(question)).contextPrompt`. | +| `lite.ask(question, opts)` → `string` | `scope.ask({ question, topK, systemPrompt })` → `{ mode: "answer", answer, sources, model }`. The per-call `llmProvider` option is removed; pass it to `createLite`. Without an LLM, `ConfigError` (was `Error`). | +| `lite.askStream(question)` → tokens (`string`) | `scope.askStream(question)` → `{ token }` events, then `{ done: true, sources, model }`. A provider without `completeStream` now works (one token). | +| `lite.rate(docId, score)` | `scope.docs.rate({ documentId, rating })` | +| `lite.deleteByLibrary(library)` | `scope.admin.bulkDelete({ library })` (up to 1000 per call; repeat until `affected` is 0) | +| Types `LiteOptions`, `LiteDoc`, `LiteSearchOptions`, `LiteSearchResult`, `LiteContextOptions`, `LiteAskOptions` | `LiteOptions` (new shape), `AddInput`, `SearchInput`, `SearchResult`, `AskInput` | +| `NormalizedInput.chunks` | Removed. `normalizeRawInput` returns `{ title, content }`. | + +### Package `exports` + +`libscope` and `libscope/lite` list `types` first and use `default` (no separate `import`/`require` entries). diff --git a/docs/reference/lite-api.md b/docs/reference/lite-api.md index 2f683e54..e5478224 100644 --- a/docs/reference/lite-api.md +++ b/docs/reference/lite-api.md @@ -1,590 +1,150 @@ # LibScope Lite API Reference -Complete TypeScript API reference for `libscope/lite`. +TypeScript API of `libscope/lite`. It exports everything from `libscope` (the `LibScope` class, its types and the error classes) and the items on this page. For the `LibScope` methods, see [Programmatic Usage](/guide/programmatic-usage). ## Import ```ts -import { LibScopeLite, TreeSitterChunker } from "libscope/lite"; +import { + createLite, + createCodeChunker, + TreeSitterChunker, + normalizeRawInput, + LibScope, + ConfigError, +} from "libscope/lite"; import type { LiteOptions, - LiteDoc, - RawInput, - LiteSearchOptions, - LiteSearchResult, - LiteContextOptions, - LiteAskOptions, + Chunker, CodeChunk, + RawInput, + NormalizedInput, + EmbeddingProvider, + LlmProvider, + SearchOutput, + AskOutput, } from "libscope/lite"; ``` --- -## `LibScopeLite` - -The main class. Creates and manages its own SQLite database, embedding provider, and search engine. - -### Constructor +## `createLite(options)` ```ts -new LibScopeLite(opts?: LiteOptions) -``` - -**`LiteOptions`** +function createLite(options: LiteOptions): LibScope; -```ts interface LiteOptions { - /** - * Path to the SQLite database file. - * - Use ":memory:" for in-process ephemeral storage (lost on close()) - * - Use a file path for persistent cross-session storage - * - Defaults to ~/.libscope/lite.db - */ - dbPath?: string; - - /** - * Inject a pre-configured better-sqlite3 Database instance. - * When provided, dbPath is ignored. No migrations, no sqlite-vec - * setup, and no extension loading are performed — the caller is - * responsible for schema initialization. - * - * Useful for tests and for callers that already manage their own - * database connection. - */ - db?: Database; - - /** - * Embedding provider used for indexing and similarity search. - * Defaults to LocalEmbeddingProvider (all-MiniLM-L6-v2, ~80 MB download). - */ - provider?: EmbeddingProvider; - - /** - * LLM provider used by ask() and askStream(). - * Required to call those methods; other methods work without it. - */ - llmProvider?: LlmProvider; -} -``` - -**Throws** `DatabaseError` if the database file cannot be opened or migrations fail. - ---- - -### `index(docs)` - -```ts -async index(docs: LiteDoc[]): Promise<void> -``` - -Index an array of pre-parsed documents. Each document is chunked using the markdown-aware chunker (or code-aware tree-sitter chunker when `language` is set), embedded, and stored. - -**`LiteDoc`** - -```ts -interface LiteDoc { - /** Document title. Required. Used in search result display and title boosting. */ - title: string; - - /** Full document text. Required. Will be chunked before embedding. */ - content: string; - - /** Source URL. Used for deduplication: if a document with this URL exists, - * it is replaced if the content hash changed, skipped if unchanged. */ - url?: string; - - /** - * Source type for provenance tracking. - * @default "manual" - */ - sourceType?: "manual" | "library" | "topic" | "model-generated"; - - /** Library namespace. Allows scoping search to a specific library. */ - library?: string; - - /** Library version. Used with library for version-scoped search. */ - version?: string; - - /** Topic ID to associate the document with for topic-scoped search. */ - topicId?: string; - - /** - * Language alias for code-aware tree-sitter chunking. - * When set and tree-sitter is available, chunks at function/class boundaries - * instead of text boundaries. Falls back silently to the standard text chunker - * if tree-sitter is not installed or parsing fails. - * - * Supported languages and aliases: - * - `"typescript"` (aliases: `"ts"`, `"tsx"`) - * - `"javascript"` (aliases: `"js"`, `"jsx"`, `"mjs"`, `"cjs"`) - * - `"python"` (alias: `"py"`) - * - `"csharp"` (alias: `"cs"`) - * - `"cpp"` (aliases: `"cc"`, `"cxx"`, `"hpp"`, `"h"`) - * - `"c"` - * - `"go"` - */ - language?: string; + dbPath?: string; // SQLite file or ":memory:" (give dbPath or db) + db?: Database.Database; // open better-sqlite3 database; close() leaves it open + provider?: EmbeddingProvider; // default: local all-MiniLM-L6-v2 + llmProvider?: LlmProvider; // LLM for ask() and askStream() + chunker?: Chunker; // custom chunker, e.g. createCodeChunker() + config?: ConfigOverrides; // config values over the defaults } ``` -**`LiteDoc` properties:** - -| Property | Type | Required | Description | -|---|---|---|---| -| `title` | `string` | Yes | Document title. Used in search result display and title boosting. | -| `content` | `string` | Yes | Full document text. Will be chunked before embedding. | -| `url` | `string` | No | Source URL for deduplication — replaced if content hash changed, skipped if unchanged. | -| `sourceType` | `string` | No | `"manual"` (default), `"library"`, `"topic"`, or `"model-generated"`. | -| `library` | `string` | No | Library namespace for scoped search. | -| `version` | `string` | No | Library version. Used with `library` for version-scoped search. | -| `topicId` | `string` | No | Topic ID to associate the document with for topic-scoped search. | -| `language` | `string` | No | Language alias for code-aware tree-sitter chunking (e.g. `"typescript"`, `"cpp"`, `"go"`). When set and tree-sitter is available, chunks at function/class boundaries instead of text boundaries. | - -**Example:** - -```ts -await lite.index([ - { - title: "Rate Limiting", - content: "Apply rate limiting using the X-RateLimit-* headers...", - library: "api", - version: "3.2", - url: "https://docs.example.com/rate-limiting", - }, -]); -``` - ---- - -### `indexRaw(input)` - -```ts -async indexRaw(input: RawInput): Promise<string> -``` - -Index from a raw input source. The input is passed through the parser pipeline (same parsers as the CLI `add` command), normalized to markdown, then chunked and indexed. - -Returns the document ID of the newly created document. - -**`RawInput`** - -```ts -type RawInput = - | { type: "file"; path: string; title?: string } - | { type: "url"; url: string; title?: string } - | { type: "text"; content: string; title: string } - | { type: "buffer"; buffer: Buffer; filename: string; title?: string }; -``` - -| `type` | Description | Format detection | -|---|---|---| -| `"file"` | Read from local filesystem | File extension (`.md`, `.pdf`, `.docx`, etc.) | -| `"url"` | Fetch and parse a web page | Content-Type header | -| `"text"` | Plain text or markdown string | Always treated as markdown | -| `"buffer"` | In-memory buffer (e.g., upload) | `filename` extension | - -**Supported formats:** Markdown, plain text, HTML, PDF (`pdf-parse`), DOCX (`mammoth`), EPUB (`epub2`), PPTX (`pizzip`), CSV, JSON, YAML. - -**Example:** - -```ts -const id1 = await lite.indexRaw({ type: "file", path: "./README.md" }); -const id2 = await lite.indexRaw({ type: "url", url: "https://docs.example.com" }); -const id3 = await lite.indexRaw({ type: "text", title: "Notes", content: "# My Notes\n..." }); -``` - ---- - -### `indexBatch(docs, opts)` +Returns `LibScope.create({ ...options, useConfigFile: false })`: config files, `secrets.json` and `LIBSCOPE_*` config variables are not read. -```ts -async indexBatch(docs: LiteDoc[], opts: { concurrency: number }): Promise<void> -``` +**Throws** `ValidationError` when neither `dbPath` nor `db` is given. Throws `ConfigError` when the database was built with a different embedding model. -Index multiple documents with concurrency control. Documents are embedded in parallel up to `concurrency` at a time. Each document's database write is still atomic. +### Main methods -| Parameter | Type | Description | -|---|---|---| -| `docs` | `LiteDoc[]` | Documents to index | -| `opts.concurrency` | `number` | Max parallel embedding calls. Recommended: 4–8. | +| Method | Result | +| ------------------------------ | ---------------------------------------------------------------------------------------------------------- | +| `add(input \| source)` | `{ kind, documents: [{ documentId, title, chunkCount, source }], errors, skipped }` | +| `search(input \| query)` | `{ items: SearchResult[], total, limit, offset }` | +| `ask(input \| question)` | `{ mode: "answer", answer, sources, model, tokensUsed? }` or `{ mode: "context", contextPrompt, sources }` | +| `askStream(input \| question)` | Async iterator of `{ token }`, then `{ done: true, sources, model }` | +| `overview()` | `{ stats, topics, packs, index, health }` | +| `close()` | Closes the database (unless it was passed as `db`) | -**Example:** +`SearchResult` has `documentId`, `chunkId`, `title`, `content`, `score`, `sourceType`, `library`, `version`, `topicId`, `url`, `avgRating` and `scoreExplanation`. -```ts -await lite.indexBatch( - files.map((f) => ({ title: f.name, content: f.text, library: "docs" })), - { concurrency: 6 }, -); -``` +`docs`, `topics`, `tags`, `links`, `searches`, `packs`, `connectors`, `tasks`, `admin`, `analytics` and `webhooks` have the same methods as on `LibScope.create()`. --- -### `search(query, opts?)` +## `createCodeChunker(options?)` ```ts -async search(query: string, opts?: LiteSearchOptions): Promise<LiteSearchResult[]> +function createCodeChunker(options?: { language?: string }): Chunker; ``` -Hybrid vector + FTS5 search using Reciprocal Rank Fusion. Returns chunks ranked by relevance. - -**`LiteSearchOptions`** - -```ts -interface LiteSearchOptions { - /** Maximum number of results. Default: 10. Max: 1000. */ - limit?: number; - - /** Restrict results to a specific library namespace. */ - library?: string; - - /** Restrict results to documents in this topic. */ - topic?: string; - - /** Restrict to documents with all of these tags. */ - tags?: string[]; - - /** - * MMR diversity reranking coefficient (0–1). - * 0 = pure relevance order. 1 = maximum diversity (no two similar chunks). - * Default: no reranking. - */ - diversity?: number; -} -``` +Returns a `Chunker` that splits code with tree-sitter. The language is `options.language`, else the extension of the document's source path or URL, else the extension of its title. It returns `undefined` (use the built-in chunker) for unsupported languages, and when tree-sitter is not installed or cannot parse the code. -**`LiteSearchResult`** +## `Chunker` ```ts -interface LiteSearchResult { - /** Document ID. Use with rate() to record feedback. */ - docId: string; - - /** Chunk ID within the document. */ - chunkId: string; - - /** Document title. */ - title: string; - - /** Chunk text (the actual content that matched). */ +type Chunker = (doc: { content: string; - - /** - * Relevance score. Higher is better. - * Combines vector similarity, BM25, and title boost. - */ - score: number; - - /** Source URL if set at index time, otherwise null. */ - url: string | null; -} -``` - -**Example:** - -```ts -const results = await lite.search("JWT token validation", { - limit: 5, - library: "auth-service", - diversity: 0.2, -}); - -for (const r of results) { - console.log(`[${r.score.toFixed(3)}] ${r.title}`); - console.log(r.content.slice(0, 200)); -} -``` - ---- - -### `getContext(question, opts?)` - -```ts -async getContext(question: string, opts?: LiteContextOptions): Promise<string> -``` - -Retrieve top-K relevant chunks and return them as a formatted context string. Does not call an LLM — returns the context ready for injection into an external prompt. - -This is the primary integration point for external LLM pipelines. - -**`LiteContextOptions`** - -```ts -interface LiteContextOptions { - /** Number of chunks to retrieve. Default: 5. */ - topK?: number; - - /** Restrict retrieval to a specific library. */ - library?: string; - - /** Restrict retrieval to a specific topic. */ - topic?: string; -} -``` - -**Returns:** A formatted string containing the retrieved chunks with their titles. The exact format is: - -``` -[Document Title] -Chunk text here... - -[Another Document] -More chunk text... -``` - -**Example:** - -```ts -const context = await lite.getContext("How do I handle auth errors?", { topK: 3 }); -const prompt = `You are a helpful assistant. Answer using only this context: - -${context} - -Question: How do I handle auth errors?`; -``` - ---- - -### `ask(question, opts?)` - -```ts -async ask(question: string, opts?: LiteAskOptions): Promise<string> -``` - -Full RAG: retrieves context then calls an LLM to produce a grounded answer. - -Requires an `llmProvider` configured in the constructor or passed in `opts`. - -**`LiteAskOptions`** - -```ts -interface LiteAskOptions { - /** Number of context chunks to retrieve. Default: 5. */ - topK?: number; - - /** Scope retrieval to a library. */ - library?: string; - - /** Scope retrieval to a topic. */ - topic?: string; - - /** Custom system prompt. Overrides the default "answer using context" instruction. */ - systemPrompt?: string; - - /** - * LLM provider for this request. - * Overrides the instance-level llmProvider for this single call. - */ - llmProvider?: LlmProvider; -} -``` - -**Returns:** The LLM's answer as a plain string. - -**Throws** `Error` if no `llmProvider` is configured. - -**Example:** - -```ts -const answer = await lite.ask("What authentication methods does the API support?", { - library: "api-docs", - topK: 8, - systemPrompt: "You are a concise technical assistant. Answer in bullet points.", -}); -``` - ---- - -### `askStream(question, opts?)` - -```ts -async *askStream(question: string, opts?: LiteAskOptions): AsyncGenerator<string> -``` - -Streaming version of `ask()`. Yields string tokens as they arrive from the LLM. - -Requires an `llmProvider` with a `completeStream()` method. - -**Throws:** -- `Error` if no `llmProvider` is configured -- `Error` if the provider does not support streaming - -**Example:** - -```ts -process.stdout.write("Answer: "); -for await (const token of lite.askStream("Explain the rate limiting algorithm")) { - process.stdout.write(token); -} -process.stdout.write("\n"); -``` - ---- - -### `deleteByLibrary(library)` - -```ts -deleteByLibrary(library: string): void -``` - -Delete all documents and chunks belonging to the given library namespace. Loops internally until all matching documents are removed (batch size is capped at 1000 per iteration). - -| Parameter | Type | Description | -|---|---|---| -| `library` | `string` | Library name to clear — matches the `library` field set at index time | - -Use this before a full reindex to avoid stale content accumulating: - -```ts -// Wipe the old index for this repo, then rebuild -lite.deleteByLibrary("my-repo"); -await lite.indexBatch(freshDocs, { concurrency: 4 }); -``` - ---- - -### `rate(docId, score)` - -```ts -rate(docId: string, score: number): void -``` - -Record a quality rating for a document. Ratings are stored persistently and influence subsequent search rankings — highly-rated documents are boosted. - -| Parameter | Type | Description | -|---|---|---| -| `docId` | `string` | Document ID (from `LiteSearchResult.docId`) | -| `score` | `number` | Rating 1–5 (1 = poor, 5 = excellent) | - -**Throws** `ValidationError` for invalid scores or unknown document IDs. - -**Example:** - -```ts -const results = await lite.search("error handling patterns"); -if (results[0]) { - lite.rate(results[0].docId, 4); // this result was useful -} -``` - ---- - -### `close()` - -```ts -close(): void + title: string; + source: string; // file path or URL; "" for inline content +}) => Promise<string[] | undefined> | string[] | undefined; ``` -Close the database connection and release all resources. Must be called when the `LibScopeLite` instance is no longer needed. - -After `close()`, all other methods will throw if called. +Used by `add` for inline content and local files. Return the chunk texts, or `undefined` to use the built-in markdown chunker. --- ## `TreeSitterChunker` -Code-aware chunker using tree-sitter AST parsing. Optional — requires `tree-sitter` and at least one grammar package. +Code-aware chunker using tree-sitter AST parsing. Requires `tree-sitter` and at least one grammar package. The parser and grammars load on the first `chunk()` call and are cached. Create one instance and reuse it. -### Constructor +### `supports(language)` ```ts -new TreeSitterChunker() +supports(language: string): boolean ``` -The parser and grammar instances are lazily initialized on first `chunk()` call and cached for the lifetime of the instance. Create one `TreeSitterChunker` and reuse it across all files. - ---- +Returns `true` if the language name or alias is supported (case-insensitive): `typescript`/`ts`/`tsx`, `javascript`/`js`/`jsx`/`mjs`/`cjs`, `python`/`py`, `csharp`/`cs`, `cpp`/`cc`/`cxx`/`hpp`/`h`, `c`, `go`. It does not throw. -### `supports(language)` +### `chunk(source, language, maxChunkSize?)` ```ts -supports(language: string): boolean +async chunk(source: string, language: string, maxChunkSize?: number): Promise<CodeChunk[]> ``` -Returns `true` if the given language alias is supported. Case-insensitive. +| Parameter | Type | Default | Description | +| -------------- | -------- | ------- | ---------------------------- | +| `source` | `string` | — | Source code to chunk | +| `language` | `string` | — | Language name or alias | +| `maxChunkSize` | `number` | `1500` | Maximum characters per chunk | ```ts -chunker.supports("ts"); // true -chunker.supports("TypeScript"); // true -chunker.supports("go"); // true -chunker.supports("cs"); // true -chunker.supports("rust"); // false (not supported) -chunker.supports("unknown"); // false +interface CodeChunk { + content: string; // source text of the chunk + startLine: number; // 1-based + endLine: number; // 1-based + nodeType: string; // e.g. "function_declaration", "class_declaration", "preamble", "module" +} ``` -Does not throw. Safe to call before attempting `chunk()`. +**Throws** `ValidationError` when the language is not supported, tree-sitter is not installed, or the source cannot be parsed. --- -### `chunk(source, language, maxChunkSize?)` +## `normalizeRawInput(input)` ```ts -async chunk( - source: string, - language: string, - maxChunkSize?: number, -): Promise<CodeChunk[]> -``` - -Parse `source` and return an array of semantically meaningful chunks. +function normalizeRawInput(input: RawInput): Promise<NormalizedInput>; -| Parameter | Type | Default | Description | -|---|---|---|---| -| `source` | `string` | — | Source code to chunk | -| `language` | `string` | — | Language name or alias (e.g., `"ts"`, `"python"`) | -| `maxChunkSize` | `number` | `1500` | Maximum characters per chunk | - -**`CodeChunk`** +type RawInput = + | { type: "file"; path: string; title?: string } + | { type: "url"; url: string; title?: string } + | { type: "text"; content: string; title: string } + | { type: "buffer"; buffer: Buffer; filename: string; title?: string }; -```ts -interface CodeChunk { - /** Source text of this chunk (function body, class, etc.) */ +interface NormalizedInput { + title: string; content: string; - - /** 1-based line number where this chunk starts in the original source. */ - startLine: number; - - /** 1-based line number where this chunk ends. */ - endLine: number; - - /** - * Tree-sitter node type. Common values: - * - "function_declaration" - * - "class_declaration" - * - "method_definition" - * - "export_statement" - * - "preamble" (accumulated imports/comments before first declaration) - * - "trailing" (non-declaration nodes after last declaration) - * - "module" (entire source, returned when no declarations found) - */ - nodeType: string; } ``` -**Throws** `ValidationError`: -- If `language` is not in the supported list -- If `tree-sitter` is not installed (with install instructions) -- If the source file cannot be parsed - -**Example:** - -```ts -const chunks = await chunker.chunk( - await readFile("src/api.ts", "utf8"), - "typescript", - 2000, -); - -console.log(`${chunks.length} chunks`); -chunks.forEach((c) => { - console.log(` ${c.nodeType} (lines ${c.startLine}–${c.endLine}): ${c.content.length} chars`); -}); -``` +Files and buffers are parsed by the parser for their extension (PDF, DOCX, PPTX, EPUB, HTML, CSV, YAML, JSON, Markdown, text). Other files, for example source code, are read as UTF-8 text. The default title is the file name without its extension. URLs are fetched and converted to markdown. --- -## Type Reference - -### `EmbeddingProvider` +## Provider interfaces ```ts interface EmbeddingProvider { @@ -593,60 +153,37 @@ interface EmbeddingProvider { embed(text: string): Promise<number[]>; embedBatch(texts: string[]): Promise<number[][]>; } -``` - -Import from `libscope`: -```ts -import type { EmbeddingProvider } from "libscope"; -``` - -### `LlmProvider` - -```ts interface LlmProvider { - model: string; - complete(prompt: string, systemPrompt?: string): Promise<{ text: string }>; - completeStream?(prompt: string, systemPrompt?: string): AsyncGenerator<string>; + readonly model: string; + complete(prompt: string, systemPrompt?: string): Promise<{ text: string; tokensUsed?: number }>; + completeStream?(prompt: string, systemPrompt?: string): AsyncIterable<string>; } ``` -Import from `libscope`: - -```ts -import type { LlmProvider } from "libscope"; -``` - --- -## Error Types - -All errors extend `LibScopeError` with a `.code` string property: +## Errors -| Class | Code | When thrown | -|---|---|---| -| `DatabaseError` | `DATABASE_ERROR` | SQLite failures, schema errors | -| `ValidationError` | `VALIDATION_ERROR` | Bad input, unsupported language, missing tree-sitter | -| `EmbeddingError` | `EMBEDDING_ERROR` | Embedding provider failures | -| `DocumentNotFoundError` | `DOCUMENT_NOT_FOUND` | `rate()` with unknown docId | +All errors extend `LibScopeError` and have a `code`: -```ts -import { ValidationError, DatabaseError } from "libscope"; - -try { - await lite.index([{ title: "", content: "..." }]); -} catch (err) { - if (err instanceof ValidationError) { - console.error("Invalid input:", err.message); // "Document title is required" - } -} -``` +| Class | Code | When | +| ----------------------- | ------------------------------ | ----------------------------------------------------------- | +| `ValidationError` | `VALIDATION_ERROR` | Invalid input, unsupported language, tree-sitter missing | +| `ConfigError` | `CONFIG_ERROR` | No LLM for `ask`, missing API key, embedding model mismatch | +| `NotFoundError` | `NOT_FOUND` or a specific code | Unknown link, webhook, saved search, version, pack, task | +| `DocumentNotFoundError` | `DOCUMENT_NOT_FOUND` | Unknown `documentId` | +| `ChunkNotFoundError` | `CHUNK_NOT_FOUND` | Unknown `chunkId` | +| `TopicNotFoundError` | `TOPIC_NOT_FOUND` | Unknown topic | +| `DatabaseError` | `DATABASE_ERROR` | SQLite failures | +| `EmbeddingError` | `EMBEDDING_ERROR` | Embedding provider failures | +| `FetchError` | `FETCH_ERROR` | URL fetch failures | --- -## See Also +## See also -- [LibScope Lite Guide](/guide/lite) — usage guide with examples -- [Code Indexing Guide](/guide/code-indexing) — tree-sitter chunking in depth -- [How Search Works](/guide/how-search-works) — hybrid vector + FTS5 explained -- [Configuration Reference](/reference/configuration) — embedding providers, LLM setup +- [LibScope Lite Guide](/guide/lite) +- [Programmatic Usage](/guide/programmatic-usage) +- [Code Indexing Guide](/guide/code-indexing) +- [Configuration Reference](/reference/configuration) From 15d8185545a5ed6b79cced0562dff8cbd48e0f2e Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:15:24 +0000 Subject: [PATCH 49/83] feat(core): generic pieces for the CLI redesign - saveConnection(): create/update a saved connection, validated against the connector's schema, with a cron schedule (set, keep or remove) - ingest stores a local file's absolute path as its URL, so re-adding a file replaces it instead of duplicating it (and matches the file watcher) - FileWatcher takes document metadata (topic, library, version, source type) - LocalEmbeddingProvider.onDownloadProgress for first-run model download progress - install-pack accepts resumeFrom - hints name 'libscope admin reindex --rebuild' and 'connect <type> --name' --- src/connectors/index.ts | 2 +- src/connectors/registry.ts | 41 ++++++++++++++++ src/connectors/saved-config.ts | 6 +-- src/core/indexing.ts | 3 ++ src/core/ingest.ts | 4 +- src/core/operations/packs.ts | 7 +++ src/core/watcher.ts | 17 ++++++- src/db/index-meta.ts | 2 +- src/db/schema.ts | 4 +- src/providers/local.ts | 20 +++++++- tests/unit/chunk-writer.test.ts | 2 +- tests/unit/connector-registry.test.ts | 67 ++++++++++++++++++++++++++- tests/unit/index-meta.test.ts | 2 +- tests/unit/ingest.test.ts | 13 ++++++ tests/unit/watcher-document.test.ts | 58 +++++++++++++++++++++++ 15 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 tests/unit/watcher-document.test.ts diff --git a/src/connectors/index.ts b/src/connectors/index.ts index a7ec0968..ca5e5aa3 100644 --- a/src/connectors/index.ts +++ b/src/connectors/index.ts @@ -161,7 +161,7 @@ export function loadNamedConnectorConfig<T>(name: string): T { const filePath = namedConfigPath(name); if (!existsSync(filePath)) { throw new ConfigError( - `No connector config found for "${name}". Run 'libscope connect ${name}' first.`, + `No connector config found for "${name}". Run 'libscope connect <type> --name ${name}' first.`, ); } const raw = readFileSync(filePath, "utf-8"); diff --git a/src/connectors/registry.ts b/src/connectors/registry.ts index 20ed67a0..431b9914 100644 --- a/src/connectors/registry.ts +++ b/src/connectors/registry.ts @@ -5,6 +5,7 @@ * Connector settings are the saved configs in ~/.libscope/connectors/<name>.json. */ import type Database from "better-sqlite3"; +import cron from "node-cron"; import { z } from "zod"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { NotFoundError, ValidationError } from "../errors.js"; @@ -13,6 +14,7 @@ import { hasNamedConnectorConfig, listNamedConnectorConfigs, loadNamedConnectorConfig, + saveNamedConnectorConfig, } from "./index.js"; import { isConnectorType, @@ -289,6 +291,45 @@ export interface ConnectionInfo { lastRun?: { status: string; startedAt: string; error: string | null } | undefined; } +/** + * Create or update saved connection `name` of type `type`: `settings` (undefined values + * ignored) are merged over the connection's saved settings and checked against the type's + * config schema. `schedule`: a cron expression sets it, null removes it, undefined keeps it. + * Returns the merged settings (including secrets). + */ +export function saveConnection( + name: string, + type: string, + settings: Record<string, unknown>, + options: { schedule?: string | null | undefined } = {}, +): Record<string, unknown> { + const connector = getConnector(type); + const saved = hasNamedConnectorConfig(name) + ? loadSavedConnectorConfig<Record<string, unknown>>(connector.type, name) + : {}; + const merged: Record<string, unknown> = { ...saved }; + for (const [key, value] of Object.entries(settings)) { + if (value !== undefined) merged[key] = value; + } + const parsed = connector.configSchema.safeParse(merged); + if (!parsed.success) { + const fields = [...new Set(parsed.error.issues.map((issue) => issue.path.join(".")))]; + throw new ValidationError(`Invalid ${connector.label} settings: ${fields.join(", ")}`); + } + if (options.schedule === null) { + delete merged["schedule"]; + } else if (options.schedule !== undefined) { + if (!cron.validate(options.schedule)) { + throw new ValidationError( + `Invalid cron expression "${options.schedule}" (example: "0 */6 * * *" = every 6 hours)`, + ); + } + merged["schedule"] = { cronExpression: options.schedule }; + } + saveNamedConnectorConfig(name, { ...merged, connectorType: connector.type }); + return merged; +} + /** Hide secret values of a saved config (the field stays so users see it is set). */ export function maskConnectorSecrets( type: string, diff --git a/src/connectors/saved-config.ts b/src/connectors/saved-config.ts index cd904a27..1131226a 100644 --- a/src/connectors/saved-config.ts +++ b/src/connectors/saved-config.ts @@ -1,7 +1,7 @@ /** * Saved connector configs: one file per named connector in ~/.libscope/connectors/<name>.json - * (mode 0600). `libscope connect <type>` writes them, and `connect <type> --sync`, the - * scheduler and the MCP sync tools read them. + * (mode 0600). `libscope connect <type>` writes them; `libscope sync`, the scheduler and the + * sync operation read them. */ import type Database from "better-sqlite3"; import { ConfigError, ValidationError } from "../errors.js"; @@ -33,7 +33,7 @@ export function isConnectorType(value: unknown): value is ConnectorType { export interface SavedConnectorFields { /** Connector type the config belongs to. */ connectorType?: ConnectorType | undefined; - /** Cron schedule set by `libscope schedule set`. */ + /** Cron schedule set by `libscope connect <type> --schedule`. */ schedule?: { cronExpression: string } | undefined; } diff --git a/src/core/indexing.ts b/src/core/indexing.ts index 2d029f79..2cf727ee 100644 --- a/src/core/indexing.ts +++ b/src/core/indexing.ts @@ -572,6 +572,8 @@ export interface IndexFileOptions { sourceType?: IndexDocumentInput["sourceType"] | undefined; /** ISO 8601 expiry timestamp (see IndexDocumentInput.expiresAt). */ expiresAt?: string | undefined; + /** Stored as the document URL; re-indexing a file with the same URL updates that document. */ + url?: string | undefined; } /** Document source types. */ @@ -638,5 +640,6 @@ export async function indexFile( topicId: options.topic, dedup: options.dedup, expiresAt: options.expiresAt, + url: options.url, }); } diff --git a/src/core/ingest.ts b/src/core/ingest.ts index 659a7949..da841c25 100644 --- a/src/core/ingest.ts +++ b/src/core/ingest.ts @@ -7,7 +7,7 @@ */ import type Database from "better-sqlite3"; import { readdirSync, statSync } from "node:fs"; -import { basename, join } from "node:path"; +import { basename, join, resolve } from "node:path"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { ValidationError } from "../errors.js"; import { globToRegExp, toPosixPath } from "../utils/glob.js"; @@ -225,6 +225,7 @@ async function ingestFile(ctx: IngestContext, input: IngestInput): Promise<Inges const path = input.source ?? ""; if (input.dryRun) return { ...result, planned: [path] }; const doc = await indexFile(ctx.db, ctx.provider, path, { + url: resolve(path), title: input.title, topic: input.topic, library: input.library, @@ -252,6 +253,7 @@ async function ingestDirectory(ctx: IngestContext, input: IngestInput): Promise< ctx.signal?.throwIfAborted(); try { const doc = await indexFile(ctx.db, ctx.provider, file, { + url: resolve(file), topic: input.topic, library: input.library, version: input.version, diff --git a/src/core/operations/packs.ts b/src/core/operations/packs.ts index 843bd4fe..f040410b 100644 --- a/src/core/operations/packs.ts +++ b/src/core/operations/packs.ts @@ -48,6 +48,12 @@ export const installPackOperation = defineOperation({ .max(16) .optional() .describe("Batches embedded in parallel (default 4)"), + resumeFrom: z + .number() + .int() + .min(0) + .optional() + .describe("Skip the first N documents (resume a partial install)"), }), annotations: { longRunning: true, idempotent: true }, http: { method: "POST", path: "/packs" }, @@ -57,6 +63,7 @@ export const installPackOperation = defineOperation({ registryUrl: input.registryUrl, batchSize: input.batchSize, concurrency: input.concurrency, + resumeFrom: input.resumeFrom, signal: ctx.signal, onProgress: (done, total, message) => ctx.onProgress?.({ done, total, message }), }); diff --git a/src/core/watcher.ts b/src/core/watcher.ts index d1363dd2..b193eaae 100644 --- a/src/core/watcher.ts +++ b/src/core/watcher.ts @@ -3,7 +3,7 @@ import { extname, resolve, basename } from "node:path"; import { createHash } from "node:crypto"; import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { indexDocument } from "./indexing.js"; +import { indexDocument, resolveSourceType, type SourceType } from "./indexing.js"; import { deleteChunkEmbeddings } from "./documents.js"; import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; import { createChildLogger } from "../logger.js"; @@ -37,6 +37,15 @@ export interface WatchOptions { onIndex?: (path: string) => void; onRemove?: (path: string) => void; onError?: (err: Error) => void; + /** Metadata for documents the watcher indexes (default: none, source type "manual"). */ + document?: + | { + topicId?: string | undefined; + library?: string | undefined; + version?: string | undefined; + sourceType?: SourceType | undefined; + } + | undefined; } export class FileWatcher { @@ -134,10 +143,14 @@ export class FileWatcher { } const title = basename(fullPath).replace(/\.[^.]+$/, ""); + const doc = this.options.document ?? {}; const result = await indexDocument(this.db, this.provider, { title, content, - sourceType: "manual", + sourceType: resolveSourceType({ ...doc, topic: doc.topicId }), + library: doc.library, + version: doc.version, + topicId: doc.topicId, url: fullPath, }); diff --git a/src/db/index-meta.ts b/src/db/index-meta.ts index 0b6de705..e20a580b 100644 --- a/src/db/index-meta.ts +++ b/src/db/index-meta.ts @@ -14,7 +14,7 @@ export const INDEX_META_DDL = ` /** The guidance shown whenever the vector index does not match the embedding model. */ export const REBUILD_VECTOR_INDEX_HINT = - "Run `libscope reindex --rebuild` to rebuild the vector index with the configured embedding model."; + "Run `libscope admin reindex --rebuild` to rebuild the vector index with the configured embedding model."; /** * sqlite-vec error raised when a vector's size differs from the vector table's diff --git a/src/db/schema.ts b/src/db/schema.ts index c4809350..0e84fcb4 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -404,7 +404,7 @@ export function runMigrations(db: Database.Database, targetVersion: number = SCH * is detected, not only a change of vector size. * * @throws ConfigError when the existing vector table was built for a different provider, - * model or vector size. The message tells the user to run `libscope reindex --rebuild`. + * model or vector size. The message tells the user to run `libscope admin reindex --rebuild`. * @throws DatabaseError when the vector size is invalid, or unknown (0) and no table exists. */ export function createVectorTable( @@ -424,7 +424,7 @@ export function createVectorTable( if (dimensions === 0) { throw new DatabaseError( "Invalid vector dimensions: the vector size of the embedding model is not known yet. " + - "Set embedding.dimensions in the config, or run `libscope reindex --rebuild` to detect it.", + "Set embedding.dimensions in the config, or run `libscope admin reindex --rebuild` to detect it.", ); } diff --git a/src/providers/local.ts b/src/providers/local.ts index 57f8effa..37a97130 100644 --- a/src/providers/local.ts +++ b/src/providers/local.ts @@ -1,3 +1,5 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; import { EmbeddingError } from "../errors.js"; import type { EmbeddingProvider } from "./embedding.js"; import { getLogger } from "../logger.js"; @@ -13,6 +15,15 @@ type FeatureExtractionPipeline = ( options: { pooling: string; normalize: boolean }, ) => Promise<TransformersOutput>; +/** Download events from @xenova/transformers while the model is fetched (first use only). */ +export interface ModelDownloadEvent { + /** "initiate" | "download" | "progress" | "done" per file, then "ready". */ + status: string; + file?: string | undefined; + /** Percent of `file` downloaded (status "progress"). */ + progress?: number | undefined; +} + /** * Local embedding provider using @xenova/transformers (all-MiniLM-L6-v2). * Downloads the model on first use (~80MB). Runs entirely in-process. @@ -25,6 +36,9 @@ export class LocalEmbeddingProvider implements EmbeddingProvider { private pipeline: FeatureExtractionPipeline | null = null; private initPromise: Promise<void> | null = null; + /** Called while the model is downloaded (not when it is already cached), e.g. to show progress. */ + onDownloadProgress: ((event: ModelDownloadEvent) => void) | undefined; + private async ensureInitialized(): Promise<void> { this.initPromise ??= this.doInitialize(); await this.initPromise; @@ -35,11 +49,15 @@ export class LocalEmbeddingProvider implements EmbeddingProvider { log.info("Loading local embedding model (all-MiniLM-L6-v2)..."); try { // Dynamic import to avoid loading transformers until needed - const { pipeline } = await import("@xenova/transformers"); + const { pipeline, env } = await import("@xenova/transformers"); + const cacheDir: unknown = env.cacheDir; + const cached = typeof cacheDir === "string" && existsSync(join(cacheDir, this.model)); + const onDownload = cached ? undefined : this.onDownloadProgress; // Cast to the typed interface; @xenova/transformers lacks precise TS generics for pipeline output this.pipeline = (await pipeline( "feature-extraction", this.model, + onDownload ? { progress_callback: onDownload } : undefined, )) as unknown as FeatureExtractionPipeline; log.info("Local embedding model loaded successfully"); } catch (err) { diff --git a/tests/unit/chunk-writer.test.ts b/tests/unit/chunk-writer.test.ts index c7d20961..0679b48a 100644 --- a/tests/unit/chunk-writer.test.ts +++ b/tests/unit/chunk-writer.test.ts @@ -52,6 +52,6 @@ describe.skipIf(!hasSqliteVec())("createChunkWriter (sqlite-vec)", () => { error = err; } expect(error).toBeInstanceOf(EmbeddingError); - expect((error as Error).message).toMatch(/3 dimensions.*libscope reindex --rebuild/); + expect((error as Error).message).toMatch(/3 dimensions.*libscope admin reindex --rebuild/); }); }); diff --git a/tests/unit/connector-registry.test.ts b/tests/unit/connector-registry.test.ts index 46ab4b2f..986de824 100644 --- a/tests/unit/connector-registry.test.ts +++ b/tests/unit/connector-registry.test.ts @@ -1,9 +1,28 @@ -import { describe, it, expect } from "vitest"; -import { CONNECTORS, getConnector, maskConnectorSecrets } from "../../src/connectors/registry.js"; +import { describe, it, expect, vi, afterAll } from "vitest"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + CONNECTORS, + getConnector, + maskConnectorSecrets, + saveConnection, +} from "../../src/connectors/registry.js"; import { CONNECTOR_TYPES } from "../../src/connectors/saved-config.js"; import { createTestDb } from "../fixtures/test-db.js"; import { ValidationError } from "../../src/errors.js"; +// Saved connections live under a temp HOME, never the real ~/.libscope. +const tempHome = mkdtempSync(join(tmpdir(), "libscope-registry-")); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); + +afterAll(() => { + rmSync(tempHome, { recursive: true, force: true }); +}); + describe("connector registry", () => { it("has one definition per connector type", () => { expect(Object.keys(CONNECTORS).sort()).toEqual([...CONNECTOR_TYPES].sort()); @@ -64,3 +83,47 @@ describe("connector registry", () => { } }); }); + +describe("saveConnection", () => { + const saved = (name: string): Record<string, unknown> => + JSON.parse( + readFileSync(join(tempHome, ".libscope", "connectors", `${name}.json`), "utf-8"), + ) as Record<string, unknown>; + + it("validates against the type's schema and tags the connector type", () => { + expect(() => saveConnection("n1", "notion", {})).toThrow(/Invalid Notion settings: token/); + saveConnection("n1", "notion", { token: "secret_x", excludePages: undefined }); + expect(saved("n1")).toEqual({ token: "secret_x", connectorType: "notion" }); + }); + + it("merges over the saved settings and sets, keeps or removes the schedule", () => { + saveConnection( + "s1", + "slack", + { token: "a", channels: ["all"], threadMode: "aggregate" }, + { + schedule: "0 * * * *", + }, + ); + saveConnection("s1", "slack", { token: "b" }); + expect(saved("s1")).toMatchObject({ + token: "b", + channels: ["all"], + schedule: { cronExpression: "0 * * * *" }, + }); + saveConnection("s1", "slack", {}, { schedule: null }); + expect(saved("s1")["schedule"]).toBeUndefined(); + expect(() => saveConnection("s1", "slack", {}, { schedule: "every hour" })).toThrow( + ValidationError, + ); + }); + + it("refuses to change a saved connection into another type", () => { + saveConnection("v1", "obsidian", { + vaultPath: "/v", + topicMapping: "folder", + excludePatterns: [], + }); + expect(() => saveConnection("v1", "notion", { token: "t" })).toThrow(/obsidian connector/); + }); +}); diff --git a/tests/unit/index-meta.test.ts b/tests/unit/index-meta.test.ts index 9c31f47e..aa360850 100644 --- a/tests/unit/index-meta.test.ts +++ b/tests/unit/index-meta.test.ts @@ -109,7 +109,7 @@ describe.skipIf(!vecAvailable)("vector index metadata (sqlite-vec)", () => { expect(change).toThrow( /configured embedding model \(ollama\/mxbai-embed-large, 1024 dimensions\)/, ); - expect(change).toThrow(/libscope reindex --rebuild/); + expect(change).toThrow(/libscope admin reindex --rebuild/); // A bare vector size is checked against the table too. expect(() => createVectorTable(db, 1024)).toThrow( /does not match the configured embedding model \(1024 dimensions\)/, diff --git a/tests/unit/ingest.test.ts b/tests/unit/ingest.test.ts index a4b8b31d..a5ee2b0f 100644 --- a/tests/unit/ingest.test.ts +++ b/tests/unit/ingest.test.ts @@ -153,6 +153,19 @@ describe("ingest", () => { expect(row).toEqual({ source_type: "library", library: "lib" }); }); + it("stores a file's absolute path as its URL, so adding it again replaces it", async () => { + const file = join(dir, "a.md"); + writeFileSync(file, "# A\n\nAlpha"); + const first = await ingest({ db, provider }, { source: file }); + const unchanged = await ingest({ db, provider }, { source: file }); + expect(unchanged.documents[0]?.documentId).toBe(first.documents[0]?.documentId); + writeFileSync(file, "# A\n\nAlpha, changed"); + await ingest({ db, provider }, { source: file }); + expect(db.prepare("SELECT url, content FROM documents").all()).toEqual([ + { url: file, content: "# A\n\nAlpha, changed" }, + ]); + }); + it("plans without indexing on a dry run", async () => { writeFileSync(join(dir, "a.md"), "# A"); const result = await ingest({ db, provider }, { source: dir, dryRun: true }); diff --git a/tests/unit/watcher-document.test.ts b/tests/unit/watcher-document.test.ts new file mode 100644 index 00000000..e269a382 --- /dev/null +++ b/tests/unit/watcher-document.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect, vi } from "vitest"; +import type Database from "better-sqlite3"; +import type { EmbeddingProvider } from "../../src/providers/embedding.js"; + +vi.mock("node:fs", async (importOriginal) => { + const original = await importOriginal<typeof import("node:fs")>(); + return { + ...original, + watch: vi.fn(() => ({ on: vi.fn(), close: vi.fn() })), + readFileSync: vi.fn(() => "# Doc\n\nBody"), + statSync: vi.fn(() => ({ isFile: (): boolean => true })), + }; +}); + +vi.mock("../../src/core/indexing.js", async (importOriginal) => { + const original = await importOriginal<typeof import("../../src/core/indexing.js")>(); + return { ...original, indexDocument: vi.fn(() => Promise.resolve({ id: "d1", chunkCount: 1 })) }; +}); + +const { watch } = await import("node:fs"); +const { indexDocument } = await import("../../src/core/indexing.js"); +const { FileWatcher } = await import("../../src/core/watcher.js"); + +describe("FileWatcher document metadata", () => { + it("indexes changed files with the given topic, library and source type", async () => { + vi.useFakeTimers(); + const db = { + prepare: vi.fn(() => ({ get: vi.fn(() => undefined), run: vi.fn() })), + } as unknown as Database.Database; + const onIndex = vi.fn(); + const watcher = new FileWatcher(db, {} as EmbeddingProvider, { + directory: "/docs", + extensions: [".md"], + document: { topicId: "t1", library: "lib", version: "2" }, + onIndex, + }); + watcher.start(); + const callback = vi.mocked(watch).mock.calls[0]?.[2] as (e: string, f: string) => void; + callback("change", "guide.md"); + await vi.advanceTimersByTimeAsync(500); + vi.useRealTimers(); + + expect(indexDocument).toHaveBeenCalledWith( + db, + expect.anything(), + expect.objectContaining({ + title: "guide", + sourceType: "library", + library: "lib", + version: "2", + topicId: "t1", + url: "/docs/guide.md", + }), + ); + expect(onIndex).toHaveBeenCalledWith("/docs/guide.md"); + watcher.stop(); + }); +}); From 46cc802c13526123dab94697eed8abe93d61d30e Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:15:37 +0000 Subject: [PATCH 50/83] feat(cli)!: one command per task over the operation layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 3200-line src/cli/index.ts becomes program setup plus one register() per group in src/cli/commands/. Every command parses flags, calls an operation (shared context from bootstrap(); commands that do not embed never require the embedding provider) and prints the result: JSON with the global --json flag, otherwise a human formatter that shows document/chunk IDs. - add <sources...> covers files, directories (--include/--exclude/--watch), URLs (--spider), repositories and --dry-run - search [query] (interactive with no query on a terminal; --related <id>), ask - docs list|show|update|delete|history|rollback|rate|link|unlink|links| prereqs|tag|untag|suggest-tags; topics; tags; searches; bulk - connect <type> [source] (--schedule, --no-sync), sync, disconnect, connections - admin reindex|dedupe|backup|restore|stats|prune; doctor [--fix] - one error handler: '✗ message' + next-step hint, stack only with --verbose - shared filter flags --topic --library --lib-version --source-type --tags -n - first-run local model download prints one progress line to stderr BREAKING CHANGE: init, import, import-batch, add-repo, repl, related, ratings, stats, search-analytics, watch, dedupe, reindex, export, import-backup, link, links, unlink, prereqs, tag, schedule, update and the per-type connect / disconnect subcommands are removed. See docs/migration-v2.md. --- src/cli/commands/add.ts | 168 ++ src/cli/commands/admin.ts | 173 ++ src/cli/commands/bulk.ts | 94 + src/cli/commands/config.ts | 66 + src/cli/commands/connectors.ts | 251 ++ src/cli/commands/docs.ts | 321 +++ src/cli/commands/doctor.ts | 193 ++ src/cli/commands/pack.ts | 220 ++ src/cli/commands/registry.ts | 2 +- src/cli/commands/search.ts | 136 + src/cli/commands/searches.ts | 77 + src/cli/commands/serve.ts | 85 + src/cli/commands/topics.ts | 71 + src/cli/commands/webhooks.ts | 59 + src/cli/commands/workspace.ts | 55 + src/cli/context.ts | 139 + src/cli/errors.ts | 40 + src/cli/index.ts | 3241 +----------------------- src/cli/options.ts | 81 + src/cli/repl.ts | 61 +- src/cli/reporter.ts | 124 +- src/cli/run.ts | 68 + tests/unit/cli-commands.test.ts | 452 ++++ tests/unit/cli-run.test.ts | 160 ++ tests/unit/cli-version-options.test.ts | 6 +- tests/unit/repl.test.ts | 157 +- tests/unit/reporter.test.ts | 207 +- 27 files changed, 3134 insertions(+), 3573 deletions(-) create mode 100644 src/cli/commands/add.ts create mode 100644 src/cli/commands/admin.ts create mode 100644 src/cli/commands/bulk.ts create mode 100644 src/cli/commands/config.ts create mode 100644 src/cli/commands/connectors.ts create mode 100644 src/cli/commands/docs.ts create mode 100644 src/cli/commands/doctor.ts create mode 100644 src/cli/commands/pack.ts create mode 100644 src/cli/commands/search.ts create mode 100644 src/cli/commands/searches.ts create mode 100644 src/cli/commands/serve.ts create mode 100644 src/cli/commands/topics.ts create mode 100644 src/cli/commands/webhooks.ts create mode 100644 src/cli/commands/workspace.ts create mode 100644 src/cli/context.ts create mode 100644 src/cli/errors.ts create mode 100644 src/cli/options.ts create mode 100644 src/cli/run.ts create mode 100644 tests/unit/cli-commands.test.ts create mode 100644 tests/unit/cli-run.test.ts diff --git a/src/cli/commands/add.ts b/src/cli/commands/add.ts new file mode 100644 index 00000000..f562f6f0 --- /dev/null +++ b/src/cli/commands/add.ts @@ -0,0 +1,168 @@ +/** `libscope add`: files, directories, URLs (optionally crawled) and repositories. */ +import type { Command } from "commander"; +import { statSync } from "node:fs"; +import { addOperation } from "../../core/operations/index.js"; +import type { IngestResult } from "../../core/ingest.js"; +import { resolveTopicId } from "../../core/topics.js"; +import { FileWatcher } from "../../core/watcher.js"; +import type { SourceType } from "../../core/indexing.js"; +import { ValidationError } from "../../errors.js"; +import { getContext, untilInterrupted } from "../context.js"; +import { + addDocumentOptions, + defined, + documentInput, + splitList, + toNumber, + type DocumentFlags, +} from "../options.js"; +import { call, plural, print } from "../run.js"; + +interface AddFlags extends DocumentFlags { + title?: string; + format?: string; + dedup?: string; + expires?: string; + include?: string; + exclude?: string; + watch?: boolean; + spider?: boolean; + maxPages?: number; + maxDepth?: number; + sameDomain: boolean; + pathPrefix?: string; + excludeUrls?: string; + branch?: string; + path?: string; + extensions?: string; + token?: string; + dryRun?: boolean; +} + +const list = (value: string | undefined): string[] | undefined => + value === undefined ? undefined : splitList(value); + +/** add-operation input for one source. */ +export function addInput(source: string, flags: AddFlags): Record<string, unknown> { + return defined({ + source, + ...documentInput(flags), + title: flags.title, + format: flags.format, + dedup: flags.dedup, + expiresAt: flags.expires, + include: list(flags.include), + exclude: list(flags.exclude), + spider: flags.spider, + maxPages: flags.maxPages, + maxDepth: flags.maxDepth, + sameDomain: flags.spider ? flags.sameDomain : undefined, + pathPrefix: flags.pathPrefix, + excludePatterns: list(flags.excludeUrls), + branch: flags.branch, + paths: list(flags.path), + extensions: list(flags.extensions), + token: flags.token, + dryRun: flags.dryRun, + }); +} + +function printResult(result: IngestResult): void { + if (result.planned) { + console.log(`Would add ${plural(result.planned.length, "item")}:`); + for (const source of result.planned) console.log(` ${source}`); + } + for (const doc of result.documents) { + console.log(`✓ ${doc.title} (${plural(doc.chunkCount, "chunk")}) ${doc.documentId}`); + if (doc.source) console.log(` ${doc.source}`); + } + for (const s of result.skipped) console.log(` skipped ${s.source}: ${s.reason}`); + for (const e of result.errors) console.log(`✗ ${e.source}: ${e.error}`); + if (result.crawl) { + const stop = result.crawl.abortReason ? ` (stopped: ${result.crawl.abortReason})` : ""; + console.log( + `Crawled ${plural(result.crawl.pagesCrawled, "page")}, skipped ${result.crawl.pagesSkipped}${stop}`, + ); + } + if (!result.planned && result.documents.length !== 1) { + const errors = result.errors.length > 0 ? `, ${plural(result.errors.length, "error")}` : ""; + console.log(`Added ${plural(result.documents.length, "document")}${errors}`); + } +} + +/** Keep re-indexing changed files in `directories` until Ctrl+C. */ +async function watchDirectories(directories: string[], flags: AddFlags): Promise<void> { + const ctx = getContext({ embeddings: true }); + const document = { + topicId: flags.topic === undefined ? undefined : resolveTopicId(ctx.db, flags.topic), + library: flags.library, + version: flags.libVersion, + sourceType: flags.sourceType as SourceType | undefined, + }; + const watchers = directories.map( + (directory) => + new FileWatcher(ctx.db, ctx.provider, { + directory, + document, + onIndex: (path): void => console.log(`✓ indexed ${path}`), + onRemove: (path): void => console.log(`✗ removed ${path}`), + onError: (err): void => console.error(`⚠ ${err.message}`), + }), + ); + for (const w of watchers) w.start(); + console.error(`Watching ${directories.join(", ")} for changes (Ctrl+C to stop)...`); + await untilInterrupted(); + for (const w of watchers) w.stop(); +} + +function isDirectory(path: string): boolean { + try { + return statSync(path).isDirectory(); + } catch { + return false; + } +} + +export function register(program: Command): void { + const cmd = program + .command("add <sources...>") + .description( + "Add files, directories, URLs or GitHub/GitLab repository URLs to the knowledge base", + ); + addDocumentOptions(cmd, { mode: "assign", limit: false }) + .option("--title <title>", "Title (one file or URL; default: detected)") + .option("--format <ext>", "Read files as this format, e.g. .pdf") + .option("--dedup <mode>", "Duplicate handling: skip, warn or force") + .option("--expires <time>", "ISO 8601 time after which `admin prune` deletes the documents") + .option("--include <globs>", "Directory: only files matching these globs (comma-separated)") + .option("--exclude <globs>", "Directory: skip files matching these globs (comma-separated)") + .option("--watch", "Directory: keep running and re-index files when they change") + .option("--spider", "URL: also crawl linked pages") + .option("--max-pages <n>", "Crawl: page limit (default 25, max 200)", toNumber) + .option("--max-depth <n>", "Crawl: link depth (default 2, max 5)", toNumber) + .option("--no-same-domain", "Crawl: also follow links to other domains") + .option("--path-prefix <path>", "Crawl: only follow links under this path") + .option("--exclude-urls <globs>", "Crawl: skip URLs matching these globs (comma-separated)") + .option("--branch <name>", "Repository: branch (default: from the URL, else main)") + .option("--path <dirs>", "Repository: only these subdirectories (comma-separated)") + .option("--extensions <exts>", "Repository: file extensions (default .md,.mdx,.txt,.rst)") + .option("--token <token>", "Repository: access token for a private repository") + .option("--dry-run", "List what would be added without adding it") + .action(async (sources: string[], flags: AddFlags) => { + const directories = sources.filter(isDirectory); + if (flags.watch && directories.length !== sources.length) { + throw new ValidationError("--watch works with directories only"); + } + const results: IngestResult[] = []; + for (const source of sources) { + results.push( + await call(addOperation, addInput(source, flags), { embeddings: !flags.dryRun }), + ); + } + print(results, (all) => all.forEach(printResult)); + if (results.every((r) => r.documents.length === 0 && r.errors.length > 0)) { + process.exitCode = 1; + } + if (flags.watch && !flags.dryRun) await watchDirectories(directories, flags); + }); +} diff --git a/src/cli/commands/admin.ts b/src/cli/commands/admin.ts new file mode 100644 index 00000000..b1b8ba74 --- /dev/null +++ b/src/cli/commands/admin.ts @@ -0,0 +1,173 @@ +/** `libscope admin reindex|dedupe|backup|restore|stats|prune`. */ +import type { Command } from "commander"; +import { + backupOperation, + dedupeOperation, + overviewOperation, + popularOperation, + pruneExpiredOperation, + reindexOperation, + restoreOperation, + searchAnalyticsOperation, + staleOperation, +} from "../../core/operations/index.js"; +import type { Overview } from "../../core/overview.js"; +import { describeEmbeddingIdentity } from "../../db/index-meta.js"; +import { confirmOrCancel } from "../confirm.js"; +import { defined, toNumber } from "../options.js"; +import { call, plural, print, printList, run } from "../run.js"; + +/** Counts, topics, packs and index identity (shared with `doctor`). */ +export function printOverview(o: Overview): void { + const s = o.stats; + console.log( + `Documents: ${s.totalDocuments} | chunks: ${s.totalChunks} | topics: ${s.totalTopics} | ` + + `database: ${(s.databaseSizeBytes / 1024).toFixed(1)} KB`, + ); + console.log(`Searches: ${s.totalSearches} (avg ${s.avgLatencyMs} ms)`); + const stored = describeEmbeddingIdentity(o.index.stored); + console.log(`Index: ${o.index.vectorTableDimensions === undefined ? "none" : stored}`); + if (o.packs.length > 0) { + console.log(`Packs: ${o.packs.map((p) => `${p.name} v${p.version}`).join(", ")}`); + } +} + +type Stats = { + overview: Overview; + popular: Awaited<ReturnType<typeof popularOperation.handler>>; + stale: Awaited<ReturnType<typeof staleOperation.handler>>; + searches: Awaited<ReturnType<typeof searchAnalyticsOperation.handler>>; +}; + +function printStats(r: Stats): void { + printOverview(r.overview); + console.log("\nMost returned documents:"); + printList(r.popular.items, " (no searches yet)", (d) => + console.log(` ${String(d.hitCount).padStart(5)} ${d.title} ${d.documentId}`), + ); + console.log("\nDocuments no search returned:"); + printList(r.stale.items, " (none)", (d) => + console.log(` ${d.updatedAt.slice(0, 10)} ${d.title} ${d.documentId}`), + ); + const q = r.searches; + console.log(`\nSearches: ${q.totalSearches} (avg ${q.avgResultCount} results)`); + for (const t of q.topQueries) console.log(` ${String(t.count).padStart(5)}x ${t.query}`); + if (q.zeroResultQueries.length > 0) { + console.log("\nQueries with no results:"); + for (const t of q.zeroResultQueries) { + console.log(` ${String(t.count).padStart(5)}x ${t.query}`); + } + } +} + +function registerMaintenance(admin: Command): void { + admin + .command("reindex") + .description("Re-embed chunks with the configured embedding model") + .option("--rebuild", "Recreate the vector index for the configured model, then re-embed all") + .option("--doc <documentIds...>", "Only these documents") + .option("--since <date>", "Only documents created on or after (ISO 8601)") + .option("--before <date>", "Only documents created on or before (ISO 8601)") + .option("--batch-size <n>", "Chunks per embedding call (default 50)", toNumber) + .action( + async (flags: { + rebuild?: boolean; + doc?: string[]; + since?: string; + before?: string; + batchSize?: number; + }) => { + const input = defined({ + rebuild: flags.rebuild, + documentIds: flags.doc, + since: flags.since, + before: flags.before, + batchSize: flags.batchSize, + }); + await run( + reindexOperation, + input, + (r) => { + if (r.rebuiltIndex) console.log(`✓ Rebuilt vector index (${r.rebuiltIndex})`); + console.log(`✓ Re-embedded ${r.completed} of ${plural(r.total, "chunk")}`); + for (const id of r.failedChunkIds) console.log(`✗ failed: chunk ${id}`); + }, + // The operation creates or rebuilds the vector table itself. + { embeddings: true, vectorTable: "skip" }, + ); + }, + ); + + admin + .command("dedupe") + .description("Find duplicate and near-duplicate documents") + .option("--threshold <n>", "Similarity threshold 0-1 (default 0.95)", toNumber) + .option("--strategy <strategy>", "exact, semantic or both (default)") + .action(async (flags: { threshold?: number; strategy?: string }) => { + await run( + dedupeOperation, + defined({ ...flags }), + (r) => { + let group = 0; + printList(r.items, "No duplicates.", (g) => { + console.log(`Group ${++group} (${g.matchType}):`); + g.documentIds.forEach((id, i) => console.log(` ${id} ${g.titles[i] ?? ""}`)); + }); + }, + { embeddings: true }, + ); + }); + + admin + .command("prune") + .description("Delete documents whose expiry time has passed") + .action(async () => { + await run(pruneExpiredOperation, {}, (r) => + console.log(`✓ Pruned ${plural(r.pruned, "expired document")}`), + ); + }); +} + +function registerBackup(admin: Command): void { + admin + .command("backup <file>") + .description("Write the whole knowledge base to a JSON file") + .action(async (file: string) => { + await run(backupOperation, { outputPath: file }, (r) => + console.log(`✓ Backed up ${plural(r.counts.documents, "document")} to ${r.path}`), + ); + }); + + admin + .command("restore <file>") + .description("Import a file written by `admin backup`") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (file: string, flags: { yes?: boolean }) => { + const question = `Import ${file} into the current knowledge base?`; + if (!(await confirmOrCancel(question, flags.yes))) return; + await run(restoreOperation, { backupPath: file }, (r) => + console.log(`✓ Restored ${plural(r.counts.documents, "document")} from ${r.path}`), + ); + }); +} + +export function register(program: Command): void { + const admin = program.command("admin").description("Maintenance: index, backups, statistics"); + registerMaintenance(admin); + registerBackup(admin); + + admin + .command("stats") + .description("Counts, index, most returned and stale documents, and search analytics") + .option("--days <n>", "Look-back days for stale documents and search analytics", toNumber) + .action(async (flags: { days?: number }) => { + const days = defined({ days: flags.days }); + const stats: Stats = { + overview: await call(overviewOperation, {}), + popular: await call(popularOperation, {}), + stale: await call(staleOperation, days), + searches: await call(searchAnalyticsOperation, days), + }; + print(stats, printStats); + }); +} diff --git a/src/cli/commands/bulk.ts b/src/cli/commands/bulk.ts new file mode 100644 index 00000000..dd48fb11 --- /dev/null +++ b/src/cli/commands/bulk.ts @@ -0,0 +1,94 @@ +/** `libscope bulk delete|retag|move`: change every document that matches the filters. */ +import type { Command } from "commander"; +import type { z } from "zod"; +import { + bulkDeleteOperation, + bulkMoveOperation, + bulkRetagOperation, + type Operation, +} from "../../core/operations/index.js"; +import type { BulkResult } from "../../core/bulk.js"; +import { confirmOrCancel } from "../confirm.js"; +import { + addDocumentOptions, + defined, + documentInput, + splitList, + type DocumentFlags, +} from "../options.js"; +import { call, plural, print } from "../run.js"; + +interface BulkFlags extends DocumentFlags { + since?: string; + before?: string; + dryRun?: boolean; + yes?: boolean; +} + +function selectorInput(flags: BulkFlags): Record<string, unknown> { + return { ...documentInput(flags), ...defined({ dateFrom: flags.since, dateTo: flags.before }) }; +} + +function bulkCommand(parent: Command, name: string, description: string): Command { + return addDocumentOptions(parent.command(name).description(description), { + limit: false, + version: false, + }) + .option("--since <date>", "Only documents created on or after (ISO 8601)") + .option("--before <date>", "Only documents created on or before (ISO 8601)") + .option("--dry-run", "List the matching documents without changing them") + .option("-y, --yes", "Do not ask for confirmation"); +} + +/** Preview the matches, confirm, then apply (the operation itself refuses empty filters). */ +async function runBulk<S extends z.ZodObject>( + op: Operation<S, BulkResult>, + input: Record<string, unknown>, + flags: BulkFlags, + verb: string, +): Promise<void> { + const preview = await call(op, { ...input, dryRun: true }); + if (flags.dryRun || preview.affected === 0) { + print(preview, (r) => { + console.log(`${plural(r.affected, "document")} match:`); + for (const id of r.documentIds) console.log(` ${id}`); + }); + return; + } + if (!(await confirmOrCancel(`${verb} ${plural(preview.affected, "document")}?`, flags.yes))) { + return; + } + const result = await call(op, input); + print(result, (r) => console.log(`✓ ${verb}: ${plural(r.affected, "document")}`)); +} + +export function register(program: Command): void { + const bulk = program + .command("bulk") + .description("Change every document that matches the filters (at least one filter required)"); + + bulkCommand(bulk, "delete", "Delete matching documents").action(async (flags: BulkFlags) => { + await runBulk(bulkDeleteOperation, selectorInput(flags), flags, "Delete"); + }); + + bulkCommand(bulk, "retag", "Add and/or remove tags on matching documents") + .option("--add <tags>", "Tags to add (comma-separated)") + .option("--remove <tags>", "Tags to remove (comma-separated)") + .action(async (flags: BulkFlags & { add?: string; remove?: string }) => { + const input = { + ...selectorInput(flags), + ...defined({ + addTags: flags.add === undefined ? undefined : splitList(flags.add), + removeTags: flags.remove === undefined ? undefined : splitList(flags.remove), + }), + }; + await runBulk(bulkRetagOperation, input, flags, "Retag"); + }); + + bulkCommand(bulk, "move", "Move matching documents to another topic") + .requiredOption("--to <topic>", "Destination topic (ID or name)") + .action(async (flags: BulkFlags & { to: string }) => { + const input = { ...selectorInput(flags), targetTopic: flags.to }; + await runBulk(bulkMoveOperation, input, flags, `Move to "${flags.to}"`); + }); +} diff --git a/src/cli/commands/config.ts b/src/cli/commands/config.ts new file mode 100644 index 00000000..0e64e00a --- /dev/null +++ b/src/cli/commands/config.ts @@ -0,0 +1,66 @@ +/** `libscope config show|get|set|unset|path`. */ +import type { Command } from "commander"; +import { + CONFIG_KEY_NAMES, + getConfigFileFor, + getConfigValue, + getSecretsPath, + getUserConfigPath, + loadConfig, + maskConfigSecrets, + maskSecret, + setUserConfigValue, + unsetUserConfigValue, +} from "../../config.js"; +import { print } from "../run.js"; + +export function register(program: Command): void { + const config = program + .command("config") + .description(`Show and change settings (user file: ${getUserConfigPath()})`); + + config + .command("show") + .description("Show the effective configuration (API keys are masked)") + .action(() => { + console.log(JSON.stringify(maskConfigSecrets(loadConfig()), null, 2)); + }); + + config + .command("get <key>") + .description("Print the effective value of a key (API keys are masked)") + .action((key: string) => { + const value = getConfigValue(loadConfig(), key); + print({ key, value: value ?? null }, (r) => + console.log(r.value === null ? "" : String(r.value)), + ); + }); + + config + .command("set <key> <value>") + .description( + `Set a key in the user config file. Keys: ${CONFIG_KEY_NAMES.join(", ")}. ` + + `API keys are written to ${getSecretsPath()} (mode 0600), never to config.json.`, + ) + .action((key: string, value: string) => { + const stored = setUserConfigValue(key, value); + const file = getConfigFileFor(key); + const shown = file === getSecretsPath() ? maskSecret(String(stored)) : String(stored); + console.log(`✓ ${key} set to: ${shown} (${file})`); + }); + + config + .command("unset <key>") + .description("Remove a key from the user config file (or secrets file for API keys)") + .action((key: string) => { + const removed = unsetUserConfigValue(key); + console.log(removed ? `✓ ${key} removed` : `${key} is not set in ${getConfigFileFor(key)}`); + }); + + config + .command("path") + .description("Print the path of the user config file") + .action(() => { + console.log(getUserConfigPath()); + }); +} diff --git a/src/cli/commands/connectors.ts b/src/cli/commands/connectors.ts new file mode 100644 index 00000000..4d24d4e4 --- /dev/null +++ b/src/cli/commands/connectors.ts @@ -0,0 +1,251 @@ +/** `libscope connect <type>`, `sync`, `disconnect`, `connections`. */ +import type { Command } from "commander"; +import { resolve } from "node:path"; +import { + disconnectOperation, + listConnectionsOperation, + syncOperation, +} from "../../core/operations/index.js"; +import { hasNamedConnectorConfig } from "../../connectors/index.js"; +import { getConnector, saveConnection } from "../../connectors/registry.js"; +import { + CONNECTOR_TYPES, + findSavedOneNoteConfig, + type ConnectorType, +} from "../../connectors/saved-config.js"; +import { authenticateDeviceCode } from "../../connectors/onenote.js"; +import { ValidationError } from "../../errors.js"; +import { confirmOrCancel } from "../confirm.js"; +import { defined, splitList, toNumber } from "../options.js"; +import { plural, printList, run } from "../run.js"; + +export interface ConnectFlags { + name?: string; + schedule?: string; + sync: boolean; + token?: string; + exclude?: string; + channels?: string; + threadMode?: string; + email?: string; + server?: boolean; + spaces?: string; + topicMapping?: string; + notebook?: string; + clientId?: string; + tenantId?: string; + siteType?: string; + library?: string; + libVersion?: string; + maxPages?: number; + maxDepth?: number; + pathPrefix?: string; +} + +const list = (value: string | undefined): string[] | undefined => + value === undefined ? undefined : splitList(value); + +function requireSource(type: string, source: string | undefined, what: string): string { + if (!source) throw new ValidationError(`connect ${type} needs the ${what}`); + return source; +} + +/** + * Settings for connector `type` from the command line. Unset values are undefined, so the + * saved settings of the connection (if any) are kept; `defaults` apply to a new connection. + */ +export function connectSettings( + type: ConnectorType, + source: string | undefined, + flags: ConnectFlags, + isNew: boolean, +): Record<string, unknown> { + const exclude = list(flags.exclude); + switch (type) { + case "notion": + return { token: flags.token, excludePages: exclude }; + case "slack": + return { + token: flags.token, + channels: list(flags.channels) ?? (isNew ? ["all"] : undefined), + excludeChannels: exclude, + threadMode: flags.threadMode ?? (isNew ? "aggregate" : undefined), + }; + case "confluence": + return { + baseUrl: isNew ? requireSource(type, source, "base URL") : source, + type: flags.server ? "server" : isNew ? "cloud" : undefined, + email: flags.email, + token: flags.token, + spaces: list(flags.spaces) ?? (isNew ? ["all"] : undefined), + excludeSpaces: exclude, + }; + case "obsidian": + return { + vaultPath: isNew + ? resolve(requireSource(type, source, "vault path")) + : source && resolve(source), + topicMapping: flags.topicMapping ?? (isNew ? "folder" : undefined), + excludePatterns: exclude ?? (isNew ? [] : undefined), + }; + case "onenote": + return { + clientId: flags.clientId ?? (isNew ? "" : undefined), + tenantId: flags.tenantId ?? (isNew ? "common" : undefined), + notebooks: flags.notebook === undefined ? (isNew ? ["all"] : undefined) : [flags.notebook], + excludeSections: isNew ? [] : undefined, + // A token given by hand cannot be refreshed. + ...(flags.token === undefined + ? {} + : { accessToken: flags.token, refreshToken: undefined, tokenExpiry: undefined }), + }; + case "docs": + return { + url: isNew ? requireSource(type, source, "site URL") : source, + type: flags.siteType, + library: flags.library, + version: flags.libVersion, + maxPages: flags.maxPages, + maxDepth: flags.maxDepth, + pathPrefix: flags.pathPrefix, + }; + } +} + +/** OneNote without a usable token: sign in with a device code (interactive). */ +async function oneNoteSignIn( + name: string, + settings: Record<string, unknown>, +): Promise<Record<string, unknown>> { + const saved = findSavedOneNoteConfig(name); + if (settings["accessToken"] !== undefined || saved?.refreshToken) return settings; + const clientId = (settings["clientId"] as string | undefined) ?? saved?.clientId; + if (!clientId) { + throw new ValidationError("OneNote sign-in needs --client-id (an Azure app client ID)"); + } + const tenantId = (settings["tenantId"] as string | undefined) ?? saved?.tenantId ?? "common"; + const auth = await authenticateDeviceCode(clientId, tenantId); + return { + ...settings, + clientId, + tenantId, + accessToken: auth.accessToken, + refreshToken: auth.refreshToken, + tokenExpiry: auth.expiresAt, + }; +} + +type SyncResult = Awaited<ReturnType<typeof syncOperation.handler>>; + +function printSync(result: SyncResult): void { + printList(result.items, "No saved connections. Add one with: libscope connect <type>", (item) => { + if (item.status === "failed") { + console.log(`✗ ${item.name}: ${item.error}`); + return; + } + const s = item.summary; + console.log( + `✓ ${item.name}: ${s.added} added, ${s.updated} updated, ${s.deleted} deleted, ${plural(s.errors.length, "error")}`, + ); + for (const e of s.errors) console.log(` ${e}`); + }); +} + +function registerConnect(program: Command): void { + program + .command("connect <type> [source]") + .summary("Save a connection to an external service and sync it") + .description( + `Save a connection and sync it. Types: ${CONNECTOR_TYPES.join(", ")}. ` + + "[source] is the vault path (obsidian) or the URL (confluence, docs). " + + "Running it again for a saved connection changes only the given settings.", + ) + .option("--name <name>", "Connection name (default: the type)") + .option( + "--schedule <cron>", + 'Sync on this cron schedule while `libscope serve api` runs ("off" removes it)', + ) + .option("--no-sync", "Save the connection without syncing now") + .option("--token <token>", "notion, slack, confluence: API token; onenote: access token") + .option( + "--exclude <list>", + "Comma-separated: slack channels, notion page IDs, confluence spaces, obsidian globs", + ) + .option("--channels <list>", "slack: channel names or IDs (default all)") + .option("--thread-mode <mode>", "slack: aggregate (one document per thread) or separate") + .option("--email <email>", "confluence: user email (Cloud)") + .option("--server", "confluence: Server/Data Center instead of Cloud") + .option("--spaces <keys>", "confluence: space keys (default all)") + .option("--topic-mapping <mode>", "obsidian: topics from folder (default) or frontmatter") + .option("--notebook <name>", "onenote: one notebook (default all)") + .option("--client-id <id>", "onenote: Azure app client ID for device sign-in") + .option("--tenant-id <id>", "onenote: Azure tenant ID (default common)") + .option("--site-type <type>", "docs: auto, sphinx, vitepress, doxygen or generic") + .option("--library <name>", "docs: library name for the pages") + .option("--lib-version <version>", "docs: library version for the pages") + .option("--max-pages <n>", "docs: page limit (default 500)", toNumber) + .option("--max-depth <n>", "docs: link depth (default 10)", toNumber) + .option("--path-prefix <path>", "docs: only pages under this path") + .action(async (type: string, source: string | undefined, flags: ConnectFlags) => { + const connector = getConnector(type); + const name = flags.name ?? connector.type; + const isNew = !hasNamedConnectorConfig(name); + let settings = connectSettings(connector.type, source, flags, isNew); + if (connector.type === "onenote") settings = await oneNoteSignIn(name, settings); + const schedule = flags.schedule === "off" ? null : flags.schedule; + saveConnection(name, connector.type, defined(settings), { schedule }); + console.error(`✓ Saved ${connector.label} connection "${name}"`); + if (!flags.sync) return; + await run(syncOperation, { name }, printSync, { embeddings: true }); + }); +} + +export function register(program: Command): void { + registerConnect(program); + + program + .command("sync [name]") + .description("Sync a saved connection with its saved settings, or all with --all") + .option("--all", "Sync every saved connection") + .action(async (name: string | undefined, flags: { all?: boolean }) => { + await run(syncOperation, defined({ name, all: flags.all }), printSync, { embeddings: true }); + }); + + program + .command("disconnect <name>") + .description("Delete a connection's documents and its saved settings (including credentials)") + .option("--type <type>", "Connector type, when no saved connection has this name") + .option("--keep-documents", "Only delete the saved settings") + .option("-y, --yes", "Do not ask for confirmation") + .action( + async (name: string, flags: { type?: string; keepDocuments?: boolean; yes?: boolean }) => { + const what = flags.keepDocuments ? "its saved settings" : "its documents and settings"; + if (!(await confirmOrCancel(`Disconnect "${name}" and delete ${what}?`, flags.yes))) { + return; + } + const input = defined({ name, type: flags.type, keepDocuments: flags.keepDocuments }); + await run(disconnectOperation, input, (r) => + console.log( + `✓ Disconnected ${r.type} "${r.name}": ${plural(r.documentsRemoved, "document")} removed`, + ), + ); + }, + ); + + program + .command("connections") + .description("List saved connections with their schedule and last sync") + .action(async () => { + await run(listConnectionsOperation, {}, (r) => + printList(r.items, "No saved connections. Add one with: libscope connect <type>", (c) => { + console.log(`${c.name} (${c.type})`); + if (c.schedule) console.log(` schedule: ${c.schedule}`); + if (c.lastSync) console.log(` last sync: ${c.lastSync}`); + if (c.lastRun) { + const error = c.lastRun.error ? ` — ${c.lastRun.error}` : ""; + console.log(` last run: ${c.lastRun.status} at ${c.lastRun.startedAt}${error}`); + } + }), + ); + }); +} diff --git a/src/cli/commands/docs.ts b/src/cli/commands/docs.ts new file mode 100644 index 00000000..0265941a --- /dev/null +++ b/src/cli/commands/docs.ts @@ -0,0 +1,321 @@ +/** `libscope docs ...`: one document at a time (show, update, links, tags, ratings, versions). */ +import type { Command } from "commander"; +import { readFileSync } from "node:fs"; +import { + addTagsOperation, + deleteDocumentOperation, + documentHistoryOperation, + getDocumentOperation, + linkDocumentsOperation, + listDocumentsOperation, + listLinksOperation, + prerequisitesOperation, + rateDocumentOperation, + removeTagsOperation, + rollbackDocumentOperation, + suggestTagsOperation, + unlinkDocumentsOperation, + updateDocumentOperation, + type DocumentSummary, + type ListResult, +} from "../../core/operations/index.js"; +import type { DocumentView } from "../../core/document-view.js"; +import { LINK_TYPES } from "../../core/links.js"; +import { confirmOrCancel } from "../confirm.js"; +import { + addDocumentOptions, + defined, + documentInput, + splitList, + toNumber, + type DocumentFlags, +} from "../options.js"; +import { call, plural, printList, run } from "../run.js"; + +/** Tags given as separate words and/or comma-separated. */ +function tagList(tags: string[]): string[] { + return tags.flatMap(splitList); +} + +function printDocumentList(result: ListResult<DocumentSummary>): void { + if (result.items.length === 0) { + console.log("No documents."); + return; + } + const last = result.offset + result.items.length; + console.log(`Documents ${result.offset + 1}-${last} of ${result.total}:\n`); + for (const d of result.items) { + console.log(`${d.documentId} ${d.title}`); + const lib = d.library ? ` | ${d.library}${d.version ? ` ${d.version}` : ""}` : ""; + const url = d.url ? ` | ${d.url}` : ""; + console.log(` ${d.sourceType}${lib}${url} | updated ${d.updatedAt}`); + } +} + +function printDocument(view: DocumentView): void { + const d = view.document; + console.log(`# ${d.title}\n`); + console.log(`ID: ${d.documentId}`); + console.log(`Type: ${d.sourceType}`); + if (d.library) console.log(`Library: ${d.library}${d.version ? ` ${d.version}` : ""}`); + if (d.topicId) console.log(`Topic: ${d.topicId}`); + if (d.url) console.log(`URL: ${d.url}`); + console.log(`Added by: ${d.submittedBy} (${d.createdAt}, updated ${d.updatedAt})`); + if (view.tags.length > 0) console.log(`Tags: ${view.tags.join(", ")}`); + const r = view.ratings; + if (r.totalRatings > 0) { + console.log( + `Rating: ${r.averageRating.toFixed(1)}/5 (${plural(r.totalRatings, "rating")}, ${plural(r.corrections, "correction")})`, + ); + } + for (const l of view.links.outgoing) { + console.log(`Link: → [${l.linkType}] ${l.targetTitle} (${l.targetId}) link ${l.id}`); + } + for (const l of view.links.incoming) { + console.log(`Link: ← [${l.linkType}] ${l.sourceTitle} (${l.sourceId}) link ${l.id}`); + } + console.log("\n---\n"); + console.log(view.content); + if (view.nextOffset !== null) { + console.log( + `\n... ${view.contentLength - view.nextOffset} more characters (--offset ${view.nextOffset})`, + ); + } +} + +interface UpdateFlags { + title?: string; + content?: string; + contentFile?: string; + library?: string; + libVersion?: string; + url?: string; + topic?: string; + tags?: string; +} + +export function updateInput(documentId: string, flags: UpdateFlags): Record<string, unknown> { + const content = + flags.contentFile === undefined ? flags.content : readFileSync(flags.contentFile, "utf-8"); + return defined({ + documentId, + title: flags.title, + content, + library: flags.library, + version: flags.libVersion, + url: flags.url, + topic: flags.topic, + tags: flags.tags === undefined ? undefined : splitList(flags.tags), + }); +} + +function registerViewCommands(docs: Command): void { + addDocumentOptions(docs.command("list").description("List documents, newest first")) + .option("--offset <n>", "Documents to skip (paging)", toNumber) + .action(async (flags: DocumentFlags & { offset?: number }) => { + await run( + listDocumentsOperation, + { ...documentInput(flags), ...defined({ offset: flags.offset }) }, + printDocumentList, + ); + }); + + docs + .command("show <documentId>") + .description("Show a document with its tags, links and ratings") + .option("--offset <n>", "Start the content at this character", toNumber) + .option("--max-length <n>", "Show at most this many characters of content", toNumber) + .action(async (documentId: string, flags: { offset?: number; maxLength?: number }) => { + await run( + getDocumentOperation, + defined({ documentId, offset: flags.offset, maxLength: flags.maxLength }), + printDocument, + ); + }); + + docs + .command("history <documentId>") + .description("List saved versions of a document") + .action(async (documentId: string) => { + await run(documentHistoryOperation, { documentId }, (r) => + printList(r.items, "No saved versions.", (v) => + console.log(`v${v.version} ${v.title} (${v.createdAt})`), + ), + ); + }); +} + +function registerEditCommands(docs: Command): void { + docs + .command("update <documentId>") + .description("Change a document's title, content, metadata or tags") + .option("--title <title>", "New title") + .option("--content <text>", "New content (re-chunked and re-embedded)") + .option("--content-file <path>", "Read the new content from a file") + .option("--library <name>", "New library name") + .option("--lib-version <version>", "New library version") + .option("--url <url>", "New URL") + .option("--topic <topic>", "New topic (ID or name)") + .option("--tags <tags>", "Replace the tags with these (comma-separated)") + .action(async (documentId: string, flags: UpdateFlags) => { + await run( + updateDocumentOperation, + updateInput(documentId, flags), + (view) => console.log(`✓ Updated "${view.document.title}" (${documentId})`), + { embeddings: true }, + ); + }); + + docs + .command("delete <documentId>") + .description("Delete a document with its chunks, links and ratings") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (documentId: string, flags: { yes?: boolean }) => { + const view = await call(getDocumentOperation, { documentId, maxLength: 0 }); + const title = view.document.title; + const question = `Delete "${title}" (${documentId})? This cannot be undone.`; + if (!(await confirmOrCancel(question, flags.yes))) return; + await run(deleteDocumentOperation, { documentId }, () => + console.log(`✓ Deleted "${title}" (${documentId})`), + ); + }); + + docs + .command("rollback <documentId> <version>") + .description("Restore a saved version (the current state is saved first)") + .action(async (documentId: string, version: string) => { + await run( + rollbackDocumentOperation, + { documentId, version: Number(version) }, + (v) => console.log(`✓ Restored version ${version}; the previous state is v${v.version}`), + { embeddings: true }, + ); + }); + + docs + .command("rate <documentId> <rating>") + .description("Rate a document from 1 (poor) to 5 (excellent)") + .option("--feedback <text>", "What is good or wrong") + .option("--correction <text>", "Suggested replacement text") + .option("--chunk <chunkId>", "Rate one chunk of the document") + .action( + async ( + documentId: string, + rating: string, + flags: { feedback?: string; correction?: string; chunk?: string }, + ) => { + const input = defined({ + documentId, + rating: Number(rating), + feedback: flags.feedback, + suggestedCorrection: flags.correction, + chunkId: flags.chunk, + }); + await run(rateDocumentOperation, input, (r) => + console.log(`✓ Rated ${documentId} ${r.rating}/5 rating ${r.id}`), + ); + }, + ); +} + +function registerLinkCommands(docs: Command): void { + docs + .command("link <documentId> <targetDocumentId>") + .description("Link one document to another") + .requiredOption("--type <type>", `Link type: ${LINK_TYPES.join(", ")}`) + .option("--label <text>", "Short description of the relationship") + .action( + async ( + documentId: string, + targetDocumentId: string, + flags: { type: string; label?: string }, + ) => { + const input = defined({ + documentId, + targetDocumentId, + linkType: flags.type, + label: flags.label, + }); + await run(linkDocumentsOperation, input, (l) => + console.log( + `✓ Linked ${documentId} -[${l.linkType}]-> ${targetDocumentId} link ${l.linkId}`, + ), + ); + }, + ); + + docs + .command("unlink <linkId>") + .description("Delete a link (link IDs are shown by `docs links`)") + .action(async (linkId: string) => { + await run(unlinkDocumentsOperation, { linkId }, () => + console.log(`✓ Deleted link ${linkId}`), + ); + }); + + docs + .command("links [documentId]") + .description("List the links of a document, or every link") + .option("--type <type>", `Only this link type: ${LINK_TYPES.join(", ")}`) + .action(async (documentId: string | undefined, flags: { type?: string }) => { + await run(listLinksOperation, defined({ documentId, linkType: flags.type }), (r) => + printList(r.items, "No links.", (l) => { + const dir = "direction" in l && l.direction === "incoming" ? "←" : "→"; + const label = l.label ? ` — ${l.label}` : ""; + console.log(`${l.linkId} ${l.sourceId} ${dir} [${l.linkType}] ${l.targetId}${label}`); + }), + ); + }); + + docs + .command("prereqs <documentId>") + .description("List the documents to read first (following prerequisite links)") + .action(async (documentId: string) => { + await run(prerequisitesOperation, { documentId }, (r) => { + let step = 0; + printList(r.items, "No prerequisites.", (d) => + console.log(`${++step}. ${d.title} ${d.documentId}`), + ); + }); + }); +} + +function registerTagCommands(docs: Command): void { + docs + .command("tag <documentId> <tags...>") + .description("Add tags to a document (space- or comma-separated)") + .action(async (documentId: string, tags: string[]) => { + await run(addTagsOperation, { documentId, tags: tagList(tags) }, (r) => + console.log(`✓ ${documentId} tags: ${r.tags.join(", ")}`), + ); + }); + + docs + .command("untag <documentId> <tags...>") + .description("Remove tags from a document") + .action(async (documentId: string, tags: string[]) => { + await run(removeTagsOperation, { documentId, tags: tagList(tags) }, (r) => + console.log( + `✓ Removed ${plural(r.removed.length, "tag")}; ${documentId} tags: ${r.tags.join(", ") || "(none)"}`, + ), + ); + }); + + docs + .command("suggest-tags <documentId>") + .description("Suggest tags from a document's content") + .option("-n, --limit <n>", "Maximum suggestions", toNumber) + .action(async (documentId: string, flags: { limit?: number }) => { + await run(suggestTagsOperation, defined({ documentId, limit: flags.limit }), (r) => + printList(r.suggestions, "No suggestions.", (t) => console.log(` ${t}`)), + ); + }); +} + +export function register(program: Command): void { + const docs = program.command("docs").description("Show and change documents"); + registerViewCommands(docs); + registerEditCommands(docs); + registerLinkCommands(docs); + registerTagCommands(docs); +} diff --git a/src/cli/commands/doctor.ts b/src/cli/commands/doctor.ts new file mode 100644 index 00000000..9f4e936f --- /dev/null +++ b/src/cli/commands/doctor.ts @@ -0,0 +1,193 @@ +/** + * `libscope doctor [--fix]`: config files, workspace and database, embedding model and index, + * LLM resolution, and health checks. With --fix it creates the database and vector index. + */ +import type { Command } from "commander"; +import { existsSync } from "node:fs"; +import { + getSecretsPath, + getUserConfigPath, + loadConfig, + resolveLlmProviderName, + type LibScopeConfig, +} from "../../config.js"; +import { overviewOperation } from "../../core/operations/index.js"; +import type { Overview } from "../../core/overview.js"; +import { getActiveWorkspace } from "../../core/workspace.js"; +import { resolveDatabasePath } from "../../db/connection.js"; +import { + describeEmbeddingIdentity, + toEmbeddingIdentity, + type EmbeddingIndexIdentity, +} from "../../db/index-meta.js"; +import { createEmbeddingProvider } from "../../providers/index.js"; +import { getGlobalOptions } from "../context.js"; +import { call, print } from "../run.js"; +import { printOverview } from "./admin.js"; + +export interface DoctorCheck { + name: string; + status: "ok" | "warn" | "error"; + detail: string; + /** What to run to fix a warning or error. */ + fix?: string | undefined; +} + +export interface DoctorReport { + configFile: { path: string; exists: boolean }; + secretsFile: { path: string; exists: boolean }; + workspace: string; + database: { path: string; exists: boolean }; + embedding: EmbeddingIndexIdentity; + llm: string; + overview?: Overview | undefined; + checks: DoctorCheck[]; +} + +const message = (err: unknown): string => (err instanceof Error ? err.message : String(err)); + +/** True when a field is known on both sides and differs. */ +function conflicts(a: EmbeddingIndexIdentity, b: EmbeddingIndexIdentity): boolean { + return (["provider", "model", "dimensions"] as const).some( + (k) => a[k] !== undefined && b[k] !== undefined && a[k] !== b[k], + ); +} + +function embeddingCheck(config: LibScopeConfig): { + identity: EmbeddingIndexIdentity; + check: DoctorCheck; +} { + try { + const identity = toEmbeddingIdentity(createEmbeddingProvider(config)); + return { + identity, + check: { name: "embedding", status: "ok", detail: describeEmbeddingIdentity(identity) }, + }; + } catch (err) { + return { + identity: { provider: config.embedding.provider }, + check: { + name: "embedding", + status: "error", + detail: message(err), + fix: "libscope config show", + }, + }; + } +} + +function llmCheck(config: LibScopeConfig): { llm: string; check: DoctorCheck } { + const name = resolveLlmProviderName(config, { surface: "cli" }); + if (name === null) { + return { + llm: "none", + check: { + name: "llm", + status: "warn", + detail: "No LLM configured: `ask` will not work (search does)", + fix: "libscope config set openai.apiKey <key> (or anthropic.apiKey, or llm.provider ollama)", + }, + }; + } + const model = name === "passthrough" ? "" : ` (${config.llm?.model ?? "default model"})`; + return { llm: name, check: { name: "llm", status: "ok", detail: `${name}${model}` } }; +} + +function indexChecks(overview: Overview, configured: EmbeddingIndexIdentity): DoctorCheck[] { + const checks: DoctorCheck[] = [ + { name: "database", status: overview.health.database, detail: "opens and is migrated" }, + { name: "keyword index", status: overview.health.fts, detail: "full-text index readable" }, + ]; + const stored: EmbeddingIndexIdentity = { + ...overview.index.stored, + dimensions: overview.index.vectorTableDimensions ?? overview.index.stored.dimensions, + }; + if (overview.index.vectorTableDimensions === undefined) { + checks.push({ + name: "vector index", + status: "warn", + detail: "not created yet (it is created when content is first added)", + fix: "libscope doctor --fix", + }); + } else if (conflicts(stored, configured)) { + checks.push({ + name: "vector index", + status: "error", + detail: `built with ${describeEmbeddingIdentity(stored)}, but ${describeEmbeddingIdentity(configured)} is configured`, + fix: "libscope admin reindex --rebuild", + }); + } else { + checks.push({ name: "vector index", status: "ok", detail: describeEmbeddingIdentity(stored) }); + } + return checks; +} + +export async function runDoctor(fix: boolean): Promise<DoctorReport> { + const config = loadConfig(); + const workspace = getGlobalOptions().workspace ?? getActiveWorkspace(); + const dbPath = resolveDatabasePath({ explicitPath: config.database.path, workspace }); + const embedding = embeddingCheck(config); + const llm = llmCheck(config); + const checks: DoctorCheck[] = [embedding.check, llm.check]; + let overview: Overview | undefined; + + if (existsSync(dbPath) || fix) { + try { + overview = await call(overviewOperation, {}, { vectorTable: fix ? "best-effort" : "skip" }); + checks.push(...indexChecks(overview, embedding.identity)); + } catch (err) { + checks.push({ + name: "database", + status: "error", + detail: message(err), + fix: "libscope admin reindex --rebuild", + }); + } + } else { + checks.push({ + name: "database", + status: "warn", + detail: "does not exist yet (it is created when content is first added)", + fix: "libscope doctor --fix", + }); + } + + return { + configFile: { path: getUserConfigPath(), exists: existsSync(getUserConfigPath()) }, + secretsFile: { path: getSecretsPath(), exists: existsSync(getSecretsPath()) }, + workspace, + database: { path: dbPath, exists: existsSync(dbPath) }, + embedding: embedding.identity, + llm: llm.llm, + overview, + checks, + }; +} + +const MARK = { ok: "✓", warn: "⚠", error: "✗" } as const; + +function printReport(r: DoctorReport): void { + const exists = (e: boolean): string => (e ? "" : " (not created)"); + console.log(`Config: ${r.configFile.path}${exists(r.configFile.exists)}`); + console.log(`Secrets: ${r.secretsFile.path}${exists(r.secretsFile.exists)}`); + console.log(`Workspace: ${r.workspace}`); + console.log(`Database: ${r.database.path}${exists(r.database.exists)}`); + if (r.overview) printOverview(r.overview); + console.log(""); + for (const c of r.checks) { + console.log(`${MARK[c.status]} ${c.name}: ${c.detail}`); + if (c.fix && c.status !== "ok") console.log(` fix: ${c.fix}`); + } +} + +export function register(program: Command): void { + program + .command("doctor") + .description("Check the setup: config, workspace, database, embedding model, index and LLM") + .option("--fix", "Create what is missing (database and vector index)") + .action(async (flags: { fix?: boolean }) => { + const report = await runDoctor(flags.fix === true); + print(report, printReport); + if (report.checks.some((c) => c.status === "error")) process.exitCode = 1; + }); +} diff --git a/src/cli/commands/pack.ts b/src/cli/commands/pack.ts new file mode 100644 index 00000000..eaf0fe66 --- /dev/null +++ b/src/cli/commands/pack.ts @@ -0,0 +1,220 @@ +/** `libscope pack install|remove|list|create`, over the pack operations. */ +import type { Command } from "commander"; +import { createInterface } from "node:readline/promises"; +import { + createPackOperation, + installPackOperation, + listPacksOperation, + removePackOperation, +} from "../../core/operations/index.js"; +import type { InstallResult } from "../../core/packs.js"; +import { loadRegistries } from "../../registry/config.js"; +import { + parsePackSpecifier, + resolvePackFromRegistries, + verifyResolvedPackChecksum, + type ResolvedPack, +} from "../../registry/resolve.js"; +import { ValidationError } from "../../errors.js"; +import { confirmOrCancel } from "../confirm.js"; +import { defined, splitList, toNumber } from "../options.js"; +import { call, plural, print, printList, run } from "../run.js"; + +interface InstallFlags { + registry?: string; + fromRegistry?: string; + packVersion?: string; + yes?: boolean; + batchSize?: number; + resumeFrom?: number; + concurrency?: number; +} + +/** Ask which registry to use when a pack is in several. */ +async function chooseRegistry( + packName: string, + sources: Array<{ registryName: string; version: string; priority: number }>, +): Promise<string> { + console.log(`Pack "${packName}" is in several registries:`); + sources.forEach((s, i) => + console.log(` [${i + 1}] ${s.registryName} (v${s.version}, priority ${s.priority})`), + ); + const rl = createInterface({ input: process.stdin, output: process.stdout }); + try { + const answer = (await rl.question("Select registry [1]: ")).trim() || "1"; + const chosen = sources[Number.parseInt(answer, 10) - 1]; + if (!chosen) throw new ValidationError(`Invalid selection: ${answer}`); + return chosen.registryName; + } finally { + rl.close(); + } +} + +/** + * Find `nameOrPath` in the git registries. Returns the resolved pack (checksum verified), or + * undefined for a local file, when no git registry is configured, or when none has the pack. + */ +async function resolveFromGitRegistries( + nameOrPath: string, + flags: InstallFlags, +): Promise<ResolvedPack | undefined> { + const isFile = nameOrPath.endsWith(".json") || nameOrPath.endsWith(".json.gz"); + if (isFile || loadRegistries().length === 0) return undefined; + + const { name, version: specVersion } = parsePackSpecifier(nameOrPath); + const version = flags.packVersion ?? specVersion; + const resolve = ( + registryName: string | undefined, + ): ReturnType<typeof resolvePackFromRegistries> => + resolvePackFromRegistries(name, { + version, + registryName, + conflictResolution: registryName + ? { strategy: "explicit", registryName } + : flags.yes + ? { strategy: "priority" } + : undefined, + }); + + let result = resolve(flags.fromRegistry); + for (const w of result.warnings) console.error(`⚠ ${w}`); + if (result.conflict && !result.resolved) { + result = resolve(await chooseRegistry(name, result.conflict.sources)); + } + if (!result.resolved) return undefined; + await verifyResolvedPackChecksum(result.resolved); + return result.resolved; +} + +function printInstall(result: InstallResult): void { + if (result.alreadyInstalled) { + console.log(`Pack "${result.packName}" is already installed.`); + return; + } + const errors = result.errors > 0 ? ` (${plural(result.errors, "error")})` : ""; + console.log( + `✓ Installed pack "${result.packName}": ${plural(result.documentsInstalled, "document")}${errors}`, + ); +} + +interface CreateFlags { + name: string; + from?: string[]; + topic?: string; + packVersion?: string; + description?: string; + author?: string; + license?: string; + output?: string; + extensions?: string; + exclude?: string[]; + recursive: boolean; +} + +export function createPackInput(flags: CreateFlags): Record<string, unknown> { + const fromSources = flags.from !== undefined && flags.from.length > 0; + return defined({ + name: flags.name, + from: fromSources ? flags.from : undefined, + topic: flags.topic, + version: flags.packVersion, + description: flags.description, + author: flags.author, + license: flags.license, + // Packs built from files can be large: gzip them by default. + outputPath: flags.output ?? `${flags.name}.json${fromSources ? ".gz" : ""}`, + extensions: flags.extensions === undefined ? undefined : splitList(flags.extensions), + exclude: flags.exclude, + recursive: flags.recursive, + }); +} + +export function register(program: Command): void { + const pack = program + .command("pack") + .description("Install, remove, list and create knowledge packs"); + + pack + .command("install <nameOrPath>") + .description( + "Install a pack from a git registry, the pack registry URL, or a local .json/.json.gz file (name@version works)", + ) + .option("--registry <url>", "Pack registry URL (URL registries)") + .option("--from-registry <name>", "Use this git registry") + .option("--pack-version <semver>", "Version to install (git registries)") + .option("-y, --yes", "Do not prompt: with several registries, use the highest priority") + .option("--batch-size <n>", "Documents embedded per batch (default 10)", toNumber) + .option("--resume-from <n>", "Skip the first N documents (resume a partial install)", toNumber) + .option("--concurrency <n>", "Batches embedded in parallel (default 4)", toNumber) + .action(async (nameOrPath: string, flags: InstallFlags) => { + const resolved = await resolveFromGitRegistries(nameOrPath, flags); + const input = defined({ + pack: resolved?.dataPath ?? nameOrPath, + registryUrl: resolved ? undefined : flags.registry, + batchSize: flags.batchSize, + resumeFrom: flags.resumeFrom, + concurrency: flags.concurrency, + }); + await run(installPackOperation, input, printInstall, { embeddings: true }); + }); + + pack + .command("remove <name>") + .description("Remove an installed pack and its documents") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (name: string, flags: { yes?: boolean }) => { + const question = `Remove pack "${name}" and its documents? This cannot be undone.`; + if (!(await confirmOrCancel(question, flags.yes))) return; + await run(removePackOperation, { pack: name }, () => console.log(`✓ Removed pack "${name}"`)); + }); + + pack + .command("list") + .description("List installed packs, or packs in the registry with --available") + .option("--available", "List packs available in the registry") + .option("--registry <url>", "Pack registry URL") + .action(async (flags: { available?: boolean; registry?: string }) => { + const input = defined({ available: flags.available, registryUrl: flags.registry }); + await run(listPacksOperation, input, (r) => + printList<{ name: string; version: string; description: string | null; docCount: number }>( + r.items, + flags.available ? "No packs available." : "No packs installed.", + (p) => + console.log( + `${p.name} v${p.version} — ${p.description ?? ""} (${plural(p.docCount, "document")})`, + ), + ), + ); + }); + + pack + .command("create") + .description("Create a pack from indexed documents, or from files, folders and URLs (--from)") + .requiredOption("--name <name>", "Pack name") + .option("--from <sources...>", "Files, folders or URLs to build the pack from") + .option("--topic <topic>", "Only this topic's documents (without --from)") + .option("--pack-version <version>", "Pack version (default 1.0.0)") + .option("--description <text>", "Pack description") + .option("--author <name>", "Pack author") + .option("--license <license>", "Pack license") + .option("--output <path>", "Output file (default <name>.json, or <name>.json.gz with --from)") + .option("--extensions <exts>", "With --from: file extensions to include (comma-separated)") + .option("--exclude <globs...>", "With --from: globs to skip") + .option("--no-recursive", "With --from: do not walk subdirectories") + .action(async (flags: CreateFlags) => { + const input = createPackInput(flags); + const created = await call(createPackOperation, input); + // The pack holds every document; print a summary instead of the whole pack. + const summary = { + name: created.name, + version: created.version, + documents: created.documents.length, + outputPath: input["outputPath"], + }; + print(summary, (s) => + console.log( + `✓ Created pack "${s.name}" with ${plural(s.documents, "document")} → ${String(s.outputPath)}`, + ), + ); + }); +} diff --git a/src/cli/commands/registry.ts b/src/cli/commands/registry.ts index 6cd2ab83..f49ddfb4 100644 --- a/src/cli/commands/registry.ts +++ b/src/cli/commands/registry.ts @@ -157,7 +157,7 @@ async function initialSync(name: string, url: string): Promise<void> { } /** Register all `registry` subcommands on the given Commander program. */ -export function registerRegistryCommands(program: Command): void { +export function register(program: Command): void { const registryCmd = program .command("registry") .description("Manage pack registries (git-backed)"); diff --git a/src/cli/commands/search.ts b/src/cli/commands/search.ts new file mode 100644 index 00000000..fcb8a794 --- /dev/null +++ b/src/cli/commands/search.ts @@ -0,0 +1,136 @@ +/** `libscope search` (including interactive mode) and `libscope ask`. */ +import type { Command } from "commander"; +import type { ListResult } from "../../core/operations/index.js"; +import { askOperation, searchOperation } from "../../core/operations/index.js"; +import type { AnswerResult } from "../../core/rag.js"; +import type { SearchResult } from "../../core/search.js"; +import { ValidationError } from "../../errors.js"; +import { startInteractiveSearch } from "../repl.js"; +import { + addDocumentOptions, + defined, + documentInput, + toNumber, + type DocumentFlags, +} from "../options.js"; +import { isJsonOutput, preview, run } from "../run.js"; + +/** One search hit: title, score, IDs, metadata and a content preview. */ +export function printSearchItem(r: SearchResult): void { + console.log(`\n── ${r.title} (score ${r.score.toFixed(2)})`); + console.log(` document ${r.documentId} chunk ${r.chunkId}`); + const meta = [r.library ? `${r.library}${r.version ? ` ${r.version}` : ""}` : "", r.url ?? ""]; + const metaLine = meta.filter(Boolean).join(" | "); + if (metaLine) console.log(` ${metaLine}`); + for (const c of r.contextBefore ?? []) console.log(` ↑ ${preview(c.content, 120)}`); + console.log(` ${preview(r.content)}`); + for (const c of r.contextAfter ?? []) console.log(` ↓ ${preview(c.content, 120)}`); +} + +export function printSearchResults(result: ListResult<SearchResult>): void { + if (result.items.length === 0) { + console.log("No results."); + return; + } + const first = result.offset + 1; + const last = result.offset + result.items.length; + console.log(`Results ${first}-${last} of ${result.total}:`); + result.items.forEach(printSearchItem); +} + +function printAnswer(result: AnswerResult): void { + if (result.mode === "context") { + console.log("llm.provider is passthrough: no answer is generated. Context for the question:\n"); + console.log(result.contextPrompt); + } else { + console.log(`\n${result.answer}\n`); + } + if (result.sources.length > 0) { + console.log("Sources:"); + for (const s of result.sources) { + console.log(` • ${s.title} (score ${s.score.toFixed(2)}) ${s.documentId}`); + } + } + if (result.mode === "answer") { + const tokens = result.tokensUsed === undefined ? "" : ` | tokens: ${result.tokensUsed}`; + console.log(`\nModel: ${result.model}${tokens}`); + } +} + +interface SearchFlags extends DocumentFlags { + related?: string; + offset?: number; + minRating?: number; + maxPerDoc?: number; + context?: number; +} + +export function searchInput( + query: string | undefined, + flags: SearchFlags, +): Record<string, unknown> { + return defined({ + query, + relatedTo: flags.related, + ...documentInput(flags), + offset: flags.offset, + minRating: flags.minRating, + maxChunksPerDocument: flags.maxPerDoc, + contextChunks: flags.context, + }); +} + +interface AskFlags extends DocumentFlags { + minRating?: number; + model?: string; +} + +export function askInput(question: string, flags: AskFlags): Record<string, unknown> { + const { limit, ...filters } = documentInput(flags); + return defined({ question, ...filters, minRating: flags.minRating, topK: limit }); +} + +export function register(program: Command): void { + const search = program + .command("search [query]") + .description( + "Search by meaning and keywords. With no query on a terminal, start interactive search", + ) + .option("--related <id>", "Find content similar to this document or chunk ID instead"); + addDocumentOptions(search) + .option("--offset <n>", "Results to skip (paging)", toNumber) + .option("--min-rating <n>", "Only documents rated at least this (1-5)", toNumber) + .option("--max-per-doc <n>", "At most this many chunks per document", toNumber) + .option("--context <n>", "Neighbouring chunks to show around each result (0-2)", toNumber) + .action(async (query: string | undefined, flags: SearchFlags) => { + if (query === undefined && flags.related === undefined) { + if (!process.stdin.isTTY || isJsonOutput()) { + throw new ValidationError("Give a search query, or --related <documentId|chunkId>"); + } + await startInteractiveSearch({ + onQuery: async (q) => { + await run(searchOperation, searchInput(q, flags), printSearchResults, { + embeddings: true, + }); + }, + }); + return; + } + await run(searchOperation, searchInput(query, flags), printSearchResults, { + embeddings: true, + }); + }); + + const ask = program + .command("ask <question>") + .description("Answer a question from the knowledge base with the configured LLM"); + addDocumentOptions(ask, { limit: "Chunks to use as context (default 5)" }) + .option("--min-rating <n>", "Only documents rated at least this (1-5)", toNumber) + .option("--model <model>", "LLM model for this question (default: llm.model)"); + ask.action(async (question: string, flags: AskFlags) => { + await run(askOperation, askInput(question, flags), printAnswer, { + embeddings: true, + configOverrides: flags.model === undefined ? undefined : { llm: { model: flags.model } }, + }); + }); +} diff --git a/src/cli/commands/searches.ts b/src/cli/commands/searches.ts new file mode 100644 index 00000000..dd3b4d1b --- /dev/null +++ b/src/cli/commands/searches.ts @@ -0,0 +1,77 @@ +/** `libscope searches ...`: saved searches. */ +import type { Command } from "commander"; +import { + deleteSavedSearchOperation, + listSavedSearchesOperation, + runSavedSearchOperation, + saveSearchOperation, +} from "../../core/operations/index.js"; +import { + addDocumentOptions, + defined, + documentInput, + toNumber, + type DocumentFlags, +} from "../options.js"; +import { printList, run } from "../run.js"; +import { printSearchItem } from "./search.js"; + +export function register(program: Command): void { + const searches = program.command("searches").description("Save and re-run searches"); + + addDocumentOptions( + searches.command("save <name> <query>").description("Save a query and its filters"), + ) + .option("--min-rating <n>", "Only documents rated at least this (1-5)", toNumber) + .action(async (name: string, query: string, flags: DocumentFlags & { minRating?: number }) => { + const input = { + name, + query, + ...documentInput(flags), + ...defined({ minRating: flags.minRating }), + }; + await run(saveSearchOperation, input, (s) => + console.log(`✓ Saved search "${s.name}" ${s.id}`), + ); + }); + + searches + .command("list") + .description("List saved searches") + .option("-n, --limit <n>", "Maximum results", toNumber) + .option("--offset <n>", "Results to skip (paging)", toNumber) + .action(async (flags: { limit?: number; offset?: number }) => { + await run(listSavedSearchesOperation, defined({ ...flags }), (r) => + printList(r.items, "No saved searches.", (s) => { + const filters = s.filters ? ` filters: ${JSON.stringify(s.filters)}` : ""; + console.log(`${s.name} ${s.id}`); + console.log(` query: "${s.query}"${filters}`); + if (s.lastRunAt) console.log(` last run ${s.lastRunAt} (${s.resultCount} results)`); + }), + ); + }); + + searches + .command("run <search>") + .description("Run a saved search (name or ID)") + .action(async (search: string) => { + await run( + runSavedSearchOperation, + { search }, + (r) => { + console.log(`Saved search "${r.search.name}": ${r.search.query}`); + printList(r.items, "No results.", printSearchItem); + }, + { embeddings: true }, + ); + }); + + searches + .command("delete <search>") + .description("Delete a saved search (name or ID)") + .action(async (search: string) => { + await run(deleteSavedSearchOperation, { search }, () => + console.log(`✓ Deleted saved search "${search}"`), + ); + }); +} diff --git a/src/cli/commands/serve.ts b/src/cli/commands/serve.ts new file mode 100644 index 00000000..e7dba3bb --- /dev/null +++ b/src/cli/commands/serve.ts @@ -0,0 +1,85 @@ +/** `libscope serve [mcp|api|dashboard]`. */ +import type { Command } from "commander"; +import type { Server } from "node:http"; +import type Database from "better-sqlite3"; +import type { LibScopeConfig } from "../../config.js"; +import type { EmbeddingProvider } from "../../providers/embedding.js"; +import { ValidationError } from "../../errors.js"; +import { + getContext, + getGlobalOptions, + leaveInterruptsToCommand, + untilInterrupted, +} from "../context.js"; +import { toNumber } from "../options.js"; + +const MODES = ["mcp", "api", "dashboard"] as const; + +interface App { + db: Database.Database; + provider: EmbeddingProvider; + config: LibScopeConfig; +} + +// The server modules' entry points, typed here so this file compiles both before and after +// the stage-2 MCP/REST changes are merged. TODO(merge): import them directly and drop these. +interface McpModule { + runStdioServer(options: { logLevel?: string; workspace?: string }): Promise<void>; +} +interface ApiModule { + startApiServer( + app: App, + options: { port?: number | undefined; host?: string }, + ): Promise<{ close(): Promise<void>; port: number }>; +} +interface WebModule { + startWebServer(app: App, options: { port?: number | undefined; host?: string }): Promise<Server>; +} + +async function serveMcp(): Promise<void> { + const { workspace, logLevel, verbose } = getGlobalOptions(); + // stdout carries the MCP protocol: print nothing here. The server handles Ctrl+C itself. + leaveInterruptsToCommand(); + const mcp = (await import("../../mcp/server.js")) as unknown as McpModule; + await mcp.runStdioServer({ + ...(workspace ? { workspace } : {}), + ...(verbose ? { logLevel: "debug" } : logLevel ? { logLevel } : {}), + }); +} + +export function register(program: Command): void { + program + .command("serve [mode]") + .description( + "Start the MCP server on stdio (mcp, default), the REST API (api, port 3378) or the web dashboard (dashboard, port 3377)", + ) + .option("--port <n>", "Port (api and dashboard)", toNumber) + .option("--host <host>", "Host to listen on (default localhost)") + .action(async (mode: string | undefined, flags: { port?: number; host?: string }) => { + const selected = mode ?? "mcp"; + if (!(MODES as readonly string[]).includes(selected)) { + throw new ValidationError(`Unknown serve mode "${selected}": use ${MODES.join(", ")}`); + } + if (selected === "mcp") { + await serveMcp(); + return; + } + const ctx = getContext({ embeddings: true }); + const app: App = { db: ctx.db, provider: ctx.provider, config: ctx.config }; + const host = flags.host ?? "localhost"; + if (selected === "api") { + const api = (await import("../../api/server.js")) as unknown as ApiModule; + const server = await api.startApiServer(app, { port: flags.port, host }); + console.error(`LibScope API listening on http://${host}:${server.port} (Ctrl+C to stop)`); + await untilInterrupted(); + await server.close(); + } else { + const web = (await import("../../web/server.js")) as unknown as WebModule; + const port = flags.port ?? 3377; + const server = await web.startWebServer(app, { port, host }); + console.error(`LibScope dashboard at http://${host}:${port} (Ctrl+C to stop)`); + await untilInterrupted(); + server.close(); + } + }); +} diff --git a/src/cli/commands/topics.ts b/src/cli/commands/topics.ts new file mode 100644 index 00000000..44b490c2 --- /dev/null +++ b/src/cli/commands/topics.ts @@ -0,0 +1,71 @@ +/** `libscope topics ...` and `libscope tags list`. */ +import type { Command } from "commander"; +import { + createTopicOperation, + deleteTopicOperation, + listTagsOperation, + listTopicsOperation, +} from "../../core/operations/index.js"; +import { confirmOrCancel } from "../confirm.js"; +import { defined } from "../options.js"; +import { plural, printList, run } from "../run.js"; + +export function register(program: Command): void { + const topics = program.command("topics").description("Manage topics"); + + topics + .command("list") + .description("List topics with their document counts") + .option("--parent <topic>", "Only direct subtopics of this topic (ID or name)") + .action(async (flags: { parent?: string }) => { + await run(listTopicsOperation, defined({ parent: flags.parent }), (r) => + printList(r.items, "No topics. Create one with: libscope topics create <name>", (t) => + console.log(`${t.id} ${t.name} (${plural(t.documentCount, "document")})`), + ), + ); + }); + + topics + .command("create <name>") + .description("Create a topic") + .option("--description <text>", "What the topic covers") + .option("--parent <topic>", "Parent topic (ID or name)") + .action(async (name: string, flags: { description?: string; parent?: string }) => { + const input = defined({ name, description: flags.description, parent: flags.parent }); + await run(createTopicOperation, input, (t) => + console.log(`✓ Created topic ${t.name} ${t.id}`), + ); + }); + + topics + .command("delete <topic>") + .description("Delete a topic (its documents are kept without a topic)") + .option("--delete-documents", "Also delete the topic's documents") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (topic: string, flags: { deleteDocuments?: boolean; yes?: boolean }) => { + const what = flags.deleteDocuments + ? `topic "${topic}" and its documents` + : `topic "${topic}"`; + if (!(await confirmOrCancel(`Delete ${what}?`, flags.yes))) return; + await run( + deleteTopicOperation, + { topic, deleteDocuments: flags.deleteDocuments === true }, + (r) => console.log(`✓ Deleted topic ${r.topicId}`), + ); + }); + + program + .command("tags") + .description("List tags") + .command("list") + .description("List all tags with their document counts") + .action(async () => { + await run(listTagsOperation, {}, (r) => + printList( + r.items, + "No tags. Add some with: libscope docs tag <documentId> <tags...>", + (t) => console.log(`${t.name} (${plural(t.documentCount, "document")})`), + ), + ); + }); +} diff --git a/src/cli/commands/webhooks.ts b/src/cli/commands/webhooks.ts new file mode 100644 index 00000000..b499c4b4 --- /dev/null +++ b/src/cli/commands/webhooks.ts @@ -0,0 +1,59 @@ +/** `libscope webhooks list|create|delete|test`. */ +import type { Command } from "commander"; +import { + createWebhookOperation, + deleteWebhookOperation, + listWebhooksOperation, + testWebhookOperation, +} from "../../core/operations/index.js"; +import { WEBHOOK_EVENTS } from "../../core/webhooks.js"; +import { defined, splitList } from "../options.js"; +import { printList, run } from "../run.js"; + +export function register(program: Command): void { + const webhooks = program.command("webhooks").description("Send signed POSTs on document events"); + + webhooks + .command("list") + .description("List webhooks (secrets are never shown)") + .action(async () => { + await run(listWebhooksOperation, {}, (r) => + printList(r.items, "No webhooks.", (h) => { + console.log(`${h.id} ${h.url}`); + console.log(` events: ${h.events.join(", ")} | active: ${h.active ? "yes" : "no"}`); + if (h.lastTriggeredAt) console.log(` last triggered: ${h.lastTriggeredAt}`); + if (h.failureCount > 0) console.log(` failures: ${h.failureCount}`); + }), + ); + }); + + webhooks + .command("create <url>") + .description("Register a webhook") + .requiredOption("--events <events>", `Comma-separated: ${WEBHOOK_EVENTS.join(", ")}`) + .option("--secret <secret>", "HMAC-SHA256 signing secret (needs LIBSCOPE_SECRET_KEY)") + .action(async (url: string, flags: { events: string; secret?: string }) => { + const input = defined({ url, events: splitList(flags.events), secret: flags.secret }); + await run(createWebhookOperation, input, (h) => + console.log(`✓ Created webhook ${h.id} for ${h.events.join(", ")}`), + ); + }); + + webhooks + .command("delete <webhookId>") + .description("Delete a webhook") + .action(async (webhookId: string) => { + await run(deleteWebhookOperation, { webhookId }, () => + console.log(`✓ Deleted webhook ${webhookId}`), + ); + }); + + webhooks + .command("test <webhookId>") + .description("Send a test event and show the HTTP status") + .action(async (webhookId: string) => { + await run(testWebhookOperation, { webhookId }, (r) => + console.log(`Response: ${r.status} ${r.statusText}`), + ); + }); +} diff --git a/src/cli/commands/workspace.ts b/src/cli/commands/workspace.ts new file mode 100644 index 00000000..4d3b582d --- /dev/null +++ b/src/cli/commands/workspace.ts @@ -0,0 +1,55 @@ +/** `libscope workspace create|list|use|delete`. */ +import type { Command } from "commander"; +import { + createWorkspace, + deleteWorkspace, + getActiveWorkspace, + listWorkspaces, + setActiveWorkspace, +} from "../../core/workspace.js"; +import { confirmOrCancel } from "../confirm.js"; +import { print, printList } from "../run.js"; + +export function register(program: Command): void { + const workspace = program.command("workspace").description("Separate knowledge bases"); + + workspace + .command("create <name>") + .description("Create a workspace") + .action((name: string) => { + const ws = createWorkspace(name); + console.log(`✓ Created workspace "${ws.name}" at ${ws.path}`); + }); + + workspace + .command("list") + .description("List workspaces") + .action(() => { + const active = getActiveWorkspace(); + const items = listWorkspaces().map((ws) => ({ ...ws, active: ws.name === active })); + print({ items }, (r) => + printList(r.items, "No workspaces.", (ws) => + console.log(`${ws.active ? "* " : " "}${ws.name} ${ws.path}`), + ), + ); + }); + + workspace + .command("use <name>") + .description("Make a workspace the active one") + .action((name: string) => { + setActiveWorkspace(name); + console.log(`✓ Active workspace: ${name}`); + }); + + workspace + .command("delete <name>") + .description("Delete a workspace and its database") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (name: string, flags: { yes?: boolean }) => { + const question = `Delete workspace "${name}" and its database? This cannot be undone.`; + if (!(await confirmOrCancel(question, flags.yes))) return; + deleteWorkspace(name); + console.log(`✓ Deleted workspace "${name}"`); + }); +} diff --git a/src/cli/context.ts b/src/cli/context.ts new file mode 100644 index 00000000..c0cf1a37 --- /dev/null +++ b/src/cli/context.ts @@ -0,0 +1,139 @@ +/** + * Process-wide CLI state: the global options and the one operation context a command uses. + * The context is built once, on first use, with `bootstrap()`; commands that do not embed + * text open it without requiring the embedding provider, so they never load a model. + */ +import { bootstrap, type Bootstrapped, type ConfigOverrides } from "../core/bootstrap.js"; +import { createOperationContext, type OperationContext } from "../core/operations/index.js"; +import { LocalEmbeddingProvider, type ModelDownloadEvent } from "../providers/local.js"; +import type { EmbeddingProvider } from "../providers/embedding.js"; + +export interface GlobalOptions { + json?: boolean | undefined; + verbose?: boolean | undefined; + logLevel?: string | undefined; + workspace?: string | undefined; +} + +let globals: GlobalOptions = {}; + +export function setGlobalOptions(options: GlobalOptions): void { + globals = options; +} + +export function getGlobalOptions(): GlobalOptions { + return globals; +} + +export interface ContextOptions { + /** The command embeds text: the embedding provider must be usable. Default false. */ + embeddings?: boolean | undefined; + /** Vector table handling (default: "required" with embeddings, else "skip"). */ + vectorTable?: "required" | "best-effort" | "skip" | undefined; + /** Per-section config values merged over the loaded config. */ + configOverrides?: ConfigOverrides | undefined; +} + +interface OpenContext { + boot: Bootstrapped; + ctx: OperationContext; +} + +let open: OpenContext | undefined; +let controller = new AbortController(); + +/** Print one stderr line while the local model downloads (first run only). */ +function reportModelDownload(provider: EmbeddingProvider): void { + if (!(provider instanceof LocalEmbeddingProvider)) return; + const text = `Downloading embedding model ${provider.model} (first run only)...`; + const tty = process.stderr.isTTY; + let started = false; + provider.onDownloadProgress = (event: ModelDownloadEvent): void => { + if (!started) { + started = true; + process.stderr.write(tty ? text : `${text}\n`); + } + if (!tty) return; + if (event.status === "progress" && event.progress !== undefined) { + process.stderr.write(`\r${text} ${Math.round(event.progress)}% ${event.file ?? ""}`); + } else if (event.status === "ready") { + process.stderr.write(`\r${text} done\x1b[K\n`); + } + }; +} + +/** The operation context for this command, opened on first call. */ +export function getContext(options: ContextOptions = {}): OperationContext { + if (open) return open.ctx; + const embeddings = options.embeddings ?? false; + const boot = bootstrap({ + workspace: globals.workspace, + configOverrides: options.configOverrides, + requireProvider: embeddings, + vectorTable: options.vectorTable ?? (embeddings ? "required" : "skip"), + warn: (message) => process.stderr.write(`⚠ ${message}\n`), + }); + reportModelDownload(boot.provider); + const ctx = createOperationContext({ + db: boot.db, + provider: boot.provider, + config: boot.config, + surface: "cli", + signal: controller.signal, + }); + open = { boot, ctx }; + return ctx; +} + +/** Database file of the open context, if any. */ +export function getOpenDbPath(): string | undefined { + return open?.boot.dbPath; +} + +/** Close the database (safe to call when nothing is open). */ +export function closeContext(): void { + open?.boot.close(); + open = undefined; + controller = new AbortController(); +} + +/** Ask a running operation to stop (Ctrl+C). */ +export function abortContext(): void { + controller.abort(); +} + +/** True after abortContext() for the current command. */ +export function isAborted(): boolean { + return controller.signal.aborted; +} + +let waiters = 0; + +/** Resolves when the user presses Ctrl+C (or the process gets SIGTERM). For long-running commands. */ +export function untilInterrupted(): Promise<void> { + if (controller.signal.aborted) return Promise.resolve(); + const signal = controller.signal; + waiters++; + return new Promise((resolve) => + signal.addEventListener("abort", () => { + waiters--; + resolve(); + }), + ); +} + +/** True while a command waits in untilInterrupted() (it cleans up itself on Ctrl+C). */ +export function isWaitingForInterrupt(): boolean { + return waiters > 0; +} + +let commandHandlesInterrupts = false; + +/** The running command (the MCP server) handles Ctrl+C itself; the CLI must not exit. */ +export function leaveInterruptsToCommand(): void { + commandHandlesInterrupts = true; +} + +export function isInterruptLeftToCommand(): boolean { + return commandHandlesInterrupts; +} diff --git a/src/cli/errors.ts b/src/cli/errors.ts new file mode 100644 index 00000000..79b0a788 --- /dev/null +++ b/src/cli/errors.ts @@ -0,0 +1,40 @@ +/** The one CLI error handler: "✗ message", a next-step hint, and the stack only with --verbose. */ +import { ConfigError, NotFoundError, ValidationError } from "../errors.js"; + +const NOT_FOUND_HINTS: Record<string, string> = { + DOCUMENT_NOT_FOUND: "List document IDs with `libscope docs list`.", + CHUNK_NOT_FOUND: "Chunk IDs are shown in `libscope search` results.", + TOPIC_NOT_FOUND: "List topics with `libscope topics list`.", + CONNECTION_NOT_FOUND: "List saved connections with `libscope connections`.", + SAVED_SEARCH_NOT_FOUND: "List saved searches with `libscope searches list`.", + WEBHOOK_NOT_FOUND: "List webhooks with `libscope webhooks list`.", + LINK_NOT_FOUND: "Link IDs are shown by `libscope docs links <documentId>`.", + PACK_NOT_FOUND: "List installed packs with `libscope pack list`.", +}; + +/** Next step to suggest for an error, if any. */ +export function errorHint(err: unknown): string | undefined { + if (err instanceof NotFoundError) return NOT_FOUND_HINTS[err.code] ?? "Check the name or ID."; + if (err instanceof ConfigError) return "Run `libscope doctor` to check your setup."; + if (err instanceof ValidationError) return "Add --help to the command to see its arguments."; + return undefined; +} + +/** Lines to print for an error. */ +export function formatError(err: unknown, verbose: boolean): string[] { + const message = err instanceof Error ? err.message : String(err); + const lines = [`✗ ${message}`]; + const hint = errorHint(err); + if (hint) lines.push(` ${hint}`); + if (verbose && err instanceof Error && err.stack) { + lines.push("", err.stack); + if (err.cause instanceof Error && err.cause.stack) lines.push("Caused by:", err.cause.stack); + } + return lines; +} + +/** Print an error to stderr and set exit code 1. */ +export function reportError(err: unknown, verbose: boolean): void { + process.stderr.write(`${formatError(err, verbose).join("\n")}\n`); + process.exitCode = 1; +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 26a37f14..74bc2b96 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,475 +1,49 @@ #!/usr/bin/env node - +/** + * libscope CLI: program setup, global options, the error handler, and one register() call + * per command group (src/cli/commands/<group>.ts). Commands are thin adapters over the + * operations in src/core/operations. + */ import { Command } from "commander"; -import { - loadConfig, - maskConfigSecrets, - getUserConfigPath, - getSecretsPath, - getConfigFileFor, - maskSecret, - CONFIG_KEY_NAMES, - setUserConfigValue, - unsetUserConfigValue, - getConfigValue, -} from "../config.js"; -import { getDatabase, runMigrations, createVectorTable, closeDatabase } from "../db/index.js"; -import { resolveDatabasePath } from "../db/connection.js"; -import { createEmbeddingProvider, type EmbeddingProvider } from "../providers/index.js"; -import { indexDocument, indexFile } from "../core/indexing.js"; -import { getSupportedExtensions } from "../core/parsers/index.js"; -import { searchDocuments, getRelatedChunks } from "../core/search.js"; -import { askQuestion, createLlmProvider } from "../core/rag.js"; -import { getDocumentRatings, listRatings } from "../core/ratings.js"; -import { createTopic, listTopics } from "../core/topics.js"; -import { getDocument, listDocuments, deleteDocument, updateDocument } from "../core/documents.js"; -import { createLink, getDocumentLinks, deleteLink, getPrerequisiteChain } from "../core/links.js"; -import { LINK_TYPES, type LinkType } from "../core/links.js"; -import { getVersionHistory, rollbackToVersion } from "../core/versioning.js"; -import { initLogger, type LogLevel } from "../logger.js"; -import { readdirSync, realpathSync, statSync } from "node:fs"; -import { join, extname, basename, resolve as resolvePath } from "node:path"; -import { globToRegExp, toPosixPath } from "../utils/glob.js"; -import { fileURLToPath } from "node:url"; -import { fetchAndConvert } from "../core/url-fetcher.js"; -import { spiderUrl } from "../core/spider.js"; -import type { SpiderOptions } from "../core/spider.js"; -import { exportKnowledgeBase, importFromBackup } from "../core/export.js"; -import { batchImport } from "../core/batch.js"; -import { findDuplicates } from "../core/dedup.js"; -import { - getStats, - getPopularDocuments, - getStaleDocuments, - getTopQueries, - getSearchTrends, - getSearchAnalytics, - getKnowledgeGaps, -} from "../core/analytics.js"; -import { startRepl } from "./repl.js"; -import { confirmOrCancel } from "./confirm.js"; -import { createReporter, isVerbose } from "./reporter.js"; -import { - addTagsToDocument, - removeTagsFromDocument, - listTags, - getDocumentTags, - suggestTags, -} from "../core/tags.js"; -import { bulkDelete, bulkRetag, bulkMove } from "../core/bulk.js"; -import type { BulkResult, BulkSelector } from "../core/bulk.js"; -import { - createWorkspace, - deleteWorkspace, - listWorkspaces, - getActiveWorkspace, - setActiveWorkspace, -} from "../core/workspace.js"; -import { - installPack, - removePack, - listInstalledPacks, - listAvailablePacks, - createPack, - createPackFromSource, -} from "../core/packs.js"; - -import { spawnSync } from "node:child_process"; +import { realpathSync } from "node:fs"; import { createRequire } from "node:module"; -import { FileWatcher, DEFAULT_WATCH_EXTENSIONS, readWatchedFile } from "../core/watcher.js"; -import { indexRepository, parseRepoUrl } from "../core/repo.js"; -import { - authenticateDeviceCode, - syncOneNote, - disconnectOneNote, - type OneNoteConfig, -} from "../connectors/onenote.js"; -import { - saveNamedConnectorConfig, - loadNamedConnectorConfig, - hasNamedConnectorConfig, - deleteNamedConnectorConfig, -} from "../connectors/index.js"; -import type { ConnectorSyncOptions } from "../connectors/sync-tracker.js"; -import { - findSavedConnectorConfig, - findSavedOneNoteConfig, - runAndSaveConnector, - saveConnectorSettings, - resolveConnectorType, - type ConnectorType, -} from "../connectors/saved-config.js"; -import { syncNotion, disconnectNotion } from "../connectors/notion.js"; -import type { NotionConfig } from "../connectors/notion.js"; -import { syncSlack, disconnectSlack, type SlackConfig } from "../connectors/slack.js"; -import { syncConfluence, type ConfluenceConfig } from "../connectors/confluence.js"; -import { syncObsidianVault, type ObsidianConfig } from "../connectors/obsidian.js"; -import { - createSavedSearch, - listSavedSearches, - runSavedSearch, - deleteSavedSearch, -} from "../core/saved-searches.js"; -import { - createWebhook, - listWebhooks, - deleteWebhook, - getWebhook, - deliverWebhook, - TEST_PING_DATA, -} from "../core/webhooks.js"; -import type { WebhookEvent } from "../core/webhooks.js"; -import { registerRegistryCommands } from "./commands/registry.js"; +import { fileURLToPath } from "node:url"; +import { initLogger, type LogLevel } from "../logger.js"; import { - parsePackSpecifier, - resolvePackFromRegistries, - verifyResolvedPackChecksum, - type ResolvedPack, -} from "../registry/resolve.js"; -import { loadRegistries } from "../registry/config.js"; -import { createInterface } from "node:readline"; - -// Graceful shutdown -const handleShutdown = (): void => { - closeDatabase(); - process.exit(0); -}; -process.on("SIGINT", handleShutdown); -process.on("SIGTERM", handleShutdown); - -const _require = createRequire(import.meta.url); -const _pkg = _require("../../package.json") as { version: string }; - -/** Parse a CLI option string as an integer, exiting with an error if the value is not a valid number. */ -function parseIntOption(value: string, name: string): number { - const n = Number.parseInt(value, 10); - if (Number.isNaN(n)) { - console.error(`Error: "${value}" is not a valid number for ${name}`); - process.exit(1); - } - return n; -} - -// --------------------------------------------------------------------------- -// Extracted helpers to reduce cognitive complexity (SonarCloud S3776) -// --------------------------------------------------------------------------- - -/** Truncate a string and add "..." suffix if it exceeds maxLen. */ -function previewText(text: string, maxLen: number): string { - return text.length > maxLen ? text.slice(0, maxLen) + "..." : text; -} - -/** Print context-before chunks for a search result. */ -function printContextBefore(contextBefore: Array<{ content: string }> | undefined): void { - if (!contextBefore || contextBefore.length === 0) return; - for (const c of contextBefore) { - console.log(` \u2191 ${previewText(c.content, 120)}`); - } - console.log(" \u2500 \u2500 \u2500"); -} - -/** Print context-after chunks for a search result. */ -function printContextAfter(contextAfter: Array<{ content: string }> | undefined): void { - if (!contextAfter || contextAfter.length === 0) return; - console.log(" \u2500 \u2500 \u2500"); - for (const c of contextAfter) { - console.log(` \u2193 ${previewText(c.content, 120)}`); - } -} - -/** Print a single search result with optional context chunks. */ -function printSearchResult(r: { - title: string; - score: number; - library?: string | null; - url?: string | null; - content: string; - contextBefore?: Array<{ content: string }> | undefined; - contextAfter?: Array<{ content: string }> | undefined; -}): void { - console.log(`\n\u2500\u2500 ${r.title} (score: ${r.score.toFixed(2)}) \u2500\u2500`); - if (r.library) console.log(` Library: ${r.library}`); - if (r.url) console.log(` Source: ${r.url}`); - printContextBefore(r.contextBefore); - console.log(` ${previewText(r.content, 200)}`); - printContextAfter(r.contextAfter); -} - -/** Print a single related-chunk result. */ -function printRelatedChunk(r: { - title: string; - score: number; - chunkId: string; - library?: string | null; - url?: string | null; - content: string; -}): void { - console.log(`\n\u2500\u2500 ${r.title} (score: ${r.score.toFixed(2)}) \u2500\u2500`); - console.log(` Chunk ID: ${r.chunkId}`); - if (r.library) console.log(` Library: ${r.library}`); - if (r.url) console.log(` Source: ${r.url}`); - console.log(` ${previewText(r.content, 200)}`); -} - -/** Build search filters from CLI options. */ -function buildSearchFilters(opts: { - topic?: string; - library?: string; - source?: string; -}): Record<string, unknown> | undefined { - const filters: Record<string, unknown> = {}; - if (opts.topic) filters.topic = opts.topic; - if (opts.library) filters.library = opts.library; - if (opts.source) filters.source = opts.source; - return Object.keys(filters).length > 0 ? filters : undefined; -} - -/** Report the result of a pack install operation. */ -function reportInstallResult( - result: { - packName: string; - documentsInstalled: number; - alreadyInstalled: boolean; - errors: number; - }, - reporter: { log(msg: string): void; success(msg: string): void }, - registryName?: string, -): void { - if (result.alreadyInstalled) { - reporter.log(`Pack "${result.packName}" is already installed.`); - return; - } - const errMsg = result.errors > 0 ? ` (${result.errors} errors)` : ""; - const from = registryName ? ` from ${registryName}` : ""; - reporter.success( - `Pack "${result.packName}" installed${from}: ${result.documentsInstalled} documents${errMsg}.`, - ); -} - -/** Install a resolved registry pack -- verify checksum, install, and report. */ -async function installResolvedPack( - db: ReturnType<typeof getDatabase>, - provider: EmbeddingProvider, - resolved: ResolvedPack, - installOpts: { - batchSize?: number | undefined; - resumeFrom?: number | undefined; - concurrency?: number | undefined; - }, - reporter: { - log(msg: string): void; - success(msg: string): void; - progress(c: number, t: number, l: string): void; - clearProgress(): void; - }, -): Promise<void> { - await verifyResolvedPackChecksum(resolved); - const result = await installPack(db, provider, resolved.dataPath, { - ...installOpts, - onProgress: (current, total, docTitle) => { - reporter.progress(current, total, docTitle); - }, - }); - reporter.clearProgress(); - reportInstallResult(result, reporter, resolved.registryName); -} - -/** - * Prompt user to choose a registry when there is a conflict. - * Returns the chosen source index or -1 on invalid input. - */ -async function promptRegistryChoice( - packName: string, - sources: Array<{ registryName: string; version: string; priority: number }>, -): Promise<number> { - console.log(`Pack "${packName}" found in multiple registries:`); - for (let i = 0; i < sources.length; i++) { - const s = sources[i]!; - console.log(` [${i + 1}] ${s.registryName} (v${s.version}, priority: ${s.priority})`); - } - - const rl = createInterface({ input: process.stdin, output: process.stdout }); - const answer = await new Promise<string>((resolve) => { - rl.question("Select registry [1]: ", (ans) => { - rl.close(); - resolve(ans.trim() || "1"); - }); - }); - - const choice = Number.parseInt(answer, 10) - 1; - if (Number.isNaN(choice) || choice < 0 || choice >= sources.length) return -1; - return choice; -} - -/** - * Handle the conflict case where a pack exists in multiple registries. - * Returns true if the install was handled (or process exited). - */ -async function handlePackConflict( - conflict: { - sources: Array<{ registryName: string; version: string; priority: number }>; - }, - packName: string, - version: string | undefined, - opts: { yes?: boolean }, - db: ReturnType<typeof getDatabase>, - provider: EmbeddingProvider, - installOpts: { - batchSize?: number | undefined; - resumeFrom?: number | undefined; - concurrency?: number | undefined; - }, - reporter: { - log(msg: string): void; - success(msg: string): void; - progress(c: number, t: number, l: string): void; - clearProgress(): void; - }, -): Promise<boolean> { - if (opts.yes) { - const names = conflict.sources.map((s) => s.registryName).join(", "); - reporter.log( - `Error: Pack "${packName}" found in multiple registries: ${names}. ` + - "Use --from-registry <name> to disambiguate.", - ); - closeDatabase(); - process.exit(1); - return true; - } - - const choice = await promptRegistryChoice(packName, conflict.sources); - if (choice < 0) { - reporter.log("Invalid selection. Cancelled."); - closeDatabase(); - process.exit(1); - return true; - } - - const chosen = conflict.sources[choice]!; - const retryResult = resolvePackFromRegistries(packName, { - version, - registryName: chosen.registryName, - conflictResolution: { strategy: "explicit", registryName: chosen.registryName }, - }); - - if (retryResult.resolved) { - await installResolvedPack(db, provider, retryResult.resolved, installOpts, reporter); - return true; - } - - return false; -} - -/** Try to resolve a pack from git registries; returns true if installed. */ -async function tryGitRegistryInstall( - nameOrPath: string, - opts: { fromRegistry?: string; packVersion?: string; yes?: boolean }, - db: ReturnType<typeof getDatabase>, - provider: EmbeddingProvider, - installOpts: { - batchSize: number | undefined; - resumeFrom: number | undefined; - concurrency: number | undefined; - }, - reporter: ReturnType<typeof createReporter>, -): Promise<boolean> { - const isLocalFile = nameOrPath.endsWith(".json") || nameOrPath.endsWith(".json.gz"); - if (isLocalFile || loadRegistries().length === 0) return false; - - const { name: packName, version: specVersion } = parsePackSpecifier(nameOrPath); - const version = opts.packVersion ?? specVersion; - - const { resolved, conflict, warnings } = resolvePackFromRegistries(packName, { - version, - registryName: opts.fromRegistry, - conflictResolution: opts.fromRegistry - ? { strategy: "explicit", registryName: opts.fromRegistry } - : opts.yes - ? { strategy: "priority" } - : undefined, - }); - - for (const w of warnings) { - reporter.log(`Warning: ${w}`); - } - - if (conflict && !resolved) { - const handled = await handlePackConflict( - conflict, - packName, - version, - opts, - db, - provider, - installOpts, - reporter, - ); - if (handled) return true; - } - - if (resolved) { - await installResolvedPack(db, provider, resolved, installOpts, reporter); - return true; - } - - return false; -} - -/** Execute a pack install from git registry, URL, or local file. */ -async function executePackInstall( - nameOrPath: string, - opts: { - registry?: string; - fromRegistry?: string; - packVersion?: string; - yes?: boolean; - batchSize?: string; - resumeFrom?: string; - concurrency?: string; - }, -): Promise<void> { - const { db, provider } = initializeAppWithEmbedding(); - const globalOpts = program.opts<ProgramOpts>(); - const reporter = createReporter(globalOpts.verbose); - - const batchSize = opts.batchSize ? parseIntOption(opts.batchSize, "--batch-size") : undefined; - const resumeFrom = opts.resumeFrom ? parseIntOption(opts.resumeFrom, "--resume-from") : undefined; - const concurrency = opts.concurrency - ? parseIntOption(opts.concurrency, "--concurrency") - : undefined; - - if (concurrency !== undefined && concurrency < 1) { - reporter.log('Error: "--concurrency" must be an integer greater than or equal to 1.'); - closeDatabase(); - process.exit(1); - return; - } - - const installOpts = { batchSize, resumeFrom, concurrency }; - - try { - const installed = await tryGitRegistryInstall( - nameOrPath, - opts, - db, - provider, - installOpts, - reporter, - ); - if (installed) return; - - const result = await installPack(db, provider, nameOrPath, { - registryUrl: opts.registry, - ...installOpts, - onProgress: (current, total, docTitle) => { - reporter.progress(current, total, docTitle); - }, - }); - - reporter.clearProgress(); - reportInstallResult(result, reporter); - } finally { - closeDatabase(); + abortContext, + closeContext, + getGlobalOptions, + isAborted, + isInterruptLeftToCommand, + isWaitingForInterrupt, + setGlobalOptions, + type GlobalOptions, +} from "./context.js"; +import { reportError } from "./errors.js"; +import { register as registerAdd } from "./commands/add.js"; +import { register as registerAdmin } from "./commands/admin.js"; +import { register as registerBulk } from "./commands/bulk.js"; +import { register as registerConfig } from "./commands/config.js"; +import { register as registerConnectors } from "./commands/connectors.js"; +import { register as registerDoctor } from "./commands/doctor.js"; +import { register as registerDocs } from "./commands/docs.js"; +import { register as registerPack } from "./commands/pack.js"; +import { register as registerRegistry } from "./commands/registry.js"; +import { register as registerSearch } from "./commands/search.js"; +import { register as registerSearches } from "./commands/searches.js"; +import { register as registerServe } from "./commands/serve.js"; +import { register as registerTopics } from "./commands/topics.js"; +import { register as registerWebhooks } from "./commands/webhooks.js"; +import { register as registerWorkspace } from "./commands/workspace.js"; + +const pkg = createRequire(import.meta.url)("../../package.json") as { version: string }; + +function setupLogging(options: GlobalOptions): void { + if (options.verbose === true || process.env["LIBSCOPE_VERBOSE"] === "1") { + initLogger("debug"); + } else { + // The CLI prints its own output; structured logs only with --verbose or --log-level. + initLogger((options.logLevel ?? "silent") as LogLevel); } } @@ -478,2700 +52,59 @@ const program = new Command(); program .name("libscope") .description("AI-powered knowledge base with MCP integration") - .version(_pkg.version) - .option("-v, --verbose", "Enable verbose logging") - .option("--log-level <level>", "Set log level (debug, info, warn, error, silent)") - .option("--workspace <name>", "Use a specific workspace"); - -// init -program - .command("init") - .description("Initialize the LibScope database") - .action(() => { - const { config, db } = initializeApp(); - try { - // Only create vector table if provider can be initialized without download - try { - const provider = createEmbeddingProvider(config); - createVectorTable(db, provider); - } catch { - console.log(" ℹ Vector table skipped (embedding provider not available)"); - } - console.log(`✓ Database initialized at ${db.name}`); - } finally { - closeDatabase(); - } - }); - -function resolveSourceType(library?: string, topic?: string): "library" | "topic" | "manual" { - if (library) return "library"; - if (topic) return "topic"; - return "manual"; -} - -function buildSpiderOptions( - opts: { - maxPages?: number; - maxDepth?: number; - sameDomain?: boolean; - pathPrefix?: string; - excludePatterns?: string[]; - }, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): SpiderOptions { - const options: SpiderOptions = { fetchOptions }; - if (opts.maxPages !== undefined) options.maxPages = opts.maxPages; - if (opts.maxDepth !== undefined) options.maxDepth = opts.maxDepth; - if (opts.sameDomain !== undefined) options.sameDomain = opts.sameDomain; - if (opts.pathPrefix !== undefined) options.pathPrefix = opts.pathPrefix; - if (opts.excludePatterns !== undefined) options.excludePatterns = opts.excludePatterns; - return options; -} - -async function runSpiderAdd( - db: ReturnType<typeof getDatabase>, - provider: EmbeddingProvider, - url: string, - opts: { - topic?: string; - library?: string; - libVersion?: string; - dedup?: string; - maxPages?: number; - maxDepth?: number; - sameDomain?: boolean; - pathPrefix?: string; - excludePatterns?: string[]; - }, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): Promise<void> { - const spiderOptions = buildSpiderOptions(opts, fetchOptions); - const sourceType = resolveSourceType(opts.library, opts.topic); - console.log(`Spidering ${url}...`); - let indexed = 0; - let failed = 0; - - const gen = spiderUrl(url, spiderOptions); - let next = await gen.next(); - while (!next.done) { - const page = next.value; - try { - await indexDocument(db, provider, { - title: page.title, - content: page.content, - sourceType, - library: opts.library, - version: opts.libVersion, - topicId: opts.topic, - url: page.url, - dedup: opts.dedup as "skip" | "warn" | "force" | undefined, - }); - indexed++; - console.log(` ✓ [${indexed}] ${page.title} (${page.url})`); - } catch (err) { - failed++; - const msg = err instanceof Error ? err.message : String(err); - console.error(` ✗ ${page.url}: ${msg}`); - } - next = await gen.next(); - } - - const stats = next.value; - console.log( - `\nDone. Indexed: ${indexed}, Failed: ${failed}, Skipped: ${stats.pagesSkipped}` + - (stats.abortReason ? ` (stopped: ${stats.abortReason})` : ""), - ); -} - -// add -program - .command("add <fileOrUrl>") - .description( - "Index a document from a file or URL (supports: " + getSupportedExtensions().join(", ") + ")", - ) - .option("--topic <topicId>", "Assign to a topic") - .option("--library <name>", "Mark as library documentation") - .option("--lib-version <version>", "Library version") - .option("--title <title>", "Override document title") - .option("--format <ext>", "Force file format (e.g. .pdf, .csv, .yaml)") - .option("--dedup <mode>", "Dedup mode: skip, warn, or force") - .option("--spider", "Crawl linked pages from the URL") - .option("--max-pages <n>", "Max pages to crawl (default: 25, hard cap: 200)", Number.parseInt) - .option( - "--max-depth <n>", - "Max link-hop depth from seed URL (default: 2, hard cap: 5)", - Number.parseInt, - ) - .option("--no-same-domain", "Follow links outside the seed domain") - .option("--path-prefix <path>", "Only follow links under this path prefix") - .option("--exclude-patterns <globs...>", "Glob patterns for URLs to skip") - .action( - async ( - fileOrUrl: string, - opts: { - topic?: string; - library?: string; - libVersion?: string; - title?: string; - format?: string; - dedup?: string; - spider?: boolean; - maxPages?: number; - maxDepth?: number; - sameDomain?: boolean; - pathPrefix?: string; - excludePatterns?: string[]; - }, - ) => { - const { config, db, provider } = initializeAppWithEmbedding(); - const fetchOptions = { - allowPrivateUrls: config.indexing.allowPrivateUrls, - allowSelfSignedCerts: config.indexing.allowSelfSignedCerts, - }; - try { - if (fileOrUrl.startsWith("http://") || fileOrUrl.startsWith("https://")) { - if (opts.spider) { - await runSpiderAdd(db, provider, fileOrUrl, opts, fetchOptions); - } else { - console.log(`Fetching ${fileOrUrl}...`); - const fetched = await fetchAndConvert(fileOrUrl, fetchOptions); - const title = opts.title ?? fetched.title; - const result = await indexDocument(db, provider, { - title, - content: fetched.content, - sourceType: resolveSourceType(opts.library, opts.topic), - library: opts.library, - version: opts.libVersion, - topicId: opts.topic, - url: fileOrUrl, - dedup: opts.dedup as "skip" | "warn" | "force" | undefined, - }); - console.log(`✓ Indexed "${title}" (${result.chunkCount} chunks)`); - console.log(` ID: ${result.id}`); - } - } else { - const result = await indexFile(db, provider, fileOrUrl, { - title: opts.title, - topic: opts.topic, - library: opts.library, - version: opts.libVersion, - format: opts.format, - dedup: opts.dedup as "skip" | "warn" | "force" | undefined, - }); - const title = opts.title ?? basename(fileOrUrl).replace(/\.[^.]+$/, ""); - console.log(`✓ Indexed "${title}" (${result.chunkCount} chunks)`); - console.log(` ID: ${result.id}`); - } - } finally { - closeDatabase(); - } - }, - ); - -// import -program - .command("import <directory>") - .description("Bulk import files from a directory (supports all document formats)") - .option("--topic <topicId>", "Assign all to a topic") - .option("--library <name>", "Mark all as library documentation") - .option("--lib-version <version>", "Library version") - .option( - "--extensions <exts>", - "Comma-separated file extensions to include", - getSupportedExtensions().join(","), - ) - .option("--dedup <mode>", "Dedup mode: skip, warn, or force") - .action( - async ( - directory: string, - opts: { - topic?: string; - library?: string; - libVersion?: string; - extensions: string; - dedup?: string; - }, - ) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const extensions = new Set(opts.extensions.split(",").map((e) => e.trim().toLowerCase())); - const files = findFiles(directory, extensions); - - if (files.length === 0) { - console.log(`No matching files found in ${directory}`); - return; - } - - console.log(`Found ${files.length} files to import...`); - const startTime = Date.now(); - let indexed = 0; - let failed = 0; - - for (const file of files) { - try { - const result = await indexFile(db, provider, file, { - topic: opts.topic, - library: opts.library, - version: opts.libVersion, - dedup: opts.dedup as "skip" | "warn" | "force" | undefined, - }); - - indexed++; - console.log( - ` [${indexed + failed}/${files.length}] ✓ ${file} (${result.chunkCount} chunks)`, - ); - } catch (err) { - failed++; - console.error( - ` [${indexed + failed}/${files.length}] ✗ ${file}: ${err instanceof Error ? err.message : String(err)}`, - ); - } - } - - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log(`\nDone: ${indexed} indexed, ${failed} failed in ${elapsed}s`); - } finally { - closeDatabase(); - } - }, - ); - -// import-batch -program - .command("import-batch <directory>") - .description("Batch import files with parallel processing") - .option("--concurrency <n>", "Number of parallel imports", "5") - .option( - "--filter <glob>", - "Glob pattern for file selection", - `**/*.{${getSupportedExtensions() - .map((e) => e.slice(1)) - .join(",")}}`, - ) - .option("--dry-run", "Preview files without importing") - .option("--topic <topicId>", "Assign all to a topic") - .option("--library <name>", "Mark all as library documentation") - .option("--lib-version <version>", "Library version") - .action( - async ( - directory: string, - opts: { - concurrency: string; - filter: string; - dryRun?: boolean; - topic?: string; - library?: string; - libVersion?: string; - }, - ) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const matcher = globToRegExp(opts.filter); - const files = readdirSync(directory, { recursive: true, encoding: "utf8" }) - .filter((f) => matcher.test(toPosixPath(f))) - .map((f) => join(directory, f)) - .filter((f) => statSync(f).isFile()); - files.sort(); - - if (files.length === 0) { - console.log(`No files matching "${opts.filter}" found in ${directory}`); - return; - } - - if (opts.dryRun) { - console.log(`Found ${files.length} files (dry run):\n`); - for (const f of files) { - console.log(` ${f}`); - } - return; - } - - console.log(`Importing ${files.length} files (concurrency: ${opts.concurrency})...`); - const startTime = Date.now(); - - const result = await batchImport(db, provider, files, { - concurrency: parseIntOption(opts.concurrency, "--concurrency"), - library: opts.library, - version: opts.libVersion, - topicId: opts.topic, - onProgress: (progress) => { - const done = progress.completed + progress.failed + progress.skipped; - const file = progress.currentFile ?? ""; - console.log(` [${done}/${progress.total}] ${file}`); - }, - }); - - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log( - `\nDone: ${result.completed} indexed, ${result.failed} failed, ${result.skipped} skipped in ${elapsed}s`, - ); - - if (result.skipped > 0) { - console.warn("\nSkipped files (unsupported format):"); - for (const r of result.results) { - if (r.skipped) { - console.warn(` - ${r.file}`); - } - } - } - - if (result.failed > 0) { - console.log("\nFailed files:"); - for (const r of result.results) { - if (!r.success && !r.skipped) { - console.log(` ✗ ${r.file}: ${r.error}`); - } - } - } - } finally { - closeDatabase(); - } - }, - ); - -// search -program - .command("search <query>") - .description("Search indexed documents") - .option("--topic <topicId>", "Filter by topic") - .option("--library <name>", "Filter by library") - .option( - "--source <type>", - "Filter by source type (e.g., 'library', 'topic', 'manual', 'model-generated')", - ) - .option("--limit <n>", "Max results", "5") - .option("--offset <n>", "Offset for pagination", "0") - .option("--max-chunks-per-doc <n>", "Max chunks per document in results (default: no limit)") - .option("--context <n>", "Include N neighboring chunks before/after each result (0-2)", "0") - .option("--save <name>", "Save this search with the given name for later re-use") - .action( - async ( - query: string, - opts: { - topic?: string; - library?: string; - source?: string; - limit: string; - offset: string; - maxChunksPerDoc?: string; - context: string; - save?: string; - }, - ) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const contextChunks = parseIntOption(opts.context, "--context"); - const maxChunksPerDoc = opts.maxChunksPerDoc - ? parseIntOption(opts.maxChunksPerDoc, "--max-chunks-per-doc") - : undefined; - const { results, totalCount } = await searchDocuments(db, provider, { - query, - topic: opts.topic, - library: opts.library, - source: opts.source, - limit: parseIntOption(opts.limit, "--limit"), - offset: parseIntOption(opts.offset, "--offset"), - maxChunksPerDocument: maxChunksPerDoc, - contextChunks: contextChunks > 0 ? contextChunks : undefined, - }); - - if (results.length === 0) { - console.log("No results found."); - } else { - console.log(`\nShowing ${results.length} of ${totalCount} results:\n`); - for (const r of results) { - printSearchResult(r); - } - } - - if (opts.save) { - const filters = buildSearchFilters(opts); - const saved = createSavedSearch(db, opts.save, query, filters); - console.log(`\n✓ Search saved as "${saved.name}" (${saved.id})`); - } - } finally { - closeDatabase(); - } - }, - ); - -// related -program - .command("related <chunkId>") - .description("Find chunks related to a given chunk by vector similarity") - .option("--limit <n>", "Number of results", "10") - .option("--topic <topic>", "Filter by topic") - .option("--library <lib>", "Filter by library") - .option("--min-score <n>", "Minimum similarity score (0-1)") - .option("--tags <tags>", "Comma-separated tags to filter by") - .action( - ( - chunkId: string, - opts: { - limit: string; - topic?: string; - library?: string; - minScore?: string; - tags?: string; - }, - ) => { - const { db } = initializeApp(); - try { - const limit = parseIntOption(opts.limit, "--limit"); - const minScore = opts.minScore !== undefined ? Number.parseFloat(opts.minScore) : undefined; - const tags = opts.tags ? splitCsv(opts.tags) : undefined; - - let result; - try { - result = getRelatedChunks(db, { - chunkId, - ...(limit !== undefined && { limit }), - ...(opts.topic !== undefined && { topic: opts.topic }), - ...(opts.library !== undefined && { library: opts.library }), - ...(tags !== undefined && { tags }), - ...(minScore !== undefined && { minScore }), - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - console.error(`Error: ${message}`); - process.exit(1); - } - - const { sourceChunk, chunks } = result; - console.log( - `\nRelated to chunk: ${sourceChunk.id} from document ${sourceChunk.documentId}`, - ); - - if (chunks.length === 0) { - console.log("No related chunks found."); - } else { - console.log(`\nShowing ${chunks.length} related chunks:\n`); - for (const r of chunks) { - printRelatedChunk(r); - } - } - } finally { - closeDatabase(); - } - }, - ); - -// saved searches -const searchesCmd = program.command("searches").description("Manage saved searches"); - -searchesCmd - .command("list") - .description("List all saved searches") - .action(() => { - const { db } = initializeApp(); - try { - const searches = listSavedSearches(db); - if (searches.length === 0) { - console.log("No saved searches."); - } else { - console.log(`Found ${searches.length} saved searches:\n`); - for (const s of searches) { - console.log(` ${s.name}`); - console.log(` ID: ${s.id}`); - console.log(` Query: "${s.query}"`); - if (s.filters) console.log(` Filters: ${JSON.stringify(s.filters)}`); - if (s.lastRunAt) console.log(` Last run: ${s.lastRunAt} (${s.resultCount} results)`); - console.log(); - } - } - } finally { - closeDatabase(); - } - }); - -searchesCmd - .command("run <nameOrId>") - .description("Run a saved search") - .action(async (nameOrId: string) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const { search, results } = await runSavedSearch(db, provider, nameOrId); - console.log(`\nRunning saved search "${search.name}" (query: "${search.query}"):\n`); - if (results.length === 0) { - console.log("No results found."); - } else { - console.log(`Found ${results.length} results:\n`); - for (const r of results) { - console.log(`── ${r.title} (score: ${r.score.toFixed(2)}) ──`); - if (r.library) console.log(` Library: ${r.library}`); - console.log(` ${r.content.slice(0, 200)}${r.content.length > 200 ? "..." : ""}`); - } - } - } finally { - closeDatabase(); - } - }); - -searchesCmd - .command("delete <nameOrId>") - .description("Delete a saved search") - .action((nameOrId: string) => { - const { db } = initializeApp(); - try { - deleteSavedSearch(db, nameOrId); - console.log(`✓ Saved search "${nameOrId}" deleted.`); - } finally { - closeDatabase(); - } - }); - -// ask (RAG question answering) -program - .command("ask <question>") - .description("Ask a question and get an LLM-synthesized answer using RAG") - .option("--top-k <n>", "Number of chunks to retrieve", "5") - .option("--topic <topic>", "Filter by topic") - .option("--library <lib>", "Filter by library") - .option("--model <model>", "Override LLM model") - .action( - async ( - question: string, - opts: { topK: string; topic?: string; library?: string; model?: string }, - ) => { - const { config, db, provider } = initializeAppWithEmbedding(); - try { - let llmProvider; - try { - if (opts.model) { - config.llm = { ...config.llm, model: opts.model }; - } - llmProvider = createLlmProvider(config); - } catch (err) { - console.error( - `\u2717 ${err instanceof Error ? err.message : String(err)}\n\n` + - `To configure an LLM provider, run:\n` + - ` libscope config set llm.provider openai # or ollama\n` + - ` export LIBSCOPE_LLM_PROVIDER=openai\n`, - ); - process.exit(1); - } - - const result = await askQuestion(db, provider, llmProvider, { - question, - topK: parseIntOption(opts.topK, "--top-k"), - topic: opts.topic, - library: opts.library, - }); - - console.log(`\n${result.answer}\n`); - - if (result.sources.length > 0) { - console.log("\u2500\u2500 Sources \u2500\u2500"); - for (const src of result.sources) { - console.log( - ` \u2022 ${src.title} (score: ${src.score.toFixed(2)}) [${src.documentId}]`, - ); - } - } - - if (result.tokensUsed != null) { - console.log(`\n Model: ${result.model} | Tokens: ${result.tokensUsed}`); - } - } finally { - closeDatabase(); - } - }, - ); - -// topics -const topicsCmd = program.command("topics").description("Manage topics"); - -topicsCmd - .command("list") - .description("List all topics") - .action(() => { - const { db } = initializeApp(); - try { - const topics = listTopics(db); - if (topics.length === 0) { - console.log("No topics found. Create one with: libscope topics create <name>"); - } else { - for (const t of topics) { - const topicDesc = t.description ? ` (${t.description})` : ""; - console.log(` ${t.id} — ${t.name}${topicDesc}`); - } - } - } finally { - closeDatabase(); - } - }); - -topicsCmd - .command("create <name>") - .description("Create a new topic") - .option("--description <desc>", "Topic description") - .option("--parent <parentId>", "Parent topic ID") - .action((name: string, opts: { description?: string; parent?: string }) => { - const { db } = initializeApp(); - try { - const topic = createTopic(db, { - name, - description: opts.description, - parentId: opts.parent, - }); - console.log(`✓ Topic created: ${topic.id} (${topic.name})`); - } finally { - closeDatabase(); - } - }); - -// ratings -const ratingsCmd = program.command("ratings").description("View document ratings"); - -ratingsCmd - .command("show <documentId>") - .description("Show ratings for a document") - .action((documentId: string) => { - const { db } = initializeApp(); - try { - const doc = getDocument(db, documentId); - const summary = getDocumentRatings(db, documentId); - const ratings = listRatings(db, documentId); - - console.log(`\nRatings for: ${doc.title}`); - console.log( - ` Average: ${summary.averageRating.toFixed(1)}/5 (${summary.totalRatings} ratings)`, - ); - console.log(` Corrections suggested: ${summary.corrections}`); - - if (ratings.length > 0) { - console.log("\nRecent ratings:"); - for (const r of ratings.slice(0, 10)) { - console.log(` ${r.rating}/5 by ${r.ratedBy} — ${r.feedback ?? "(no feedback)"}`); - } - } - } finally { - closeDatabase(); - } - }); - -// links (document cross-references) -program - .command("link <sourceId> <targetId>") - .description("Create a cross-reference link between two documents") - .option("--type <type>", `Link type: ${LINK_TYPES.join(", ")}`, "related") - .option("--label <text>", "Human-readable description of the link") - .action((sourceId: string, targetId: string, opts: { type: string; label?: string }) => { - const validTypes: readonly string[] = LINK_TYPES; - if (!validTypes.includes(opts.type)) { - console.error(`Invalid link type: ${opts.type}. Must be one of: ${validTypes.join(", ")}`); - process.exit(1); - } - const { db } = initializeApp(); - try { - const link = createLink(db, sourceId, targetId, opts.type as LinkType, opts.label); - console.log(`✓ Link created: ${link.linkType} (${link.id})`); - } finally { - closeDatabase(); - } - }); - -program - .command("links <documentId>") - .description("Show all cross-reference links for a document") - .action((documentId: string) => { - const { db } = initializeApp(); - try { - const { outgoing, incoming } = getDocumentLinks(db, documentId); - if (outgoing.length === 0 && incoming.length === 0) { - console.log("No links found for this document."); - return; - } - if (outgoing.length > 0) { - console.log("\nOutgoing links:"); - for (const l of outgoing) { - const outLinkLabel = l.label ? ` — ${l.label}` : ""; - console.log(` → [${l.linkType}] ${l.targetTitle}${outLinkLabel}`); - console.log(` ID: ${l.id} Target: ${l.targetId}`); - } - } - if (incoming.length > 0) { - console.log("\nIncoming links:"); - for (const l of incoming) { - const inLinkLabel = l.label ? ` — ${l.label}` : ""; - console.log(` ← [${l.linkType}] ${l.sourceTitle}${inLinkLabel}`); - console.log(` ID: ${l.id} Source: ${l.sourceId}`); - } - } - } finally { - closeDatabase(); - } - }); - -program - .command("unlink <linkId>") - .description("Remove a cross-reference link") - .action((linkId: string) => { - const { db } = initializeApp(); - try { - deleteLink(db, linkId); - console.log(`✓ Link ${linkId} deleted.`); - } finally { - closeDatabase(); - } - }); - -program - .command("prereqs <documentId>") - .description("Show prerequisite reading chain for a document") - .action((documentId: string) => { - const { db } = initializeApp(); - try { - const chain = getPrerequisiteChain(db, documentId); - if (chain.length === 0) { - console.log("No prerequisites found for this document."); - return; - } - console.log("\nPrerequisite reading order:"); - for (let i = 0; i < chain.length; i++) { - const doc = chain[i]!; - console.log(` ${i + 1}. ${doc.title} (${doc.id})`); - } - console.log(` ${chain.length + 1}. [this document]`); - } finally { - closeDatabase(); - } - }); - -// docs -const docsCmd = program.command("docs").description("Manage documents"); - -docsCmd - .command("list") - .description("List indexed documents") - .option("--library <name>", "Filter by library") - .option("--topic <topicId>", "Filter by topic") - .option("--source-type <type>", "Filter by source type") - .option("--limit <n>", "Max results", "50") - .action((opts: { library?: string; topic?: string; sourceType?: string; limit: string }) => { - const { db } = initializeApp(); - try { - const docs = listDocuments(db, { - library: opts.library, - topicId: opts.topic, - sourceType: opts.sourceType, - limit: parseIntOption(opts.limit, "--limit"), - }); - - if (docs.length === 0) { - console.log("No documents found."); - } else { - console.log(`Found ${docs.length} documents:\n`); - for (const d of docs) { - console.log(` ${d.id} ${d.title}`); - if (d.library) { - const listDocVersion = d.version ? ` v${d.version}` : ""; - console.log(` Library: ${d.library}${listDocVersion}`); - } - if (d.url) console.log(` URL: ${d.url}`); - console.log(` Type: ${d.sourceType} | Updated: ${d.updatedAt}`); - console.log(); - } - } - } finally { - closeDatabase(); - } - }); - -docsCmd - .command("show <documentId>") - .description("Show a specific document") - .action((documentId: string) => { - const { db } = initializeApp(); - try { - const doc = getDocument(db, documentId); - console.log(`\n# ${doc.title}\n`); - console.log(`ID: ${doc.id}`); - console.log(`Type: ${doc.sourceType}`); - if (doc.library) { - const showDocVersion = doc.version ? ` v${doc.version}` : ""; - console.log(`Library: ${doc.library}${showDocVersion}`); - } - if (doc.url) console.log(`URL: ${doc.url}`); - console.log(`Submitted by: ${doc.submittedBy}`); - console.log(`Created: ${doc.createdAt}`); - console.log(`Updated: ${doc.updatedAt}`); - const tags = getDocumentTags(db, documentId); - if (tags.length > 0) { - console.log(`Tags: ${tags.map((t) => t.name).join(", ")}`); - } - console.log(`\n---\n`); - console.log(doc.content); - } finally { - closeDatabase(); - } - }); - -docsCmd - .command("delete <documentId>") - .description("Delete a document by ID") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (documentId: string, opts: { yes?: boolean }) => { - const { db } = initializeApp(); - try { - const doc = getDocument(db, documentId); - if ( - !(await confirmOrCancel( - `Delete document "${doc.title}" (${documentId})? This cannot be undone.`, - opts.yes, - )) - ) { - return; - } - deleteDocument(db, documentId); - console.log(`✓ Deleted "${doc.title}" (${documentId})`); - } finally { - closeDatabase(); - } - }); - -docsCmd - .command("update <documentId>") - .description("Update an existing document") - .option("--title <title>", "New title") - .option("--content <content>", "New content (will re-chunk and re-index)") - .option("--library <name>", "New library name") - .option("--lib-version <ver>", "New library version") - .option("--url <url>", "New URL") - .option("--topic <topicId>", "New topic ID") - .action( - async ( - documentId: string, - opts: { - title?: string; - content?: string; - library?: string; - libVersion?: string; - url?: string; - topic?: string; - }, - ) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const metadata: Record<string, string | null | undefined> = {}; - if (opts.library !== undefined) metadata.library = opts.library; - if (opts.libVersion !== undefined) metadata.version = opts.libVersion; - if (opts.url !== undefined) metadata.url = opts.url; - if (opts.topic !== undefined) metadata.topicId = opts.topic; - - const doc = await updateDocument(db, provider, documentId, { - title: opts.title, - content: opts.content, - metadata: - Object.keys(metadata).length > 0 - ? (metadata as { - library?: string | null; - version?: string | null; - url?: string | null; - topicId?: string | null; - }) - : undefined, - }); - console.log(`✓ Updated "${doc.title}" (${doc.id})`); - } finally { - closeDatabase(); - } - }, - ); - -// tag -const tagCmd = program.command("tag").description("Manage document tags"); - -tagCmd - .command("add <docId> <tags...>") - .description("Add tags to a document (comma or space separated)") - .action((docId: string, tags: string[]) => { - const { db } = initializeApp(); - try { - const tagNames = tags - .flatMap((t) => t.split(",")) - .map((t) => t.trim()) - .filter(Boolean); - const created = addTagsToDocument(db, docId, tagNames); - console.log(`✓ Added ${created.length} tag(s) to document ${docId}`); - for (const t of created) { - console.log(` • ${t.name}`); - } - } finally { - closeDatabase(); - } - }); - -tagCmd - .command("remove <docId> <tag>") - .description("Remove a tag from a document") - .action((docId: string, tagName: string) => { - const { db } = initializeApp(); - try { - const [removed] = removeTagsFromDocument(db, docId, [tagName]); - if (!removed) { - console.log(`Tag "${tagName}" not found on document ${docId}`); - return; - } - console.log(`✓ Removed tag "${removed}" from document ${docId}`); - } finally { - closeDatabase(); - } - }); - -tagCmd - .command("list") - .description("List all tags with document counts") - .action(() => { - const { db } = initializeApp(); - try { - const allTags = listTags(db); - if (allTags.length === 0) { - console.log("No tags found. Add tags with: libscope tag add <docId> <tags...>"); - } else { - for (const t of allTags) { - console.log(` ${t.name} (${t.documentCount} doc${t.documentCount === 1 ? "" : "s"})`); - } - } - } finally { - closeDatabase(); - } - }); - -tagCmd - .command("suggest <documentId>") - .description("Suggest tags for a document based on content analysis") - .option("-l, --limit <n>", "Max number of suggestions", "5") - .action((documentId: string, opts: { limit: string }) => { - const { db } = initializeApp(); - try { - const limit = Number.parseInt(opts.limit, 10); - const suggestions = suggestTags(db, documentId, limit); - if (suggestions.length === 0) { - console.log("No tag suggestions found for this document."); - } else { - console.log(`Suggested tags for ${documentId}:`); - for (const tag of suggestions) { - console.log(` • ${tag}`); - } - } - } finally { - closeDatabase(); - } - }); - -docsCmd - .command("history <documentId>") - .description("Show version history of a document") - .action((documentId: string) => { - const { db } = initializeApp(); - try { - const versions = getVersionHistory(db, documentId); - if (versions.length === 0) { - console.log("No version history found."); - } else { - console.log(`Version history for ${documentId}:\n`); - for (const v of versions) { - console.log(` v${v.version} ${v.title} (${v.createdAt})`); - } - } - } finally { - closeDatabase(); - } - }); - -docsCmd - .command("rollback <documentId> <version>") - .description("Rollback a document to a specific version") - .action(async (documentId: string, version: string) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const restored = await rollbackToVersion( - db, - provider, - documentId, - parseIntOption(version, "<version>"), - ); - console.log(`✓ Rolled back to version ${version}, saved as v${restored.version}`); - } finally { - closeDatabase(); - } - }); - -// export -program - .command("export <outputPath>") - .description("Export the knowledge base to a JSON file") - .action((outputPath: string) => { - const { db } = initializeApp(); - try { - const data = exportKnowledgeBase(db, outputPath); - console.log(`✓ Exported ${data.metadata.counts.documents} documents to ${outputPath}`); - } finally { - closeDatabase(); - } - }); - -// import-backup -program - .command("import-backup <backupPath>") - .description("Import knowledge base data from a backup file") - .action((backupPath: string) => { - const { db } = initializeApp(); - try { - const data = importFromBackup(db, backupPath); - console.log(`✓ Imported ${data.metadata.counts.documents} documents from ${backupPath}`); - } finally { - closeDatabase(); - } - }); - -// serve -program - .command("serve") - .description("Start the MCP server, REST API (--api), or web dashboard (--dashboard)") - .option("--api", "Start the REST API server instead of MCP") - .option("--dashboard", "Start the web dashboard UI") - .option("--port <port>", "Server port (default: 3378 for API, 3377 for dashboard)") - .option("--host <host>", "Server host", "localhost") - .action(async (opts: { api?: boolean; dashboard?: boolean; port?: string; host: string }) => { - if (opts.dashboard) { - const { db, provider } = initializeAppWithEmbedding(); - const { startWebServer } = await import("../web/server.js"); - const defaultPort = 3377; - const port = opts.port ? parseIntOption(opts.port, "--port") : defaultPort; - await startWebServer(db, provider, { port, host: opts.host }); - console.log(`LibScope dashboard running at http://${opts.host}:${port}`); - console.log("Press Ctrl+C to stop"); - } else if (opts.api) { - const { db, provider } = initializeAppWithEmbedding(); - const { startApiServer } = await import("../api/server.js"); - const defaultPort = 3378; - const port = opts.port ? parseIntOption(opts.port, "--port") : defaultPort; - const result = await startApiServer(db, provider, { port, host: opts.host }); - console.log(`LibScope API server listening on http://${opts.host}:${result.port}`); - console.log("Press Ctrl+C to stop"); - } else { - await import("../mcp/server.js"); - } - }); - -// config -const configCmd = program - .command("config") - .description(`Manage configuration (user file: ${getUserConfigPath()})`); - -/** Run a config subcommand: set up logging, print errors without a stack trace. */ -function runConfigAction(fn: () => void): void { - setupLogging(program.opts<ProgramOpts>()); + .version(pkg.version) + .option("--json", "Print results as JSON") + .option("-v, --verbose", "Debug logging, and stack traces on errors") + .option("--log-level <level>", "Log level: debug, info, warn, error, silent (default silent)") + .option("--workspace <name>", "Use this workspace instead of the active one") + .hook("preAction", () => { + setGlobalOptions(program.opts<GlobalOptions>()); + setupLogging(getGlobalOptions()); + }); + +registerAdd(program); +registerSearch(program); +registerDocs(program); +registerTopics(program); +registerSearches(program); +registerBulk(program); +registerConnectors(program); +registerPack(program); +registerRegistry(program); +registerServe(program); +registerConfig(program); +registerWorkspace(program); +registerWebhooks(program); +registerAdmin(program); +registerDoctor(program); + +/** Parse `argv` and run the command; errors are printed by the one handler (exit code 1). */ +export async function main(argv: readonly string[] = process.argv): Promise<void> { try { - fn(); + await program.parseAsync(argv); } catch (err) { - console.error(err instanceof Error ? err.message : String(err)); - process.exit(1); - } -} - -configCmd - .command("set <key> <value>") - .description( - `Set a value in the user config file. Keys: ${CONFIG_KEY_NAMES.join(", ")}. ` + - `API keys are written to ${getSecretsPath()} (mode 0600), never to config.json.`, - ) - .action((key: string, value: string) => { - runConfigAction(() => { - const stored = setUserConfigValue(key, value); - const file = getConfigFileFor(key); - const shown = file === getSecretsPath() ? maskSecret(String(stored)) : String(stored); - console.log(`✓ ${key} set to: ${shown} (${file})`); - }); - }); - -configCmd - .command("get <key>") - .description("Print the effective value of a config key (API keys are masked)") - .action((key: string) => { - runConfigAction(() => { - const value = getConfigValue(loadConfig(), key); - console.log(value === undefined ? "" : String(value)); - }); - }); - -configCmd - .command("unset <key>") - .description( - "Remove a key from the user config file, or from the secrets file for API keys (the default or other layers apply)", - ) - .action((key: string) => { - runConfigAction(() => { - const removed = unsetUserConfigValue(key); - console.log(removed ? `✓ ${key} removed` : `${key} is not set in ${getConfigFileFor(key)}`); - }); - }); - -configCmd - .command("path") - .description("Print the path of the user config file") - .action(() => { - console.log(getUserConfigPath()); - }); - -configCmd - .command("show") - .description("Show the effective configuration (API keys are masked)") - .action(() => { - runConfigAction(() => { - console.log(JSON.stringify(maskConfigSecrets(loadConfig()), null, 2)); - }); - }); - -interface ProgramOpts { - verbose?: boolean; - logLevel?: string; - workspace?: string; -} - -function setupLogging(opts: ProgramOpts): void { - if (isVerbose(opts.verbose)) { - initLogger("debug"); - } else if (opts.logLevel) { - initLogger(opts.logLevel as LogLevel); - } else { - // Default to silent in CLI mode — pretty reporter handles user-facing output. - // Set LIBSCOPE_VERBOSE=1 or pass --verbose to see structured JSON logs. - initLogger("silent"); - } -} - -/** Shared CLI initialization: loadConfig → setupLogging → getDatabase → runMigrations. */ -function initializeApp(): { - config: ReturnType<typeof loadConfig>; - db: ReturnType<typeof getDatabase>; -} { - // Configure logging first so warnings emitted while loading config honour the CLI log level. - const opts = program.opts<ProgramOpts>(); - setupLogging(opts); - const config = loadConfig(); - - if (opts.workspace) { - process.env["LIBSCOPE_WORKSPACE"] = opts.workspace; - } - - const dbPath = resolveDatabasePath({ - explicitPath: config.database.path, - warn: (message) => console.error(`⚠ ${message}`), - }); - const db = getDatabase(dbPath); - runMigrations(db); - return { config, db }; -} - -/** Initialization with an embedding provider and vector table. */ -function initializeAppWithEmbedding(): { - config: ReturnType<typeof loadConfig>; - db: ReturnType<typeof getDatabase>; - provider: EmbeddingProvider; -} { - const { config, db } = initializeApp(); - const provider = createEmbeddingProvider(config); - createVectorTable(db, provider); - return { config, db, provider }; -} - -/** Split a comma-separated option value and trim each entry. */ -function splitCsv(value: string): string[] { - return value.split(",").map((t) => t.trim()); -} - -/** Recursively find files matching given extensions. */ -function findFiles(dir: string, extensions: Set<string>): string[] { - const results: string[] = []; - - function walk(currentDir: string): void { - const entries = readdirSync(currentDir); - for (const entry of entries) { - if (entry.startsWith(".")) continue; - const fullPath = join(currentDir, entry); - const stat = statSync(fullPath); - if (stat.isDirectory()) { - walk(fullPath); - } else if (extensions.has(extname(entry).toLowerCase())) { - results.push(fullPath); - } - } - } - - walk(dir); - return results.sort((a, b) => a.localeCompare(b)); -} - -// watch -program - .command("watch <directory>") - .description("Watch a directory for file changes and automatically re-index") - .option( - "--extensions <exts>", - "Comma-separated file extensions to watch", - DEFAULT_WATCH_EXTENSIONS.join(","), - ) - .option("--debounce <ms>", "Debounce interval in milliseconds", "300") - .action(async (directory: string, opts: { extensions: string; debounce: string }) => { - const { db, provider } = initializeAppWithEmbedding(); - const extensions = opts.extensions.split(",").map((e) => e.trim().toLowerCase()); - const debounceMs = parseIntOption(opts.debounce, "--debounce"); - - // Initial scan - const extensionSet = new Set(extensions); - const files = findFiles(directory, extensionSet); - console.log(`Found ${files.length} matching files in ${directory}`); - - let indexed = 0; - let skipped = 0; - for (const file of files) { - try { - const content = await readWatchedFile(file); - if (content === null) { - console.warn(` ⚠ Skipping ${file}: unsupported file format`); - skipped++; - continue; - } - const title = basename(file).replace(/\.[^.]+$/, ""); - const result = await indexDocument(db, provider, { - title, - content, - sourceType: "manual", - url: file, - }); - if (result.chunkCount > 0) { - indexed++; - } else { - skipped++; - } - } catch { - skipped++; - } - } - console.log( - `Initial scan: ${indexed} indexed, ${skipped} skipped (unchanged, unsupported, or failed)`, - ); - console.log(`\nWatching for changes (extensions: ${extensions.join(", ")})...\n`); - - const watcher = new FileWatcher(db, provider, { - directory, - extensions, - debounceMs, - onIndex: (path: string): void => { - console.log(` ✓ Indexed: ${path}`); - }, - onRemove: (path: string): void => { - console.log(` ✗ Removed: ${path}`); - }, - onError: (err: Error): void => { - console.error(` ⚠ Error: ${err.message}`); - }, - }); - - const cleanup = (): void => { - watcher.stop(); - closeDatabase(); - process.exit(0); - }; - - process.on("SIGINT", cleanup); - process.on("SIGTERM", cleanup); - - watcher.start(); - - // Keep the process alive - await new Promise(() => {}); - }); - -// reindex -interface ReindexCliOptions { - doc?: string[]; - since?: string; - before?: string; - batchSize: string; - rebuild?: boolean; -} - -program - .command("reindex") - .description("Re-embed all chunks with the current embedding provider") - .option("--doc <documentId...>", "Only reindex specific document IDs") - .option("--since <date>", "Only reindex documents created on or after this ISO-8601 date") - .option("--before <date>", "Only reindex documents created on or before this ISO-8601 date") - .option("--batch-size <n>", "Chunks per embedding batch", "50") - .option( - "--rebuild", - "Drop and recreate the vector table for the configured embedding model, then re-embed every chunk (use after changing provider, model or dimensions)", - ) - .action(async (opts: ReindexCliOptions) => { - if (opts.rebuild && (opts.doc ?? opts.since ?? opts.before)) { - console.error( - "Error: --rebuild re-embeds every chunk; it cannot be combined with --doc, --since or --before.", - ); - process.exit(1); - } - const { reindex } = await import("../core/reindex.js"); - const { rebuildVectorTable, describeEmbeddingIdentity } = await import("../db/index-meta.js"); - const { config, db } = initializeApp(); - try { - const provider = createEmbeddingProvider(config); - if (opts.rebuild) { - const identity = await rebuildVectorTable(db, provider); - console.log(`✓ Vector table recreated (${describeEmbeddingIdentity(identity)})`); - } else { - createVectorTable(db, provider); - } - console.log("Re-embedding chunks..."); - const startTime = Date.now(); - - const result = await reindex(db, provider, { - documentIds: opts.doc, - since: opts.since, - before: opts.before, - batchSize: parseIntOption(opts.batchSize, "--batch-size"), - onProgress: (progress) => { - const done = progress.completed + progress.failed; - console.log(` [${done}/${progress.total}] chunks processed`); - }, - }); - - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log( - `\n✓ Reindex complete: ${result.completed} updated, ${result.failed} failed (${result.total} total) in ${elapsed}s`, - ); - - if (result.failedChunkIds.length > 0) { - console.log("\nFailed chunks:"); - for (const id of result.failedChunkIds) { - console.log(` ✗ ${id}`); - } - } - } finally { - closeDatabase(); - } - }); - -// repl (interactive search) -program - .command("repl") - .description("Start an interactive search REPL") - .option("--limit <n>", "Max results per search", "5") - .action(async (opts: { limit: string }) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - await startRepl({ db, provider, limit: parseIntOption(opts.limit, "--limit") }); - } finally { - closeDatabase(); - } - }); - -// dedupe — scan and report duplicates -program - .command("dedupe") - .description("Scan the knowledge base for duplicate documents") - .option("--threshold <n>", "Similarity threshold (0-1)", "0.95") - .option("--strategy <type>", "Detection strategy: exact, semantic, or both", "both") - .action(async (opts: { threshold: string; strategy: string }) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - console.log("Scanning for duplicates...\n"); - const groups = await findDuplicates(db, provider, { - threshold: Number.parseFloat(opts.threshold), - strategy: opts.strategy as "exact" | "semantic" | "both", - }); - - if (groups.length === 0) { - console.log("No duplicates found."); - return; - } - - console.log(`Found ${groups.length} duplicate group(s):\n`); - - const groupCol = 7; - const typeCol = 10; - const idCol = 38; - const titleCol = 40; - - const header = - "Group".padEnd(groupCol) + "Type".padEnd(typeCol) + "Document ID".padEnd(idCol) + "Title"; - console.log(header); - console.log("\u2500".repeat(header.length + titleCol)); - - for (let i = 0; i < groups.length; i++) { - const group = groups[i]!; - for (let j = 0; j < group.documentIds.length; j++) { - const gLabel = j === 0 ? String(i + 1) : ""; - const tLabel = j === 0 ? group.matchType : ""; - const docId = group.documentIds[j] ?? ""; - const title = group.titles[j] ?? ""; - console.log( - gLabel.padEnd(groupCol) + - tLabel.padEnd(typeCol) + - docId.padEnd(idCol) + - title.slice(0, titleCol), - ); - } - console.log(); - } - } finally { - closeDatabase(); - } - }); - -// stats -const statsCmd = program.command("stats").description("Usage analytics and content health metrics"); - -statsCmd - .command("overview", { isDefault: true }) - .description("Show overview dashboard") - .action(() => { - const { db } = initializeApp(); - try { - const s = getStats(db); - console.log("\n\u{1f4ca} Knowledge Base Overview\n"); - console.log(` Documents: ${s.totalDocuments}`); - console.log(` Chunks: ${s.totalChunks}`); - console.log(` Topics: ${s.totalTopics}`); - console.log(` Database size: ${(s.databaseSizeBytes / 1024).toFixed(1)} KB`); - console.log(` Total searches: ${s.totalSearches}`); - console.log(` Avg latency: ${s.avgLatencyMs} ms`); - - const trends = getSearchTrends(db, 7); - if (trends.length > 0) { - console.log("\n Recent search activity (last 7 days):"); - for (const t of trends) { - const bar = "\u2588".repeat(Math.min(t.count, 40)); - console.log(` ${t.date} ${bar} ${t.count}`); - } - } - console.log(); - } finally { - closeDatabase(); - } - }); - -statsCmd - .command("popular") - .description("Most-returned documents in search results") - .option("--limit <n>", "Number of results", "10") - .action((opts: { limit: string }) => { - const { db } = initializeApp(); - try { - const docs = getPopularDocuments(db, parseIntOption(opts.limit, "--limit")); - if (docs.length === 0) { - console.log("No search data yet."); - return; - } - console.log("\n\u{1f525} Most Popular Documents\n"); - console.log(" Hits Document"); - console.log(" " + "\u2500".repeat(50)); - for (const d of docs) { - console.log( - ` ${String(d.hitCount).padStart(4)} ${d.title} (${d.documentId.slice(0, 8)}\u2026)`, - ); - } - console.log(); - } finally { - closeDatabase(); - } - }); - -statsCmd - .command("stale") - .description("Documents with no search hits") - .option("--days <n>", "Look-back period in days", "90") - .action((opts: { days: string }) => { - const { db } = initializeApp(); - try { - const docs = getStaleDocuments(db, parseIntOption(opts.days, "--days")); - if (docs.length === 0) { - console.log("No stale documents found."); - return; - } - console.log(`\n\u{1f4ed} Stale Documents (no hits in ${opts.days} days)\n`); - console.log(" Updated Title"); - console.log(" " + "\u2500".repeat(50)); - for (const d of docs) { - const date = d.updatedAt.slice(0, 10); - console.log(` ${date} ${d.title}`); - } - console.log(); - } finally { - closeDatabase(); - } - }); - -statsCmd - .command("queries") - .description("Top search queries") - .option("--limit <n>", "Number of results", "10") - .action((opts: { limit: string }) => { - const { db } = initializeApp(); - try { - const queries = getTopQueries(db, parseIntOption(opts.limit, "--limit")); - if (queries.length === 0) { - console.log("No search data yet."); - return; - } - console.log("\n\u{1f50d} Top Search Queries\n"); - console.log(" Count Avg ms Query"); - console.log(" " + "\u2500".repeat(50)); - for (const q of queries) { - console.log( - ` ${String(q.count).padStart(5)} ${String(q.avgLatencyMs).padStart(6)} ${q.query}`, - ); - } - console.log(); - } finally { - closeDatabase(); - } - }); - -statsCmd - .command("search-analytics") - .description("Search analytics dashboard with knowledge gap detection") - .option("--days <n>", "Look-back period in days", "30") - .action((opts: { days: string }) => { - const { db } = initializeApp(); - try { - const days = parseIntOption(opts.days, "--days"); - const analytics = getSearchAnalytics(db, days); - const gaps = getKnowledgeGaps(db, days); - - console.log(`\n\u{1f50d} Search Analytics (last ${days} days)\n`); - console.log(` Total searches: ${analytics.totalSearches}`); - console.log(` Avg result count: ${analytics.avgResultCount}`); - - if (analytics.topQueries.length > 0) { - console.log("\n Top queries:"); - for (const q of analytics.topQueries) { - console.log(` ${String(q.count).padStart(5)}x ${q.query}`); - } - } - - if (analytics.zeroResultQueries.length > 0) { - console.log("\n Zero-result queries:"); - for (const q of analytics.zeroResultQueries) { - console.log(` ${String(q.count).padStart(5)}x ${q.query}`); - } - } - - if (gaps.length > 0) { - console.log("\n \u26a0\ufe0f Knowledge Gaps:"); - for (const g of gaps) { - console.log(` ${String(g.count).padStart(5)}x ${g.query} (last: ${g.lastSearched})`); - } - } - - if (analytics.queriesPerDay.length > 0) { - console.log("\n Queries per day:"); - for (const d of analytics.queriesPerDay) { - const bar = "\u2588".repeat(Math.min(d.count, 40)); - console.log(` ${d.date} ${bar} ${d.count}`); - } - } - console.log(); - } finally { - closeDatabase(); - } - }); - -// add-repo -program - .command("add-repo <url>") - .description("Index documentation from a GitHub or GitLab repository") - .option("--branch <name>", "Branch to index (default: main or from URL)") - .option("--path <subdir>", "Only index files under this subdirectory") - .option("--extensions <ext1,ext2>", "Comma-separated file extensions", ".md,.mdx,.txt,.rst") - .option("--token <pat>", "Personal access token for private repos") - .action( - async ( - url: string, - opts: { branch?: string; path?: string; extensions: string; token?: string }, - ) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const parsed = parseRepoUrl(url); - console.log(`Repository: ${parsed.owner}/${parsed.repo} (${parsed.host})`); - - const extensions = opts.extensions.split(",").map((e) => { - const trimmed = e.trim(); - return trimmed.startsWith(".") ? trimmed : `.${trimmed}`; - }); - - const startTime = Date.now(); - const result = await indexRepository( - db, - provider, - { - url, - branch: opts.branch, - paths: opts.path ? [opts.path] : undefined, - extensions, - token: opts.token, - }, - (message: string) => { - console.log(` ${message}`); - }, - ); - - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log(``); - console.log(`┌─────────────────────────────┐`); - console.log(`│ Repository Indexing Summary │`); - console.log(`├─────────────────────────────┤`); - console.log(`│ Indexed: ${String(result.indexed).padStart(17)} │`); - console.log(`│ Skipped: ${String(result.skipped).padStart(17)} │`); - console.log(`│ Errors: ${String(result.errors.length).padStart(17)} │`); - console.log(`│ Time: ${(elapsed + "s").padStart(17)} │`); - console.log(`└─────────────────────────────┘`); - - if (result.errors.length > 0) { - console.log("\nErrors:"); - for (const err of result.errors) { - console.log(` ✗ ${err}`); - } - } - } finally { - closeDatabase(); - } - }, - ); - -// workspace -const workspaceCmd = program.command("workspace").description("Manage workspaces"); - -workspaceCmd - .command("create <name>") - .description("Create a new workspace") - .action((name: string) => { - const ws = createWorkspace(name); - console.log(`✓ Workspace "${ws.name}" created at ${ws.path}`); - }); - -workspaceCmd - .command("list") - .description("List all workspaces") - .action(() => { - const workspaces = listWorkspaces(); - const active = getActiveWorkspace(); - - if (workspaces.length === 0) { - console.log("No workspaces found."); - return; - } - - console.log("Workspaces:\n"); - for (const ws of workspaces) { - const marker = ws.name === active ? " (active)" : ""; - console.log(` ${ws.name}${marker}`); - console.log(` Path: ${ws.path}`); - console.log(` Created: ${ws.createdAt}`); - console.log(); - } - }); - -workspaceCmd - .command("use <name>") - .description("Switch active workspace") - .action((name: string) => { - setActiveWorkspace(name); - console.log(`✓ Switched to workspace "${name}"`); - }); - -workspaceCmd - .command("delete <name>") - .description("Delete a workspace") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (name: string, opts: { yes?: boolean }) => { - if (!(await confirmOrCancel(`Delete workspace "${name}"? This cannot be undone.`, opts.yes))) { - return; - } - deleteWorkspace(name); - console.log(`✓ Workspace "${name}" deleted.`); - }); - -// pack -const packCmd = program.command("pack").description("Manage knowledge packs"); - -packCmd - .command("install <nameOrPath>") - .description( - "Install a knowledge pack from a git registry, URL registry, or local .json/.json.gz file. " + - "Supports name@version syntax.", - ) - .option("--registry <url>", "Custom registry URL (for URL-based registries)") - .option("--from-registry <name>", "Install from a specific git registry by name") - .option( - "--pack-version <semver>", - "Install a specific version (for git registries; or use name@version)", - ) - .option("-y, --yes", "Non-interactive mode (fail on conflicts instead of prompting)") - .option("--batch-size <n>", "Number of documents to embed per batch (default: 10)") - .option("--resume-from <n>", "Skip the first N documents (resume a partial install)") - .option("--concurrency <n>", "Number of batches to embed in parallel (default: 4)") - .action( - async ( - nameOrPath: string, - opts: { - registry?: string; - fromRegistry?: string; - packVersion?: string; - yes?: boolean; - batchSize?: string; - resumeFrom?: string; - concurrency?: string; - }, - ) => { - await executePackInstall(nameOrPath, opts); - }, - ); - -packCmd - .command("remove <name>") - .description("Remove a pack and its documents") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (name: string, opts: { yes?: boolean }) => { - if ( - !(await confirmOrCancel( - `Remove pack "${name}" and its documents? This cannot be undone.`, - opts.yes, - )) - ) { - return; - } - const { db } = initializeApp(); - try { - removePack(db, name); - console.log(`✓ Pack "${name}" removed.`); - } finally { - closeDatabase(); - } - }); - -packCmd - .command("list") - .description("List installed or available packs") - .option("--available", "List packs available in the registry") - .option("--registry <url>", "Custom registry URL") - .action(async (opts: { available?: boolean; registry?: string }) => { - if (opts.available) { - const packs = await listAvailablePacks(opts.registry); - if (packs.length === 0) { - console.log("No packs available in the registry."); - return; - } - console.log("Available packs:\n"); - for (const p of packs) { - console.log(` ${p.name} v${p.version} — ${p.description} (${p.docCount} docs)`); - } + if (isAborted()) { + process.stderr.write("Cancelled.\n"); + process.exitCode = 130; } else { - const { db } = initializeApp(); - try { - const packs = listInstalledPacks(db); - if (packs.length === 0) { - console.log("No packs installed."); - return; - } - console.log("Installed packs:\n"); - for (const p of packs) { - console.log(` ${p.name} v${p.version} — ${p.description ?? ""} (${p.docCount} docs)`); - } - } finally { - closeDatabase(); - } + reportError(err, getGlobalOptions().verbose === true); } - }); - -packCmd - .command("create") - .description("Create a pack from the database, a local folder, or a URL") - .requiredOption("--name <name>", "Pack name") - .option("--from <sources...>", "Source folder(s), file(s), or URL(s) to build pack from") - .option("--topic <topic>", "Filter documents by topic ID (database mode only)") - .option("--pack-version <version>", "Pack version (default: 1.0.0)") - .option("--description <desc>", "Pack description") - .option("--author <author>", "Pack author") - .option("--output <path>", "Output file path") - .option( - "--extensions <exts>", - "Comma-separated file extensions to include (e.g. .md,.html). Default: all supported", - ) - .option("--exclude <patterns...>", "Glob patterns to exclude (e.g. *.min.js, assets/**)") - .option("--no-recursive", "Do not recurse into subdirectories") - .action( - async (opts: { - name: string; - from?: string[]; - topic?: string; - packVersion?: string; - description?: string; - author?: string; - output?: string; - extensions?: string; - exclude?: string[]; - recursive: boolean; - }) => { - if (opts.from && opts.from.length > 0) { - // Source mode: build pack directly from files/URLs (no database needed) - // Default to .json.gz for source packs (they can be large) - const outputPath = opts.output ?? `${opts.name}.json.gz`; - const extensionList = opts.extensions ? splitCsv(opts.extensions) : undefined; - - const pack = await createPackFromSource({ - name: opts.name, - from: opts.from, - version: opts.packVersion, - description: opts.description, - author: opts.author, - outputPath, - extensions: extensionList, - exclude: opts.exclude, - recursive: opts.recursive, - onProgress: ({ file, index, total }) => { - const pct = Math.round(((index + 1) / total) * 100); - const short = file.length > 60 ? `...${file.slice(-57)}` : file; - process.stdout.write(`\r [${pct}%] ${index + 1}/${total} ${short}`.padEnd(80)); - }, - }); - // Clear progress line - process.stdout.write("\r" + " ".repeat(80) + "\r"); - console.log( - `✓ Pack "${pack.name}" created with ${pack.documents.length} documents → ${outputPath}`, - ); - } else { - // Database mode: export existing documents - const outputPath = opts.output ?? `${opts.name}.json`; - const { db } = initializeApp(); - try { - const pack = createPack(db, { - name: opts.name, - version: opts.packVersion, - description: opts.description, - author: opts.author, - topic: opts.topic, - outputPath, - }); - console.log( - `✓ Pack "${pack.name}" created with ${pack.documents.length} documents → ${outputPath}`, - ); - } finally { - closeDatabase(); - } - } - }, - ); - -/** Description for the `--name` option of connect/disconnect commands. */ -const CONNECTOR_NAME_HELP = "Saved connector config name (default: the connector type)"; - -/** Return the saved config for `connect <type> --sync`, or exit with a hint. */ -function requireSavedConfig<T>(saved: T | undefined, type: ConnectorType, name: string): T { - if (saved === undefined) { - console.error( - `No saved ${type} connector named "${name}". Run 'libscope connect ${type}' without --sync first.`, - ); - process.exit(1); - } - return saved; -} - -/** Sync with `config` and save it as connector `name` (with the new lastSync). */ -function runConnector<C extends { lastSync?: string | undefined }, R>( - type: ConnectorType, - name: string, - config: C, - sync: (config: C, options: ConnectorSyncOptions) => Promise<R>, -): Promise<R> { - return runAndSaveConnector(type, name, config, (c) => sync(c, { syncName: name })); -} - -/** Device-code sign-in for OneNote. Returns `base` with the new client and token fields. */ -async function onenoteDeviceAuth(base: Partial<OneNoteConfig>): Promise<OneNoteConfig> { - const clientId = base.clientId ?? process.env.ONENOTE_CLIENT_ID; - if (!clientId) { - console.error("Error: No client ID. Set ONENOTE_CLIENT_ID env var or provide --token."); - process.exit(1); - } - const tenantId = base.tenantId ?? process.env.ONENOTE_TENANT_ID ?? "common"; - - console.log("Starting device code authentication..."); - const auth = await authenticateDeviceCode(clientId, tenantId); - console.log("\u2713 Authenticated successfully"); - return { - notebooks: ["all"], - excludeSections: [], - ...base, - clientId, - tenantId, - accessToken: auth.accessToken, - refreshToken: auth.refreshToken, - tokenExpiry: auth.expiresAt, - }; -} - -/** - * Delete saved connector config `name` (it may hold credentials) when it belongs to `type` - * and passes the optional `matches` check. - */ -function forgetConnector( - type: ConnectorType, - name: string, - matches: (saved: Record<string, unknown>) => boolean = () => true, -): void { - if (!hasNamedConnectorConfig(name)) return; - const saved = loadNamedConnectorConfig<Record<string, unknown>>(name); - if (resolveConnectorType(name, saved) !== type || !matches(saved)) return; - deleteNamedConnectorConfig(name); - console.log(` Removed saved connector config "${name}".`); -} - -/** Absolute, normalised vault path (relative paths resolve against the working directory). */ -function resolveVaultPath(vaultPath: string): string { - return resolvePath(vaultPath); -} - -/** Build the OneNote config for a first-time (non --sync) connect. */ -async function buildOneNoteConnectConfig( - name: string, - opts: { token?: string; notebook?: string }, -): Promise<OneNoteConfig> { - const saved: Partial<OneNoteConfig> = findSavedOneNoteConfig(name) ?? {}; - const notebooks = opts.notebook ? [opts.notebook] : ["all"]; - if (opts.token) { - // A manually supplied token cannot be refreshed; drop any stored refresh token. - return { - clientId: "", - tenantId: "common", - excludeSections: [], - ...saved, - notebooks, - accessToken: opts.token, - refreshToken: undefined, - tokenExpiry: undefined, - lastSync: undefined, - }; + } finally { + closeContext(); } - const config = await onenoteDeviceAuth({ ...saved, notebooks, lastSync: undefined }); - // Keep the credentials even if the first sync fails. - saveConnectorSettings("onenote", name, config); - return config; -} - -// connect onenote -const connectCmd = program.command("connect").description("Connect external services"); - -connectCmd - .command("onenote") - .description("Connect and sync OneNote notebooks via Microsoft Graph API") - .option("--token <accessToken>", "Use a pre-existing access token (cannot be refreshed)") - .option("--sync", "Re-sync using the saved configuration (refreshes the access token)") - .option("--notebook <name>", "Sync a specific notebook") - .option("--name <name>", CONNECTOR_NAME_HELP) - .action(async (opts: { token?: string; sync?: boolean; notebook?: string; name?: string }) => { - const { db, provider } = initializeAppWithEmbedding(); - - try { - const name = opts.name ?? "onenote"; - const oneNoteConfig = opts.sync - ? requireSavedConfig(findSavedOneNoteConfig(name), "onenote", name) - : await buildOneNoteConnectConfig(name, opts); - - const syncResult = await runConnector("onenote", name, oneNoteConfig, (c, o) => - syncOneNote(db, provider, c, o), - ); - - console.log(`\n✓ OneNote sync complete:`); - console.log(` Notebooks: ${syncResult.notebooks}`); - console.log(` Sections: ${syncResult.sections}`); - console.log(` Added: ${syncResult.pagesAdded}`); - console.log(` Updated: ${syncResult.pagesUpdated}`); - console.log(` Deleted: ${syncResult.pagesDeleted}`); - if (syncResult.errors.length > 0) { - console.log(` Errors: ${syncResult.errors.length}`); - for (const e of syncResult.errors) { - console.log(` - ${e.page}: ${e.error}`); - } - } - } finally { - closeDatabase(); - } - }); - -// disconnect commands -const disconnectCmd = program.command("disconnect").description("Disconnect external sources"); - -disconnectCmd - .command("onenote") - .description("Disconnect OneNote and remove its data") - .option("-y, --yes", "Skip confirmation prompt") - .option("--name <name>", CONNECTOR_NAME_HELP) - .action(async (opts: { yes?: boolean; name?: string }) => { - if ( - !(await confirmOrCancel( - "Disconnect OneNote and remove all its data? This cannot be undone.", - opts.yes, - )) - ) { - return; - } - const { db } = initializeApp(); - - try { - const removed = disconnectOneNote(db); - console.log(`✓ Disconnected OneNote. Removed ${removed} documents.`); - forgetConnector("onenote", opts.name ?? "onenote"); - } finally { - closeDatabase(); - } - }); - -connectCmd - .command("obsidian [vault-path]") - .description("Sync an Obsidian vault into the knowledge base") - .option("--sync", "Re-sync using the saved configuration (only changed files are re-indexed)") - .option( - "--topic-mapping <mode>", - "Map topics from 'folder' or 'frontmatter' (default: folder)", - "folder", - ) - .option("--exclude <patterns...>", "Additional exclude patterns") - .option("--name <name>", CONNECTOR_NAME_HELP) - .action( - async ( - vaultPath: string | undefined, - cmdOpts: { sync?: boolean; topicMapping?: string; exclude?: string[]; name?: string }, - ) => { - const name = cmdOpts.name ?? "obsidian"; - let obsConfig: ObsidianConfig; - if (cmdOpts.sync) { - obsConfig = requireSavedConfig( - findSavedConnectorConfig<ObsidianConfig>("obsidian", name), - "obsidian", - name, - ); - } else { - if (!vaultPath) { - console.error("Error: <vault-path> is required (or use --sync with a saved config)."); - process.exit(1); - } - obsConfig = { - vaultPath: resolveVaultPath(vaultPath), - topicMapping: cmdOpts.topicMapping === "frontmatter" ? "frontmatter" : "folder", - excludePatterns: cmdOpts.exclude ?? [], - }; - } - - const { db, provider } = initializeAppWithEmbedding(); - try { - console.log(`Syncing Obsidian vault: ${obsConfig.vaultPath}`); - const result = await runConnector("obsidian", name, obsConfig, (c, o) => - syncObsidianVault(db, provider, c, o), - ); - - console.log(`✓ Sync complete:`); - console.log(` Added: ${result.added}`); - console.log(` Updated: ${result.updated}`); - console.log(` Deleted: ${result.deleted}`); - if (result.errors.length > 0) { - console.log(` Errors: ${result.errors.length}`); - for (const e of result.errors) { - console.log(` - ${e.file}: ${e.error}`); - } - } - } finally { - closeDatabase(); - } - }, - ); - -connectCmd - .command("slack") - .description("Connect a Slack workspace") - .option("--token <token>", "Slack bot or user token (xoxb-... or xoxp-...)") - .option("--channels <channels>", "Comma-separated channel names or IDs, or 'all'", "all") - .option("--exclude <channels>", "Comma-separated channel names to exclude") - .option( - "--thread-mode <mode>", - "Thread handling: aggregate (full thread as one doc) or separate", - "aggregate", - ) - .option("--sync", "Re-sync using the saved configuration") - .option("--name <name>", CONNECTOR_NAME_HELP) - .action( - async (opts: { - token?: string; - channels: string; - exclude?: string; - threadMode: string; - sync?: boolean; - name?: string; - }) => { - const name = opts.name ?? "slack"; - let slackConfig: SlackConfig; - if (opts.sync) { - slackConfig = requireSavedConfig( - findSavedConnectorConfig<SlackConfig>("slack", name), - "slack", - name, - ); - } else { - if (!opts.token) { - console.error("--token is required for initial Slack connection."); - process.exit(1); - } - slackConfig = { - token: opts.token, - channels: splitCsv(opts.channels), - threadMode: opts.threadMode === "separate" ? "separate" : "aggregate", - ...(opts.exclude ? { excludeChannels: splitCsv(opts.exclude) } : {}), - }; - } - - const { db, provider } = initializeAppWithEmbedding(); - try { - console.log("Syncing Slack messages..."); - const result = await runConnector("slack", name, slackConfig, (c, o) => - syncSlack(db, provider, c, o), - ); - - console.log(`✓ Slack sync complete:`); - console.log(` Channels: ${result.channels}`); - console.log(` Messages indexed: ${result.messagesIndexed}`); - console.log(` Threads indexed: ${result.threadsIndexed}`); - if (result.errors.length > 0) { - console.log(` Errors: ${result.errors.length}`); - for (const err of result.errors) { - console.log(` - #${err.channel}: ${err.error}`); - } - } - } finally { - closeDatabase(); - } - }, - ); - -connectCmd - .command("confluence") - .description("Sync Confluence spaces and pages") - .option("--url <url>", "Confluence base URL (e.g. https://acme.atlassian.net)") - .option("--type <type>", "Confluence type: cloud or server (Data Center)", "cloud") - .option("--email <email>", "Confluence user email (Cloud only)") - .option("--token <token>", "API token (Cloud) or Personal Access Token (Server/Data Center)") - .option("--spaces <keys>", "Comma-separated space keys, or 'all'", "all") - .option("--exclude-spaces <keys>", "Comma-separated space keys to exclude") - .option("--sync", "Re-sync using the saved configuration") - .option("--name <name>", CONNECTOR_NAME_HELP) - .action( - async (opts: { - url?: string; - type?: string; - email?: string; - token?: string; - spaces?: string; - excludeSpaces?: string; - sync?: boolean; - name?: string; - }) => { - const name = opts.name ?? "confluence"; - let confluenceConfig: ConfluenceConfig; - if (opts.sync) { - confluenceConfig = requireSavedConfig( - findSavedConnectorConfig<ConfluenceConfig>("confluence", name), - "confluence", - name, - ); - } else { - const email = opts.email ?? process.env["CONFLUENCE_EMAIL"]; - confluenceConfig = { - baseUrl: opts.url ?? process.env["CONFLUENCE_URL"] ?? "", - type: opts.type === "server" ? "server" : "cloud", - ...(email ? { email } : {}), - token: opts.token ?? process.env["CONFLUENCE_TOKEN"] ?? "", - spaces: splitCsv(opts.spaces ?? "all"), - excludeSpaces: opts.excludeSpaces ? splitCsv(opts.excludeSpaces) : undefined, - }; - } - - const { db, provider } = initializeAppWithEmbedding(); - try { - const result = await runConnector("confluence", name, confluenceConfig, (c, o) => - syncConfluence(db, provider, c, o), - ); - - console.log(`✓ Confluence sync complete`); - console.log(` Spaces: ${result.spaces}`); - console.log(` Pages indexed: ${result.pagesIndexed}`); - console.log(` Pages updated: ${result.pagesUpdated}`); - if (result.errors.length > 0) { - console.log(` Errors: ${result.errors.length}`); - for (const e of result.errors) { - console.log(` - ${e.page}: ${e.error}`); - } - } - } finally { - closeDatabase(); - } - }, - ); - -disconnectCmd - .command("obsidian <vault-path>") - .description("Remove all documents from an Obsidian vault") - .option("-y, --yes", "Skip confirmation prompt") - .option("--name <name>", CONNECTOR_NAME_HELP) - .action(async (vaultPath: string, opts: { yes?: boolean; name?: string }) => { - if ( - !(await confirmOrCancel( - `Disconnect Obsidian vault "${vaultPath}" and remove its documents? This cannot be undone.`, - opts.yes, - )) - ) { - return; - } - const { db } = initializeApp(); - try { - const { disconnectVault } = await import("../connectors/obsidian.js"); - - const resolvedPath = resolveVaultPath(vaultPath); - const removed = disconnectVault(db, resolvedPath); - console.log(`✓ Disconnected vault. Removed ${removed} documents.`); - forgetConnector( - "obsidian", - opts.name ?? "obsidian", - (saved) => saved["vaultPath"] === resolvedPath, - ); - } finally { - closeDatabase(); - } - }); - -connectCmd - .command("notion") - .description("Connect and sync Notion pages and databases") - .option("--token <token>", "Notion integration token (secret_... or ntn_...; or NOTION_TOKEN)") - .option("--sync", "Re-sync using the saved configuration (pages edited since the last sync)") - .option("--exclude <ids...>", "Page/database IDs to exclude") - .option("--name <name>", CONNECTOR_NAME_HELP) - .action(async (opts: { token?: string; sync?: boolean; exclude?: string[]; name?: string }) => { - const name = opts.name ?? "notion"; - let config: NotionConfig; - if (opts.sync) { - config = requireSavedConfig( - findSavedConnectorConfig<NotionConfig>("notion", name), - "notion", - name, - ); - } else { - const token = opts.token ?? process.env["NOTION_TOKEN"]; - if (!token) { - console.error("Error: --token <token> is required, or set NOTION_TOKEN."); - process.exitCode = 1; - return; - } - config = { token, ...(opts.exclude ? { excludePages: opts.exclude } : {}) }; - } - - const { db, provider } = initializeAppWithEmbedding(); - try { - console.log("Syncing Notion..."); - const result = await runConnector("notion", name, config, (c, o) => - syncNotion(db, provider, c, o), - ); - console.log( - `✓ Synced: ${result.pagesIndexed} pages, ${result.databasesIndexed} databases` + - (result.errors.length > 0 ? `, ${result.errors.length} errors` : ""), - ); - for (const err of result.errors) { - console.log(` ⚠ ${err.page}: ${err.error}`); - } - } finally { - closeDatabase(); - } - }); - -disconnectCmd - .command("notion") - .description("Remove all Notion documents") - .option("-y, --yes", "Skip confirmation prompt") - .option("--name <name>", CONNECTOR_NAME_HELP) - .action(async (opts: { yes?: boolean; name?: string }) => { - if ( - !(await confirmOrCancel( - "Disconnect Notion and remove all its documents? This cannot be undone.", - opts.yes, - )) - ) { - return; - } - const { db } = initializeApp(); - try { - const removed = await disconnectNotion(db); - console.log(`✓ Removed ${removed} Notion documents.`); - forgetConnector("notion", opts.name ?? "notion"); - } finally { - closeDatabase(); - } - }); - -disconnectCmd - .command("slack") - .description("Remove all Slack data from the knowledge base") - .option("-y, --yes", "Skip confirmation prompt") - .option("--name <name>", CONNECTOR_NAME_HELP) - .action(async (opts: { yes?: boolean; name?: string }) => { - if ( - !(await confirmOrCancel( - "Disconnect Slack and remove all its data? This cannot be undone.", - opts.yes, - )) - ) { - return; - } - const { db } = initializeApp(); - try { - const count = disconnectSlack(db); - console.log(`✓ Removed ${count} Slack documents from the knowledge base.`); - forgetConnector("slack", opts.name ?? "slack"); - } finally { - closeDatabase(); - } - }); - -disconnectCmd - .command("confluence") - .description("Remove all Confluence-synced content") - .option("-y, --yes", "Skip confirmation prompt") - .option("--name <name>", CONNECTOR_NAME_HELP) - .action(async (opts: { yes?: boolean; name?: string }) => { - if ( - !(await confirmOrCancel( - "Disconnect Confluence and remove all synced content? This cannot be undone.", - opts.yes, - )) - ) { - return; - } - const { disconnectConfluence } = await import("../connectors/confluence.js"); - const { db } = initializeApp(); - try { - const removed = disconnectConfluence(db); - console.log(`✓ Removed ${removed} Confluence documents`); - forgetConnector("confluence", opts.name ?? "confluence"); - } finally { - closeDatabase(); - } - }); - -// update -program - .command("update") - .description("Update libscope to the latest version") - .action(() => { - const currentVersion = _pkg.version; - console.log(`Current version: ${currentVersion}`); - - try { - const versionResult = spawnSync("npm", ["view", "libscope", "version"], { - encoding: "utf-8", - }); - if (versionResult.error || versionResult.status !== 0) { - throw new Error(versionResult.stderr?.trim() || "Failed to check latest version"); - } - const latest = versionResult.stdout.trim(); - if (latest === currentVersion) { - console.log("✓ Already up to date."); - return; - } - console.log(`Latest version: ${latest}`); - console.log("Updating..."); - const installResult = spawnSync("npm", ["install", "-g", "libscope@latest"], { - stdio: "inherit", - }); - if (installResult.error || installResult.status !== 0) { - throw new Error("npm install failed"); - } - console.log(`✓ Updated to ${latest}`); - } catch (err) { - console.error("Failed to update. Try manually: npm install -g libscope@latest"); - if (err instanceof Error) console.error(err.message); - process.exit(1); - } - }); - -// bulk -const bulkCmd = program.command("bulk").description("Bulk operations on documents"); - -interface BulkFilterOpts { - topic?: string; - library?: string; - sourceType?: string; - tags?: string; - dryRun?: boolean; - yes?: boolean; -} - -/** Add the document filter options shared by all bulk commands. */ -function addBulkFilterOptions(cmd: Command): Command { - return cmd - .option("--topic <topicId>", "Filter by topic ID") - .option("--library <name>", "Filter by library name") - .option("--source-type <type>", "Filter by source type") - .option("--tags <tags>", "Filter by tags (comma-separated)"); -} - -/** Add the --dry-run and --yes options shared by all bulk commands. */ -function addBulkRunOptions(cmd: Command): Command { - return cmd - .option("--dry-run", "Show what would be affected without making changes") - .option("-y, --yes", "Skip confirmation prompt"); -} - -function selectorFromOpts(opts: BulkFilterOpts): BulkSelector { - const selector: BulkSelector = {}; - if (opts.topic) selector.topicId = opts.topic; - if (opts.library) selector.library = opts.library; - if (opts.sourceType) selector.sourceType = opts.sourceType; - if (opts.tags) selector.tags = splitCsv(opts.tags); - return selector; } -/** - * Shared bulk flow: preview matches, list them on --dry-run, confirm, then apply. - * `run(db, dryRun)` performs the bulk operation (as a preview when dryRun is true). - */ -async function runBulk( - opts: BulkFilterOpts, - run: (db: ReturnType<typeof getDatabase>, dryRun?: boolean) => BulkResult, - confirmMessage: (affected: number) => string, - doneMessage: (affected: number) => string, -): Promise<void> { - const { db } = initializeApp(); - try { - const result = run(db, true); - console.log(`Found ${result.affected} document(s) matching filters.`); - - if (result.affected === 0) return; - - if (opts.dryRun) { - for (const id of result.documentIds) { - console.log(` - ${id}`); - } - console.log("(dry run — no changes made)"); - return; - } - - if (!(await confirmOrCancel(confirmMessage(result.affected), opts.yes))) { - return; - } - - const actual = run(db); - console.log(doneMessage(actual.affected)); - } finally { - closeDatabase(); +/** Ctrl+C: stop the running operation; a second Ctrl+C (or nothing to stop) exits now. */ +function onInterrupt(): void { + if (isInterruptLeftToCommand()) return; + const waiting = isWaitingForInterrupt(); + if (isAborted() || !waiting) { + closeContext(); + process.exit(130); } + abortContext(); } -addBulkRunOptions( - addBulkFilterOptions( - bulkCmd.command("delete").description("Delete multiple documents matching filters"), - ), -).action(async (opts: BulkFilterOpts) => { - const selector = selectorFromOpts(opts); - await runBulk( - opts, - (db, dryRun) => bulkDelete(db, selector, dryRun), - (n) => `Delete ${n} document(s)? This cannot be undone.`, - (n) => `✓ Deleted ${n} document(s).`, - ); -}); - -addBulkRunOptions( - addBulkFilterOptions( - bulkCmd.command("retag").description("Add or remove tags from multiple documents"), - ) - .option("--add-tags <tags>", "Tags to add (comma-separated)") - .option("--remove-tags <tags>", "Tags to remove (comma-separated)"), -).action(async (opts: BulkFilterOpts & { addTags?: string; removeTags?: string }) => { - const selector = selectorFromOpts(opts); - const addTags = opts.addTags ? splitCsv(opts.addTags) : undefined; - const removeTags = opts.removeTags ? splitCsv(opts.removeTags) : undefined; - await runBulk( - opts, - (db, dryRun) => bulkRetag(db, selector, addTags, removeTags, dryRun), - (n) => `Retag ${n} document(s)?`, - (n) => `✓ Retagged ${n} document(s).`, - ); -}); - -addBulkRunOptions( - addBulkFilterOptions( - bulkCmd.command("move").description("Move multiple documents to a different topic"), - ).requiredOption("--to <targetTopicId>", "Target topic ID to move documents to"), -).action(async (opts: BulkFilterOpts & { to: string }) => { - const selector = selectorFromOpts(opts); - await runBulk( - opts, - (db, dryRun) => bulkMove(db, selector, opts.to, dryRun), - (n) => `Move ${n} document(s) to topic "${opts.to}"?`, - (n) => `✓ Moved ${n} document(s) to topic "${opts.to}".`, - ); -}); - -// Webhooks -const webhookCmd = program.command("webhooks").description("Manage webhooks"); - -webhookCmd - .command("list") - .description("List all registered webhooks") - .action(() => { - const { db } = initializeApp(); - try { - const hooks = listWebhooks(db); - if (hooks.length === 0) { - console.log("No webhooks registered."); - } else { - console.log(`Found ${hooks.length} webhook(s):\n`); - for (const h of hooks) { - console.log(` ${h.url}`); - console.log(` ID: ${h.id}`); - console.log(` Events: ${h.events.join(", ")}`); - console.log(` Active: ${h.active ? "yes" : "no"}`); - if (h.lastTriggeredAt) console.log(` Last triggered: ${h.lastTriggeredAt}`); - if (h.failureCount > 0) console.log(` Failures: ${h.failureCount}`); - console.log(); - } - } - } finally { - closeDatabase(); - } - }); - -webhookCmd - .command("create <url>") - .description("Register a new webhook") - .requiredOption( - "--events <events>", - "Comma-separated event types (e.g. document.created,document.updated)", - ) - .option("--secret <secret>", "Secret for HMAC-SHA256 signature") - .action(async (url: string, opts: { events: string; secret?: string }) => { - const { db } = initializeApp(); - try { - const events = opts.events.split(",").map((e) => e.trim()) as WebhookEvent[]; - const webhook = await createWebhook(db, url, events, opts.secret); - console.log(`✓ Webhook created: ${webhook.id}`); - console.log(` URL: ${webhook.url}`); - console.log(` Events: ${webhook.events.join(", ")}`); - } finally { - closeDatabase(); - } - }); - -webhookCmd - .command("delete <id>") - .description("Delete a webhook") - .action((id: string) => { - const { db } = initializeApp(); - try { - deleteWebhook(db, id); - console.log(`✓ Webhook "${id}" deleted.`); - } finally { - closeDatabase(); - } - }); - -webhookCmd - .command("test <id>") - .description("Send a test ping to a webhook") - .action(async (id: string) => { - const { db } = initializeApp(); - try { - const webhook = getWebhook(db, id); - console.log(`Sending test ping to ${webhook.url}...`); - const resp = await deliverWebhook(webhook, "document.created", TEST_PING_DATA); - console.log(`✓ Response: ${resp.status} ${resp.statusText}`); - } finally { - closeDatabase(); - } - }); - -// schedule -const scheduleCmd = program.command("schedule").description("Manage connector sync schedules"); - -scheduleCmd - .command("list") - .description("List all configured connector sync schedules") - .action(async () => { - const { loadScheduleEntries: loadEntries } = await import("../core/scheduler.js"); - const entries = loadEntries(); - if (entries.length === 0) { - console.log("No scheduled syncs configured."); - console.log( - 'Add a schedule to a connector config: libscope schedule set <connector> "<cron>"', - ); - return; - } - console.log("Connector sync schedules:\n"); - for (const entry of entries) { - console.log(` ${entry.connectorName} (${entry.connectorType})`); - console.log(` Cron: ${entry.cronExpression}`); - console.log(); - } - }); - -scheduleCmd - .command("set <connector> <cron>") - .description('Set a sync schedule for a connector (e.g. schedule set notion "0 */6 * * *")') - .action(async (connector: string, cronExpr: string) => { - const nodeCron = await import("node-cron"); - if (!nodeCron.validate(cronExpr)) { - console.error(`Invalid cron expression: "${cronExpr}"`); - console.error("Examples: '0 */6 * * *' (every 6h), '0 0 * * *' (daily at midnight)"); - process.exit(1); - } - - if (!hasNamedConnectorConfig(connector)) { - console.error( - `No connector config found for "${connector}". Run 'libscope connect ${connector}' first.`, - ); - process.exit(1); - } - - const config = loadNamedConnectorConfig<Record<string, unknown>>(connector); - config.schedule = { cronExpression: cronExpr }; - saveNamedConnectorConfig(connector, config); - console.log(`✓ Schedule set for ${connector}: ${cronExpr}`); - console.log( - "The schedule will be active when the API server is running (libscope serve --api)", - ); - }); - -scheduleCmd - .command("remove <connector>") - .description("Remove the sync schedule for a connector") - .action((connector: string) => { - if (!hasNamedConnectorConfig(connector)) { - console.error(`No connector config found for "${connector}".`); - process.exit(1); - } - - const config = loadNamedConnectorConfig<Record<string, unknown>>(connector); - if (!config.schedule) { - console.error(`No schedule configured for "${connector}". Nothing to remove.`); - process.exit(1); - } - delete config.schedule; - saveNamedConnectorConfig(connector, config); - console.log(`✓ Schedule removed for ${connector}`); - }); - -// Registry commands -registerRegistryCommands(program); - /** True when this module is the process entry point (the `libscope` bin), not an import. */ function isCliEntryPoint(): boolean { const entry = process.argv[1]; @@ -3188,7 +121,9 @@ function isCliEntryPoint(): boolean { } if (isCliEntryPoint()) { - program.parse(); + process.on("SIGINT", onInterrupt); + process.on("SIGTERM", onInterrupt); + void main(); } export { program }; diff --git a/src/cli/options.ts b/src/cli/options.ts new file mode 100644 index 00000000..3780e95c --- /dev/null +++ b/src/cli/options.ts @@ -0,0 +1,81 @@ +/** Option helpers shared by commands: the document filter flags and value parsers. */ +import type { Command } from "commander"; +import { SOURCE_TYPES } from "../core/indexing.js"; + +/** Commander parser for numeric options; the operation schema validates the value. */ +export function toNumber(value: string): number { + return Number(value); +} + +/** "a, b,c" -> ["a", "b", "c"] (empty entries dropped). */ +export function splitList(value: string): string[] { + return value + .split(",") + .map((v) => v.trim()) + .filter((v) => v.length > 0); +} + +/** Flags added by addDocumentOptions, as commander parses them. */ +export interface DocumentFlags { + topic?: string | undefined; + library?: string | undefined; + libVersion?: string | undefined; + sourceType?: string | undefined; + tags?: string | undefined; + limit?: number | undefined; +} + +export interface DocumentOptionsConfig { + /** "filter" (default): flags select documents. "assign": flags set new documents' metadata. */ + mode?: "filter" | "assign"; + /** false: no -n/--limit. A string replaces its help text. */ + limit?: false | string; + /** Add --lib-version (default true). */ + version?: boolean; +} + +/** + * Add the shared document flags: --topic --library --lib-version --source-type --tags and + * -n/--limit. Map them to operation input with documentInput(). + */ +export function addDocumentOptions(cmd: Command, config: DocumentOptionsConfig = {}): Command { + const assign = config.mode === "assign"; + cmd + .option("--topic <topic>", assign ? "Topic (ID or name)" : "Only this topic (ID or name)") + .option("--library <name>", assign ? "Library name" : "Only this library"); + if (config.version !== false) { + cmd.option("--lib-version <version>", assign ? "Library version" : "Only this library version"); + } + cmd + .option( + "--source-type <type>", + `${assign ? "Source type" : "Only this source type"} (${SOURCE_TYPES.join(", ")})`, + ) + .option( + "--tags <tags>", + assign + ? "Tags to add (comma-separated)" + : "Only documents with all these tags (comma-separated)", + ); + if (config.limit !== false) { + cmd.option("-n, --limit <n>", config.limit ?? "Maximum results", toNumber); + } + return cmd; +} + +/** Operation input for the shared document flags (unset flags are left out). */ +export function documentInput(flags: DocumentFlags): Record<string, unknown> { + return defined({ + topic: flags.topic, + library: flags.library, + version: flags.libVersion, + sourceType: flags.sourceType, + tags: flags.tags === undefined ? undefined : splitList(flags.tags), + limit: flags.limit, + }); +} + +/** `obj` without undefined values (so operation defaults apply). */ +export function defined(obj: Record<string, unknown>): Record<string, unknown> { + return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined)); +} diff --git a/src/cli/repl.ts b/src/cli/repl.ts index c13cd706..8e027717 100644 --- a/src/cli/repl.ts +++ b/src/cli/repl.ts @@ -1,52 +1,20 @@ import * as readline from "node:readline/promises"; import { stdin as input, stdout as output } from "node:process"; -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import { searchDocuments } from "../core/search.js"; -export interface ReplOptions { - db: Database.Database; - provider: EmbeddingProvider; - limit?: number; - /** Overridable for testing */ +export interface InteractiveOptions { + /** Called for each non-empty line the user enters. Errors are printed and the loop continues. */ + onQuery: (query: string) => Promise<void>; + /** Overridable for testing. */ createInterface?: () => readline.Interface; } -function formatResults( - results: { - title: string; - score: number; - library: string | null; - url: string | null; - content: string; - }[], - totalCount: number, -): string { - if (results.length === 0) { - return "No results found."; - } - - const lines: string[] = []; - lines.push(`\nShowing ${results.length} of ${totalCount} results:\n`); - - for (const r of results) { - lines.push(`\n── ${r.title} (score: ${r.score.toFixed(2)}) ──`); - if (r.library) lines.push(` Library: ${r.library}`); - if (r.url) lines.push(` Source: ${r.url}`); - lines.push(` ${r.content.slice(0, 200)}${r.content.length > 200 ? "..." : ""}`); - } - - return lines.join("\n"); -} - -export async function startRepl(options: ReplOptions): Promise<void> { - const { db, provider, limit = 5 } = options; - +/** Interactive search loop (`libscope search` with no query on a terminal). */ +export async function startInteractiveSearch(options: InteractiveOptions): Promise<void> { const rl = options.createInterface ? options.createInterface() : readline.createInterface({ input, output }); - console.log("LibScope interactive search (type 'quit' or 'exit' to leave)\n"); + console.log("LibScope interactive search (type 'quit' or 'exit', or press Ctrl+D, to leave)\n"); try { for (;;) { @@ -54,7 +22,7 @@ export async function startRepl(options: ReplOptions): Promise<void> { try { query = await rl.question("search> "); } catch { - // Ctrl+C or closed stream + // Ctrl+C / Ctrl+D or closed stream break; } @@ -63,20 +31,13 @@ export async function startRepl(options: ReplOptions): Promise<void> { if (trimmed === "quit" || trimmed === "exit") break; try { - const { results, totalCount } = await searchDocuments(db, provider, { - query: trimmed, - limit, - }); - console.log(formatResults(results, totalCount)); + await options.onQuery(trimmed); } catch (err) { - console.error(`Search error: ${err instanceof Error ? err.message : String(err)}`); + console.error(`✗ ${err instanceof Error ? err.message : String(err)}`); } - - console.log(); // blank line between searches + console.log(); } } finally { rl.close(); } - - console.log("Goodbye!"); } diff --git a/src/cli/reporter.ts b/src/cli/reporter.ts index 08d5e558..637769d1 100644 --- a/src/cli/reporter.ts +++ b/src/cli/reporter.ts @@ -1,99 +1,47 @@ /** - * CLI output reporter — pretty human-readable output for interactive terminals. - * In verbose/JSON mode, a SilentReporter is used so pino JSON logs handle output. + * Progress line for long-running operations. Written to stderr so `--json` output on stdout + * stays clean, and only on a terminal (nothing is written when stderr is redirected). */ +import type { ProgressEvent } from "../core/operations/index.js"; -const RESET = "\x1b[0m"; -const GREEN = "\x1b[32m"; -const YELLOW = "\x1b[33m"; -const RED = "\x1b[31m"; -const CYAN = "\x1b[36m"; -const DIM = "\x1b[2m"; - -export interface CliReporter { - log(msg: string): void; - success(msg: string): void; - warn(msg: string): void; - error(msg: string): void; - progress(current: number, total: number, label: string): void; - clearProgress(): void; -} - -function buildBar(pct: number, width = 20): string { - const filled = Math.round((pct / 100) * width); - return "\u2588".repeat(filled) + "\u2591".repeat(width - filled); +export interface ProgressLine { + update(event: ProgressEvent): void; + clear(): void; } -/** Pretty human-readable reporter. Uses ANSI colors and \r progress lines. */ -class PrettyReporter implements CliReporter { - private hasProgress = false; - - log(msg: string): void { - this.clearProgress(); - process.stdout.write(`${msg}\n`); - } - - success(msg: string): void { - this.clearProgress(); - process.stdout.write(`${GREEN}\u2713${RESET} ${msg}\n`); - } - - warn(msg: string): void { - this.clearProgress(); - process.stderr.write(`${YELLOW}\u26a0${RESET} ${msg}\n`); - } - - error(msg: string): void { - this.clearProgress(); - process.stderr.write(`${RED}\u2717${RESET} ${msg}\n`); - } - - progress(current: number, total: number, label: string): void { - const pct = total > 0 ? Math.round((current / total) * 100) : 0; - const bar = buildBar(pct); - const truncatedLabel = label.length > 40 ? `${label.slice(0, 37)}...` : label; - const line = `${CYAN}[${bar}]${RESET} ${pct}% (${current}/${total}) ${DIM}${truncatedLabel}${RESET}`; - process.stdout.write(`\r${line}`); - this.hasProgress = true; - } - - clearProgress(): void { - if (this.hasProgress) { - const width = process.stdout.columns ?? 80; - process.stdout.write(`\r${" ".repeat(width - 1)}\r`); - this.hasProgress = false; - } - } +function bar(done: number, total: number, width = 20): string { + const filled = Math.min(width, Math.round((done / total) * width)); + return "█".repeat(filled) + "░".repeat(width - filled); } -/** No-op reporter: used in verbose/JSON mode where pino logs handle output. */ -class SilentReporter implements CliReporter { - log(_msg: string): void { - // intentionally empty — silent reporter - } - success(_msg: string): void { - // intentionally empty — silent reporter - } - warn(_msg: string): void { - // intentionally empty — silent reporter +/** Text of one progress line, e.g. "[████░░…] 40% (4/10) docs/a.md". */ +export function formatProgress(event: ProgressEvent): string { + const label = event.message ?? ""; + const shortLabel = label.length > 50 ? `...${label.slice(-47)}` : label; + if (event.total !== undefined && event.total > 0) { + const pct = Math.round((event.done / event.total) * 100); + return `[${bar(event.done, event.total)}] ${pct}% (${event.done}/${event.total}) ${shortLabel}`; } - error(_msg: string): void { - // intentionally empty — silent reporter - } - progress(_current: number, _total: number, _label: string): void { - // intentionally empty — silent reporter - } - clearProgress(): void { - // intentionally empty — silent reporter - } -} - -/** Returns true if verbose mode is active (flag or env var). */ -export function isVerbose(verbose?: boolean): boolean { - return verbose === true || process.env["LIBSCOPE_VERBOSE"] === "1"; + return `(${event.done}) ${shortLabel}`; } -/** Create a reporter appropriate for the current mode. */ -export function createReporter(verbose?: boolean): CliReporter { - return isVerbose(verbose) ? new SilentReporter() : new PrettyReporter(); +export function createProgressLine(stream: NodeJS.WriteStream = process.stderr): ProgressLine { + let shown = false; + const width = (): number => stream.columns ?? 80; + return { + update(event: ProgressEvent): void { + if (!stream.isTTY) return; + stream.write( + `\r${formatProgress(event) + .slice(0, width() - 1) + .padEnd(width() - 1)}`, + ); + shown = true; + }, + clear(): void { + if (!shown) return; + stream.write(`\r${" ".repeat(width() - 1)}\r`); + shown = false; + }, + }; } diff --git a/src/cli/run.ts b/src/cli/run.ts new file mode 100644 index 00000000..68e8c6ae --- /dev/null +++ b/src/cli/run.ts @@ -0,0 +1,68 @@ +/** + * Run an operation for a command and print its result: JSON with the global --json flag, + * otherwise the command's human formatter. + */ +import type { z } from "zod"; +import { runOperation, type Operation } from "../core/operations/index.js"; +import { getContext, getGlobalOptions, type ContextOptions } from "./context.js"; +import { createProgressLine } from "./reporter.js"; + +/** Run `op` with `input` in the command's context, showing progress on a terminal. */ +export async function call<S extends z.ZodObject, O>( + op: Operation<S, O>, + input: unknown, + options: ContextOptions = {}, +): Promise<O> { + const ctx = getContext(options); + const progress = createProgressLine(); + try { + return await runOperation(op, { ...ctx, onProgress: (p) => progress.update(p) }, input); + } finally { + progress.clear(); + } +} + +export function isJsonOutput(): boolean { + return getGlobalOptions().json === true; +} + +/** Print `result` as JSON (--json) or with `human`. */ +export function print<T>(result: T, human: (result: T) => void): void { + if (isJsonOutput()) { + console.log(JSON.stringify(result, null, 2)); + } else { + human(result); + } +} + +/** call() then print(). */ +export async function run<S extends z.ZodObject, O>( + op: Operation<S, O>, + input: unknown, + human: (result: O) => void, + options: ContextOptions = {}, +): Promise<O> { + const result = await call(op, input, options); + print(result, human); + return result; +} + +/** First `max` characters of `text` on one line, with "..." when cut. */ +export function preview(text: string, max = 200): string { + const flat = text.replaceAll("\n", " ").trim(); + return flat.length > max ? `${flat.slice(0, max)}...` : flat; +} + +/** "1 document" / "3 documents". */ +export function plural(count: number, noun: string): string { + return `${count} ${noun}${count === 1 ? "" : "s"}`; +} + +/** Print "message" when `items` is empty, else print each item. */ +export function printList<T>(items: readonly T[], empty: string, each: (item: T) => void): void { + if (items.length === 0) { + console.log(empty); + return; + } + for (const item of items) each(item); +} diff --git a/tests/unit/cli-commands.test.ts b/tests/unit/cli-commands.test.ts new file mode 100644 index 00000000..849793a7 --- /dev/null +++ b/tests/unit/cli-commands.test.ts @@ -0,0 +1,452 @@ +/** + * CLI contract tests: each command line maps to one operation with the expected input. + * runOperation is replaced by a spy that returns a canned result, and bootstrap() by an + * in-memory database, so nothing touches disk or the network. + */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import type { MockInstance } from "vitest"; +import type { Command } from "commander"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Operation } from "../../src/core/operations/index.js"; +import { createTestDb } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { testConfig } from "./operations/helpers.js"; + +const db = createTestDb(); + +vi.mock("../../src/core/bootstrap.js", () => ({ + bootstrap: (): unknown => ({ + db, + provider: new MockEmbeddingProvider(), + config: testConfig(), + dbPath: ":memory:", + close: (): void => {}, + }), +})); + +const CANNED: Record<string, unknown> = { + add: { kind: "file", documents: [], errors: [], skipped: [] }, + "get-document": { document: { title: "Doc" } }, + "bulk-delete": { affected: 2, documentIds: ["d1", "d2"] }, + "bulk-retag": { affected: 2, documentIds: ["d1", "d2"] }, + "bulk-move": { affected: 2, documentIds: ["d1", "d2"] }, + "create-pack": { name: "p", version: "1.0.0", documents: [] }, +}; + +vi.mock("../../src/core/operations/index.js", async (importOriginal) => { + const actual = await importOriginal<typeof import("../../src/core/operations/index.js")>(); + return { + ...actual, + runOperation: vi.fn( + (op: Operation): Promise<unknown> => Promise.resolve(CANNED[op.name] ?? { items: [] }), + ), + }; +}); + +const { runOperation } = await import("../../src/core/operations/index.js"); +const { main, program } = await import("../../src/cli/index.js"); + +/** Run `libscope --json <args>` and return [operation name, input] of each operation call. */ +async function invoke(args: string[]): Promise<Array<[string, unknown]>> { + vi.mocked(runOperation).mockClear(); + await main(["node", "libscope", "--json", ...args]); + expect(process.exitCode ?? 0).toBe(0); + return vi.mocked(runOperation).mock.calls.map(([op, , input]) => [op.name, input]); +} + +let home: string; +const savedHome = process.env["HOME"]; + +beforeAll(() => { + home = mkdtempSync(join(tmpdir(), "libscope-cli-")); + process.env["HOME"] = home; +}); + +afterAll(() => { + process.env["HOME"] = savedHome; + rmSync(home, { recursive: true, force: true }); +}); + +let stderrWrite: MockInstance; + +beforeEach(() => { + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + stderrWrite = vi.spyOn(process.stderr, "write").mockImplementation(() => true); +}); + +afterEach(() => { + process.exitCode = undefined; + vi.restoreAllMocks(); +}); + +/** [argv, operation, expected input] for commands that call exactly one operation. */ +const SINGLE: Array<[string[], string, Record<string, unknown>]> = [ + [ + ["add", "a.md", "--topic", "t", "--library", "lib", "--lib-version", "1", "--tags", "x, y"], + "add", + { source: "a.md", topic: "t", library: "lib", version: "1", tags: ["x", "y"] }, + ], + [ + ["add", "https://e.com/docs", "--spider", "--max-pages", "5", "--no-same-domain"], + "add", + { source: "https://e.com/docs", spider: true, maxPages: 5, sameDomain: false }, + ], + [ + ["add", "https://e.com/d", "--spider", "--exclude-urls", "*.pdf", "--path-prefix", "/d"], + "add", + { spider: true, sameDomain: true, excludePatterns: ["*.pdf"], pathPrefix: "/d" }, + ], + [ + ["add", "https://github.com/o/r", "--branch", "dev", "--path", "docs,guide", "--token", "T"], + "add", + { source: "https://github.com/o/r", branch: "dev", paths: ["docs", "guide"], token: "T" }, + ], + [ + ["add", "dir", "--include", "**/*.md", "--exclude", "drafts/**", "--dry-run"], + "add", + { source: "dir", include: ["**/*.md"], exclude: ["drafts/**"], dryRun: true }, + ], + [ + [ + "add", + "a.md", + "--title", + "T", + "--format", + ".txt", + "--dedup", + "skip", + "--source-type", + "topic", + ], + "add", + { title: "T", format: ".txt", dedup: "skip", sourceType: "topic" }, + ], + [ + ["search", "how", "--topic", "t", "--source-type", "library", "--tags", "a", "-n", "3"], + "search", + { query: "how", topic: "t", sourceType: "library", tags: ["a"], limit: 3 }, + ], + [ + ["search", "q", "--offset", "2", "--min-rating", "4", "--max-per-doc", "1", "--context", "1"], + "search", + { query: "q", offset: 2, minRating: 4, maxChunksPerDocument: 1, contextChunks: 1 }, + ], + [["search", "--related", "doc-1", "-n", "4"], "search", { relatedTo: "doc-1", limit: 4 }], + [ + ["ask", "why?", "-n", "7", "--library", "lib", "--model", "m"], + "ask", + { question: "why?", topK: 7, library: "lib" }, + ], + [ + ["docs", "list", "--library", "lib", "-n", "5", "--offset", "10"], + "list-documents", + { library: "lib", limit: 5, offset: 10 }, + ], + [ + ["docs", "show", "d1", "--offset", "10", "--max-length", "100"], + "get-document", + { documentId: "d1", offset: 10, maxLength: 100 }, + ], + [ + ["docs", "update", "d1", "--title", "T", "--lib-version", "2", "--tags", "a,b", "--topic", "t"], + "update-document", + { documentId: "d1", title: "T", version: "2", tags: ["a", "b"], topic: "t" }, + ], + [["docs", "history", "d1"], "document-history", { documentId: "d1" }], + [["docs", "rollback", "d1", "3"], "rollback-document", { documentId: "d1", version: 3 }], + [ + ["docs", "rate", "d1", "4", "--feedback", "good", "--chunk", "c1"], + "rate-document", + { documentId: "d1", rating: 4, feedback: "good", chunkId: "c1" }, + ], + [ + ["docs", "link", "d1", "d2", "--type", "prerequisite", "--label", "first"], + "link-documents", + { documentId: "d1", targetDocumentId: "d2", linkType: "prerequisite", label: "first" }, + ], + [["docs", "unlink", "l1"], "unlink-documents", { linkId: "l1" }], + [ + ["docs", "links", "d1", "--type", "related"], + "list-links", + { documentId: "d1", linkType: "related" }, + ], + [["docs", "links"], "list-links", {}], + [["docs", "prereqs", "d1"], "prerequisites", { documentId: "d1" }], + [["docs", "tag", "d1", "a", "b,c"], "add-tags", { documentId: "d1", tags: ["a", "b", "c"] }], + [["docs", "untag", "d1", "a"], "remove-tags", { documentId: "d1", tags: ["a"] }], + [["docs", "suggest-tags", "d1", "-n", "3"], "suggest-tags", { documentId: "d1", limit: 3 }], + [["topics", "list", "--parent", "p"], "list-topics", { parent: "p" }], + [ + ["topics", "create", "n", "--description", "d", "--parent", "p"], + "create-topic", + { name: "n", description: "d", parent: "p" }, + ], + [ + ["topics", "delete", "t", "--delete-documents", "-y"], + "delete-topic", + { topic: "t", deleteDocuments: true }, + ], + [["tags", "list"], "list-tags", {}], + [ + ["searches", "save", "n", "q", "--topic", "t", "-n", "5", "--min-rating", "3"], + "save-search", + { name: "n", query: "q", topic: "t", limit: 5, minRating: 3 }, + ], + [ + ["searches", "list", "-n", "2", "--offset", "1"], + "list-saved-searches", + { limit: 2, offset: 1 }, + ], + [["searches", "run", "n"], "run-saved-search", { search: "n" }], + [["searches", "delete", "n"], "delete-saved-search", { search: "n" }], + [["sync", "conn"], "sync", { name: "conn" }], + [["sync", "--all"], "sync", { all: true }], + [ + ["disconnect", "conn", "--type", "slack", "--keep-documents", "-y"], + "disconnect", + { name: "conn", type: "slack", keepDocuments: true }, + ], + [["connections"], "list-connections", {}], + [ + [ + "pack", + "install", + "./p.json", + "--batch-size", + "5", + "--resume-from", + "2", + "--concurrency", + "3", + ], + "install-pack", + { pack: "./p.json", batchSize: 5, resumeFrom: 2, concurrency: 3 }, + ], + [["pack", "remove", "p", "-y"], "remove-pack", { pack: "p" }], + [["pack", "list"], "list-packs", {}], + [ + ["pack", "list", "--available", "--registry", "https://r.example"], + "list-packs", + { available: true, registryUrl: "https://r.example" }, + ], + [ + ["pack", "create", "--name", "p", "--from", "./src", "--exclude", "*.min.js"], + "create-pack", + { name: "p", from: ["./src"], outputPath: "p.json.gz", exclude: ["*.min.js"], recursive: true }, + ], + [ + ["pack", "create", "--name", "p", "--topic", "t", "--pack-version", "2.0.0"], + "create-pack", + { name: "p", topic: "t", version: "2.0.0", outputPath: "p.json" }, + ], + [ + ["webhooks", "create", "https://h.example", "--events", "document.created,document.deleted"], + "create-webhook", + { url: "https://h.example", events: ["document.created", "document.deleted"] }, + ], + [["webhooks", "list"], "list-webhooks", {}], + [["webhooks", "delete", "w1"], "delete-webhook", { webhookId: "w1" }], + [["webhooks", "test", "w1"], "test-webhook", { webhookId: "w1" }], + [["admin", "reindex", "--rebuild"], "reindex", { rebuild: true }], + [ + ["admin", "reindex", "--doc", "d1", "d2", "--since", "2024-01-01", "--batch-size", "10"], + "reindex", + { documentIds: ["d1", "d2"], since: "2024-01-01", batchSize: 10 }, + ], + [ + ["admin", "dedupe", "--threshold", "0.9", "--strategy", "exact"], + "dedupe", + { threshold: 0.9, strategy: "exact" }, + ], + [["admin", "backup", "kb.json"], "backup", { outputPath: "kb.json" }], + [["admin", "restore", "kb.json", "-y"], "restore", { backupPath: "kb.json" }], + [["admin", "prune"], "prune-expired", {}], +]; + +describe("CLI command -> operation contract", () => { + it.each(SINGLE)("%j calls %s", async (argv, opName, expected) => { + const calls = await invoke(argv); + expect(calls).toHaveLength(1); + expect(calls[0]?.[0]).toBe(opName); + expect(calls[0]?.[1]).toMatchObject(expected); + }); + + it("leaves unset flags out so operation defaults apply", async () => { + const [[, input]] = (await invoke(["search", "q"])) as [[string, unknown]]; + expect(input).toEqual({ query: "q" }); + }); + + it("adds each source with its own operation call", async () => { + const calls = await invoke(["add", "a.md", "b.md", "--library", "lib"]); + expect(calls.map(([name, input]) => [name, (input as { source: string }).source])).toEqual([ + ["add", "a.md"], + ["add", "b.md"], + ]); + }); + + it("docs delete looks the document up, then deletes it", async () => { + const calls = await invoke(["docs", "delete", "d1", "-y"]); + expect(calls).toEqual([ + ["get-document", { documentId: "d1", maxLength: 0 }], + ["delete-document", { documentId: "d1" }], + ]); + }); + + it.each([ + [["bulk", "delete", "--topic", "t", "-y"], "bulk-delete", { topic: "t" }], + [ + ["bulk", "retag", "--tags", "old", "--add", "a,b", "--remove", "c", "-y"], + "bulk-retag", + { tags: ["old"], addTags: ["a", "b"], removeTags: ["c"] }, + ], + [ + ["bulk", "move", "--library", "lib", "--since", "2024-01-01", "--to", "t2", "-y"], + "bulk-move", + { library: "lib", dateFrom: "2024-01-01", targetTopic: "t2" }, + ], + ])("%j previews (dryRun) and then applies %s", async (argv, opName, expected) => { + const calls = await invoke(argv); + expect(calls.map(([name]) => name)).toEqual([opName, opName]); + expect(calls[0]?.[1]).toEqual({ ...expected, dryRun: true }); + expect(calls[1]?.[1]).toEqual(expected); + }); + + it("bulk --dry-run only previews", async () => { + const calls = await invoke(["bulk", "delete", "--topic", "t", "--dry-run"]); + expect(calls).toEqual([["bulk-delete", { topic: "t", dryRun: true }]]); + }); + + it("admin stats combines overview, popular, stale and search analytics", async () => { + const calls = await invoke(["admin", "stats", "--days", "7"]); + expect(calls).toEqual([ + ["overview", {}], + ["popular", {}], + ["stale", { days: 7 }], + ["search-analytics", { days: 7 }], + ]); + }); + + it("connect saves the connection (with schedule) and skips the sync with --no-sync", async () => { + const vault = join(home, "vault"); + const calls = await invoke([ + "connect", + "obsidian", + vault, + "--name", + "notes", + "--schedule", + "0 * * * *", + "--no-sync", + ]); + expect(calls).toEqual([]); + const saved = JSON.parse( + readFileSync(join(home, ".libscope", "connectors", "notes.json"), "utf-8"), + ) as Record<string, unknown>; + expect(saved).toMatchObject({ + connectorType: "obsidian", + vaultPath: vault, + topicMapping: "folder", + excludePatterns: [], + schedule: { cronExpression: "0 * * * *" }, + }); + + // Re-running changes only what is given; "off" removes the schedule; then it syncs. + const again = await invoke(["connect", "obsidian", "--name", "notes", "--schedule", "off"]); + expect(again).toEqual([["sync", { name: "notes" }]]); + const updated = JSON.parse( + readFileSync(join(home, ".libscope", "connectors", "notes.json"), "utf-8"), + ) as Record<string, unknown>; + expect(updated["schedule"]).toBeUndefined(); + expect(updated["vaultPath"]).toBe(vault); + }); + + it("connect rejects missing required settings", async () => { + await main(["node", "libscope", "connect", "slack", "--no-sync"]); + expect(process.exitCode).toBe(1); + expect(stderrWrite.mock.calls.join("")).toContain("Invalid Slack settings: token"); + }); +}); + +/** Every leaf command path, e.g. "docs show". */ +function leafCommands(cmd: Command, prefix: string[] = []): string[] { + return cmd.commands.flatMap((sub) => { + const path = [...prefix, sub.name()]; + return sub.commands.length > 0 ? leafCommands(sub, path) : [path.join(" ")]; + }); +} + +/** The command path an argv selects, e.g. ["docs", "show", "d1"] -> "docs show". */ +function commandPath(argv: string[]): string { + const path: string[] = []; + let cmd: Command = program; + for (const word of argv) { + const sub = cmd.commands.find((c) => c.name() === word); + if (!sub) break; + path.push(word); + cmd = sub; + } + return path.join(" "); +} + +describe("CLI command set", () => { + /** Commands not covered above: they do not call operations (files, servers, registries). */ + const NOT_OPERATIONS = new Set([ + "serve", + "doctor", + "config show", + "config get", + "config set", + "config unset", + "config path", + "workspace create", + "workspace list", + "workspace use", + "workspace delete", + ]); + + it("covers every command with a contract test (registry commands are tested elsewhere)", () => { + const tested = new Set([ + ...SINGLE.map(([argv]) => commandPath(argv)), + "docs delete", + "bulk delete", + "bulk retag", + "bulk move", + "admin stats", + "connect", + ]); + const missing = leafCommands(program).filter( + (c) => !c.startsWith("registry ") && !NOT_OPERATIONS.has(c) && !tested.has(c), + ); + expect(missing).toEqual([]); + }); + + it.each([ + "init", + "import", + "import-batch", + "add-repo", + "repl", + "related", + "ratings", + "stats", + "search-analytics", + "watch", + "dedupe", + "reindex", + "export", + "import-backup", + "link", + "links", + "unlink", + "prereqs", + "tag", + "schedule", + "update", + ])("has no top-level %s command", (name) => { + expect(program.commands.map((c) => c.name())).not.toContain(name); + }); +}); diff --git a/tests/unit/cli-run.test.ts b/tests/unit/cli-run.test.ts new file mode 100644 index 00000000..a6217e45 --- /dev/null +++ b/tests/unit/cli-run.test.ts @@ -0,0 +1,160 @@ +/** + * CLI end-to-end over an in-memory database: add -> search -> docs show, --json output, + * and the one error handler (message, hint, stack only with --verbose, exit code 1). + */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ConfigError, DocumentNotFoundError, ValidationError } from "../../src/errors.js"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { testConfig } from "./operations/helpers.js"; + +const db = createTestDbWithVec(); + +vi.mock("../../src/core/bootstrap.js", () => ({ + bootstrap: (): unknown => ({ + db, + provider: new MockEmbeddingProvider(), + config: testConfig(), + dbPath: ":memory:", + close: (): void => {}, + }), +})); + +const { main } = await import("../../src/cli/index.js"); +const { errorHint, formatError } = await import("../../src/cli/errors.js"); + +let dir: string; +let stdout: string[]; +let stderr: string[]; + +async function cli(...args: string[]): Promise<{ out: string; err: string; code: number }> { + stdout = []; + stderr = []; + process.exitCode = undefined; + await main(["node", "libscope", ...args]); + const code = Number(process.exitCode ?? 0); + process.exitCode = undefined; + return { out: stdout.join("\n"), err: stderr.join(""), code }; +} + +beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), "libscope-cli-run-")); + writeFileSync( + join(dir, "widgets.md"), + "# Widget Guide\n\nWidgets are configured with widget.yaml. Set the color key to change the color.", + ); +}); + +afterAll(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +beforeEach(() => { + vi.spyOn(console, "log").mockImplementation((...args: unknown[]) => { + stdout.push(args.map(String).join(" ")); + }); + vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => { + stderr.push(String(chunk)); + return true; + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("CLI over a database", () => { + let documentId = ""; + + it("add prints the new document ID; --json prints the operation results", async () => { + const human = await cli("add", join(dir, "widgets.md"), "--tags", "guide"); + expect(human.code).toBe(0); + expect(human.out).toMatch(/✓ widgets \(1 chunk\) {2}[0-9a-f-]{36}/); + + // Adding the same file again updates the same document (the path is its URL). + const json = await cli("--json", "add", join(dir, "widgets.md")); + const results = JSON.parse(json.out) as Array<{ documents: Array<{ documentId: string }> }>; + documentId = results[0]?.documents[0]?.documentId ?? ""; + expect(human.out).toContain(documentId); + }); + + it("search shows document and chunk IDs; --json returns the ListResult", async () => { + const human = await cli("search", "widget color"); + expect(human.out).toContain(`document ${documentId} chunk `); + + const json = await cli("--json", "search", "widget color", "-n", "1"); + const result = JSON.parse(json.out) as { + items: Array<{ documentId: string; chunkId: string }>; + total: number; + limit: number; + }; + expect(result.limit).toBe(1); + expect(result.items[0]?.documentId).toBe(documentId); + expect(result.items[0]?.chunkId).toBeTruthy(); + }); + + it("docs show prints metadata, tags and content", async () => { + const { out, code } = await cli("docs", "show", documentId); + expect(code).toBe(0); + expect(out).toContain(`ID: ${documentId}`); + expect(out).toContain("Tags: guide"); + expect(out).toContain("Widgets are configured"); + }); + + it("docs list shows IDs", async () => { + const { out } = await cli("docs", "list"); + expect(out).toContain(`${documentId} widgets`); + }); + + it("ask without an LLM fails with the configuration hint", async () => { + const { err, code } = await cli("ask", "what configures widgets?"); + expect(code).toBe(1); + expect(err).toContain("✗ No LLM provider configured"); + expect(err).toContain("libscope doctor"); + }); + + it("not found: message, hint, no stack, exit code 1", async () => { + const { err, code } = await cli("docs", "show", "nope"); + expect(code).toBe(1); + expect(err).toContain("✗ Document not found: nope"); + expect(err).toContain("libscope docs list"); + expect(err).not.toContain(" at "); + }); + + it("--verbose adds the stack trace", async () => { + const { err } = await cli("--verbose", "docs", "show", "nope"); + expect(err).toContain("DocumentNotFoundError"); + expect(err).toContain(" at "); + }); + + it("invalid input is a validation error with a --help hint", async () => { + const { err, code } = await cli("docs", "rate", documentId, "9"); + expect(code).toBe(1); + expect(err).toContain("✗ Invalid input for rate-document"); + expect(err).toContain("--help"); + }); + + it("search without a query outside a terminal asks for one", async () => { + const { err, code } = await cli("--json", "search"); + expect(code).toBe(1); + expect(err).toContain("Give a search query"); + }); +}); + +describe("error hints", () => { + it("suggests a next step per error type", () => { + expect(errorHint(new DocumentNotFoundError("x"))).toContain("docs list"); + expect(errorHint(new ConfigError("x"))).toContain("doctor"); + expect(errorHint(new ValidationError("x"))).toContain("--help"); + expect(errorHint(new Error("x"))).toBeUndefined(); + }); + + it("formats the message first and the stack only when verbose", () => { + const err = new Error("boom"); + expect(formatError(err, false)).toEqual(["✗ boom"]); + expect(formatError(err, true).join("\n")).toContain("Error: boom"); + }); +}); diff --git a/tests/unit/cli-version-options.test.ts b/tests/unit/cli-version-options.test.ts index d6f8c5b0..c16e2962 100644 --- a/tests/unit/cli-version-options.test.ts +++ b/tests/unit/cli-version-options.test.ts @@ -14,7 +14,7 @@ vi.mock("../../src/registry/publish.js", () => ({ })); import { program } from "../../src/cli/index.js"; -import { registerRegistryCommands } from "../../src/cli/commands/registry.js"; +import { register as registerRegistryCommands } from "../../src/cli/commands/registry.js"; import { unpublishPack } from "../../src/registry/publish.js"; const pkgVersion = ( @@ -73,8 +73,8 @@ describe("CLI subcommand version flags", () => { it.each([ [["add"], ["doc.md"]], - [["import"], ["./docs"]], - [["import-batch"], ["./docs"]], + [["search"], ["query"]], + [["docs", "list"], []], [["docs", "update"], ["doc-1"]], ])("%j accepts --lib-version", async (path, positional) => { const opts = await parseWithStubbedAction(path, [...positional, "--lib-version", "2.1.0"]); diff --git a/tests/unit/repl.test.ts b/tests/unit/repl.test.ts index beb4ea92..fa1eace7 100644 --- a/tests/unit/repl.test.ts +++ b/tests/unit/repl.test.ts @@ -1,129 +1,60 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import type { MockInstance } from "vitest"; -import { startRepl } from "../../src/cli/repl.js"; -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../../src/providers/embedding.js"; -import { EventEmitter } from "node:events"; +import { startInteractiveSearch } from "../../src/cli/repl.js"; +import type { Interface } from "node:readline/promises"; -/** Minimal mock readline.Interface that yields pre-programmed answers. */ -function createMockInterface(inputs: string[]) { +/** Minimal readline.Interface that yields pre-programmed answers, then "closes". */ +function createMockInterface(inputs: string[]): { iface: Interface; closeFn: MockInstance } { const queue = [...inputs]; - const emitter = new EventEmitter(); - - const questionFn = vi.fn(() => { - const next = queue.shift(); - if (next === undefined) return Promise.reject(new Error("closed")); - return Promise.resolve(next); - }); const closeFn = vi.fn(); - const iface = { - question: questionFn, + question: vi.fn(() => { + const next = queue.shift(); + return next === undefined ? Promise.reject(new Error("closed")) : Promise.resolve(next); + }), close: closeFn, - on: emitter.on.bind(emitter), - } as unknown as import("node:readline/promises").Interface; - - return { iface, questionFn, closeFn }; -} - -function createMockDb() { - return {} as unknown as Database.Database; -} - -function createMockProvider() { - return { - dimensions: 3, - generateEmbedding: vi.fn(() => Promise.resolve(new Float32Array([0.1, 0.2, 0.3]))), - } as unknown as EmbeddingProvider; + } as unknown as Interface; + return { iface, closeFn }; } -describe("startRepl", () => { - let consoleSpy: MockInstance; - let consoleErrorSpy: MockInstance; +describe("startInteractiveSearch", () => { + let log: MockInstance; + let error: MockInstance; beforeEach(() => { - consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + log = vi.spyOn(console, "log").mockImplementation(() => {}); + error = vi.spyOn(console, "error").mockImplementation(() => {}); }); afterEach(() => { - consoleSpy.mockRestore(); - consoleErrorSpy.mockRestore(); - }); - - it("exits on 'quit' command", async () => { - const { iface, closeFn } = createMockInterface(["quit"]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(closeFn).toHaveBeenCalled(); - expect(consoleSpy).toHaveBeenCalledWith("Goodbye!"); - }); - - it("exits on 'exit' command", async () => { - const { iface, closeFn } = createMockInterface(["exit"]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(closeFn).toHaveBeenCalled(); - expect(consoleSpy).toHaveBeenCalledWith("Goodbye!"); - }); - - it("skips empty input lines", async () => { - const { iface, questionFn, closeFn } = createMockInterface(["", " ", "quit"]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(questionFn).toHaveBeenCalledTimes(3); - expect(closeFn).toHaveBeenCalled(); - }); - - it("exits gracefully on readline close (Ctrl+C)", async () => { - const { iface, closeFn } = createMockInterface([]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(closeFn).toHaveBeenCalled(); - }); - - it("handles search errors gracefully", async () => { - const { iface, closeFn } = createMockInterface(["test query", "quit"]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("Search error:")); - expect(closeFn).toHaveBeenCalled(); - }); - - it("prints banner on start", async () => { - const { iface } = createMockInterface(["quit"]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("LibScope interactive search")); + log.mockRestore(); + error.mockRestore(); + }); + + it("runs each non-empty query and stops on 'quit'", async () => { + const { iface, closeFn } = createMockInterface([" first ", "", "second", "quit", "never"]); + const onQuery = vi.fn(() => Promise.resolve()); + await startInteractiveSearch({ onQuery, createInterface: () => iface }); + expect(onQuery.mock.calls).toEqual([["first"], ["second"]]); + expect(closeFn).toHaveBeenCalledOnce(); + }); + + it("stops when input closes (Ctrl+D)", async () => { + const { iface, closeFn } = createMockInterface(["one"]); + const onQuery = vi.fn(() => Promise.resolve()); + await startInteractiveSearch({ onQuery, createInterface: () => iface }); + expect(onQuery).toHaveBeenCalledTimes(1); + expect(closeFn).toHaveBeenCalledOnce(); + }); + + it("prints an error and keeps going when a query fails", async () => { + const { iface } = createMockInterface(["bad", "good", "exit"]); + const onQuery = vi + .fn<(q: string) => Promise<void>>() + .mockRejectedValueOnce(new Error("boom")) + .mockResolvedValueOnce(undefined); + await startInteractiveSearch({ onQuery, createInterface: () => iface }); + expect(error).toHaveBeenCalledWith("✗ boom"); + expect(onQuery).toHaveBeenCalledTimes(2); }); }); diff --git a/tests/unit/reporter.test.ts b/tests/unit/reporter.test.ts index b63d8d79..7fc4995e 100644 --- a/tests/unit/reporter.test.ts +++ b/tests/unit/reporter.test.ts @@ -1,172 +1,49 @@ -import { describe, it, expect, vi, afterEach } from "vitest"; -import { isVerbose, createReporter } from "../../src/cli/reporter.js"; - -describe("reporter", () => { - afterEach(() => { - delete process.env["LIBSCOPE_VERBOSE"]; - vi.restoreAllMocks(); +import { describe, it, expect, vi } from "vitest"; +import { createProgressLine, formatProgress } from "../../src/cli/reporter.js"; + +function fakeStream(isTTY: boolean): { stream: NodeJS.WriteStream; written: string[] } { + const written: string[] = []; + const stream = { + isTTY, + columns: 60, + write: vi.fn((s: string) => { + written.push(s); + return true; + }), + } as unknown as NodeJS.WriteStream; + return { stream, written }; +} + +describe("formatProgress", () => { + it("shows a bar, percent and counts when the total is known", () => { + const line = formatProgress({ done: 5, total: 10, message: "docs/a.md" }); + expect(line).toContain("50% (5/10) docs/a.md"); + expect(line).toContain("█".repeat(10) + "░".repeat(10)); }); - describe("isVerbose", () => { - it("returns true when verbose flag is set", () => { - expect(isVerbose(true)).toBe(true); - }); - - it("returns false when verbose flag is false", () => { - expect(isVerbose(false)).toBe(false); - }); - - it("returns false when verbose flag is undefined", () => { - expect(isVerbose(undefined)).toBe(false); - }); - - it("returns true when LIBSCOPE_VERBOSE=1 env var is set", () => { - process.env["LIBSCOPE_VERBOSE"] = "1"; - expect(isVerbose(false)).toBe(true); - }); - - it("returns false when LIBSCOPE_VERBOSE=0", () => { - process.env["LIBSCOPE_VERBOSE"] = "0"; - expect(isVerbose(false)).toBe(false); - }); + it("shows only the count without a total, and shortens long labels", () => { + const line = formatProgress({ done: 3, message: "x".repeat(80) }); + expect(line.startsWith("(3) ...")).toBe(true); + expect(line.length).toBeLessThan(60); }); +}); - describe("createReporter", () => { - it("returns a SilentReporter (no-op) in verbose mode", () => { - const reporter = createReporter(true); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); - - reporter.log("hello"); - reporter.success("done"); - reporter.warn("careful"); - reporter.error("bad"); - reporter.progress(1, 10, "task"); - reporter.clearProgress(); - - expect(stdout).not.toHaveBeenCalled(); - expect(stderr).not.toHaveBeenCalled(); - }); - - it("returns a SilentReporter when LIBSCOPE_VERBOSE=1", () => { - process.env["LIBSCOPE_VERBOSE"] = "1"; - const reporter = createReporter(); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.log("hello"); - expect(stdout).not.toHaveBeenCalled(); - }); - - it("PrettyReporter.log writes to stdout", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.log("test message"); - - expect(stdout).toHaveBeenCalledOnce(); - expect(String(stdout.mock.calls[0]![0])).toContain("test message"); - }); - - it("PrettyReporter.success writes green checkmark to stdout", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.success("all done"); - - const output = String(stdout.mock.calls[0]![0]); - expect(output).toContain("all done"); - // Green ANSI code - expect(output).toContain("\x1b[32m"); - }); - - it("PrettyReporter.warn writes to stderr", () => { - const reporter = createReporter(false); - const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); - - reporter.warn("watch out"); - - expect(stderr).toHaveBeenCalledOnce(); - expect(String(stderr.mock.calls[0]![0])).toContain("watch out"); - }); - - it("PrettyReporter.error writes to stderr", () => { - const reporter = createReporter(false); - const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); - - reporter.error("something failed"); - - expect(stderr).toHaveBeenCalledOnce(); - expect(String(stderr.mock.calls[0]![0])).toContain("something failed"); - }); - - it("PrettyReporter.progress writes \\r-prefixed line to stdout", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.progress(3, 10, "indexing doc"); - - const output = String(stdout.mock.calls[0]![0]); - expect(output).toMatch(/^\r/); - expect(output).toContain("3/10"); - expect(output).toContain("30%"); - }); - - it("PrettyReporter.clearProgress clears the progress line", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.progress(1, 5, "working"); - stdout.mockClear(); - - reporter.clearProgress(); - - // Should write spaces to clear the line - const output = String(stdout.mock.calls[0]![0]); - expect(output).toMatch(/^\r\s+\r$/); - }); - - it("PrettyReporter.clearProgress is a no-op when no progress shown", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.clearProgress(); - - expect(stdout).not.toHaveBeenCalled(); - }); - - it("PrettyReporter.log clears progress before writing", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.progress(1, 5, "working"); - stdout.mockClear(); - - reporter.log("a message"); - - // First call should be the clear, second the message - expect(stdout.mock.calls.length).toBeGreaterThanOrEqual(2); - const clearCall = String(stdout.mock.calls[0]![0]); - expect(clearCall).toMatch(/^\r\s+\r$/); - }); - - it("PrettyReporter.progress truncates long labels", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.progress(1, 1, "a".repeat(50)); - - const output = String(stdout.mock.calls[0]![0]); - expect(output).toContain("..."); - }); - - it("PrettyReporter.progress handles zero total gracefully", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.progress(0, 0, "starting"); +describe("createProgressLine", () => { + it("writes and clears one line on a terminal", () => { + const { stream, written } = fakeStream(true); + const progress = createProgressLine(stream); + progress.update({ done: 1, total: 2 }); + progress.clear(); + expect(written).toHaveLength(2); + expect(written[0]?.startsWith("\r[")).toBe(true); + expect(written[1]?.trim()).toBe(""); + }); - const output = String(stdout.mock.calls[0]![0]); - expect(output).toContain("0%"); - }); + it("writes nothing when the stream is not a terminal", () => { + const { stream, written } = fakeStream(false); + const progress = createProgressLine(stream); + progress.update({ done: 1, total: 2 }); + progress.clear(); + expect(written).toEqual([]); }); }); From 61215c466d5ac6c8e5c85c04863a8dbc951ca2b6 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:19:13 +0000 Subject: [PATCH 51/83] docs(cli): reference, getting started and migration table for the 2.0 CLI --- docs/guide/getting-started.md | 44 +-- docs/migration-v2.md | 76 +++++ docs/reference/cli.md | 540 +++++++++++++++++----------------- src/cli/errors.ts | 1 + 4 files changed, 374 insertions(+), 287 deletions(-) diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index bb9fbc6f..fca72ab5 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -12,15 +12,13 @@ npm install -g libscope Requires Node.js 20 or later. -## Initialize - -Create the database: +## Check the Setup ```bash -libscope init +libscope doctor ``` -This sets up a SQLite database at `~/.libscope/workspaces/default/libscope.db` (the active workspace's database) with vector search and full-text indexing. The command prints the path it used. On first run, it also downloads the local embedding model (~80MB). +This shows the config files, the active workspace and its database (`~/.libscope/workspaces/default/libscope.db`), the embedding model, and which LLM `ask` will use. Each warning comes with the command that fixes it. You do not have to create anything first: the database is created when you add the first document (or run `libscope doctor --fix`). The first command that embeds text downloads the local embedding model (~80MB, once) and prints one line while it does. ## Index Some Documents @@ -31,10 +29,15 @@ libscope add ./docs/getting-started.md --library my-lib # A URL (fetches and converts to markdown automatically) libscope add https://docs.example.com/guide -# An entire directory -libscope import ./docs/ --library my-lib --extensions .md,.mdx +# An entire directory (keep it in sync with --watch) +libscope add ./docs/ --library my-lib --include "*.md,*.mdx" + +# A GitHub or GitLab repository +libscope add https://github.com/org/repo --path docs ``` +Each added document is printed with its ID. Add `--dry-run` to see what would be added. + LibScope supports **Markdown** (`.md`, `.mdx`) and **plain text** natively. Additional formats — **PDF** (`.pdf`), **Word** (`.docx`), **EPUB** (`.epub`), and **PowerPoint** (`.pptx`) — are available via optional dependencies that install automatically. See the [README](/) for the full format table. Each document gets chunked by heading, embedded into vectors, and stored in the database. @@ -46,20 +49,27 @@ Each document gets chunked by heading, embedded into vectors, and stored in the libscope search "how to authenticate" # Filtered by library and topic -libscope search "API rate limiting" --library my-lib --topic security --limit 10 +libscope search "API rate limiting" --library my-lib --topic security -n 10 + +# Interactive: type queries one after another +libscope search + +# JSON for scripts +libscope search "API rate limiting" --json ``` -Results are ranked by vector similarity, with optional FTS5 boosting for keyword matches. +Results are ranked by vector similarity, with optional FTS5 boosting for keyword matches. Each result shows its document ID and chunk ID; `libscope docs show <documentId>` prints the whole document. ## Ask Questions If you have an LLM provider configured (OpenAI, Ollama, or Anthropic), you can ask questions and get synthesized answers with source citations: ```bash +libscope config set openai.apiKey sk-... # or anthropic.apiKey; stored in ~/.libscope/secrets.json libscope ask "What is the recommended auth flow?" ``` -See [Configuration](/guide/configuration) for LLM setup. +Without an LLM, `ask` stops with a hint and `libscope doctor` shows what to set. See [Configuration](/guide/configuration) for LLM setup. ## Start the MCP Server @@ -74,7 +84,7 @@ This starts a stdio-based MCP server that any compatible AI assistant can connec Run the local web dashboard to browse, search, and manage your knowledge base in a browser: ```bash -libscope serve --dashboard +libscope serve dashboard # opens at http://localhost:3377 ``` @@ -86,22 +96,22 @@ Once you have content indexed you can enrich it: ```bash # Tag documents -libscope tag add <doc-id> typescript,api,v2 +libscope docs tag <documentId> typescript,api,v2 # Group into topics libscope topics create "backend" libscope topics create "auth" --parent backend # Save frequent searches -libscope search "auth best practices" --save "Auth Docs" +libscope searches save "Auth Docs" "auth best practices" --topic auth libscope searches run "Auth Docs" # Cross-reference documents -libscope link <source-id> <target-id> --type prerequisite +libscope docs link <documentId> <targetDocumentId> --type prerequisite # Bulk operations -libscope bulk retag --library react --add-tags deprecated --dry-run -libscope bulk move --library react --to new-topic-id +libscope bulk retag --library react --add deprecated --dry-run +libscope bulk move --library react --to frontend ``` ## REST API @@ -109,7 +119,7 @@ libscope bulk move --library react --to new-topic-id For programmatic access, start the REST API instead of the MCP server: ```bash -libscope serve --api --port 3378 +libscope serve api --port 3378 ``` The OpenAPI 3.0 spec is served at `GET /openapi.json`. See [REST API Reference](/reference/rest-api) for full documentation. diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 1d83e537..171380c1 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -69,3 +69,79 @@ Every "not found" error is now a `NotFoundError` (`DocumentNotFoundError`, `Chun | `getVersion`, `rollbackToVersion` with an unknown version | `DocumentNotFoundError` | `NotFoundError` (`VERSION_NOT_FOUND`) | | `removePack` for a pack that is not installed | `ValidationError` | `NotFoundError` (`PACK_NOT_FOUND`) | | `getRelatedChunks` with an unknown chunk ID | `Error` | `ChunkNotFoundError` (`CHUNK_NOT_FOUND`) | + +## CLI + +The CLI has one command per task, and every command calls the same operation as the MCP server, REST API and SDK. The 1.x commands below are removed without aliases. See the [CLI reference](reference/cli.md). + +### Removed and renamed commands + +| 1.x command | 2.0 command | +| ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | +| `init` | Not needed: the database is created on first use. `doctor` shows the setup; `doctor --fix` creates the database. | +| `add <fileOrUrl>` | `add <sources...>` (several sources at once) | +| `import <directory> [--extensions .md,.txt]` | `add <directory> [--include "*.md,*.txt"]` | +| `import-batch <directory> [--filter <glob>] [--dry-run]` | `add <directory> [--include <globs>] [--dry-run]` | +| `add-repo <url> [--path <dir>]` | `add <repo-url> [--path <dirs>]` (`--branch`, `--extensions`, `--token` are unchanged) | +| `watch <directory>` | `add <directory> --watch` | +| `search <query>` (query required) | `search [query]` | +| `repl` | `search` with no query (on a terminal) | +| `related <chunkId>` | `search --related <documentId or chunkId>` | +| `search <query> --save <name>` | `searches save <name> <query> [filters]` | +| `ratings show <documentId>` | `docs show <documentId>` (shows the rating summary) | +| `link <sourceId> <targetId>` | `docs link <documentId> <targetDocumentId> --type <type>` | +| `links <documentId>` | `docs links [documentId]` | +| `unlink <linkId>` | `docs unlink <linkId>` | +| `prereqs <documentId>` | `docs prereqs <documentId>` | +| `tag add <docId> <tags...>` | `docs tag <documentId> <tags...>` | +| `tag remove <docId> <tag>` | `docs untag <documentId> <tags...>` | +| `tag list` | `tags list` | +| `tag suggest <documentId>` | `docs suggest-tags <documentId>` | +| `stats`, `stats overview`, `stats popular`, `stats stale`, `stats queries`, `stats search-analytics` | `admin stats` (one report) | +| `reindex` | `admin reindex` | +| `dedupe` | `admin dedupe` | +| `export <outputPath>` | `admin backup <file>` | +| `import-backup <backupPath>` | `admin restore <file>` | +| `serve --api` | `serve api` | +| `serve --dashboard` | `serve dashboard` | +| `connect notion`, `slack`, `confluence`, `onenote`, `obsidian` with `--sync` | `sync <name>` (or `sync --all`) | +| `connect confluence --url <url> --type server` | `connect confluence <url> --server` | +| `disconnect notion`, `slack`, `confluence`, `onenote` `[--name <n>]` | `disconnect <name>` (the connection name; the default name is the type) | +| `disconnect obsidian <vault-path>` | `disconnect <name>` (the vault path comes from the saved connection) | +| `schedule set <connector> <cron>` | `connect <type> --name <connector> --schedule <cron> --no-sync` | +| `schedule remove <connector>` | `connect <type> --name <connector> --schedule off --no-sync` | +| `schedule list` | `connections` | +| `update` (self-update) | `npm install -g libscope@latest` | + +New in 2.0: `doctor`, `docs rate`, `topics delete`, `admin prune`, `connect docs <site-url>`, `disconnect --keep-documents`, `connections`. + +### Removed and renamed options + +| 1.x option | 2.0 option | +| ------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------- | +| `search --source <type>` | `--source-type <type>` (the same on every command that filters documents) | +| `search --limit <n>` (default 5) | `-n, --limit <n>` (default 10) | +| `search --max-chunks-per-doc <n>` | `search --max-per-doc <n>` | +| `ask --top-k <n>` | `ask -n <n>` | +| `related --min-score <n>` | Removed | +| `add --exclude-patterns <globs...>` (crawl) | `add --exclude-urls <globs>` (comma-separated) | +| `bulk retag --add-tags`, `--remove-tags` | `bulk retag --add`, `--remove` | +| `link --type` defaulted to `related` | `docs link --type <type>` is required | +| `stats stale --days`, `stats search-analytics --days` | `admin stats --days <n>` | +| `connect slack --exclude`, `connect confluence --exclude-spaces` | `connect <type> --exclude <list>` for every type | +| `connect onenote` read `ONENOTE_CLIENT_ID` and `ONENOTE_TENANT_ID` | `connect onenote --client-id <id> --tenant-id <id>` | +| `connect notion` read `NOTION_TOKEN`; `connect confluence` read `CONFLUENCE_URL`, `CONFLUENCE_EMAIL`, `CONFLUENCE_TOKEN` | Give `--token`, the URL and `--email` on the command line | +| `-v, --verbose` only enabled debug logs | `-v, --verbose` also prints the stack trace when a command fails | + +### Behavior changes + +| 1.x | 2.0 | +| ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| Errors were printed in several formats, some with a stack trace. | Every error prints `✗ <message>` and a next step on stderr, and exits with code 1. Add `--verbose` for the stack trace. | +| No machine-readable output. | The global `--json` option prints the operation result as JSON. | +| Some results did not show IDs. | Every result shows document IDs; search results also show chunk IDs. | +| `add <file>` stored no URL, so adding a file twice made two documents. | The file's absolute path is stored as its URL. Adding a changed file again replaces the old version. | +| `--topic` took a topic ID. | `--topic` takes a topic ID or name on every command. | +| `connect <type>` always started from scratch. | `connect <type>` creates or updates a saved connection (only the given settings change) and syncs it. | +| `import-backup` did not ask for confirmation. | `admin restore` asks; add `-y` to skip the question. | +| Import progress was printed on stdout. | Progress is one line on stderr, only on a terminal. | diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a62ef2f7..a2d61ba8 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1,269 +1,271 @@ # CLI Commands -## Core Commands +Every command is a thin wrapper over one LibScope operation, so the CLI, the MCP server, the REST API and the SDK use the same parameters and defaults. Upgrading from 1.x? See [Migrating to LibScope 2.0](/migration-v2#cli). -| Command | Description | -| ----------------------------------- | ---------------------------------------------------------------------- | -| `libscope init` | Initialize the database | -| `libscope add <fileOrUrl>` | Index a document from a file or URL | -| `libscope import <directory>` | Bulk import files from a directory | -| `libscope import-batch <directory>` | Batch import with parallel processing | -| `libscope search <query>` | Semantic search | -| `libscope ask <question>` | RAG question-answering | -| `libscope repl` | Interactive search REPL | -| `libscope serve` | Start MCP server, REST API (`--api`), or web dashboard (`--dashboard`) | +| Command | What it does | +| ------------------------------------------------------------- | -------------------------------------------------------------- | +| [`add`](#libscope-add) | Add files, directories, URLs (optionally crawled) or repos | +| [`search`](#libscope-search) | Search by meaning and keywords; interactive with no query | +| [`ask`](#libscope-ask) | Answer a question with the configured LLM | +| [`docs`](#libscope-docs) | Show, change, link, tag, rate and version single documents | +| [`topics`, `tags`](#topics-and-tags) | Manage topics; list tags | +| [`searches`](#libscope-searches) | Save and re-run searches | +| [`bulk`](#libscope-bulk) | Delete, retag or move every document that matches filters | +| [`connect`, `sync`, `disconnect`, `connections`](#connectors) | Notion, Slack, Confluence, Obsidian, OneNote and doc sites | +| [`pack`](#knowledge-packs), [`registry`](#pack-registries) | Knowledge packs and git pack registries | +| [`serve`](#libscope-serve) | MCP server, REST API or web dashboard | +| [`config`](#libscope-config), [`workspace`](#workspaces) | Settings and separate knowledge bases | +| [`webhooks`](#libscope-webhooks) | Signed POSTs on document events | +| [`admin`](#libscope-admin) | Reindex, find duplicates, backup/restore, statistics, prune | +| [`doctor`](#libscope-doctor) | Check the setup; `--fix` creates the database and vector index | -### `libscope add` +## Global options -Index a single file or URL. +These go before or after the command name. -```bash -libscope add ./path/to/doc.md --topic deployment --title "Deploy Guide" -libscope add https://reactjs.org/docs/hooks-intro.html --library react -``` +| Option | Description | +| --------------------- | --------------------------------------------------------------- | +| `--json` | Print the operation result as JSON (for scripts) | +| `-v, --verbose` | Debug logging, and the stack trace when a command fails | +| `--log-level <level>` | `debug`, `info`, `warn`, `error` or `silent` (default `silent`) | +| `--workspace <name>` | Use this workspace instead of the active one | -| Option | Description | -| --------------------- | --------------------------- | -| `--library <name>` | Assign to a library | -| `--topic <name>` | Assign to a topic | -| `--title <title>` | Override the document title | -| `--lib-version <ver>` | Library version tag | +Every result shows document IDs (and chunk IDs for search hits), so you can pass them to the next command. -### `libscope import` +**Errors.** A failed command prints `✗ <message>` and a next step on stderr, and exits with code 1. For example, an unknown document ID suggests `libscope docs list`, and a configuration problem suggests `libscope doctor`. Add `--verbose` to see the stack trace. -Recursively import files from a directory. +**Progress.** Long operations (directory imports, crawls, reindexing, pack installs) show a progress line on stderr when it is a terminal. The first command that needs the local embedding model prints one line while the model downloads (about 80 MB, first run only). -```bash -libscope import ./wiki/ --topic internal --extensions .md,.mdx,.txt -``` +### Document filters -| Option | Description | -| --------------------- | ---------------------------------------------------------------- | -| `--library <name>` | Assign all docs to a library | -| `--topic <name>` | Assign all docs to a topic | -| `--lib-version <ver>` | Library version tag | -| `--extensions <exts>` | Comma-separated file extensions (default: all supported formats) | +Commands that select documents (`search`, `ask`, `docs list`, `searches save`, `bulk`) share these flags: -### `libscope import-batch` +| Flag | Description | +| ---------------------- | -------------------------------------------------------------- | +| `--topic <topic>` | Topic ID or name | +| `--library <name>` | Library name | +| `--lib-version <ver>` | Library version (not on `bulk`) | +| `--source-type <type>` | `library`, `topic`, `manual` or `model-generated` | +| `--tags <a,b>` | Only documents with all these tags | +| `-n, --limit <n>` | Maximum results (`ask`: chunks used as context; not on `bulk`) | -Parallel batch import with progress tracking. Files are parsed by format (PDF, Word, EPUB, PowerPoint, HTML, and so on). Files with an unsupported extension are skipped with a warning. +## `libscope add` ```bash -libscope import-batch ./docs/ --concurrency 10 --filter "**/*.md" --library my-lib +libscope add <sources...> [options] ``` -| Option | Description | -| --------------------- | ------------------------------------------------------------ | -| `--concurrency <n>` | Number of parallel workers | -| `--filter <glob>` | Glob pattern to match files (default: all supported formats) | -| `--library <name>` | Assign all docs to a library | -| `--lib-version <ver>` | Library version tag | - -### `libscope search` +Each source can be a file, a directory, a URL or a GitHub/GitLab repository URL. LibScope detects which one it is. A local file's absolute path is stored as its URL, so adding a changed file again replaces the old version. ```bash -libscope search "authentication best practices" --library my-lib --limit 10 -libscope search "deploy process" --context 1 # include neighboring chunks +libscope add ./guide.md --topic deployment --tags ops,runbook +libscope add ./docs --include "**/*.md" --exclude "drafts/**" --library my-lib --lib-version 2.1 +libscope add ./docs --watch # keep re-indexing changed files (Ctrl+C to stop) +libscope add https://react.dev/learn --library react +libscope add https://docs.example.com --spider --max-pages 50 --path-prefix /guide +libscope add https://github.com/org/repo --branch main --path docs +libscope add ./docs --dry-run # list what would be added ``` -| Option | Description | -| -------------------------- | ----------------------------------------------------------------------------- | -| `--library <name>` | Filter by library | -| `--topic <name>` | Filter by topic | -| `--source <type>` | Filter by source type (e.g., `library`, `topic`, `manual`, `model-generated`) | -| `--limit <n>` | Max results (default: 10) | -| `--min-rating <n>` | Minimum average rating | -| `--max-chunks-per-doc <n>` | Max chunks per document in results (default: no limit) | -| `--context <n>` | Include N neighboring chunks before/after each result (0-2, default: 0) | -| `--save <name>` | Save this search with the given name for later re-use | - -### `libscope searches` - -Manage saved searches. +| Option | Applies to | Description | +| ------------------------ | ---------- | ---------------------------------------------------------------------------------------------------- | +| `--topic <topic>` | all | Topic ID or name (the topic must exist) | +| `--library <name>` | all | Library name | +| `--lib-version <ver>` | all | Library version | +| `--source-type <type>` | all | Default: `library` with `--library`, `topic` with `--topic`, else `manual` (repositories: `library`) | +| `--tags <a,b>` | all | Tags to add to every new document | +| `--title <title>` | file, URL | Title (default: file name or page title) | +| `--format <ext>` | files | Read files as this format, e.g. `.pdf` | +| `--dedup <mode>` | all | `skip`, `warn` or `force` | +| `--expires <time>` | all | ISO 8601 time after which `admin prune` deletes the documents | +| `--include <globs>` | directory | Only files matching these globs (comma-separated) | +| `--exclude <globs>` | directory | Skip files matching these globs (comma-separated) | +| `--watch` | directory | Keep running and re-index files when they change | +| `--spider` | URL | Also crawl linked pages | +| `--max-pages <n>` | crawl | Page limit (default 25, max 200) | +| `--max-depth <n>` | crawl | Link depth (default 2, max 5) | +| `--no-same-domain` | crawl | Also follow links to other domains | +| `--path-prefix <path>` | crawl | Only follow links under this path | +| `--exclude-urls <globs>` | crawl | Skip URLs matching these globs (comma-separated) | +| `--branch <name>` | repository | Branch (default: from the URL, else `main`) | +| `--path <dirs>` | repository | Only these subdirectories (comma-separated) | +| `--extensions <exts>` | repository | File extensions (default `.md,.mdx,.txt,.rst`) | +| `--token <token>` | repository | Access token for a private repository | +| `--dry-run` | all | List what would be added without adding it | + +With `--json`, `add` prints one result per source: `{ kind, documents: [{ documentId, title, chunkCount, source }], errors, skipped, planned?, crawl? }`. + +## `libscope search` ```bash -libscope searches list # list all saved searches -libscope searches run "My Search" # re-run a saved search by name or ID -libscope searches delete "My Search" # delete a saved search -libscope search "auth best practices" --save "Auth Docs" # save a search while running it +libscope search [query] [options] ``` -| Subcommand | Description | -| ---------------------------- | -------------------------------- | -| `searches list` | List all saved searches | -| `searches run <nameOrId>` | Run a saved search by name or ID | -| `searches delete <nameOrId>` | Delete a saved search | - -### `libscope ask` - ```bash -libscope ask "How do I configure OAuth2?" --library my-lib --top-k 8 +libscope search "authentication best practices" --library my-lib -n 10 +libscope search "deploy process" --context 1 # show neighbouring chunks +libscope search --related <documentId|chunkId> # content similar to a document or chunk +libscope search # interactive search (on a terminal) ``` -| Option | Description | -| ------------------ | ---------------------------------------- | -| `--library <name>` | Filter source documents by library | -| `--topic <name>` | Filter source documents by topic | -| `--top-k <n>` | Number of chunks to retrieve for context | +Takes the [document filters](#document-filters) and: + +| Option | Description | +| ------------------- | ------------------------------------------------------ | +| `--related <id>` | Find content similar to this document or chunk ID | +| `--offset <n>` | Results to skip (paging) | +| `--min-rating <n>` | Only documents with at least this average rating (1-5) | +| `--max-per-doc <n>` | At most this many chunks per document | +| `--context <n>` | Neighbouring chunks to show around each result (0-2) | -### `libscope serve` +With no query and no `--related`, on a terminal, `search` starts interactive mode: type a query, see results, repeat. Type `quit`, `exit` or press Ctrl+D to leave. The filters given on the command line apply to every query. -Start the MCP server, REST API, or web dashboard. +## `libscope ask` ```bash -libscope serve # MCP server (stdio) -libscope serve --api # REST API (port 3378) -libscope serve --dashboard # Web dashboard UI (port 3377) -libscope serve --dashboard --port 8080 +libscope ask "How do I configure OAuth2?" --library my-lib -n 8 +libscope ask "What changed in v2?" --model gpt-4o-mini ``` -| Option | Description | -| ------------- | ------------------------------------------------------- | -| `--api` | Start REST API instead of MCP server | -| `--dashboard` | Start the web dashboard UI | -| `--port <n>` | Server port (default: 3378 for API, 3377 for dashboard) | -| `--host <h>` | Server host (default: localhost) | +Takes the [document filters](#document-filters) (`-n` is the number of chunks used as context, default 5), `--min-rating <n>` and `--model <model>` (LLM model for this question). -## Document Management +`ask` needs an LLM. With `llm.provider` set to `auto` (the default), LibScope uses OpenAI when an OpenAI API key is set, else Anthropic when an Anthropic key is set, else Ollama when `llm.url` is set or the embedding provider is Ollama. Without an LLM, `ask` fails with a hint; `libscope doctor` shows what is resolved. With `llm.provider passthrough`, `ask` prints the retrieved context instead of an answer. -| Command | Description | -| --------------------------------------- | ------------------------------ | -| `libscope docs list` | List indexed documents | -| `libscope docs show <id>` | Show a specific document | -| `libscope docs update <id>` | Update a document | -| `libscope docs delete <id>` | Delete a document | -| `libscope docs history <id>` | View version history | -| `libscope docs rollback <id> <version>` | Rollback to a previous version | +## `libscope docs` -## Document Updates +| Command | Description | +| -------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| `docs list [filters] [--offset <n>]` | List documents, newest first | +| `docs show <documentId> [--offset <n>] [--max-length <n>]` | Show a document with its tags, links and rating summary | +| `docs update <documentId> [options]` | Change title, content, metadata or tags | +| `docs delete <documentId> [-y]` | Delete a document with its chunks, links and ratings | +| `docs history <documentId>` | List saved versions | +| `docs rollback <documentId> <version>` | Restore a saved version (the current state is saved first) | +| `docs rate <documentId> <1-5> [--feedback <text>] [--correction <text>] [--chunk <chunkId>]` | Rate a document or one chunk | +| `docs link <documentId> <targetDocumentId> --type <type> [--label <text>]` | Link two documents (`see_also`, `prerequisite`, `supersedes`, `related`, `references`) | +| `docs unlink <linkId>` | Delete a link | +| `docs links [documentId] [--type <type>]` | Links of one document (both directions), or every link | +| `docs prereqs <documentId>` | Documents to read first, following prerequisite links | +| `docs tag <documentId> <tags...>` | Add tags (space- or comma-separated) | +| `docs untag <documentId> <tags...>` | Remove tags | +| `docs suggest-tags <documentId> [-n <n>]` | Suggest tags from the content | -### `libscope docs update` - -Update an existing document's title, content, or metadata. Changing content triggers re-chunking and re-indexing of embeddings. +`docs update` options: `--title`, `--content <text>`, `--content-file <path>`, `--library`, `--lib-version`, `--url`, `--topic <topic>` and `--tags <a,b>` (replaces all tags). Changing the content re-chunks and re-embeds the document. ```bash -libscope docs update <documentId> --title "New Title" -libscope docs update <documentId> --content "Updated content here" -libscope docs update <documentId> --library vue --lib-version 3.0.0 +libscope docs update <documentId> --title "New Title" --tags guide,v2 +libscope docs update <documentId> --content-file ./updated.md ``` -| Option | Description | -| --------------------- | ---------------------------------- | -| `--title <title>` | New document title | -| `--content <content>` | New content (triggers re-chunking) | -| `--library <name>` | New library name | -| `--lib-version <ver>` | New library version | -| `--url <url>` | New source URL | -| `--topic <topicId>` | New topic ID | +## Topics and tags + +| Command | Description | +| ---------------------------------------------------------------- | ------------------------------------------------ | +| `topics list [--parent <topic>]` | Topics with document counts | +| `topics create <name> [--description <text>] [--parent <topic>]` | Create a topic | +| `topics delete <topic> [--delete-documents] [-y]` | Delete a topic (documents are kept unless asked) | +| `tags list` | Tags with document counts | + +`<topic>` is a topic ID or name everywhere. -## Bulk Operations +## `libscope searches` -Perform operations on multiple documents at once using filter criteria. +| Command | Description | +| ----------------------------------------------------------- | ---------------------------------- | +| `searches save <name> <query> [filters] [--min-rating <n>]` | Save a query with its filters | +| `searches list [-n <n>] [--offset <n>]` | List saved searches | +| `searches run <name>` | Run a saved search (name or ID) | +| `searches delete <name>` | Delete a saved search (name or ID) | -### `libscope bulk delete` +## `libscope bulk` -Delete all documents matching the specified filters. +`bulk delete`, `bulk retag` and `bulk move` change every document that matches the filters. At least one filter is required, and one call changes at most 1000 documents. Each command first shows how many documents match and asks for confirmation. ```bash libscope bulk delete --library react --dry-run -libscope bulk delete --topic topic-1 --source-type manual --yes +libscope bulk retag --topic guides --add important,v2 --remove draft -y +libscope bulk move --library react --since 2024-01-01 --to frontend -y ``` -### `libscope bulk retag` +| Option | Description | +| ------------------------------------------------- | ------------------------------------------------- | +| `--topic`, `--library`, `--source-type`, `--tags` | Filters ([document filters](#document-filters)) | +| `--since <date>` | Created on or after (ISO 8601) | +| `--before <date>` | Created on or before (ISO 8601) | +| `--add <tags>` | `retag`: tags to add (comma-separated) | +| `--remove <tags>` | `retag`: tags to remove (comma-separated) | +| `--to <topic>` | `move`: destination topic (required) | +| `--dry-run` | List the matching documents without changing them | +| `-y, --yes` | Do not ask for confirmation | -Add or remove tags from all matching documents. +## Connectors -```bash -libscope bulk retag --library react --add-tags important,v2 --dry-run -libscope bulk retag --topic topic-1 --remove-tags deprecated --yes -``` +A connection is a saved connector configuration in `~/.libscope/connectors/<name>.json` (mode `0600`, credentials included). -### `libscope bulk move` +| Command | Description | +| ----------------------------------------------------------- | ------------------------------------------------------------- | +| `connect <type> [source] [options]` | Save a connection and sync it | +| `sync [name]` / `sync --all` | Sync one or every saved connection with its saved settings | +| `disconnect <name> [--keep-documents] [--type <type>] [-y]` | Delete the connection's documents and its saved settings | +| `connections` | List connections with schedule, last sync and last run status | -Move all matching documents to a different topic. +Connector types: `notion`, `slack`, `confluence`, `obsidian`, `onenote`, `docs` (a documentation site). `[source]` is the vault path for `obsidian` and the URL for `confluence` and `docs`. ```bash -libscope bulk move --library react --to new-topic-id --dry-run -libscope bulk move --topic old-topic --to new-topic --yes +libscope connect notion --token secret_xxx +libscope connect slack --token xoxb-xxx --channels general,eng --exclude random +libscope connect confluence https://acme.atlassian.net --email me@acme.com --token xxx --spaces ENG +libscope connect obsidian ~/notes --topic-mapping frontmatter +libscope connect onenote --client-id <azure-app-id> # device-code sign-in +libscope connect docs https://docs.example.com --library example --max-pages 200 +libscope connect notion --name work --schedule "0 */6 * * *" # second Notion connection, synced every 6 hours ``` -| Option | Description | -| ---------------------- | ------------------------------------------------- | -| `--topic <topicId>` | Filter by topic ID | -| `--library <name>` | Filter by library name | -| `--source-type <type>` | Filter by source type | -| `--tags <tags>` | Filter by tags (comma-separated) | -| `--to <targetTopicId>` | Target topic (move only) | -| `--add-tags <tags>` | Tags to add (retag only, comma-separated) | -| `--remove-tags <tags>` | Tags to remove (retag only, comma-separated) | -| `--dry-run` | Preview affected documents without making changes | -| `-y, --yes` | Skip confirmation prompt | - -## Document Links (Cross-references) - -| Command | Description | -| ------------------------------------- | ------------------------------- | -| `libscope link <sourceId> <targetId>` | Create a cross-reference link | -| `libscope links <documentId>` | Show all links for a document | -| `libscope unlink <linkId>` | Remove a link | -| `libscope prereqs <documentId>` | Show prerequisite reading chain | - -### `libscope link` +Running `connect` again for a saved connection changes only the settings you give and keeps the rest. For example, `libscope connect notion --schedule off --no-sync` removes the schedule. + +| Option | Types | Description | +| --------------------------------------------- | ----------------------------------- | ----------------------------------------------------------------------- | +| `--name <name>` | all | Connection name (default: the type) | +| `--schedule <cron>` | all | Sync on this schedule while `libscope serve api` runs; `off` removes it | +| `--no-sync` | all | Save without syncing now | +| `--token <token>` | notion, slack, confluence, onenote | API token (onenote: an access token that cannot be refreshed) | +| `--exclude <list>` | slack, notion, confluence, obsidian | Channels, page IDs, space keys or file globs to skip | +| `--channels <list>` | slack | Channel names or IDs (default `all`) | +| `--thread-mode <mode>` | slack | `aggregate` (default) or `separate` | +| `--email <email>` | confluence | User email (Cloud) | +| `--server` | confluence | Server/Data Center instead of Cloud | +| `--spaces <keys>` | confluence | Space keys (default `all`) | +| `--topic-mapping <mode>` | obsidian | `folder` (default) or `frontmatter` | +| `--notebook <name>` | onenote | One notebook (default: all) | +| `--client-id <id>` | onenote | Azure app client ID for device-code sign-in | +| `--tenant-id <id>` | onenote | Azure tenant ID (default `common`) | +| `--site-type <type>` | docs | `auto`, `sphinx`, `vitepress`, `doxygen` or `generic` | +| `--library`, `--lib-version` | docs | Library name and version for the pages | +| `--max-pages`, `--max-depth`, `--path-prefix` | docs | Crawl limits (default 500 pages, depth 10) | + +## Knowledge packs + +| Command | Description | +| -------------------------------------------- | -------------------------------------------------------------------------------- | +| `pack install <nameOrPath>` | Install from a git registry, the pack registry URL, or a `.json`/`.json.gz` file | +| `pack remove <name> [-y]` | Remove a pack and its documents | +| `pack list [--available] [--registry <url>]` | Installed packs, or packs in the registry | +| `pack create --name <name> [options]` | Create a pack from indexed documents, or from files and URLs (`--from`) | ```bash -libscope link <sourceId> <targetId> --type see_also --label "Background context" +libscope pack install react-docs # latest from any configured git registry +libscope pack install react-docs@1.2.0 +libscope pack install react-docs --from-registry official +libscope pack install ./react-docs.json.gz +libscope pack create --name team-docs --topic engineering +libscope pack create --name react-docs --from ./react/docs --exclude "*.min.js" ``` -| Option | Description | -| ---------------- | ----------------------------------------------------------------------------------- | -| `--type <type>` | Link type: `see_also`, `prerequisite`, `supersedes`, `related` (default: `related`) | -| `--label <text>` | Optional description of the relationship | - -## Topics & Tags - -| Command | Description | -| ------------------------------------ | -------------------------------------------- | -| `libscope topics list` | List all topics | -| `libscope topics create <name>` | Create a topic (`--parent`, `--description`) | -| `libscope tag add <docId> <tags...>` | Add tags to a document | -| `libscope tag remove <docId> <tag>` | Remove a tag | -| `libscope tag list` | List all tags with document counts | - -## Workspaces - -| Command | Description | -| ---------------------------------- | ----------------------- | -| `libscope workspace create <name>` | Create a new workspace | -| `libscope workspace list` | List all workspaces | -| `libscope workspace use <name>` | Switch active workspace | -| `libscope workspace delete <name>` | Delete a workspace | - -## Knowledge Packs - -| Command | Description | -| ------------------------------------ | ------------------------------------------------- | -| `libscope pack install <nameOrPath>` | Install a pack (from registry, file, or by name) | -| `libscope pack remove <name>` | Remove a pack and its documents | -| `libscope pack list` | List installed packs (`--available` for registry) | -| `libscope pack create` | Export documents as a pack file | - -### `libscope pack install` (registry support) - -When no local file path is given, `pack install` searches configured registries: - -```bash -libscope pack install react-docs # latest from any registry -libscope pack install react-docs@1.2.0 # specific version -libscope pack install react-docs --pack-version 1.2.0 -libscope pack install react-docs --registry official -``` +`pack install` options: `--registry <url>` (URL registry), `--from-registry <name>` and `--pack-version <semver>` (git registries), `-y` (with several registries, use the highest priority instead of asking), `--batch-size <n>`, `--concurrency <n>`, `--resume-from <n>`. -| Option | Description | -| ------------------------- | -------------------------------------------- | -| `--pack-version <semver>` | Install a specific version (default: latest) | -| `--registry <name>` | Install from a specific registry | +`pack create` options: `--from <sources...>`, `--topic`, `--pack-version`, `--description`, `--author`, `--license`, `--output <path>` (default `<name>.json`, or `<name>.json.gz` with `--from`), `--extensions <exts>`, `--exclude <globs...>`, `--no-recursive`. -## Pack Registries +## Pack registries | Command | Description | | ----------------------------------------------------------------- | ------------------------------------------------------- | @@ -291,24 +293,12 @@ libscope registry add https://github.com/org/registry.git --sync-interval 86400 | `--sync-interval <seconds>` | Auto-sync interval in seconds, 0 = manual (default: 0) | | `--no-sync` | Skip initial sync after adding | -### `libscope registry search` - -```bash -libscope registry search "react" -libscope registry search "kubernetes" -r official -``` - ### `libscope registry publish` ```bash -# Direct publish (you have write access) libscope registry publish ./my-pack.json -r my-registry --pack-version 1.0.0 - -# Auto-bump patch version -libscope registry publish ./my-pack.json -r my-registry - -# Submit via feature branch (for PR workflow) -libscope registry publish ./my-pack.json -r community --submit +libscope registry publish ./my-pack.json -r my-registry # auto-bump patch version +libscope registry publish ./my-pack.json -r community --submit # push to a feature branch ``` | Option | Description | @@ -331,64 +321,74 @@ libscope registry unpublish my-pack -r my-registry --pack-version 1.0.0 | `-m, --message <msg>` | Git commit message | | `-y, --yes` | Skip confirmation prompt | -### `libscope registry create` +`libscope registry create ./my-registry` creates a git repo with the registry folder structure. See the [Registry Reference](/reference/registry). + +## `libscope serve` ```bash -libscope registry create ./my-registry +libscope serve # MCP server on stdio +libscope serve api # REST API on http://localhost:3378 +libscope serve dashboard --port 8080 # web dashboard (default port 3377) ``` -Creates a git repo with the canonical registry folder structure. See the [Registry Reference](/reference/registry) for full schema details. +| Option | Description | +| --------------- | --------------------------------------- | +| `--port <n>` | Port (`api` and `dashboard`) | +| `--host <host>` | Host to listen on (default `localhost`) | -## Connectors +Connection schedules (`connect --schedule`) run while `serve api` runs. + +## `libscope config` + +| Command | Description | +| -------------------------- | ----------------------------------------- | +| `config show` | Effective configuration (API keys masked) | +| `config get <key>` | One effective value (API keys masked) | +| `config set <key> <value>` | Set a value in `~/.libscope/config.json` | +| `config unset <key>` | Remove a value from the user config file | +| `config path` | Print the user config file path | + +Keys: `embedding.provider`, `embedding.model`, `embedding.url`, `embedding.dimensions`, `llm.provider`, `llm.model`, `llm.url`, `openai.apiKey`, `anthropic.apiKey`, `database.path`, `indexing.maxDocumentSize`, `indexing.allowPrivateUrls`, `indexing.allowSelfSignedCerts`, `logging.level`. `openai.apiKey` and `anthropic.apiKey` are written to `~/.libscope/secrets.json` (mode `0600`), never to `config.json`; the `LIBSCOPE_OPENAI_API_KEY` / `OPENAI_API_KEY` and `LIBSCOPE_ANTHROPIC_API_KEY` / `ANTHROPIC_API_KEY` environment variables take precedence. See the [configuration reference](/reference/configuration). + +## Workspaces + +| Command | Description | +| ------------------------------ | ------------------------------------------ | +| `workspace create <name>` | Create a workspace | +| `workspace list` | List workspaces (`*` marks the active one) | +| `workspace use <name>` | Make a workspace the active one | +| `workspace delete <name> [-y]` | Delete a workspace and its database | + +## `libscope webhooks` + +| Command | Description | +| ------------------------------------------------------ | ------------------------------------------ | +| `webhooks create <url> --events <list> [--secret <s>]` | Register a webhook | +| `webhooks list` | List webhooks (secrets are never shown) | +| `webhooks delete <webhookId>` | Delete a webhook | +| `webhooks test <webhookId>` | Send a test event and show the HTTP status | + +Events: `document.created`, `document.updated`, `document.deleted`, `document.rated`, `search.executed`. A secret signs each POST with HMAC-SHA256; storing it needs `LIBSCOPE_SECRET_KEY`. + +## `libscope admin` + +| Command | Description | +| --------------------------------------------------------------------- | --------------------------------------------------------------------------- | +| `admin reindex` | Re-embed chunks with the configured embedding model | +| `admin reindex --rebuild` | Recreate the vector index for a new model or vector size, then re-embed all | +| `admin dedupe [--threshold <0-1>] [--strategy exact\|semantic\|both]` | Find duplicate and near-duplicate documents | +| `admin backup <file>` | Write the whole knowledge base to a JSON file | +| `admin restore <file> [-y]` | Import a file written by `admin backup` | +| `admin stats [--days <n>]` | Counts, index, most returned and stale documents, search analytics | +| `admin prune` | Delete documents whose expiry time (`add --expires`) has passed | + +`admin reindex` also takes `--doc <documentIds...>`, `--since <date>`, `--before <date>` and `--batch-size <n>` (not with `--rebuild`). + +## `libscope doctor` + +```bash +libscope doctor # report only +libscope doctor --fix # also create the database and vector index +``` -| Command | Description | -| ---------------------------------- | -------------------------------- | -| `libscope connect obsidian <path>` | Sync an Obsidian vault | -| `libscope connect onenote` | Sync OneNote notebooks | -| `libscope connect notion` | Sync Notion pages and databases | -| `libscope connect confluence` | Sync Confluence spaces | -| `libscope connect slack` | Sync Slack channels | -| `libscope add-repo <url>` | Index a GitHub/GitLab repository | -| `libscope disconnect <connector>` | Remove connector data | - -## Developer Tools - -| Command | Description | -| ------------------------------------- | --------------------------------------------------------------------------------------------- | -| `libscope watch <directory>` | Watch for file changes and auto-reindex (all supported formats, plus `.rst`) | -| `libscope reindex` | Re-embed chunks with current provider | -| `libscope reindex --rebuild` | Recreate the vector index for a new embedding model or vector size, then re-embed every chunk | -| `libscope dedupe` | Scan for duplicate documents | -| `libscope export <outputPath>` | Export knowledge base to JSON | -| `libscope import-backup <backupPath>` | Import from a backup file | - -## Analytics - -| Command | Description | -| ------------------------ | --------------------------------- | -| `libscope stats` | Overview dashboard | -| `libscope stats popular` | Most-returned documents in search | -| `libscope stats stale` | Documents with no search hits | -| `libscope stats queries` | Top search queries | - -## Configuration - -| Command | Description | -| ----------------------------------- | --------------------------------------------- | -| `libscope config set <key> <value>` | Set a configuration value | -| `libscope config get <key>` | Show one configuration value (secrets masked) | -| `libscope config unset <key>` | Remove a value from the user config file | -| `libscope config show` | Show current configuration (secrets masked) | -| `libscope config path` | Print the user config file path | - -Supported config keys: `embedding.provider`, `embedding.model`, `embedding.url`, `embedding.dimensions`, `llm.provider`, `llm.model`, `llm.url`, `openai.apiKey`, `anthropic.apiKey`, `database.path`, `indexing.maxDocumentSize`, `indexing.allowPrivateUrls`, `indexing.allowSelfSignedCerts`, `logging.level`. `openai.apiKey` and `anthropic.apiKey` are written to `~/.libscope/secrets.json` (mode `0600`), never to `config.json`; the `LIBSCOPE_OPENAI_API_KEY` / `OPENAI_API_KEY` and `LIBSCOPE_ANTHROPIC_API_KEY` / `ANTHROPIC_API_KEY` environment variables take precedence. - -## Global Options - -These work with any command: - -| Flag | Description | -| --------------------- | -------------------------------------------------------------- | -| `--verbose` | Enable debug logging | -| `--log-level <level>` | Set log level (`debug` / `info` / `warn` / `error` / `silent`) | -| `--workspace <name>` | Use a specific workspace | +`doctor` shows the config and secrets files, the active workspace and database path, the configured embedding model and the model the vector index was built with, and which LLM `ask` will use. It then lists checks with a fix for each warning or error, for example `libscope admin reindex --rebuild` when the index was built with a different embedding model. It exits with code 1 when a check fails. diff --git a/src/cli/errors.ts b/src/cli/errors.ts index 79b0a788..b90424a1 100644 --- a/src/cli/errors.ts +++ b/src/cli/errors.ts @@ -10,6 +10,7 @@ const NOT_FOUND_HINTS: Record<string, string> = { WEBHOOK_NOT_FOUND: "List webhooks with `libscope webhooks list`.", LINK_NOT_FOUND: "Link IDs are shown by `libscope docs links <documentId>`.", PACK_NOT_FOUND: "List installed packs with `libscope pack list`.", + VERSION_NOT_FOUND: "List saved versions with `libscope docs history <documentId>`.", }; /** Next step to suggest for an error, if any. */ From 3de35d7e2bec4cabc95d469ef05fa26f484a6ae6 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:28:16 +0000 Subject: [PATCH 52/83] feat(config)!: mcp.toolsets config key replaces the raw LIBSCOPE_MCP_TOOLSETS read The MCP server reads optional toolsets from config (mcp.toolsets: array or comma list of core|admin|all; env LIBSCOPE_MCP_TOOLSETS is derived by the schema). Config schema gains a list leaf type. parseToolsets(string) is replaced by resolveToolsets(settings). --- docs/guide/mcp-setup.md | 10 ++++++-- docs/reference/configuration.md | 11 ++++++++ docs/reference/mcp-tools.md | 4 +-- src/config-schema.ts | 43 ++++++++++++++++++++++++++++---- src/config.ts | 25 ++++++++++++++++--- src/mcp/server.ts | 31 ++++++++--------------- tests/unit/config-layers.test.ts | 15 +++++++++++ tests/unit/config-save.test.ts | 12 +++++++++ tests/unit/mcp-server.test.ts | 31 +++++++++++++---------- 9 files changed, 137 insertions(+), 45 deletions(-) diff --git a/docs/guide/mcp-setup.md b/docs/guide/mcp-setup.md index 0853966e..8daff622 100644 --- a/docs/guide/mcp-setup.md +++ b/docs/guide/mcp-setup.md @@ -80,7 +80,13 @@ If you're using [workspaces](/guide/configuration#workspaces), pass the workspac ## Admin Tools -The server starts with 11 core tools. To also give the assistant `sync`, `install-pack`, `list-packs` and `reindex-documents`, set `LIBSCOPE_MCP_TOOLSETS` in the server's environment: +The server starts with 11 core tools. To also give the assistant `sync`, `install-pack`, `list-packs` and `reindex-documents`, enable the `admin` toolset with the `mcp.toolsets` config key: + +```bash +libscope config set mcp.toolsets admin +``` + +You can also set the environment variable `LIBSCOPE_MCP_TOOLSETS` (a comma-separated list) in the server's environment: ```json { @@ -112,7 +118,7 @@ See the [MCP Tools Reference](/reference/mcp-tools) for all parameters. - **`link-documents`**: create (`action: "create"`) or delete (`action: "delete"`) a link between documents. - **`task`**: status, cancel or list of background tasks started with `async: true`. -**Admin tools** (with `LIBSCOPE_MCP_TOOLSETS=admin`) +**Admin tools** (with `mcp.toolsets` set to `admin` or `all`) - **`sync`**: sync one saved connector connection, or all of them. Set up connections with `libscope connect`; the tool does not accept credentials. - **`install-pack`**, **`list-packs`**: manage knowledge packs. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index f7864c5e..4c560e5d 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -68,6 +68,14 @@ Each API key is read in this order: `LIBSCOPE_<PROVIDER>_API_KEY`, then `<PROVID Booleans accept `true`, `false`, `1`, and `0` in environment variables and in `config set`. +### MCP + +| Key | Type | Default | Env var | Description | +| -------------- | ------------------------------ | ------- | ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `mcp.toolsets` | list of `core`, `admin`, `all` | none | `LIBSCOPE_MCP_TOOLSETS` | Optional MCP toolsets. `admin` adds `sync`, `install-pack`, `list-packs` and `reindex-documents`. `all` enables every optional toolset. The core tools are always on. | + +A list key accepts a JSON array in a config file, or a comma-separated list in a config file, an environment variable, and `config set` (for example `libscope config set mcp.toolsets admin`). + ## Automatic LLM selection When `llm.provider` is `auto` (the default), LibScope selects the LLM when `ask` runs. It does not make network calls to decide: @@ -129,6 +137,9 @@ libscope config set indexing.allowSelfSignedCerts true # Size limit (bytes) libscope config set indexing.maxDocumentSize 52428800 +# MCP server: enable the admin toolset +libscope config set mcp.toolsets admin + # Read, remove, and locate libscope config get embedding.provider # effective value (API keys masked) libscope config unset llm.model # remove from the file that holds the key diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 235a8fbb..43603e2f 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -38,7 +38,7 @@ Tasks are kept in memory for one hour after they finish. They are lost when the ## Admin toolset -Set `LIBSCOPE_MCP_TOOLSETS` in the server's environment to enable optional toolsets. The value is a comma-separated list. `admin` enables `sync`, `install-pack`, `list-packs` and `reindex-documents`. `all` enables every optional toolset. +The `mcp.toolsets` config key enables optional toolsets (`libscope config set mcp.toolsets admin`, or the environment variable `LIBSCOPE_MCP_TOOLSETS` as a comma-separated list). `admin` enables `sync`, `install-pack`, `list-packs` and `reindex-documents`. `all` enables every optional toolset. `core` (the default tools) is always on. ```json { @@ -229,7 +229,7 @@ Annotations: not destructive, idempotent. ## Admin tools -These tools are registered only when `LIBSCOPE_MCP_TOOLSETS` contains `admin` or `all`. +These tools are registered only when `mcp.toolsets` contains `admin` or `all`. ### sync diff --git a/src/config-schema.ts b/src/config-schema.ts index 3d94a2ab..0e7b4d30 100644 --- a/src/config-schema.ts +++ b/src/config-schema.ts @@ -34,6 +34,8 @@ function leaf<T extends z.ZodType>(schema: T, meta: ConfigKeyMeta = {}): T { export const EMBEDDING_PROVIDERS = ["local", "ollama", "openai"] as const; export const LLM_PROVIDERS = ["auto", "openai", "anthropic", "ollama", "passthrough"] as const; export const LOG_LEVELS = ["debug", "info", "warn", "error", "silent"] as const; +/** Values of `mcp.toolsets`. "core" is always on; "all" enables every optional toolset. */ +export const MCP_TOOLSET_SETTINGS = ["core", "admin", "all"] as const; /** Ollama server URL used when embedding.url and llm.url are not set. */ export const DEFAULT_OLLAMA_URL = "http://localhost:11434"; @@ -135,6 +137,17 @@ export const ConfigSchema = z.object({ logging: z.object({ level: leaf(z.enum(LOG_LEVELS).default("info").describe("Log level.")), }), + mcp: z.object({ + toolsets: leaf( + z + .array(z.enum(MCP_TOOLSET_SETTINGS)) + .default([]) + .describe( + "Optional MCP toolsets (comma list). admin adds sync, install-pack, list-packs and " + + "reindex-documents; all enables every optional toolset. The core tools are always on.", + ), + ), + }), }); type SchemaOutput = z.output<typeof ConfigSchema>; @@ -159,8 +172,10 @@ export interface ConfigKeySpec { /** Default value, or undefined when the key has no fixed default. */ defaultValue: unknown; defaultText: string | undefined; - /** "string", "integer", "boolean", or the enum values. */ - type: "string" | "integer" | "boolean" | readonly string[]; + /** "string", "integer", "boolean", "list" (of `listValues`), or the enum values. */ + type: "string" | "integer" | "boolean" | "list" | readonly string[]; + /** Allowed items of a "list" key. */ + listValues?: readonly string[] | undefined; } /** `indexing.allowPrivateUrls` -> `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS`. */ @@ -178,14 +193,25 @@ function baseSchema(schema: z.ZodType): z.ZodType { return current; } +function enumValues(schema: z.ZodType): string[] | undefined { + return schema instanceof z.ZodEnum ? schema.options.map(String) : undefined; +} + function leafType(schema: z.ZodType): ConfigKeySpec["type"] { const base = baseSchema(schema); - if (base instanceof z.ZodEnum) return base.options.map(String); + if (base instanceof z.ZodArray) return "list"; + const values = enumValues(base); + if (values) return values; if (base instanceof z.ZodBoolean) return "boolean"; if (base instanceof z.ZodNumber) return "integer"; return "string"; } +function listItemValues(schema: z.ZodType): string[] | undefined { + const base = baseSchema(schema); + return base instanceof z.ZodArray ? enumValues(base.element as z.ZodType) : undefined; +} + function buildSpecs(): ConfigKeySpec[] { const specs: ConfigKeySpec[] = []; for (const [section, sectionSchema] of Object.entries(ConfigSchema.shape)) { @@ -205,6 +231,7 @@ function buildSpecs(): ConfigKeySpec[] { defaultValue: parsedDefault.success ? parsedDefault.data : undefined, defaultText: meta.defaultText, type: leafType(schema), + listValues: listItemValues(schema), }); } } @@ -229,7 +256,7 @@ export function getConfigKeySpec(key: string): ConfigKeySpec | undefined { /** One row of the generated configuration reference. */ export interface ConfigKeyTableRow { key: ConfigKey; - /** "string", "integer", "boolean", or the allowed values joined with " | ". */ + /** "string", "integer", "boolean", "list of a | b", or the allowed values joined with " | ". */ type: string; /** Default value as text ("" when none). */ default: string; @@ -245,14 +272,20 @@ function defaultAsText(spec: ConfigKeySpec): string { if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { return String(value); } + if (Array.isArray(value)) return value.length > 0 ? value.join(",") : "none"; return ""; } +function typeAsText(spec: ConfigKeySpec): string { + if (spec.type === "list") return `list of ${(spec.listValues ?? ["string"]).join(" | ")}`; + return typeof spec.type === "string" ? spec.type : spec.type.join(" | "); +} + /** The key table (for docs and `config` help): key, type, default, env vars, secret, description. */ export function getConfigKeyTable(): ConfigKeyTableRow[] { return CONFIG_KEY_SPECS.map((spec) => ({ key: spec.key, - type: typeof spec.type === "string" ? spec.type : spec.type.join(" | "), + type: typeAsText(spec), default: defaultAsText(spec), env: [...spec.env], secret: spec.secret, diff --git a/src/config.ts b/src/config.ts index e1dfeda0..84e8b096 100644 --- a/src/config.ts +++ b/src/config.ts @@ -20,6 +20,7 @@ import { type EMBEDDING_PROVIDERS, type LLM_PROVIDERS, type LOG_LEVELS, + type MCP_TOOLSET_SETTINGS, } from "./config-schema.js"; export { @@ -70,8 +71,15 @@ export interface LibScopeConfig { logging: { level: (typeof LOG_LEVELS)[number]; }; + mcp?: { + /** Optional MCP toolsets to enable. Unset: none (core tools only). */ + toolsets?: McpToolsetSetting[] | undefined; + }; } +/** Value of one `mcp.toolsets` item. */ +export type McpToolsetSetting = (typeof MCP_TOOLSET_SETTINGS)[number]; + /** Expand a leading `~` (or `~/`) to the user's home directory. */ export function expandHomeDir(path: string): string { if (path === "~") return homedir(); @@ -98,7 +106,7 @@ function getProjectConfigPath(): string { } type RawConfig = Record<string, unknown>; -type ConfigValue = string | number | boolean; +type ConfigValue = string | number | boolean | string[]; /** A config layer flattened to `section.field` keys. */ type FlatLayer = Record<string, unknown>; @@ -214,6 +222,13 @@ export function coerceConfigValue(key: ConfigKey, raw: string): ConfigValue { if (raw.trim() === "") throw new ValidationError(`${key} must not be empty`); return validateWithSchema(spec, raw); } + if (type === "list") { + const items = raw + .split(",") + .map((item) => item.trim()) + .filter((item) => item !== ""); + return validateWithSchema(spec, items); + } if (!type.includes(raw)) { throw new ValidationError(`${key} must be one of: ${type.join(", ")} (got "${raw}")`); } @@ -227,12 +242,16 @@ function warnIgnored(what: string, err: unknown): void { /** * Validate a value read from a config file. Invalid values are logged and ignored, so one bad - * key does not stop libscope from starting. Strings are accepted for booleans and integers. + * key does not stop libscope from starting. Strings are accepted for booleans, integers and + * lists (comma-separated). */ function readFileValue(spec: ConfigKeySpec, value: unknown, source: string): unknown { if (spec.openEnum && typeof value === "string" && value.trim() !== "") return value; try { - if (typeof value === "string" && (spec.type === "boolean" || spec.type === "integer")) { + if ( + typeof value === "string" && + (spec.type === "boolean" || spec.type === "integer" || spec.type === "list") + ) { return coerceConfigValue(spec.key, value); } return validateWithSchema(spec, value); diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 430baaae..b44a9c5d 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -11,7 +11,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import type { ToolAnnotations } from "@modelcontextprotocol/sdk/types.js"; import { z } from "zod"; -import { loadConfig } from "../config.js"; +import { loadConfig, type McpToolsetSetting } from "../config.js"; import { bootstrap, type BootstrapOptions, type Bootstrapped } from "../core/bootstrap.js"; import { addOperation, @@ -39,19 +39,16 @@ import { type OperationAnnotations, type OperationContext, } from "../core/operations/index.js"; -import { getLogger, initLogger, type LogLevel } from "../logger.js"; +import { initLogger, type LogLevel } from "../logger.js"; import { textResult, withErrorHandling } from "./errors.js"; import * as fmt from "./format.js"; export { errorResponse, textResult, withErrorHandling, type ToolResult } from "./errors.js"; -/** Optional toolsets, enabled with LIBSCOPE_MCP_TOOLSETS (comma list, or "all"). */ +/** Optional toolsets, enabled with the `mcp.toolsets` config key. */ export const MCP_TOOLSETS = ["admin"] as const; export type McpToolset = (typeof MCP_TOOLSETS)[number]; -/** Environment variable that enables optional toolsets. */ -export const TOOLSETS_ENV = "LIBSCOPE_MCP_TOOLSETS"; - /** Formats an operation result as tool output text. */ type Formatter<O> = (result: O) => string; @@ -178,20 +175,14 @@ export function registerActionTool( return tool.name; } -/** Toolsets named in `value` (comma list; "all" enables every optional toolset). */ -export function parseToolsets(value: string | undefined): Set<McpToolset> { +/** Optional toolsets enabled by `settings` ("all" enables every one; "core" is always on). */ +export function resolveToolsets(settings: readonly McpToolsetSetting[]): Set<McpToolset> { const enabled = new Set<McpToolset>(); - for (const raw of (value ?? "").split(",")) { - const name = raw.trim().toLowerCase(); - if (name === "" || name === "core") continue; + for (const name of settings) { if (name === "all") { for (const t of MCP_TOOLSETS) enabled.add(t); - } else if ((MCP_TOOLSETS as readonly string[]).includes(name)) { - enabled.add(name as McpToolset); - } else { - getLogger().warn( - `Unknown MCP toolset "${name}" in ${TOOLSETS_ENV} (known: ${MCP_TOOLSETS.join(", ")}, all)`, - ); + } else if (name !== "core") { + enabled.add(name); } } return enabled; @@ -245,8 +236,8 @@ function packageVersion(): string { export interface CreateMcpServerOptions extends BootstrapOptions { /** Use this operation context instead of bootstrapping one. The caller owns its database. */ ctx?: OperationContext | undefined; - /** Optional toolsets to enable (default: from LIBSCOPE_MCP_TOOLSETS). */ - toolsets?: readonly string[] | undefined; + /** Optional toolsets to enable (default: the `mcp.toolsets` config key). */ + toolsets?: readonly McpToolsetSetting[] | undefined; } export interface LibScopeMcpServer { @@ -323,7 +314,7 @@ export function createMcpServer(options: CreateMcpServerOptions = {}): LibScopeM }); } const opCtx = ctx; - const enabled = parseToolsets(toolsets ? toolsets.join(",") : process.env[TOOLSETS_ENV]); + const enabled = resolveToolsets(toolsets ?? opCtx.config.mcp?.toolsets ?? []); const withAsk = askAvailable(opCtx); const admin = enabled.has("admin"); diff --git a/tests/unit/config-layers.test.ts b/tests/unit/config-layers.test.ts index 65803738..a2891bae 100644 --- a/tests/unit/config-layers.test.ts +++ b/tests/unit/config-layers.test.ts @@ -91,6 +91,7 @@ describe("config layering (temp HOME and cwd)", () => { allowSelfSignedCerts: false, }); expect(config.logging.level).toBe("info"); + expect(config.mcp).toEqual({ toolsets: [] }); expect(config.database).toEqual({}); expect(config.openai).toBeUndefined(); expect(config.anthropic).toBeUndefined(); @@ -289,6 +290,20 @@ describe("config layering (temp HOME and cwd)", () => { expect(config.indexing.maxDocumentSize).toBe(4096); }); + it("reads mcp.toolsets as an array or a comma list, and from LIBSCOPE_MCP_TOOLSETS", () => { + writeUserConfig({ mcp: { toolsets: ["admin"] } }); + expect(load().mcp?.toolsets).toEqual(["admin"]); + writeUserConfig({ mcp: { toolsets: "core, all" } }); + expect(load().mcp?.toolsets).toEqual(["core", "all"]); + expect(load({ LIBSCOPE_MCP_TOOLSETS: "admin" }).mcp?.toolsets).toEqual(["admin"]); + }); + + it("ignores an unknown mcp toolset with a warning", () => { + const warn = spyWarn(); + expect(load({ LIBSCOPE_MCP_TOOLSETS: "admin,bogus" }).mcp?.toolsets).toEqual([]); + expect(messages(warn).some((m) => m.includes("LIBSCOPE_MCP_TOOLSETS"))).toBe(true); + }); + it("accepts a custom embedding provider name in a file", () => { writeUserConfig({ embedding: { provider: "my-plugin" } }); expect(load().embedding.provider).toBe("my-plugin"); diff --git a/tests/unit/config-save.test.ts b/tests/unit/config-save.test.ts index 631fd175..b41872e9 100644 --- a/tests/unit/config-save.test.ts +++ b/tests/unit/config-save.test.ts @@ -109,6 +109,7 @@ describe("config keys", () => { "llm.provider", "llm.url", "logging.level", + "mcp.toolsets", "openai.apiKey", ].sort(), ); @@ -134,6 +135,11 @@ describe("config keys", () => { type: "integer", default: String(100 * 1024 * 1024), }); + expect(table.find((r) => r.key === "mcp.toolsets")).toMatchObject({ + type: "list of core | admin | all", + default: "none", + env: ["LIBSCOPE_MCP_TOOLSETS"], + }); expect(table.find((r) => r.key === "openai.apiKey")).toMatchObject({ env: ["LIBSCOPE_OPENAI_API_KEY", "OPENAI_API_KEY"], secret: true, @@ -167,6 +173,12 @@ describe("setUserConfigValue", () => { expect(existsSync(secretsPath())).toBe(false); }); + it("stores a list key as an array", () => { + expect(setUserConfigValue("mcp.toolsets", "admin, core")).toEqual(["admin", "core"]); + expect(readSavedRaw()["mcp"]).toEqual({ toolsets: ["admin", "core"] }); + expect(() => setUserConfigValue("mcp.toolsets", "admin,bogus")).toThrow(/mcp\.toolsets/); + }); + it("rejects unknown and old keys and lists valid keys", () => { expect(() => setUserConfigValue("embedding.nope", "x")).toThrow( /Unknown config key.*llm\.model/, diff --git a/tests/unit/mcp-server.test.ts b/tests/unit/mcp-server.test.ts index 4b7fe74c..d6fed3c4 100644 --- a/tests/unit/mcp-server.test.ts +++ b/tests/unit/mcp-server.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import type { McpToolsetSetting } from "../../src/config.js"; import type { LlmProvider } from "../../src/core/rag.js"; import { createDatabase } from "../../src/db/connection.js"; import { createVectorTable, runMigrations } from "../../src/db/schema.js"; @@ -21,7 +22,7 @@ vi.mock("node:os", async (importOriginal) => { return { ...orig, homedir: (): string => tempHome }; }); -const { createMcpServer, parseToolsets, buildInstructions } = +const { createMcpServer, resolveToolsets, buildInstructions } = await import("../../src/mcp/server.js"); const { saveConnectorSettings } = await import("../../src/connectors/saved-config.js"); const { taskRegistry } = await import("../../src/core/tasks.js"); @@ -66,7 +67,7 @@ function passthroughContext(): TestContext { async function connect( t: TestContext, - toolsets: string[] = [], + toolsets: McpToolsetSetting[] = [], ): Promise<{ client: Client; close: () => Promise<void> }> { const mcp = createMcpServer({ ctx: t.ctx, toolsets }); const client = new Client({ name: "test-client", version: "1.0.0" }); @@ -231,7 +232,8 @@ describe("MCP server: tool list", () => { }); it("adds the admin toolset only when enabled", async () => { - for (const toolsets of [["admin"], ["all"], ["core", "Admin"]]) { + const settings: McpToolsetSetting[][] = [["admin"], ["all"], ["core", "admin"]]; + for (const toolsets of settings) { const { client, close } = await connect(t, toolsets); const { tools } = await client.listTools(); expect(tools.map((tool) => tool.name)).toEqual([...CORE_TOOLS, ...ADMIN_TOOLS]); @@ -242,19 +244,22 @@ describe("MCP server: tool list", () => { } }); - it("reads LIBSCOPE_MCP_TOOLSETS when no toolsets are passed", () => { - vi.stubEnv("LIBSCOPE_MCP_TOOLSETS", "admin"); - const mcp = createMcpServer({ ctx: t.ctx }); - expect(mcp.tools).toEqual([...CORE_TOOLS, ...ADMIN_TOOLS]); - vi.stubEnv("LIBSCOPE_MCP_TOOLSETS", ""); + it("reads mcp.toolsets from config when no toolsets are passed", () => { + const withAdmin = makeContext({ + db: t.db, + surface: "mcp", + llm: undefined, + config: { ...testConfig(), mcp: { toolsets: ["admin"] } }, + }); + expect(createMcpServer({ ctx: withAdmin.ctx }).tools).toEqual([...CORE_TOOLS, ...ADMIN_TOOLS]); expect(createMcpServer({ ctx: t.ctx }).tools).toEqual(CORE_TOOLS); }); - it("parseToolsets accepts a comma list, 'all' and ignores unknown names", () => { - expect([...parseToolsets("admin")]).toEqual(["admin"]); - expect([...parseToolsets(" ALL ")]).toEqual(["admin"]); - expect([...parseToolsets("bogus,core")]).toEqual([]); - expect([...parseToolsets(undefined)]).toEqual([]); + it("resolveToolsets expands 'all' and ignores 'core'", () => { + expect([...resolveToolsets(["admin"])]).toEqual(["admin"]); + expect([...resolveToolsets(["all"])]).toEqual(["admin"]); + expect([...resolveToolsets(["core"])]).toEqual([]); + expect([...resolveToolsets([])]).toEqual([]); }); it("omits ask when no LLM is configured and passthrough is off", async () => { From 26fcf2a04265ad16f789a24ba55388888a820b3b Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:31:14 +0000 Subject: [PATCH 53/83] fix(operations): validate add and sync before a background task starts - Operation.validate(ctx, input): synchronous checks that runOperation and startOperationTask run before the handler/task, so REST answers 400/404 and MCP returns the error at once instead of a task that fails later. - add: refuses local paths on mcp/api without probing the disk, requires content or a source, and resolves the topic up front. - sync: name/all check and unknown saved connection (NotFoundError) up front. - disconnect: an unknown name that is not a connector type is NotFoundError (404). - sync/disconnect name description no longer names a CLI-only command. --- src/connectors/registry.ts | 12 +++++-- src/core/ingest.ts | 2 +- src/core/operations/connectors.ts | 8 +++-- src/core/operations/documents.ts | 41 +++++++++++++++++------- src/core/operations/types.ts | 24 +++++++++++--- tests/unit/api-routes.test.ts | 26 ++++++++------- tests/unit/mcp-server.test.ts | 4 +++ tests/unit/operations/connectors.test.ts | 2 +- tests/unit/operations/documents.test.ts | 8 +++++ 9 files changed, 93 insertions(+), 34 deletions(-) diff --git a/src/connectors/registry.ts b/src/connectors/registry.ts index 431b9914..d59f45bb 100644 --- a/src/connectors/registry.ts +++ b/src/connectors/registry.ts @@ -257,7 +257,7 @@ export async function runSavedConnectorSync( } /** Connector type of saved connection `name`. Throws NotFoundError when it does not exist. */ -function savedConnectionType(name: string): ConnectorType { +export function savedConnectionType(name: string): ConnectorType { if (!hasNamedConnectorConfig(name)) { throw new NotFoundError( `No saved connection named "${name}". Create one with 'libscope connect <type>'.`, @@ -372,7 +372,8 @@ export function listConnections(db: Database.Database): ConnectionInfo[] { /** * Disconnect saved connection `name`: remove the documents its connector created (unless * `keepDocuments`) and delete the saved config (it may hold credentials). Without a saved - * config, `type` must be given and only documents are removed. + * config, `type` (or a `name` that is a connector type) selects the connector and only + * documents are removed; otherwise an unknown name throws NotFoundError. */ export async function disconnectConnection( db: Database.Database, @@ -385,7 +386,12 @@ export async function disconnectConnection( configRemoved: boolean; }> { const saved = hasNamedConnectorConfig(name); - const type = saved ? savedConnectionType(name) : getConnector(options.type ?? name).type; + // Without a saved connection, `name` may be a connector type (connection saved before names). + const fallbackType = options.type ?? (isConnectorType(name) ? name : undefined); + const type = + saved || fallbackType === undefined + ? savedConnectionType(name) + : getConnector(fallbackType).type; if (options.type && options.type !== type) { throw new ValidationError(`Connection "${name}" is a ${type} connection, not ${options.type}`); } diff --git a/src/core/ingest.ts b/src/core/ingest.ts index b102a2f5..86e90b72 100644 --- a/src/core/ingest.ts +++ b/src/core/ingest.ts @@ -110,7 +110,7 @@ export interface IngestResult { | undefined; } -function isHttpUrl(value: string): boolean { +export function isHttpUrl(value: string): boolean { return value.startsWith("http://") || value.startsWith("https://"); } diff --git a/src/core/operations/connectors.ts b/src/core/operations/connectors.ts index 178e8359..9c5ec8ee 100644 --- a/src/core/operations/connectors.ts +++ b/src/core/operations/connectors.ts @@ -4,6 +4,7 @@ import { listNamedConnectorConfigs } from "../../connectors/index.js"; import { disconnectConnection, listConnections, + savedConnectionType, syncConnection, type ConnectorSyncSummary, } from "../../connectors/registry.js"; @@ -14,7 +15,7 @@ import { defineOperation } from "./types.js"; const connectionName = z .string() .regex(/^[\w-]+$/, "letters, digits, '_' and '-' only") - .describe("Saved connection name (see list-connections)"); + .describe("Name of a saved connection"); export const listConnectionsOperation = defineOperation({ name: "list-connections", @@ -40,10 +41,13 @@ export const syncOperation = defineOperation({ }), annotations: { longRunning: true }, http: { method: "POST", path: "/sync" }, - async handler(ctx, input): Promise<{ items: SyncOutcome[] }> { + validate(_ctx, input) { if ((input.name === undefined) === !input.all) { throw new ValidationError("Give either a connection name or all: true"); } + if (input.name !== undefined) savedConnectionType(input.name); + }, + async handler(ctx, input): Promise<{ items: SyncOutcome[] }> { if (input.name !== undefined) { const result = await syncConnection(ctx.db, ctx.provider, input.name, { signal: ctx.signal }); return { items: [{ ...result, status: "completed" }] }; diff --git a/src/core/operations/documents.ts b/src/core/operations/documents.ts index 0bac8d28..3b2004c5 100644 --- a/src/core/operations/documents.ts +++ b/src/core/operations/documents.ts @@ -9,7 +9,7 @@ import { type Document, } from "../documents.js"; import { getDocumentView } from "../document-view.js"; -import { detectIngestKind, ingest } from "../ingest.js"; +import { detectIngestKind, ingest, isHttpUrl } from "../ingest.js"; import { rateDocument } from "../ratings.js"; import { addTagsToDocument, getDocumentTags, removeTagsFromDocument } from "../tags.js"; import { findTopicId, resolveTopicId } from "../topics.js"; @@ -41,6 +41,15 @@ export function topicForFilter( return topic === undefined ? undefined : (findTopicId(ctx.db, topic) ?? topic); } +/** What `add` reads: `source`, else `url` when no inline content is given. */ +function addSource(input: { + source?: string | undefined; + content?: string | undefined; + url?: string | undefined; +}): string | undefined { + return input.source ?? (input.content === undefined ? input.url : undefined); +} + export const addOperation = defineOperation({ name: "add", group: "documents", @@ -117,23 +126,31 @@ export const addOperation = defineOperation({ }), annotations: { longRunning: true }, http: { method: "POST", path: "/documents" }, + validate(ctx, input) { + const source = addSource(input); + if (input.content === undefined && source === undefined) { + throw new ValidationError("Provide content, or a file, directory or URL to add"); + } + topicForWrite(ctx, input.topic); + // Remote callers must not read (or probe for) files on the server's disk. + if (ctx.surface !== "mcp" && ctx.surface !== "api") return; + const local = + input.kind === "file" || + input.kind === "directory" || + (input.kind === "auto" && input.content === undefined && !isHttpUrl(source ?? "")); + if (local) { + throw new ValidationError( + "Adding local files or directories is only available from the CLI and SDK", + ); + } + }, async handler(ctx, input) { - const source = input.source ?? (input.content === undefined ? input.url : undefined); const ingestInput = { ...input, - source, + source: addSource(input), topic: topicForWrite(ctx, input.topic), }; const kind = detectIngestKind(ingestInput); - // Remote callers must not read files from the server's disk. - if ( - (kind === "file" || kind === "directory") && - (ctx.surface === "mcp" || ctx.surface === "api") - ) { - throw new ValidationError( - "Adding local files or directories is only available from the CLI and SDK", - ); - } return ingest( { db: ctx.db, diff --git a/src/core/operations/types.ts b/src/core/operations/types.ts index 8e902050..279979fe 100644 --- a/src/core/operations/types.ts +++ b/src/core/operations/types.ts @@ -71,6 +71,11 @@ export interface Operation<S extends z.ZodObject = z.ZodObject, O = unknown> { annotations?: OperationAnnotations; /** REST mapping; `:name` path segments are filled from input fields of the same name. */ http?: { method: HttpMethod; path: string }; + /** + * Synchronous checks that need the context (e.g. the surface or saved settings). They run + * before the handler and before a background task starts, so callers get the error at once. + */ + validate?(ctx: OperationContext, input: z.output<S>): void; handler(ctx: OperationContext, input: z.output<S>): Promise<O> | O; } @@ -101,20 +106,31 @@ export function parseOperationInput<S extends z.ZodObject>( return parsed.data; } +/** Parse `rawInput`, then run the operation's `validate` hook. */ +function prepareInput<S extends z.ZodObject>( + op: Operation<S, unknown>, + ctx: OperationContext, + rawInput: unknown, +): z.output<S> { + const input = parseOperationInput(op, rawInput); + op.validate?.(ctx, input); + return input; +} + /** Validate `rawInput` against the operation's schema and run it. ZodError -> ValidationError. */ export async function runOperation<S extends z.ZodObject, O>( op: Operation<S, O>, ctx: OperationContext, rawInput: unknown, ): Promise<O> { - const input = parseOperationInput(op, rawInput); + const input = prepareInput(op, ctx, rawInput); ctx.signal?.throwIfAborted(); return op.handler(ctx, input); } /** - * Validate input now, then run the operation as a background task in the shared task - * registry. The task's signal and progress replace any in `ctx`. Returns the task; its + * Validate input now (schema and `validate` hook), then run the operation as a background + * task in the shared task registry. The task's signal and progress replace any in `ctx`. Returns the task; its * `result` is the JSON-encoded operation result. */ export function startOperationTask<S extends z.ZodObject, O>( @@ -122,7 +138,7 @@ export function startOperationTask<S extends z.ZodObject, O>( ctx: OperationContext, rawInput: unknown, ): Task { - const input = parseOperationInput(op, rawInput); + const input = prepareInput(op, ctx, rawInput); const { task } = taskRegistry.run( "operation", async (signal, onProgress) => { diff --git a/tests/unit/api-routes.test.ts b/tests/unit/api-routes.test.ts index e14ee44b..ef45bd0a 100644 --- a/tests/unit/api-routes.test.ts +++ b/tests/unit/api-routes.test.ts @@ -218,11 +218,12 @@ const CASES: Record<string, RouteCases> = { ok: () => ({ path: `${V}/sync`, body: { all: true } }), okStatus: 202, bad: () => ({ path: `${V}/sync`, body: { name: "bad name!" } }), + missing: () => ({ path: `${V}/sync`, body: { name: "nope" } }), }, "DELETE /api/v1/connections/:name": { ok: () => ({ path: `${V}/connections/notes` }), - // An unknown name with no saved connection is a 400 (unknown connector type), not a 404. - bad: () => ({ path: `${V}/connections/nope` }), + bad: () => ({ path: `${V}/connections/notes`, query: { type: "nope" } }), + missing: () => ({ path: `${V}/connections/nope` }), }, "GET /api/v1/overview": { ok: () => ({ path: `${V}/overview` }) }, @@ -487,16 +488,19 @@ describe("REST routes generated from operations", () => { expect((tagged.body.data as { total: number }).total).toBe(1); }); - it("refuses local files over REST (the add task fails)", async () => { - const started = await send("POST", { path: `${V}/documents`, body: { source: "/etc/hosts" } }); - const { taskId } = started.body.data as { taskId: string }; - await vi.waitFor(async () => { - const polled = await send("GET", { path: `${V}/tasks/${taskId}` }); - expect(polled.body.data).toMatchObject({ - status: "failed", - error: expect.stringContaining("only available from the CLI") as string, - }); + it("refuses local paths over REST with 400 before any task starts", async () => { + const tasksBefore = taskRegistry.list().length; + for (const source of ["/etc/hosts", "/no/such/path"]) { + const reply = await send("POST", { path: `${V}/documents`, body: { source } }); + expect(reply.status).toBe(400); + expect(reply.body.error?.message).toContain("only available from the CLI"); + } + const noTopic = await send("POST", { + path: `${V}/documents`, + body: { title: "T", content: "C", topic: "nope" }, }); + expect(noTopic.status).toBe(404); + expect(taskRegistry.list()).toHaveLength(tasksBefore); }); it("answers 400 for a malformed JSON body and a non-object body", async () => { diff --git a/tests/unit/mcp-server.test.ts b/tests/unit/mcp-server.test.ts index d6fed3c4..504ddfab 100644 --- a/tests/unit/mcp-server.test.ts +++ b/tests/unit/mcp-server.test.ts @@ -424,6 +424,10 @@ describe("MCP server: tool calls", () => { expect(status).toContain("Background (documentId: "); expect(await callError(client, "submit-document", {})).toContain("Provide content"); + // Checked before a background task starts: the error comes back at once, with no taskId. + const background = await callError(client, "submit-document", { async: true }); + expect(background).toContain("Provide content"); + expect(background).not.toContain("taskId"); expect(await callError(client, "submit-document", { url: "/etc/passwd" })).toMatch(/url/); expect(await callError(client, "submit-document", { title: "T", content: "" })).toMatch( /content/, diff --git a/tests/unit/operations/connectors.test.ts b/tests/unit/operations/connectors.test.ts index da8ca829..e29aa603 100644 --- a/tests/unit/operations/connectors.test.ts +++ b/tests/unit/operations/connectors.test.ts @@ -100,7 +100,7 @@ describe("connector operations", () => { const result = await run(ops.disconnectOperation, t.ctx, { name: "slack" }); expect(result).toMatchObject({ type: "slack", documentsRemoved: 0, configRemoved: false }); await expect(run(ops.disconnectOperation, t.ctx, { name: "unknown" })).rejects.toBeInstanceOf( - ValidationError, + NotFoundError, ); saveVault("notes"); await expect( diff --git a/tests/unit/operations/documents.test.ts b/tests/unit/operations/documents.test.ts index e00d7575..a6631837 100644 --- a/tests/unit/operations/documents.test.ts +++ b/tests/unit/operations/documents.test.ts @@ -10,6 +10,7 @@ import { listDocumentsOperation, rateDocumentOperation, rollbackDocumentOperation, + startOperationTask, updateDocumentOperation, } from "../../../src/core/operations/index.js"; import { createTopic } from "../../../src/core/topics.js"; @@ -72,6 +73,13 @@ describe("document operations", () => { const mcp = makeContext({ db: t.db, surface: "mcp" }); await expectValidationError(run(addOperation, mcp.ctx, { source: dir })); + // Refused before a background task starts, also for paths that do not exist. + const api = makeContext({ db: t.db, surface: "api" }); + for (const source of [dir, join(dir, "missing")]) { + expect(() => startOperationTask(addOperation, api.ctx, { source })).toThrow( + /only available from the CLI/, + ); + } } finally { rmSync(dir, { recursive: true, force: true }); } From 7882db7f0b83f462c34440db1f1a790fa2801f0a Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:33:31 +0000 Subject: [PATCH 54/83] fix(searches)!: store saved search filters under the search operation's names save-search stores its input filters as given (sourceType, topic, library, version, tags, minRating, limit); run-saved-search runs them through the search operation. Core runSavedSearch is replaced by recordSavedSearchRun. Migration 19 renames source -> sourceType in existing rows (schema version 19). --- docs/guide/architecture.md | 2 +- src/core/operations/searches.ts | 31 ++++---- src/core/saved-searches.ts | 44 ++++++------ src/db/schema.ts | 18 ++++- .../operations/searches-analytics.test.ts | 9 ++- tests/unit/saved-searches.test.ts | 70 +++---------------- tests/unit/schema-migrations.test.ts | 32 +++++++++ tests/unit/schema.test.ts | 2 +- 8 files changed, 101 insertions(+), 107 deletions(-) diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 8f2e11db..2c6d8c54 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -272,7 +272,7 @@ The LLM integration in `src/core/rag.ts` supports OpenAI, Ollama, Anthropic, and ## Database Schema -Key tables (schema version 18): +Key tables (schema version 19): | Table | Purpose | | ------------------- | -------------------------------------------------- | diff --git a/src/core/operations/searches.ts b/src/core/operations/searches.ts index 4084e744..fa4faa30 100644 --- a/src/core/operations/searches.ts +++ b/src/core/operations/searches.ts @@ -3,13 +3,13 @@ import { countSavedSearches, createSavedSearch, deleteSavedSearch, + getSavedSearch, listSavedSearches, - runSavedSearch, + recordSavedSearchRun, } from "../saved-searches.js"; -import type { SearchOptions } from "../search.js"; -import { topicForFilter } from "./documents.js"; +import { searchOperation } from "./search.js"; import * as s from "./schemas.js"; -import { defineOperation, type ListResult } from "./types.js"; +import { defineOperation, runOperation, type ListResult } from "./types.js"; const savedSearch = z.string().min(1).describe("Saved search name or ID"); @@ -26,19 +26,12 @@ export const saveSearchOperation = defineOperation({ }), http: { method: "POST", path: "/searches" }, handler(ctx, input) { - const filters: Omit<SearchOptions, "query"> = {}; - const topic = topicForFilter(ctx, input.topic); - if (topic !== undefined) filters.topic = topic; - if (input.library !== undefined) filters.library = input.library; - if (input.version !== undefined) filters.version = input.version; - if (input.sourceType !== undefined) filters.source = input.sourceType; - if (input.tags !== undefined) filters.tags = input.tags; - if (input.minRating !== undefined) filters.minRating = input.minRating; - if (input.limit !== undefined) filters.limit = input.limit; + // Filters are stored under the search operation's field names and run through it. + const { name, query, ...filters } = input; return createSavedSearch( ctx.db, - input.name, - input.query, + name, + query, Object.keys(filters).length > 0 ? filters : undefined, ); }, @@ -67,8 +60,12 @@ export const runSavedSearchOperation = defineOperation({ annotations: { readOnly: true }, http: { method: "POST", path: "/searches/:search/run" }, async handler(ctx, input) { - const { search, results } = await runSavedSearch(ctx.db, ctx.provider, input.search); - return { search, items: results }; + const saved = getSavedSearch(ctx.db, input.search); + const { items } = await runOperation(searchOperation, ctx, { + ...saved.filters, + query: saved.query, + }); + return { search: recordSavedSearchRun(ctx.db, saved.id, items.length), items }; }, }); diff --git a/src/core/saved-searches.ts b/src/core/saved-searches.ts index 59b0e6d0..a4bc8ab5 100644 --- a/src/core/saved-searches.ts +++ b/src/core/saved-searches.ts @@ -1,16 +1,24 @@ import { randomUUID } from "node:crypto"; import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import { searchDocuments } from "./search.js"; -import type { SearchOptions, SearchResult } from "./search.js"; import { ValidationError, NotFoundError } from "../errors.js"; import { getLogger } from "../logger.js"; +/** Stored filters: input fields of the `search` operation (same names). */ +export interface SavedSearchFilters { + topic?: string | undefined; + library?: string | undefined; + version?: string | undefined; + sourceType?: string | undefined; + tags?: string[] | undefined; + minRating?: number | undefined; + limit?: number | undefined; +} + export interface SavedSearch { id: string; name: string; query: string; - filters: Omit<SearchOptions, "query"> | null; + filters: SavedSearchFilters | null; createdAt: string; lastRunAt: string | null; resultCount: number; @@ -27,10 +35,10 @@ interface SavedSearchRow { } function rowToSavedSearch(row: SavedSearchRow): SavedSearch { - let filters: Omit<SearchOptions, "query"> | null = null; + let filters: SavedSearchFilters | null = null; if (row.filters) { try { - filters = JSON.parse(row.filters) as Omit<SearchOptions, "query">; + filters = JSON.parse(row.filters) as SavedSearchFilters; } catch { getLogger().warn({ id: row.id }, "Failed to parse saved search filters JSON; using null"); } @@ -50,7 +58,7 @@ export function createSavedSearch( db: Database.Database, name: string, query: string, - filters?: Omit<SearchOptions, "query">, + filters?: SavedSearchFilters, ): SavedSearch { const trimmedName = name.trim(); const trimmedQuery = query.trim(); @@ -117,24 +125,14 @@ export function deleteSavedSearch(db: Database.Database, id: string): void { } } -export async function runSavedSearch( +/** Record a run of saved search `id` (time and result count); returns the updated search. */ +export function recordSavedSearchRun( db: Database.Database, - provider: EmbeddingProvider, id: string, -): Promise<{ search: SavedSearch; results: SearchResult[] }> { - const search = getSavedSearch(db, id); - - const options: SearchOptions = { - query: search.query, - ...search.filters, - }; - - const { results } = await searchDocuments(db, provider, options); - + resultCount: number, +): SavedSearch { db.prepare( "UPDATE saved_searches SET last_run_at = datetime('now'), result_count = ? WHERE id = ?", - ).run(results.length, search.id); - - const updated = getSavedSearch(db, search.id); - return { search: updated, results }; + ).run(resultCount, id); + return getSavedSearch(db, id); } diff --git a/src/db/schema.ts b/src/db/schema.ts index 0e84fcb4..5dd714b2 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -12,7 +12,7 @@ import { writeEmbeddingIdentity, } from "./index-meta.js"; -const SCHEMA_VERSION = 18; +const SCHEMA_VERSION = 19; /** Copy every chunk into chunks_fts, keyed by the chunk's rowid. */ const FTS_BACKFILL_SQL = ` @@ -331,6 +331,22 @@ const MIGRATIONS: Record<number, string> = { INSERT INTO schema_version (version) VALUES (18); `, + // Saved search filters use the search operation's field names: "source" -> "sourceType". + // Rows with invalid JSON or without "source" are left as they are. + 19: ` + UPDATE saved_searches + SET filters = json_remove( + json_set(filters, '$.sourceType', json_extract(filters, '$.source')), + '$.source' + ) + WHERE CASE + WHEN json_valid(filters) THEN json_type(filters, '$.source') IS NOT NULL + AND json_type(filters, '$.sourceType') IS NULL + ELSE 0 + END; + + INSERT INTO schema_version (version) VALUES (19); + `, }; /** diff --git a/tests/unit/operations/searches-analytics.test.ts b/tests/unit/operations/searches-analytics.test.ts index 42741ef8..a41123dd 100644 --- a/tests/unit/operations/searches-analytics.test.ts +++ b/tests/unit/operations/searches-analytics.test.ts @@ -36,7 +36,7 @@ describe("saved search and analytics operations", () => { library: "react", sourceType: "library", }); - expect(saved.filters).toEqual({ library: "react", source: "library" }); + expect(saved.filters).toEqual({ library: "react", sourceType: "library" }); const list = await run(listSavedSearchesOperation, t.ctx, {}); expect(list).toMatchObject({ total: 1, limit: 50, offset: 0 }); @@ -44,9 +44,14 @@ describe("saved search and analytics operations", () => { const ran = await run(runSavedSearchOperation, t.ctx, { search: "react-hooks" }); expect(ran.items.length).toBe(1); expect(ran.search.resultCount).toBe(1); + expect(ran.search.lastRunAt).not.toBeNull(); + + // Stored filters apply when the search runs. + await run(saveSearchOperation, t.ctx, { name: "vue", query: "hooks", library: "vue" }); + expect((await run(runSavedSearchOperation, t.ctx, { search: "vue" })).items).toEqual([]); await run(deleteSavedSearchOperation, t.ctx, { search: saved.id }); - expect((await run(listSavedSearchesOperation, t.ctx, {})).total).toBe(0); + expect((await run(listSavedSearchesOperation, t.ctx, {})).total).toBe(1); }); it("rejects a missing query and a duplicate name", async () => { diff --git a/tests/unit/saved-searches.test.ts b/tests/unit/saved-searches.test.ts index db01fbf0..755c194e 100644 --- a/tests/unit/saved-searches.test.ts +++ b/tests/unit/saved-searches.test.ts @@ -1,14 +1,12 @@ import { describe, it, expect, beforeEach } from "vitest"; -import { createTestDb, createTestDbWithVec } from "../fixtures/test-db.js"; -import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { createTestDb } from "../fixtures/test-db.js"; import { createSavedSearch, listSavedSearches, getSavedSearch, deleteSavedSearch, - runSavedSearch, + recordSavedSearchRun, } from "../../src/core/saved-searches.js"; -import { indexDocument } from "../../src/core/indexing.js"; import { ValidationError, NotFoundError } from "../../src/errors.js"; import type Database from "better-sqlite3"; import { initLogger } from "../../src/logger.js"; @@ -17,11 +15,9 @@ initLogger("silent"); describe("saved-searches", () => { let db: Database.Database; - let provider: MockEmbeddingProvider; beforeEach(() => { db = createTestDb(); - provider = new MockEmbeddingProvider(); }); describe("createSavedSearch", () => { @@ -112,63 +108,13 @@ describe("saved-searches", () => { }); }); - describe("runSavedSearch", () => { - it("should run a saved search and update metadata", async () => { - const vecDb = createTestDbWithVec(); - const vecProvider = new MockEmbeddingProvider(); - - await indexDocument(vecDb, vecProvider, { - title: "TypeScript Guide", - content: "# TypeScript\n\nTypeScript is a typed superset of JavaScript.", - sourceType: "manual", - }); - - const saved = createSavedSearch(vecDb, "TS Search", "typescript"); + describe("recordSavedSearchRun", () => { + it("records the run time and result count", () => { + const saved = createSavedSearch(db, "TS Search", "typescript"); expect(saved.lastRunAt).toBeNull(); - expect(saved.resultCount).toBe(0); - - const { search, results } = await runSavedSearch(vecDb, vecProvider, saved.id); - expect(search.lastRunAt).toBeTruthy(); - expect(search.resultCount).toBe(results.length); - expect(results.length).toBeGreaterThan(0); - }); - - it("should run by name", async () => { - const vecDb = createTestDbWithVec(); - const vecProvider = new MockEmbeddingProvider(); - - await indexDocument(vecDb, vecProvider, { - title: "Test Doc", - content: "# Test\n\nSome test content here.", - sourceType: "manual", - }); - - createSavedSearch(vecDb, "Run By Name", "test"); - const { search } = await runSavedSearch(vecDb, vecProvider, "Run By Name"); - expect(search.name).toBe("Run By Name"); - expect(search.lastRunAt).toBeTruthy(); - }); - - it("should apply saved filters", async () => { - const vecDb = createTestDbWithVec(); - const vecProvider = new MockEmbeddingProvider(); - - await indexDocument(vecDb, vecProvider, { - title: "Express Auth", - content: "# Auth\n\nAuthentication with Express.", - sourceType: "library", - library: "express", - }); - - const saved = createSavedSearch(vecDb, "Express Only", "auth", { library: "express" }); - const { results } = await runSavedSearch(vecDb, vecProvider, saved.id); - for (const r of results) { - expect(r.library).toBe("express"); - } - }); - - it("should throw for nonexistent search", async () => { - await expect(runSavedSearch(db, provider, "nonexistent")).rejects.toThrow(NotFoundError); + const updated = recordSavedSearchRun(db, saved.id, 3); + expect(updated.lastRunAt).toBeTruthy(); + expect(updated.resultCount).toBe(3); }); }); diff --git a/tests/unit/schema-migrations.test.ts b/tests/unit/schema-migrations.test.ts index c86e7a1e..dec1f113 100644 --- a/tests/unit/schema-migrations.test.ts +++ b/tests/unit/schema-migrations.test.ts @@ -191,6 +191,38 @@ describe("schema migrations", () => { }); }); + describe("migration 19: saved search filter names", () => { + it("renames source to sourceType and keeps other rows and keys", () => { + const db = newDb(); + runMigrations(db, 18); + const insert = db.prepare( + "INSERT INTO saved_searches (id, name, query, filters) VALUES (?, ?, 'q', ?)", + ); + insert.run("a", "a", JSON.stringify({ source: "library", topic: "t", limit: 5 })); + insert.run("b", "b", JSON.stringify({ library: "react" })); + insert.run("c", "c", null); + insert.run("d", "d", "{not json"); + + runMigrations(db); + + const filters = (id: string): string | null => + ( + db.prepare("SELECT filters FROM saved_searches WHERE id = ?").get(id) as { + filters: string | null; + } + ).filters; + expect(JSON.parse(filters("a") ?? "null")).toEqual({ + sourceType: "library", + topic: "t", + limit: 5, + }); + expect(JSON.parse(filters("b") ?? "null")).toEqual({ library: "react" }); + expect(filters("c")).toBeNull(); + expect(filters("d")).toBe("{not json"); + expect(count(db, "saved_searches")).toBe(4); + }); + }); + describe("migration 18: index_meta table", () => { it("creates an empty index_meta table on upgrade", () => { const db = newDb(); diff --git a/tests/unit/schema.test.ts b/tests/unit/schema.test.ts index 00288016..97865943 100644 --- a/tests/unit/schema.test.ts +++ b/tests/unit/schema.test.ts @@ -38,7 +38,7 @@ describe("database schema", () => { const version = db.prepare("SELECT MAX(version) as v FROM schema_version").get() as { v: number; }; - expect(version.v).toBe(18); + expect(version.v).toBe(19); }); it("should create expected indexes", () => { From f1f8fe31436768614f77352c68e8d131721f48e2 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:45:43 +0000 Subject: [PATCH 55/83] feat(sdk)!: Python and Go clients use the 2.0 REST routes and background tasks Both clients call the /api/v1 operation routes with the operation field names, return a Task for add/sync (202) and poll GET /api/v1/tasks/:taskId with wait_for_task / WaitForTask. Added: ask (Go), graph (Go), sync by saved connection name, overview, remove tags, API key option. Removed: calls to /stats, /documents/url, /search?q, /connectors/sync and the client-side min-score filter. Go request options are one Param type. --- sdk/go/README.md | 132 ++-- sdk/go/analytics.go | 19 - sdk/go/client.go | 118 ++-- sdk/go/client_test.go | 300 ++++++--- sdk/go/connectors.go | 14 +- sdk/go/documents.go | 170 +---- sdk/go/documents_test.go | 194 ------ sdk/go/errors.go | 40 +- sdk/go/examples/quickstart/main.go | 37 +- sdk/go/graph.go | 8 +- sdk/go/models.go | 217 +++++-- sdk/go/params.go | 88 +++ sdk/go/search.go | 90 +-- sdk/go/search_test.go | 113 ---- sdk/go/tags.go | 45 +- sdk/go/tasks.go | 61 ++ sdk/go/topics.go | 49 +- sdk/python/README.md | 102 +-- sdk/python/examples/quickstart.py | 39 +- sdk/python/src/pylibscope/__init__.py | 30 +- sdk/python/src/pylibscope/client.py | 795 ++++++++++++------------ sdk/python/src/pylibscope/connectors.py | 18 - sdk/python/src/pylibscope/exceptions.py | 16 + sdk/python/src/pylibscope/models.py | 163 +++-- sdk/python/tests/test_client.py | 465 ++++++-------- sdk/python/tests/test_models.py | 127 ++-- 26 files changed, 1679 insertions(+), 1771 deletions(-) delete mode 100644 sdk/go/analytics.go delete mode 100644 sdk/go/documents_test.go create mode 100644 sdk/go/params.go delete mode 100644 sdk/go/search_test.go create mode 100644 sdk/go/tasks.go delete mode 100644 sdk/python/src/pylibscope/connectors.py diff --git a/sdk/go/README.md b/sdk/go/README.md index 88e4c664..45913877 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -1,6 +1,6 @@ # libscope Go SDK -A lightweight, idiomatic Go client for the [libscope](https://github.com/RobertLD/libscope) REST API. Zero external dependencies — built entirely on the Go standard library. +A small Go client for the [libscope](https://github.com/RobertLD/libscope) REST API (`/api/v1`). It uses only the Go standard library. ## Installation @@ -8,6 +8,8 @@ A lightweight, idiomatic Go client for the [libscope](https://github.com/RobertL go get github.com/RobertLD/libscope/sdk/go ``` +Start a server with `libscope serve api` (default `http://localhost:3378`). + ## Quick Start ```go @@ -17,46 +19,36 @@ import ( "context" "fmt" "log" + "time" libscope "github.com/RobertLD/libscope/sdk/go" ) func main() { + ctx := context.Background() client := libscope.NewClient() - // Add a document from a URL - doc, err := client.AddDocument(context.Background(), "https://go.dev/doc/") + // Adding runs as a background task on the server. WaitForTask polls it. + task, err := client.AddURL(ctx, "https://go.dev/doc/", libscope.WithTags("go")) if err != nil { log.Fatal(err) } - fmt.Printf("Indexed: %s (%s)\n", doc.Title, doc.ID) - - // Add a text document - doc, err = client.AddText(context.Background(), "Go Concurrency", "Goroutines are lightweight threads...") + task, err = client.WaitForTask(ctx, task.ID, time.Second) if err != nil { log.Fatal(err) } - fmt.Printf("Added: %s\n", doc.ID) - - // Search - results, err := client.Search(context.Background(), "goroutines", - libscope.WithLimit(5), - libscope.WithMinScore(0.5), - ) - if err != nil { + var added libscope.IngestResult + if err := task.DecodeResult(&added); err != nil { log.Fatal(err) } - for _, hit := range results.Results { - fmt.Printf(" %s: %.2f\n", hit.Document.Title, hit.Score) - } + fmt.Printf("Indexed: %s (%s)\n", added.Documents[0].Title, added.Documents[0].DocumentID) - // List all topics - topics, err := client.ListTopics(context.Background()) + page, err := client.Search(ctx, "goroutines", libscope.WithLimit(5)) if err != nil { log.Fatal(err) } - for _, t := range topics { - fmt.Printf(" Topic: %s\n", t.Name) + for _, hit := range page.Items { + fmt.Printf(" %s: %.2f\n", hit.Title, hit.Score) } } ``` @@ -64,76 +56,56 @@ func main() { ## Configuration ```go -// Custom base URL client := libscope.NewClient( libscope.WithBaseURL("http://my-server:3378"), -) - -// Custom timeout -client := libscope.NewClient( - libscope.WithTimeout(10 * time.Second), -) - -// Custom HTTP client -client := libscope.NewClient( - libscope.WithHTTPClient(&http.Client{ - Transport: myTransport, - }), + libscope.WithTimeout(10*time.Second), + libscope.WithAPIKey(os.Getenv("LIBSCOPE_API_KEY")), // "Authorization: Bearer <key>" + libscope.WithHTTPClient(&http.Client{Transport: myTransport}), ) ``` -## API Reference +## Background tasks -### Documents +`AddText`, `AddURL`, `Sync` and `SyncAll` return a `*Task` at once (the server answers `202`). `WaitForTask(ctx, taskID, interval)` polls `GET /api/v1/tasks/:taskId` until the task finishes. It returns `*TaskError` when the task failed or was cancelled. Bound the wait with `context.WithTimeout`. `Task.DecodeResult` decodes the operation result (`*IngestResult` for `AddText` and `AddURL`). `GetTask` and `CancelTask` are also available. -| Method | Description | -| --------------------------------------- | ------------------------------------ | -| `AddDocument(ctx, url, opts...)` | Index a document from a URL | -| `AddText(ctx, title, content, opts...)` | Index raw text content | -| `GetDocument(ctx, id)` | Get a document by ID | -| `ListDocuments(ctx, opts...)` | List documents with optional filters | -| `DeleteDocument(ctx, id)` | Delete a document | - -### Search - -| Method | Description | -| ----------------------------- | -------------------------------- | -| `Search(ctx, query, opts...)` | Semantic search across documents | - -Search options: `WithLimit(n)`, `WithTopic(t)`, `WithTags(tags...)`, `WithMinScore(s)` - -### Topics - -| Method | Description | -| --------------------------------- | ------------------ | -| `ListTopics(ctx)` | List all topics | -| `CreateTopic(ctx, name, opts...)` | Create a new topic | - -### Tags - -| Method | Description | -| ------------------------------------- | ---------------------- | -| `ListTags(ctx)` | List all tags | -| `AddTagsToDocument(ctx, docID, tags)` | Add tags to a document | - -### Analytics +## API Reference -| Method | Description | -| --------------- | ----------------------- | -| `GetStats(ctx)` | Get instance statistics | -| `Health(ctx)` | Health check | +Optional fields are `Param` values, for example `WithTopic("go")`. The same `Param` works on every method that accepts the field. + +| Method | Description | Params | +| ----------------------------------------- | -------------------------------------------------------------------------------------- | ------------------------------------------------------- | +| `Search(ctx, query, params...)` | Search by meaning and keywords (`*Page[SearchHit]`) | `WithLimit`, `WithOffset`, `WithMinRating`, filters | +| `Ask(ctx, question, params...)` | Answer a question from the knowledge base (the server needs an LLM) | `WithTopK`, `WithMinRating`, filters | +| `AddText(ctx, title, content, params...)` | Start adding a document from text (`*Task`) | `WithURL`, filters | +| `AddURL(ctx, url, params...)` | Start adding a web page, a crawled site or a public GitHub/GitLab repository (`*Task`) | `WithSpider`, `WithMaxPages`, filters | +| `GetDocument(ctx, id)` | Document with content, tags, links and ratings (`*DocumentView`) | | +| `ListDocuments(ctx, params...)` | List documents (`*Page[Document]`) | `WithLimit`, `WithOffset`, filters | +| `DeleteDocument(ctx, id)` | Delete a document | | +| `ListTopics(ctx)` | Topics with document counts | | +| `CreateTopic(ctx, name, params...)` | Create a topic | `WithParent`, `WithDescription` | +| `ListTags(ctx)` | Tags with document counts | | +| `AddTags(ctx, id, tags)` | Add tags; returns the document's tags | | +| `RemoveTags(ctx, id, tags)` | Remove tags; returns the document's tags | | +| `GetGraph(ctx, params...)` | Knowledge graph | `WithTopic`, `WithTag`, `WithThreshold`, `WithMaxNodes` | +| `Sync(ctx, name)` / `SyncAll(ctx)` | Start syncing saved connector connections (`*Task`) | | +| `GetTask` / `CancelTask` / `WaitForTask` | Background tasks | | +| `Overview(ctx)` | Counts, topics, installed packs, index and health | | +| `Health(ctx)` | Liveness check | | + +Filters: `WithTopic`, `WithLibrary`, `WithVersion`, `WithSourceType`, `WithTags`. On `AddText` and `AddURL` they set the new document's fields. ## Error Handling -All methods return `(*T, error)`. API errors are returned as `*libscope.Error`: +API errors are returned as `*libscope.Error`. `errors.Is` matches by HTTP status (`ErrNotFound`, `ErrBadRequest`, `ErrServerError`): ```go -doc, err := client.GetDocument(ctx, "nonexistent") -if err != nil { - var apiErr *libscope.Error - if errors.As(err, &apiErr) { - fmt.Printf("API error %d: %s\n", apiErr.StatusCode, apiErr.Message) - } +_, err := client.GetDocument(ctx, "nonexistent") +if errors.Is(err, libscope.ErrNotFound) { + // ... +} +var apiErr *libscope.Error +if errors.As(err, &apiErr) { + fmt.Printf("API error %d %s: %s\n", apiErr.StatusCode, apiErr.Code, apiErr.Message) } ``` @@ -144,4 +116,4 @@ cd sdk/go go test ./... -v -count=1 ``` -All tests use `httptest.NewServer` — no running libscope instance required. +The tests use `httptest.NewServer`. They do not need a running libscope server. diff --git a/sdk/go/analytics.go b/sdk/go/analytics.go deleted file mode 100644 index 6b5d79c0..00000000 --- a/sdk/go/analytics.go +++ /dev/null @@ -1,19 +0,0 @@ -package libscope - -import ( - "context" - "net/http" -) - -// GetStats retrieves analytics/statistics about the libscope instance. -func (c *Client) GetStats(ctx context.Context) (*Stats, error) { - resp, err := c.do(ctx, http.MethodGet, "/api/v1/stats", nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[Stats](resp) - if err != nil { - return nil, err - } - return &result, nil -} diff --git a/sdk/go/client.go b/sdk/go/client.go index e2819d68..b9192928 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -1,11 +1,13 @@ package libscope import ( + "bytes" "context" "encoding/json" "fmt" "io" "net/http" + "net/url" "strings" "time" ) @@ -13,21 +15,23 @@ import ( const ( defaultBaseURL = "http://localhost:3378" defaultTimeout = 30 * time.Second + apiPrefix = "/api/v1" ) -// Client is the libscope API client. +// Client is the libscope REST API client. type Client struct { baseURL string + apiKey string httpClient *http.Client } // Option configures a Client. type Option func(*Client) -// WithBaseURL sets the base URL for the API. -func WithBaseURL(url string) Option { +// WithBaseURL sets the server URL (default http://localhost:3378). +func WithBaseURL(u string) Option { return func(c *Client) { - c.baseURL = strings.TrimRight(url, "/") + c.baseURL = strings.TrimRight(u, "/") } } @@ -45,7 +49,14 @@ func WithHTTPClient(hc *http.Client) Option { } } -// NewClient creates a new libscope API client with the given options. +// WithAPIKey sends "Authorization: Bearer <key>" (needed when the server sets LIBSCOPE_API_KEY). +func WithAPIKey(key string) Option { + return func(c *Client) { + c.apiKey = key + } +} + +// NewClient creates a libscope API client. func NewClient(opts ...Option) *Client { c := &Client{ baseURL: defaultBaseURL, @@ -57,65 +68,84 @@ func NewClient(opts ...Option) *Client { return c } -// do executes an HTTP request and returns the response body. -func (c *Client) do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { - url := c.baseURL + path - req, err := http.NewRequestWithContext(ctx, method, url, body) +// call sends a request to /api/v1+path and decodes the "data" envelope into out (if non-nil). +// Non-2xx responses are returned as *Error. +func (c *Client) call(ctx context.Context, method, path string, query url.Values, body, out any) error { + target := c.baseURL + apiPrefix + path + if len(query) > 0 { + target += "?" + query.Encode() + } + var reader io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("libscope: encoding request: %w", err) + } + reader = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, target, reader) if err != nil { - return nil, fmt.Errorf("libscope: creating request: %w", err) + return fmt.Errorf("libscope: creating request: %w", err) } if body != nil { req.Header.Set("Content-Type", "application/json") } + if c.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+c.apiKey) + } resp, err := c.httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("libscope: executing request: %w", err) + return fmt.Errorf("libscope: executing request: %w", err) } - return resp, nil -} - -// decodeResponse reads the response body and decodes the API envelope. -// On non-2xx status, it returns an *Error. -func decodeResponse[T any](resp *http.Response) (T, error) { - var zero T defer resp.Body.Close() - data, err := io.ReadAll(resp.Body) if err != nil { - return zero, fmt.Errorf("libscope: reading response: %w", err) + return fmt.Errorf("libscope: reading response: %w", err) } - if resp.StatusCode >= 400 { - var errResp apiErrorResponse - if json.Unmarshal(data, &errResp) == nil && errResp.Error.Message != "" { - return zero, &Error{ - StatusCode: resp.StatusCode, - Code: errResp.Error.Code, - Message: errResp.Error.Message, - } - } - return zero, &Error{ - StatusCode: resp.StatusCode, - Message: string(data), - } + return newError(resp.StatusCode, data) } - - var envelope apiResponse[T] + if out == nil { + return nil + } + envelope := struct { + Data any `json:"data"` + }{Data: out} if err := json.Unmarshal(data, &envelope); err != nil { - return zero, fmt.Errorf("libscope: decoding response: %w", err) + return fmt.Errorf("libscope: decoding response: %w", err) } - return envelope.Data, nil + return nil } -// Health checks the health of the libscope server. -func (c *Client) Health(ctx context.Context) (*HealthStatus, error) { - resp, err := c.do(ctx, http.MethodGet, "/api/v1/health", nil) - if err != nil { +// get decodes a GET response into a new T. +func get[T any](ctx context.Context, c *Client, path string, query url.Values) (*T, error) { + var out T + if err := c.call(ctx, http.MethodGet, path, query, nil, &out); err != nil { return nil, err } - result, err := decodeResponse[HealthStatus](resp) - if err != nil { + return &out, nil +} + +// send decodes a response with a JSON body into a new T. +func send[T any](ctx context.Context, c *Client, method, path string, body any) (*T, error) { + var out T + if err := c.call(ctx, method, path, nil, body, &out); err != nil { return nil, err } - return &result, nil + return &out, nil +} + +// segment escapes one path segment (an ID or a name). +func segment(s string) string { + return "/" + url.PathEscape(s) +} + +// Health checks that the server is up. +func (c *Client) Health(ctx context.Context) (*HealthStatus, error) { + return get[HealthStatus](ctx, c, "/health", nil) +} + +// Overview returns counts, topics, installed packs, the embedding index and health. +func (c *Client) Overview(ctx context.Context) (*Overview, error) { + return get[Overview](ctx, c, "/overview", nil) } diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 47fdd0c5..4aa46ef4 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -2,105 +2,261 @@ package libscope import ( "context" + "encoding/json" + "errors" + "io" "net/http" "net/http/httptest" + "reflect" "testing" "time" ) -func TestNewClientDefaults(t *testing.T) { - c := NewClient() - if c.baseURL != defaultBaseURL { - t.Errorf("expected base URL %q, got %q", defaultBaseURL, c.baseURL) - } - if c.httpClient.Timeout != defaultTimeout { - t.Errorf("expected timeout %v, got %v", defaultTimeout, c.httpClient.Timeout) - } -} - -func TestNewClientWithOptions(t *testing.T) { - c := NewClient( - WithBaseURL("http://example.com/"), - WithTimeout(5*time.Second), - ) - if c.baseURL != "http://example.com" { - t.Errorf("expected base URL %q, got %q", "http://example.com", c.baseURL) - } - if c.httpClient.Timeout != 5*time.Second { - t.Errorf("expected timeout 5s, got %v", c.httpClient.Timeout) - } +// request is what the fake server saw. +type request struct { + Method string + Path string + Query map[string][]string + Body map[string]any + Header http.Header } -func TestWithHTTPClient(t *testing.T) { - custom := &http.Client{Timeout: 10 * time.Second} - c := NewClient(WithHTTPClient(custom)) - if c.httpClient != custom { - t.Error("expected custom HTTP client") - } +// fakeServer answers each request with the next reply ({"data": ...} with status 200 unless +// set) and records the requests. +type reply struct { + status int + data any + raw string } -func TestHealth(t *testing.T) { +func fakeServer(t *testing.T, replies ...reply) (*Client, *[]request) { + t.Helper() + var seen []request srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/v1/health" { - t.Errorf("unexpected path: %s", r.URL.Path) + req := request{Method: r.Method, Path: r.URL.EscapedPath(), Query: r.URL.Query(), Header: r.Header} + if data, _ := io.ReadAll(r.Body); len(data) > 0 { + if err := json.Unmarshal(data, &req.Body); err != nil { + t.Errorf("request body is not a JSON object: %s", data) + } + } + seen = append(seen, req) + if len(replies) == 0 { + t.Errorf("unexpected request %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusInternalServerError) + return + } + rep := replies[0] + if len(replies) > 1 { + replies = replies[1:] } w.Header().Set("Content-Type", "application/json") - w.WriteHeader(200) - w.Write([]byte(`{"data":{"status":"ok","docCount":42,"dbSize":1024},"meta":{"took":5}}`)) + if rep.status != 0 { + w.WriteHeader(rep.status) + } + if rep.raw != "" { + _, _ = w.Write([]byte(rep.raw)) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": rep.data, "meta": map[string]any{"took": 1}}) })) - defer srv.Close() + t.Cleanup(srv.Close) + return NewClient(WithBaseURL(srv.URL + "/")), &seen +} + +func check(t *testing.T, label string, got, want any) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: got %#v, want %#v", label, got, want) + } +} - c := NewClient(WithBaseURL(srv.URL)) - health, err := c.Health(context.Background()) +func must(t *testing.T, err error) { + t.Helper() if err != nil { t.Fatalf("unexpected error: %v", err) } - if health.Status != "ok" { - t.Errorf("expected status ok, got %q", health.Status) - } - if health.DocCount != 42 { - t.Errorf("expected 42 docs, got %d", health.DocCount) - } } -func TestContextCancellation(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(2 * time.Second) - w.WriteHeader(200) - })) - defer srv.Close() +var ctx = context.Background() - c := NewClient(WithBaseURL(srv.URL)) - ctx, cancel := context.WithCancel(context.Background()) - cancel() // cancel immediately +func TestNewClientOptions(t *testing.T) { + c := NewClient() + check(t, "base URL", c.baseURL, defaultBaseURL) + check(t, "timeout", c.httpClient.Timeout, defaultTimeout) + custom := &http.Client{} + c = NewClient(WithBaseURL("http://example.com/"), WithHTTPClient(custom), WithTimeout(5*time.Second)) + check(t, "base URL", c.baseURL, "http://example.com") + check(t, "http client", c.httpClient == custom, true) + check(t, "timeout", custom.Timeout, 5*time.Second) +} - _, err := c.Health(ctx) - if err == nil { - t.Fatal("expected error from cancelled context") - } +func TestHealthAndAPIKey(t *testing.T) { + c, seen := fakeServer(t, reply{data: map[string]any{"status": "ok"}}) + WithAPIKey("secret")(c) + health, err := c.Health(ctx) + must(t, err) + check(t, "status", health.Status, "ok") + check(t, "path", (*seen)[0].Path, "/api/v1/health") + check(t, "auth", (*seen)[0].Header.Get("Authorization"), "Bearer secret") } -func TestErrorResponse(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(500) - w.Write([]byte(`{"error":{"code":"INTERNAL_ERROR","message":"something broke"}}`)) - })) - defer srv.Close() +func TestSearchSendsOperationNames(t *testing.T) { + c, seen := fakeServer(t, reply{data: map[string]any{ + "items": []any{map[string]any{"documentId": "d1", "chunkId": "c1", "title": "Go", "score": 0.9}}, + "total": 1, "limit": 5, "offset": 0, + }}) + page, err := c.Search(ctx, "goroutines", WithLimit(5), WithSourceType("library"), WithTags("a", "b"), WithMinRating(3)) + must(t, err) + q := (*seen)[0].Query + check(t, "query", q["query"], []string{"goroutines"}) + check(t, "limit", q["limit"], []string{"5"}) + check(t, "sourceType", q["sourceType"], []string{"library"}) + check(t, "tags", q["tags"], []string{"a", "b"}) + check(t, "minRating", q["minRating"], []string{"3"}) + check(t, "total", page.Total, 1) + check(t, "hit", page.Items[0].ChunkID, "c1") +} + +func TestAddTextWaitAndDecode(t *testing.T) { + result := `{"kind":"content","documents":[{"documentId":"d1","title":"Go","chunkCount":2}],"errors":[],"skipped":[]}` + c, seen := fakeServer(t, + reply{status: 202, data: map[string]any{"taskId": "t1", "operation": "add", "status": "running"}}, + reply{data: map[string]any{"id": "t1", "operation": "add", "status": "running"}}, + reply{data: map[string]any{"id": "t1", "operation": "add", "status": "completed", "result": result}}, + ) + task, err := c.AddText(ctx, "Go", "content", WithTopic("lang"), WithTags("go")) + must(t, err) + check(t, "task", *task, Task{ID: "t1", Operation: "add", Status: "running"}) + check(t, "body", (*seen)[0].Body, map[string]any{ + "title": "Go", "content": "content", "topic": "lang", "tags": []any{"go"}, + }) + done, err := c.WaitForTask(ctx, task.ID, time.Millisecond) + must(t, err) + check(t, "poll path", (*seen)[2].Path, "/api/v1/tasks/t1") + var added IngestResult + must(t, done.DecodeResult(&added)) + check(t, "document", added.Documents[0], IngestedDocument{DocumentID: "d1", Title: "Go", ChunkCount: 2}) +} + +func TestAddURLWithSpider(t *testing.T) { + c, seen := fakeServer(t, reply{status: 202, data: map[string]any{"taskId": "t", "status": "running"}}) + _, err := c.AddURL(ctx, "https://go.dev/doc/", WithSpider(), WithMaxPages(10)) + must(t, err) + check(t, "body", (*seen)[0].Body, map[string]any{"url": "https://go.dev/doc/", "spider": true, "maxPages": float64(10)}) +} - c := NewClient(WithBaseURL(srv.URL)) - _, err := c.Health(context.Background()) - if err == nil { - t.Fatal("expected error") +func TestWaitForTaskFailures(t *testing.T) { + c, _ := fakeServer(t, reply{data: map[string]any{"id": "t1", "status": "failed", "error": "boom"}}) + _, err := c.WaitForTask(ctx, "t1", time.Millisecond) + var taskErr *TaskError + if !errors.As(err, &taskErr) || taskErr.Task.Error != "boom" { + t.Fatalf("expected TaskError with boom, got %v", err) } - apiErr, ok := err.(*Error) - if !ok { - t.Fatalf("expected *Error, got %T", err) + + c, _ = fakeServer(t, reply{data: map[string]any{"id": "t1", "status": "running"}}) + short, cancel := context.WithTimeout(ctx, 20*time.Millisecond) + defer cancel() + _, err = c.WaitForTask(short, "t1", time.Millisecond) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected deadline exceeded, got %v", err) + } +} + +func TestDocuments(t *testing.T) { + doc := map[string]any{"documentId": "d/1", "title": "Go", "sourceType": "manual"} + c, seen := fakeServer(t, + reply{data: map[string]any{"document": doc, "content": "body", "tags": []string{"go"}, "nextOffset": nil}}, + reply{data: map[string]any{"items": []any{doc}, "total": 1, "limit": 10, "offset": 0}}, + reply{data: map[string]any{"documentId": "d/1", "deleted": true}}, + ) + view, err := c.GetDocument(ctx, "d/1") + must(t, err) + check(t, "escaped path", (*seen)[0].Path, "/api/v1/documents/d%2F1") + check(t, "view", []any{view.Document.Title, view.Content, view.Tags, view.NextOffset == nil}, []any{"Go", "body", []string{"go"}, true}) + page, err := c.ListDocuments(ctx, WithLimit(10), WithLibrary("react")) + must(t, err) + check(t, "library", (*seen)[1].Query["library"], []string{"react"}) + check(t, "list", page.Items[0].DocumentID, "d/1") + must(t, c.DeleteDocument(ctx, "d/1")) + check(t, "delete", (*seen)[2].Method, http.MethodDelete) +} + +func TestTopicsTagsGraph(t *testing.T) { + c, seen := fakeServer(t, + reply{data: map[string]any{"items": []any{map[string]any{"id": "go", "name": "Go", "documentCount": 2}}}}, + reply{data: map[string]any{"id": "web", "name": "Web", "parentId": "go"}}, + reply{data: map[string]any{"items": []any{map[string]any{"id": "1", "name": "a", "documentCount": 1}}}}, + reply{data: map[string]any{"documentId": "d1", "tags": []string{"a", "b"}}}, + reply{data: map[string]any{"documentId": "d1", "removed": []string{"b"}, "tags": []string{"a"}}}, + reply{data: map[string]any{ + "nodes": []any{map[string]any{"id": "d1", "label": "Doc", "type": "document"}}, + "edges": []any{map[string]any{"source": "d1", "target": "t", "type": "has_tag", "weight": 1}}, + }}, + ) + topics, err := c.ListTopics(ctx) + must(t, err) + check(t, "topic count", topics[0].DocumentCount, 2) + topic, err := c.CreateTopic(ctx, "Web", WithParent("go")) + must(t, err) + check(t, "create body", (*seen)[1].Body, map[string]any{"name": "Web", "parent": "go"}) + check(t, "topic", topic.ParentID, "go") + tags, err := c.ListTags(ctx) + must(t, err) + check(t, "tags", tags[0].Name, "a") + added, err := c.AddTags(ctx, "d1", []string{"b"}) + must(t, err) + check(t, "added", added, []string{"a", "b"}) + remaining, err := c.RemoveTags(ctx, "d1", []string{"b"}) + must(t, err) + check(t, "remove query", (*seen)[4].Query["tags"], []string{"b"}) + check(t, "remaining", remaining, []string{"a"}) + graph, err := c.GetGraph(ctx, WithThreshold(0.5)) + must(t, err) + check(t, "threshold", (*seen)[5].Query["threshold"], []string{"0.5"}) + check(t, "graph", []any{graph.Nodes[0].Type, graph.Edges[0].Weight}, []any{"document", 1.0}) +} + +func TestOverviewAskSync(t *testing.T) { + c, seen := fakeServer(t, + reply{data: map[string]any{"stats": map[string]any{"totalDocuments": 3, "databaseSizeBytes": 4096}, "health": map[string]any{"database": "ok"}}}, + reply{data: map[string]any{"mode": "answer", "answer": "42", "sources": []any{map[string]any{"documentId": "d1", "title": "Doc", "score": 0.5}}}}, + reply{status: 202, data: map[string]any{"taskId": "t1", "operation": "sync", "status": "running"}}, + reply{status: 202, data: map[string]any{"taskId": "t2", "operation": "sync", "status": "running"}}, + reply{data: map[string]any{"taskId": "t1", "cancelRequested": true, "status": "cancelled"}}, + ) + overview, err := c.Overview(ctx) + must(t, err) + check(t, "documents", overview.Stats.TotalDocuments, int64(3)) + answer, err := c.Ask(ctx, "why?", WithTopK(3)) + must(t, err) + check(t, "ask body", (*seen)[1].Body, map[string]any{"question": "why?", "topK": float64(3)}) + check(t, "answer", []any{answer.Answer, answer.Sources[0].DocumentID}, []any{"42", "d1"}) + task, err := c.Sync(ctx, "notes") + must(t, err) + check(t, "sync", []any{task.Operation, (*seen)[2].Body}, []any{"sync", map[string]any{"name": "notes"}}) + _, err = c.SyncAll(ctx) + must(t, err) + check(t, "sync all", (*seen)[3].Body, map[string]any{"all": true}) + cancelled, err := c.CancelTask(ctx, "t1") + must(t, err) + check(t, "cancelled", cancelled, true) +} + +func TestErrors(t *testing.T) { + c, _ := fakeServer(t, + reply{status: 404, raw: `{"error":{"code":"DOCUMENT_NOT_FOUND","message":"gone"}}`}, + reply{status: 502, raw: `bad gateway`}, + ) + _, err := c.GetDocument(ctx, "x") + var apiErr *Error + if !errors.As(err, &apiErr) || apiErr.Code != "DOCUMENT_NOT_FOUND" || apiErr.Message != "gone" { + t.Fatalf("expected API error, got %v", err) } - if apiErr.StatusCode != 500 { - t.Errorf("expected status 500, got %d", apiErr.StatusCode) + if !errors.Is(err, ErrNotFound) { + t.Errorf("expected errors.Is(err, ErrNotFound)") } - if apiErr.Code != "INTERNAL_ERROR" { - t.Errorf("expected code INTERNAL_ERROR, got %q", apiErr.Code) + _, err = c.Health(ctx) + if !errors.As(err, &apiErr) || apiErr.StatusCode != 502 || apiErr.Message != "bad gateway" { + t.Fatalf("expected raw error body, got %v", err) } } diff --git a/sdk/go/connectors.go b/sdk/go/connectors.go index 706d6b6b..495afc51 100644 --- a/sdk/go/connectors.go +++ b/sdk/go/connectors.go @@ -1,4 +1,14 @@ package libscope -// Connector operations will be added as the REST API exposes connector sync endpoints. -// This file is a placeholder for future connector sync operations. +import "context" + +// Sync starts syncing a saved connector connection (created with "libscope connect") and +// returns the background task. +func (c *Client) Sync(ctx context.Context, name string) (*Task, error) { + return c.startTask(ctx, "/sync", map[string]any{"name": name}) +} + +// SyncAll starts syncing every saved connection and returns the background task. +func (c *Client) SyncAll(ctx context.Context) (*Task, error) { + return c.startTask(ctx, "/sync", map[string]any{"all": true}) +} diff --git a/sdk/go/documents.go b/sdk/go/documents.go index 782f5cf2..72883783 100644 --- a/sdk/go/documents.go +++ b/sdk/go/documents.go @@ -1,166 +1,36 @@ package libscope import ( - "bytes" "context" - "encoding/json" - "fmt" "net/http" - "net/url" - "strconv" ) -// AddDocumentOption configures AddDocument. -type AddDocumentOption func(map[string]interface{}) - -// WithDocumentTopic sets the topic for a new document. -func WithDocumentTopic(topic string) AddDocumentOption { - return func(m map[string]interface{}) { - m["topic"] = topic - } -} - -// WithDocumentTags sets the tags for a new document. -func WithDocumentTags(tags ...string) AddDocumentOption { - return func(m map[string]interface{}) { - m["tags"] = tags - } -} - -// AddDocument indexes a document from a URL. -func (c *Client) AddDocument(ctx context.Context, docURL string, opts ...AddDocumentOption) (*Document, error) { - body := map[string]interface{}{ - "url": docURL, - } - for _, opt := range opts { - opt(body) - } - - data, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("libscope: encoding request: %w", err) - } - - resp, err := c.do(ctx, http.MethodPost, "/api/v1/documents/url", bytes.NewReader(data)) - if err != nil { - return nil, err - } - result, err := decodeResponse[Document](resp) - if err != nil { - return nil, err - } - return &result, nil -} - -// AddTextOption configures AddText. -type AddTextOption func(map[string]interface{}) - -// WithTextTopic sets the topic for a text document. -func WithTextTopic(topic string) AddTextOption { - return func(m map[string]interface{}) { - m["topic"] = topic - } +// AddText starts adding a document from text and returns the background task. Params: +// WithTopic, WithLibrary, WithVersion, WithSourceType, WithTags, WithURL. +// Use WaitForTask and Task.DecodeResult (*IngestResult) to get the new document. +func (c *Client) AddText(ctx context.Context, title, content string, params ...Param) (*Task, error) { + body := fields(map[string]any{"title": title, "content": content}, params) + return c.startTask(ctx, "/documents", body) } -// WithTextTags sets the tags for a text document. -func WithTextTags(tags ...string) AddTextOption { - return func(m map[string]interface{}) { - m["tags"] = tags - } +// AddURL starts adding a web page, a crawled site (WithSpider, WithMaxPages) or a public +// GitHub/GitLab repository, and returns the background task. +func (c *Client) AddURL(ctx context.Context, url string, params ...Param) (*Task, error) { + return c.startTask(ctx, "/documents", fields(map[string]any{"url": url}, params)) } -// WithTextSource sets the source type for a text document. -func WithTextSource(source string) AddTextOption { - return func(m map[string]interface{}) { - m["source"] = source - } +// GetDocument returns a document with its content, tags, links and rating summary. +func (c *Client) GetDocument(ctx context.Context, documentID string) (*DocumentView, error) { + return get[DocumentView](ctx, c, "/documents"+segment(documentID), nil) } -// AddText indexes a document from raw text content. -func (c *Client) AddText(ctx context.Context, title, content string, opts ...AddTextOption) (*Document, error) { - body := map[string]interface{}{ - "title": title, - "content": content, - } - for _, opt := range opts { - opt(body) - } - - data, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("libscope: encoding request: %w", err) - } - - resp, err := c.do(ctx, http.MethodPost, "/api/v1/documents", bytes.NewReader(data)) - if err != nil { - return nil, err - } - result, err := decodeResponse[Document](resp) - if err != nil { - return nil, err - } - return &result, nil -} - -// GetDocument retrieves a single document by ID. -func (c *Client) GetDocument(ctx context.Context, id string) (*Document, error) { - resp, err := c.do(ctx, http.MethodGet, "/api/v1/documents/"+url.PathEscape(id), nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[Document](resp) - if err != nil { - return nil, err - } - return &result, nil -} - -// ListOption configures ListDocuments. -type ListOption func(*url.Values) - -// WithListTopic filters documents by topic. -func WithListTopic(topic string) ListOption { - return func(v *url.Values) { - v.Set("topic", topic) - } -} - -// WithListLimit limits the number of documents returned. -func WithListLimit(n int) ListOption { - return func(v *url.Values) { - v.Set("limit", strconv.Itoa(n)) - } -} - -// ListDocuments lists documents with optional filters. -func (c *Client) ListDocuments(ctx context.Context, opts ...ListOption) ([]Document, error) { - params := url.Values{} - for _, opt := range opts { - opt(¶ms) - } - - path := "/api/v1/documents" - if len(params) > 0 { - path += "?" + params.Encode() - } - - resp, err := c.do(ctx, http.MethodGet, path, nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[[]Document](resp) - if err != nil { - return nil, err - } - return result, nil +// ListDocuments lists documents, newest first. Params: WithLimit, WithOffset and the filters +// WithTopic, WithLibrary, WithVersion, WithSourceType, WithTags. +func (c *Client) ListDocuments(ctx context.Context, params ...Param) (*Page[Document], error) { + return get[Page[Document]](ctx, c, "/documents", query(fields(map[string]any{}, params))) } -// DeleteDocument deletes a document by ID. -func (c *Client) DeleteDocument(ctx context.Context, id string) error { - resp, err := c.do(ctx, http.MethodDelete, "/api/v1/documents/"+url.PathEscape(id), nil) - if err != nil { - return err - } - _, err = decodeResponse[map[string]interface{}](resp) - return err +// DeleteDocument deletes a document. +func (c *Client) DeleteDocument(ctx context.Context, documentID string) error { + return c.call(ctx, http.MethodDelete, "/documents"+segment(documentID), nil, nil, nil) } diff --git a/sdk/go/documents_test.go b/sdk/go/documents_test.go deleted file mode 100644 index 6c18e7e8..00000000 --- a/sdk/go/documents_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package libscope - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" -) - -const ( - errUnexpectedPath = "unexpected path: %s" - headerContentType = "Content-Type" - mimeJSON = "application/json" - errUnexpected = "unexpected error: %v" - testGoDevDocURL = "https://go.dev/doc/" -) - -func TestAddText(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - t.Errorf("expected POST, got %s", r.Method) - } - if r.URL.Path != "/api/v1/documents" { - t.Errorf(errUnexpectedPath, r.URL.Path) - } - - var body map[string]interface{} - json.NewDecoder(r.Body).Decode(&body) - if body["title"] != "Test Doc" { - t.Errorf("expected title 'Test Doc', got %v", body["title"]) - } - if body["content"] != "hello world" { - t.Errorf("expected content 'hello world', got %v", body["content"]) - } - - w.Header().Set(headerContentType, mimeJSON) - w.WriteHeader(201) - w.Write([]byte(`{"data":{"id":"doc-1","title":"Test Doc","source":"manual"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - doc, err := c.AddText(context.Background(), "Test Doc", "hello world") - if err != nil { - t.Fatalf(errUnexpected, err) - } - if doc.ID != "doc-1" { - t.Errorf("expected id doc-1, got %q", doc.ID) - } -} - -func TestAddTextWithOptions(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var body map[string]interface{} - json.NewDecoder(r.Body).Decode(&body) - if body["topic"] != "golang" { - t.Errorf("expected topic 'golang', got %v", body["topic"]) - } - tags, ok := body["tags"].([]interface{}) - if !ok || len(tags) != 2 { - t.Errorf("expected 2 tags, got %v", body["tags"]) - } - - w.Header().Set(headerContentType, mimeJSON) - w.WriteHeader(201) - w.Write([]byte(`{"data":{"id":"doc-2","title":"Go Guide","topic":"golang","tags":["go","tutorial"]}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - doc, err := c.AddText(context.Background(), "Go Guide", "content", - WithTextTopic("golang"), - WithTextTags("go", "tutorial"), - ) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if doc.Topic != "golang" { - t.Errorf("expected topic golang, got %q", doc.Topic) - } -} - -func TestAddDocument(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/v1/documents/url" { - t.Errorf(errUnexpectedPath, r.URL.Path) - } - var body map[string]interface{} - json.NewDecoder(r.Body).Decode(&body) - if body["url"] != testGoDevDocURL { - t.Errorf("expected url, got %v", body["url"]) - } - - w.Header().Set(headerContentType, mimeJSON) - w.WriteHeader(201) - w.Write([]byte(`{"data":{"id":"doc-3","title":"Go Documentation","url":"` + testGoDevDocURL + `"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - doc, err := c.AddDocument(context.Background(), testGoDevDocURL) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if doc.URL != testGoDevDocURL { - t.Errorf("expected url, got %q", doc.URL) - } -} - -func TestGetDocument(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/v1/documents/doc-1" { - t.Errorf(errUnexpectedPath, r.URL.Path) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"id":"doc-1","title":"Test"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - doc, err := c.GetDocument(context.Background(), "doc-1") - if err != nil { - t.Fatalf(errUnexpected, err) - } - if doc.ID != "doc-1" { - t.Errorf("expected doc-1, got %q", doc.ID) - } -} - -func TestGetDocumentNotFound(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerContentType, mimeJSON) - w.WriteHeader(404) - w.Write([]byte(`{"error":{"code":"NOT_FOUND","message":"Document not found"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - _, err := c.GetDocument(context.Background(), "nonexistent") - if err == nil { - t.Fatal("expected error") - } - apiErr, ok := err.(*Error) - if !ok { - t.Fatalf("expected *Error, got %T", err) - } - if apiErr.StatusCode != 404 { - t.Errorf("expected 404, got %d", apiErr.StatusCode) - } -} - -func TestListDocuments(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("topic") != "go" { - t.Errorf("expected topic=go, got %q", r.URL.Query().Get("topic")) - } - if r.URL.Query().Get("limit") != "10" { - t.Errorf("expected limit=10, got %q", r.URL.Query().Get("limit")) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":[{"id":"doc-1","title":"Doc 1"},{"id":"doc-2","title":"Doc 2"}]}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - docs, err := c.ListDocuments(context.Background(), WithListTopic("go"), WithListLimit(10)) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if len(docs) != 2 { - t.Errorf("expected 2 docs, got %d", len(docs)) - } -} - -func TestDeleteDocument(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodDelete { - t.Errorf("expected DELETE, got %s", r.Method) - } - if r.URL.Path != "/api/v1/documents/doc-1" { - t.Errorf(errUnexpectedPath, r.URL.Path) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"deleted":true}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - err := c.DeleteDocument(context.Background(), "doc-1") - if err != nil { - t.Fatalf(errUnexpected, err) - } -} diff --git a/sdk/go/errors.go b/sdk/go/errors.go index 9c770549..2ba9ca6f 100644 --- a/sdk/go/errors.go +++ b/sdk/go/errors.go @@ -1,8 +1,11 @@ package libscope -import "fmt" +import ( + "encoding/json" + "fmt" +) -// Error represents an API error returned by libscope. +// Error is an error response from the libscope API. type Error struct { StatusCode int Code string @@ -16,8 +19,41 @@ func (e *Error) Error() string { return fmt.Sprintf("libscope: HTTP %d: %s", e.StatusCode, e.Message) } +// Is matches errors with the same HTTP status, so errors.Is(err, ErrNotFound) works. +func (e *Error) Is(target error) bool { + t, ok := target.(*Error) + return ok && t.StatusCode == e.StatusCode +} + var ( ErrNotFound = &Error{StatusCode: 404, Message: "not found"} ErrBadRequest = &Error{StatusCode: 400, Message: "bad request"} ErrServerError = &Error{StatusCode: 500, Message: "server error"} ) + +// newError builds an *Error from an error response body. +func newError(status int, body []byte) *Error { + var envelope struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if json.Unmarshal(body, &envelope) == nil && envelope.Error.Message != "" { + return &Error{StatusCode: status, Code: envelope.Error.Code, Message: envelope.Error.Message} + } + return &Error{StatusCode: status, Message: string(body)} +} + +// TaskError is returned by WaitForTask when the task failed or was cancelled. +type TaskError struct { + Task *Task +} + +func (e *TaskError) Error() string { + reason := e.Task.Error + if reason == "" { + reason = e.Task.Status + } + return fmt.Sprintf("libscope: task %s (%s) %s", e.Task.ID, e.Task.Operation, reason) +} diff --git a/sdk/go/examples/quickstart/main.go b/sdk/go/examples/quickstart/main.go index 2d1bd5b0..032cfc6e 100644 --- a/sdk/go/examples/quickstart/main.go +++ b/sdk/go/examples/quickstart/main.go @@ -1,42 +1,47 @@ +// Quick-start example for the libscope Go SDK. Start the server first: libscope serve api. package main import ( "context" "fmt" "log" + "time" libscope "github.com/RobertLD/libscope/sdk/go" ) func main() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() client := libscope.NewClient() - // Add a document from a URL - doc, err := client.AddDocument(context.Background(), "https://go.dev/doc/") + // Adding runs on the server as a background task; wait for it to get the new documents. + task, err := client.AddText(ctx, "Go Concurrency", + "Goroutines are lightweight threads managed by the Go runtime.", + libscope.WithTags("go", "concurrency"), + ) if err != nil { log.Fatal(err) } - fmt.Printf("Indexed: %s (%s)\n", doc.Title, doc.ID) - - // Add a text document - textDoc, err := client.AddText(context.Background(), "Go Concurrency", - "Goroutines are lightweight threads managed by the Go runtime.", - libscope.WithTextTags("go", "concurrency"), - ) + task, err = client.WaitForTask(ctx, task.ID, time.Second) if err != nil { log.Fatal(err) } - fmt.Printf("Added: %s (%s)\n", textDoc.Title, textDoc.ID) + var added libscope.IngestResult + if err := task.DecodeResult(&added); err != nil { + log.Fatal(err) + } + for _, doc := range added.Documents { + fmt.Printf("Added: %s (%s)\n", doc.Title, doc.DocumentID) + } // Search - results, err := client.Search(context.Background(), "goroutines", - libscope.WithLimit(5), - ) + page, err := client.Search(ctx, "goroutines", libscope.WithLimit(5)) if err != nil { log.Fatal(err) } - fmt.Printf("\nSearch results for 'goroutines' (%d total):\n", results.Total) - for _, hit := range results.Results { - fmt.Printf(" %s: %.2f\n", hit.Document.Title, hit.Score) + fmt.Printf("\nSearch results for 'goroutines' (%d total):\n", page.Total) + for _, hit := range page.Items { + fmt.Printf(" %s: %.2f\n", hit.Title, hit.Score) } } diff --git a/sdk/go/graph.go b/sdk/go/graph.go index 63b6f2c6..7ed96df6 100644 --- a/sdk/go/graph.go +++ b/sdk/go/graph.go @@ -1,4 +1,8 @@ package libscope -// Knowledge graph operations will be added as the REST API exposes graph endpoints. -// This file is a placeholder for future knowledge graph operations. +import "context" + +// GetGraph returns the knowledge graph. Params: WithTopic, WithTag, WithThreshold, WithMaxNodes. +func (c *Client) GetGraph(ctx context.Context, params ...Param) (*Graph, error) { + return get[Graph](ctx, c, "/graph", query(fields(map[string]any{}, params))) +} diff --git a/sdk/go/models.go b/sdk/go/models.go index 1547f0f6..386d60bc 100644 --- a/sdk/go/models.go +++ b/sdk/go/models.go @@ -1,81 +1,192 @@ package libscope -// Document represents an indexed document in libscope. +import "encoding/json" + +// Page is one page of a list result. +type Page[T any] struct { + Items []T `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +// Document is a document without its content (list results). type Document struct { - ID string `json:"id"` - Title string `json:"title"` - URL string `json:"url,omitempty"` - Topic string `json:"topic,omitempty"` - TopicID string `json:"topicId,omitempty"` - Tags []string `json:"tags,omitempty"` - ContentHash string `json:"content_hash,omitempty"` - Source string `json:"source,omitempty"` - CreatedAt string `json:"created_at,omitempty"` - UpdatedAt string `json:"updated_at,omitempty"` -} - -// SearchResult holds the result of a search query. -type SearchResult struct { - Results []SearchHit `json:"results"` - Total int `json:"total"` - Query string `json:"query"` -} - -// SearchHit represents a single search match. + DocumentID string `json:"documentId"` + Title string `json:"title"` + SourceType string `json:"sourceType"` + Library string `json:"library,omitempty"` + Version string `json:"version,omitempty"` + TopicID string `json:"topicId,omitempty"` + URL string `json:"url,omitempty"` + ContentHash string `json:"contentHash,omitempty"` + SubmittedBy string `json:"submittedBy,omitempty"` + ContentLength int `json:"contentLength,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + UpdatedAt string `json:"updatedAt,omitempty"` +} + +// DocumentView is a document with (a page of) its content, tags, links and ratings. +type DocumentView struct { + Document Document `json:"document"` + Content string `json:"content"` + ContentLength int `json:"contentLength"` + Offset int `json:"offset"` + NextOffset *int `json:"nextOffset"` + Tags []string `json:"tags"` + Links map[string]any `json:"links"` + Ratings RatingSummary `json:"ratings"` +} + +// RatingSummary summarizes a document's ratings. +type RatingSummary struct { + AverageRating float64 `json:"averageRating"` + TotalRatings int `json:"totalRatings"` + Corrections int `json:"corrections"` +} + +// SearchHit is a matching chunk. type SearchHit struct { - Document Document `json:"document"` - Score float64 `json:"score"` - ChunkText string `json:"chunk_text"` + DocumentID string `json:"documentId"` + ChunkID string `json:"chunkId"` + Title string `json:"title"` + Content string `json:"content"` + SourceType string `json:"sourceType"` + Library string `json:"library,omitempty"` + Version string `json:"version,omitempty"` + TopicID string `json:"topicId,omitempty"` + URL string `json:"url,omitempty"` + Score float64 `json:"score"` + AvgRating *float64 `json:"avgRating,omitempty"` } -// Topic represents a topic grouping for documents. +// Topic groups documents. type Topic struct { - ID string `json:"id"` - Name string `json:"name"` - ParentID string `json:"parentId,omitempty"` + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + ParentID string `json:"parentId,omitempty"` + DocumentCount int `json:"documentCount,omitempty"` } -// Tag represents a tag applied to documents. +// Tag is a tag with its document count. type Tag struct { - ID string `json:"id,omitempty"` - Name string `json:"name"` + ID string `json:"id"` + Name string `json:"name"` + DocumentCount int `json:"documentCount"` } -// Stats holds analytics/statistics about the libscope instance. +// Stats holds the counts of an Overview. type Stats struct { - TotalDocuments int64 `json:"totalDocuments"` - DatabaseSize int64 `json:"databaseSizeBytes"` + TotalDocuments int64 `json:"totalDocuments"` + TotalChunks int64 `json:"totalChunks"` + TotalTopics int64 `json:"totalTopics"` + DatabaseSizeBytes int64 `json:"databaseSizeBytes"` } -// HealthStatus holds the health check response. +// Overview holds counts, topics, installed packs, the embedding index and health. +type Overview struct { + Stats Stats `json:"stats"` + Topics []Topic `json:"topics"` + Packs []map[string]any `json:"packs"` + Index map[string]any `json:"index"` + Health map[string]any `json:"health"` +} + +// HealthStatus is the liveness check response. type HealthStatus struct { - Status string `json:"status"` - DocCount int64 `json:"docCount"` - DBSize int64 `json:"dbSize"` + Status string `json:"status"` } -// AskResult holds the response from the RAG ask endpoint. +// AskResult is an answer (Mode "answer"), or the context to answer from (Mode "context"). type AskResult struct { - Answer string `json:"answer"` - Sources []SearchHit `json:"sources,omitempty"` + Mode string `json:"mode"` + Answer string `json:"answer,omitempty"` + ContextPrompt string `json:"contextPrompt,omitempty"` + Sources []AskSource `json:"sources"` + Model string `json:"model,omitempty"` + TokensUsed int `json:"tokensUsed,omitempty"` +} + +// AskSource is a chunk an answer is based on. +type AskSource struct { + DocumentID string `json:"documentId"` + Title string `json:"title"` + Chunk string `json:"chunk"` + Score float64 `json:"score"` +} + +// Graph is the knowledge graph of documents, topics and tags. +type Graph struct { + Nodes []GraphNode `json:"nodes"` + Edges []GraphEdge `json:"edges"` +} + +// GraphNode is a document, topic or tag. +type GraphNode struct { + ID string `json:"id"` + Label string `json:"label"` + Type string `json:"type"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +// GraphEdge connects two nodes. +type GraphEdge struct { + Source string `json:"source"` + Target string `json:"target"` + Type string `json:"type"` + Weight float64 `json:"weight"` +} + +// Task is a background task on the server (adding documents, syncing connections). +type Task struct { + ID string `json:"id"` + Operation string `json:"operation,omitempty"` + Status string `json:"status"` + Progress *TaskProgress `json:"progress,omitempty"` + // Result is the operation result as JSON, once Status is "completed". See DecodeResult. + Result string `json:"result,omitempty"` + Error string `json:"error,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + CompletedAt string `json:"completedAt,omitempty"` +} + +// TaskProgress reports how far a task got. +type TaskProgress struct { + Current int `json:"current"` + Total int `json:"total"` + Message string `json:"message,omitempty"` } -// apiResponse is the standard success envelope from the REST API. -type apiResponse[T any] struct { - Data T `json:"data"` - Meta *apiMeta `json:"meta,omitempty"` +// Done reports whether the task completed, failed or was cancelled. +func (t *Task) Done() bool { + return t.Status == "completed" || t.Status == "failed" || t.Status == "cancelled" } -type apiMeta struct { - Took int `json:"took,omitempty"` +// DecodeResult decodes the task's result into v (for AddText and AddURL: *IngestResult). +func (t *Task) DecodeResult(v any) error { + return json.Unmarshal([]byte(t.Result), v) } -// apiErrorResponse is the standard error envelope from the REST API. -type apiErrorResponse struct { - Error apiErrorDetail `json:"error"` +// IngestResult is the result of an AddText or AddURL task. +type IngestResult struct { + Kind string `json:"kind"` + Documents []IngestedDocument `json:"documents"` + Errors []struct { + Source string `json:"source"` + Error string `json:"error"` + } `json:"errors"` + Skipped []struct { + Source string `json:"source"` + Reason string `json:"reason"` + } `json:"skipped"` } -type apiErrorDetail struct { - Code string `json:"code"` - Message string `json:"message"` +// IngestedDocument is one document added by a task. +type IngestedDocument struct { + DocumentID string `json:"documentId"` + Title string `json:"title"` + ChunkCount int `json:"chunkCount"` + Source string `json:"source,omitempty"` } diff --git a/sdk/go/params.go b/sdk/go/params.go new file mode 100644 index 00000000..031f59bc --- /dev/null +++ b/sdk/go/params.go @@ -0,0 +1,88 @@ +package libscope + +import ( + "fmt" + "net/url" +) + +// Param sets one optional request field. The same Params work for every method that +// accepts the field (for example WithTopic for Search, Ask, ListDocuments and AddText). +type Param func(map[string]any) + +func set(key string, value any) Param { + return func(m map[string]any) { m[key] = value } +} + +// WithTopic filters by topic (ID or name), or sets the topic of a new document. +func WithTopic(topic string) Param { return set("topic", topic) } + +// WithLibrary filters by library, or sets the library of a new document. +func WithLibrary(library string) Param { return set("library", library) } + +// WithVersion filters by library version, or sets the version of a new document. +func WithVersion(version string) Param { return set("version", version) } + +// WithSourceType filters by source type (manual, library, topic, model), or sets it. +func WithSourceType(sourceType string) Param { return set("sourceType", sourceType) } + +// WithTags keeps documents that carry all of these tags, or tags a new document. +func WithTags(tags ...string) Param { return set("tags", tags) } + +// WithMinRating keeps documents with at least this average rating (1-5). +func WithMinRating(rating float64) Param { return set("minRating", rating) } + +// WithLimit sets the page size. +func WithLimit(n int) Param { return set("limit", n) } + +// WithOffset skips this many results (paging). +func WithOffset(n int) Param { return set("offset", n) } + +// WithTopK sets how many chunks Ask retrieves. +func WithTopK(n int) Param { return set("topK", n) } + +// WithURL stores a source URL with inline content (AddText). +func WithURL(u string) Param { return set("url", u) } + +// WithSpider makes AddURL also crawl the pages the URL links to. +func WithSpider() Param { return set("spider", true) } + +// WithMaxPages limits a crawl (AddURL with WithSpider). +func WithMaxPages(n int) Param { return set("maxPages", n) } + +// WithParent sets the parent topic (ID or name) of a new topic. +func WithParent(parent string) Param { return set("parent", parent) } + +// WithDescription sets the description of a new topic. +func WithDescription(description string) Param { return set("description", description) } + +// WithTag limits the graph to documents with this tag. +func WithTag(tag string) Param { return set("tag", tag) } + +// WithThreshold sets the similarity threshold for graph edges (0-1). +func WithThreshold(t float64) Param { return set("threshold", t) } + +// WithMaxNodes limits the number of graph nodes. +func WithMaxNodes(n int) Param { return set("maxNodes", n) } + +// fields applies params over base. +func fields(base map[string]any, params []Param) map[string]any { + for _, p := range params { + p(base) + } + return base +} + +// query encodes fields as a query string; lists become repeated keys. +func query(m map[string]any) url.Values { + v := url.Values{} + for key, value := range m { + if list, ok := value.([]string); ok { + for _, item := range list { + v.Add(key, item) + } + continue + } + v.Set(key, fmt.Sprint(value)) + } + return v +} diff --git a/sdk/go/search.go b/sdk/go/search.go index 5ad95ea7..cd395047 100644 --- a/sdk/go/search.go +++ b/sdk/go/search.go @@ -3,89 +3,17 @@ package libscope import ( "context" "net/http" - "net/url" - "strconv" ) -// SearchOptions holds the configuration for a search query. -type SearchOptions struct { - Limit int - Topic string - Tags []string - MinScore float64 +// Search searches the knowledge base by meaning and keywords. Params: WithLimit, WithOffset, +// WithMinRating and the filters WithTopic, WithLibrary, WithVersion, WithSourceType, WithTags. +func (c *Client) Search(ctx context.Context, q string, params ...Param) (*Page[SearchHit], error) { + return get[Page[SearchHit]](ctx, c, "/search", query(fields(map[string]any{"query": q}, params))) } -// SearchOption configures a search query. -type SearchOption func(*SearchOptions) - -// WithLimit sets the maximum number of results. -func WithLimit(n int) SearchOption { - return func(o *SearchOptions) { - o.Limit = n - } -} - -// WithTopic filters results by topic. -func WithTopic(t string) SearchOption { - return func(o *SearchOptions) { - o.Topic = t - } -} - -// WithTags filters results by tags. -func WithTags(tags ...string) SearchOption { - return func(o *SearchOptions) { - o.Tags = tags - } -} - -// WithMinScore filters results below a minimum score. -func WithMinScore(s float64) SearchOption { - return func(o *SearchOptions) { - o.MinScore = s - } -} - -// Search performs a semantic search across indexed documents. -func (c *Client) Search(ctx context.Context, query string, opts ...SearchOption) (*SearchResult, error) { - options := &SearchOptions{} - for _, opt := range opts { - opt(options) - } - - params := url.Values{} - params.Set("q", query) - if options.Limit > 0 { - params.Set("limit", strconv.Itoa(options.Limit)) - } - if options.Topic != "" { - params.Set("topic", options.Topic) - } - if len(options.Tags) > 0 { - // The API supports a single "tag" query param - params.Set("tag", options.Tags[0]) - } - - resp, err := c.do(ctx, http.MethodGet, "/api/v1/search?"+params.Encode(), nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[SearchResult](resp) - if err != nil { - return nil, err - } - - // Client-side min score filtering - if options.MinScore > 0 { - filtered := make([]SearchHit, 0, len(result.Results)) - for _, hit := range result.Results { - if hit.Score >= options.MinScore { - filtered = append(filtered, hit) - } - } - result.Results = filtered - result.Total = len(filtered) - } - - return &result, nil +// Ask answers a question from the knowledge base (the server needs an LLM). Params: WithTopK, +// WithMinRating and the filters WithTopic, WithLibrary, WithVersion, WithSourceType, WithTags. +func (c *Client) Ask(ctx context.Context, question string, params ...Param) (*AskResult, error) { + body := fields(map[string]any{"question": question}, params) + return send[AskResult](ctx, c, http.MethodPost, "/ask", body) } diff --git a/sdk/go/search_test.go b/sdk/go/search_test.go deleted file mode 100644 index d84e00a7..00000000 --- a/sdk/go/search_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package libscope - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" -) - -func TestSearch(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("q") != "goroutines" { - t.Errorf("expected q=goroutines, got %q", r.URL.Query().Get("q")) - } - if r.URL.Query().Get("limit") != "5" { - t.Errorf("expected limit=5, got %q", r.URL.Query().Get("limit")) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"results":[{"document":{"id":"doc-1","title":"Go Concurrency"},"score":0.95,"chunk_text":"goroutines are..."}],"total":1,"query":"goroutines"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - result, err := c.Search(context.Background(), "goroutines", WithLimit(5)) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if result.Total != 1 { - t.Errorf("expected 1 result, got %d", result.Total) - } - if result.Results[0].Score != 0.95 { - t.Errorf("expected score 0.95, got %f", result.Results[0].Score) - } -} - -func TestSearchWithTopic(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("topic") != "golang" { - t.Errorf("expected topic=golang, got %q", r.URL.Query().Get("topic")) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"results":[],"total":0,"query":"test"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - result, err := c.Search(context.Background(), "test", WithTopic("golang")) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if result.Total != 0 { - t.Errorf("expected 0 results, got %d", result.Total) - } -} - -func TestSearchWithTags(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("tag") != "tutorial" { - t.Errorf("expected tag=tutorial, got %q", r.URL.Query().Get("tag")) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"results":[],"total":0,"query":"test"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - _, err := c.Search(context.Background(), "test", WithTags("tutorial")) - if err != nil { - t.Fatalf(errUnexpected, err) - } -} - -func TestSearchWithMinScore(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"results":[{"document":{"id":"d1","title":"High"},"score":0.9,"chunk_text":"high"},{"document":{"id":"d2","title":"Low"},"score":0.3,"chunk_text":"low"}],"total":2,"query":"test"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - result, err := c.Search(context.Background(), "test", WithMinScore(0.5)) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if len(result.Results) != 1 { - t.Errorf("expected 1 result after min score filter, got %d", len(result.Results)) - } - if result.Results[0].Document.ID != "d1" { - t.Errorf("expected d1, got %q", result.Results[0].Document.ID) - } -} - -func TestSearchError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerContentType, mimeJSON) - w.WriteHeader(400) - w.Write([]byte(`{"error":{"code":"VALIDATION_ERROR","message":"Query parameter 'q' is required"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - _, err := c.Search(context.Background(), "") - if err == nil { - t.Fatal("expected error") - } - apiErr, ok := err.(*Error) - if !ok { - t.Fatalf("expected *Error, got %T", err) - } - if apiErr.StatusCode != 400 { - t.Errorf("expected 400, got %d", apiErr.StatusCode) - } -} diff --git a/sdk/go/tags.go b/sdk/go/tags.go index 26bc441f..81fb1058 100644 --- a/sdk/go/tags.go +++ b/sdk/go/tags.go @@ -1,45 +1,40 @@ package libscope import ( - "bytes" "context" - "encoding/json" - "fmt" "net/http" "net/url" ) -// ListTags lists all tags. +type documentTags struct { + Tags []string `json:"tags"` +} + +// ListTags lists tags with their document counts. func (c *Client) ListTags(ctx context.Context) ([]Tag, error) { - resp, err := c.do(ctx, http.MethodGet, "/api/v1/tags", nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[[]Tag](resp) + page, err := get[Page[Tag]](ctx, c, "/tags", nil) if err != nil { return nil, err } - return result, nil + return page.Items, nil } -// AddTagsToDocument adds tags to a document. -func (c *Client) AddTagsToDocument(ctx context.Context, docID string, tags []string) ([]Tag, error) { - body := map[string]interface{}{ - "tags": tags, - } - data, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("libscope: encoding request: %w", err) - } - - path := "/api/v1/documents/" + url.PathEscape(docID) + "/tags" - resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(data)) +// AddTags adds tags to a document and returns the document's tags. +func (c *Client) AddTags(ctx context.Context, documentID string, tags []string) ([]string, error) { + path := "/documents" + segment(documentID) + "/tags" + out, err := send[documentTags](ctx, c, http.MethodPost, path, documentTags{Tags: tags}) if err != nil { return nil, err } - result, err := decodeResponse[[]Tag](resp) - if err != nil { + return out.Tags, nil +} + +// RemoveTags removes tags from a document and returns the document's remaining tags. +func (c *Client) RemoveTags(ctx context.Context, documentID string, tags []string) ([]string, error) { + var out documentTags + path := "/documents" + segment(documentID) + "/tags" + if err := c.call(ctx, http.MethodDelete, path, url.Values{"tags": tags}, nil, &out); err != nil { return nil, err } - return result, nil + return out.Tags, nil } diff --git a/sdk/go/tasks.go b/sdk/go/tasks.go new file mode 100644 index 00000000..a6d0b26e --- /dev/null +++ b/sdk/go/tasks.go @@ -0,0 +1,61 @@ +package libscope + +import ( + "context" + "net/http" + "time" +) + +// startTask posts body to a long-running route (answered with 202) and returns the task. +func (c *Client) startTask(ctx context.Context, path string, body any) (*Task, error) { + started, err := send[struct { + TaskID string `json:"taskId"` + Operation string `json:"operation"` + Status string `json:"status"` + }](ctx, c, http.MethodPost, path, body) + if err != nil { + return nil, err + } + return &Task{ID: started.TaskID, Operation: started.Operation, Status: started.Status}, nil +} + +// GetTask returns the status, progress and result of a background task. +func (c *Client) GetTask(ctx context.Context, taskID string) (*Task, error) { + return get[Task](ctx, c, "/tasks"+segment(taskID), nil) +} + +// CancelTask requests cancellation. It reports whether the task was still pending or running. +func (c *Client) CancelTask(ctx context.Context, taskID string) (bool, error) { + var out struct { + CancelRequested bool `json:"cancelRequested"` + } + path := "/tasks" + segment(taskID) + "/cancel" + if err := c.call(ctx, http.MethodPost, path, nil, nil, &out); err != nil { + return false, err + } + return out.CancelRequested, nil +} + +// WaitForTask polls a task every interval until it finishes or ctx is done. It returns +// *TaskError when the task failed or was cancelled. Use context.WithTimeout to bound the wait. +func (c *Client) WaitForTask(ctx context.Context, taskID string, interval time.Duration) (*Task, error) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + task, err := c.GetTask(ctx, taskID) + if err != nil { + return nil, err + } + if task.Done() { + if task.Status != "completed" { + return task, &TaskError{Task: task} + } + return task, nil + } + select { + case <-ctx.Done(): + return task, ctx.Err() + case <-ticker.C: + } + } +} diff --git a/sdk/go/topics.go b/sdk/go/topics.go index 78d8ba02..80e53f80 100644 --- a/sdk/go/topics.go +++ b/sdk/go/topics.go @@ -1,57 +1,20 @@ package libscope import ( - "bytes" "context" - "encoding/json" - "fmt" "net/http" ) -// CreateTopicOption configures CreateTopic. -type CreateTopicOption func(map[string]interface{}) - -// WithParentID sets the parent topic ID. -func WithParentID(id string) CreateTopicOption { - return func(m map[string]interface{}) { - m["parentId"] = id - } -} - -// ListTopics lists all topics. +// ListTopics lists topics with their document counts. func (c *Client) ListTopics(ctx context.Context) ([]Topic, error) { - resp, err := c.do(ctx, http.MethodGet, "/api/v1/topics", nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[[]Topic](resp) + page, err := get[Page[Topic]](ctx, c, "/topics", nil) if err != nil { return nil, err } - return result, nil + return page.Items, nil } -// CreateTopic creates a new topic. -func (c *Client) CreateTopic(ctx context.Context, name string, opts ...CreateTopicOption) (*Topic, error) { - body := map[string]interface{}{ - "name": name, - } - for _, opt := range opts { - opt(body) - } - - data, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("libscope: encoding request: %w", err) - } - - resp, err := c.do(ctx, http.MethodPost, "/api/v1/topics", bytes.NewReader(data)) - if err != nil { - return nil, err - } - result, err := decodeResponse[Topic](resp) - if err != nil { - return nil, err - } - return &result, nil +// CreateTopic creates a topic. Params: WithParent, WithDescription. +func (c *Client) CreateTopic(ctx context.Context, name string, params ...Param) (*Topic, error) { + return send[Topic](ctx, c, http.MethodPost, "/topics", fields(map[string]any{"name": name}, params)) } diff --git a/sdk/python/README.md b/sdk/python/README.md index c43a1208..f027ac96 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -1,40 +1,41 @@ # libscope Python SDK -Python client library for the [libscope](https://github.com/RobertLD/libscope) AI knowledge base. +Python client for the [libscope](https://github.com/RobertLD/libscope) REST API (`/api/v1`). ## Installation ```bash -pip install libscope +pip install pylibscope ``` -Or for development: +For development: ```bash pip install -e ".[dev]" +pytest ``` +Start a server with `libscope serve api` (default `http://localhost:3378`). + ## Quick Start ```python -from libscope import LibscopeClient +from pylibscope import LibscopeClient with LibscopeClient() as client: - # Add a document from a URL - doc = client.add_document("https://docs.python.org/3/tutorial/") + # Adding runs as a background task on the server. wait_for_task polls it. + task = client.add_url("https://docs.python.org/3/tutorial/") + done = client.wait_for_task(task.id) + doc_id = done.result["documents"][0]["documentId"] - # Add a document from raw text client.add_text("My Notes", "Some useful content...", topic="python") - # Search - results = client.search("how to use decorators") - for hit in results.results: + page = client.search("how to use decorators", limit=5) + for hit in page.items: print(f"{hit.title}: {hit.score:.2f}") - # Manage tags - client.add_tags(doc.id, ["python", "tutorial"]) + client.add_tags(doc_id, ["python", "tutorial"]) - # Ask a question (RAG) answer = client.ask("What is the best practice for error handling?") print(answer.answer) ``` @@ -43,68 +44,73 @@ with LibscopeClient() as client: ```python import asyncio -from libscope import AsyncLibscopeClient +from pylibscope import AsyncLibscopeClient async def main(): async with AsyncLibscopeClient() as client: - results = await client.search("decorators") - for hit in results.results: + page = await client.search("decorators") + for hit in page.items: print(f"{hit.title}: {hit.score:.2f}") asyncio.run(main()) ``` +`AsyncLibscopeClient` has the same methods as `LibscopeClient`. + ## Configuration ```python -# Custom server URL and timeout client = LibscopeClient( base_url="http://my-server:3378", timeout=60.0, + api_key="...", # sent as "Authorization: Bearer ..." when the server sets LIBSCOPE_API_KEY ) ``` +## Background tasks + +`add_text`, `add_url`, `sync` and `sync_all` return a `Task` at once (the server answers `202`). Call `wait_for_task(task.id, timeout=300, interval=1.0)` to poll `GET /api/v1/tasks/:taskId` until the task finishes. It returns the task, with `result` holding the operation result. It raises `TaskFailedError` when the task failed or was cancelled, and `TimeoutError` after `timeout` seconds. `get_task` and `cancel_task` are also available. + ## API Reference -### Document Operations +Filters: `topic`, `library`, `version`, `source_type`, `tags` (a document must have all of them). -| Method | Description | -| ------------------------------------------------- | ------------------------- | -| `search(query, *, limit, topic, tags, min_score)` | Semantic search | -| `add_document(url, *, topic, tags)` | Index a document from URL | -| `add_text(title, content, *, topic, tags)` | Index raw text | -| `get_document(doc_id)` | Get a document by ID | -| `list_documents(*, topic, limit, offset)` | List documents | -| `delete_document(doc_id)` | Delete a document | +### Documents and search -### Topic Operations +| Method | Description | +| -------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| `search(query, *, limit, offset, min_rating, **filters)` | Search by meaning and keywords. Returns `Page[SearchHit]` | +| `ask(question, *, top_k, min_rating, **filters)` | Answer a question from the knowledge base (needs an LLM on the server) | +| `add_text(title, content, *, url, **filters)` | Start adding a document from text. Returns `Task` | +| `add_url(url, *, spider, max_pages, **filters)` | Start adding a web page, a crawled site or a public GitHub/GitLab repo. Returns `Task` | +| `get_document(document_id)` | Document with content, tags, links and ratings (`DocumentView`) | +| `list_documents(*, limit, offset, **filters)` | List documents. Returns `Page[Document]` | +| `delete_document(document_id)` | Delete a document | -| Method | Description | -| ---------------------------------- | --------------- | -| `list_topics()` | List all topics | -| `create_topic(name, *, parent_id)` | Create a topic | +### Topics, tags and graph -### Tag Operations +| Method | Description | +| ------------------------------------------------ | ---------------------------------------- | +| `list_topics()` | Topics with document counts | +| `create_topic(name, *, parent, description)` | Create a topic | +| `add_tags(document_id, tags)` | Add tags. Returns the document's tags | +| `remove_tags(document_id, tags)` | Remove tags. Returns the document's tags | +| `list_tags()` | Tags with document counts | +| `get_graph(*, topic, tag, threshold, max_nodes)` | Knowledge graph | -| Method | Description | -| --------------------------- | --------------------------- | -| `add_tags(doc_id, tags)` | Add tags to a document | -| `remove_tags(doc_id, tags)` | Remove tags from a document | -| `list_tags()` | List all tags | +### Connectors, tasks and server -### Analytics & Q&A +| Method | Description | +| ---------------------------------------------- | ------------------------------------------------- | +| `sync(name)` / `sync_all()` | Start syncing saved connections. Returns `Task` | +| `get_task(task_id)` / `cancel_task(task_id)` | Task status / request cancellation | +| `wait_for_task(task_id, *, timeout, interval)` | Poll until the task finishes | +| `overview()` | Counts, topics, installed packs, index and health | +| `health()` | Liveness check | -| Method | Description | -| ------------------------- | ------------------------------ | -| `get_analytics()` | Get knowledge base statistics | -| `ask(question, *, topic)` | RAG-powered question answering | -| `health()` | Check server health | +Errors: `NotFoundError` (404), `ValidationError` (400), `ServerError` (5xx), `LibscopeConnectionError`, `TaskFailedError`, and `LibscopeError` (base class; `code` holds the server's error code). ## Requirements - Python 3.9+ -- A running libscope server (default: `http://localhost:3378`) - -## License - -MIT +- A running libscope server (`libscope serve api`) diff --git a/sdk/python/examples/quickstart.py b/sdk/python/examples/quickstart.py index ada3448b..a9cb1c44 100644 --- a/sdk/python/examples/quickstart.py +++ b/sdk/python/examples/quickstart.py @@ -1,38 +1,33 @@ -"""Quick-start example for the libscope Python SDK.""" +"""Quick-start example for the libscope Python SDK. + +Start the server first: ``libscope serve api`` (http://localhost:3378). +""" from pylibscope import LibscopeClient -# Connect to a local libscope server (default: http://localhost:3378) with LibscopeClient() as client: - # Add a document from a URL - doc = client.add_document("https://docs.python.org/3/tutorial/") - print(f"Indexed: {doc.title} (id={doc.id})") + # Adding runs on the server as a background task; wait for it to get the new documents. + task = client.add_url("https://docs.python.org/3/tutorial/", tags=["python"]) + done = client.wait_for_task(task.id) + for doc in done.result["documents"]: + print(f"Indexed: {doc['title']} ({doc['documentId']})") - # Add a document from raw text - text_doc = client.add_text( + task = client.add_text( "Error Handling Guide", "Use try/except blocks to handle exceptions in Python...", - topic="python", tags=["tutorial", "errors"], ) + client.wait_for_task(task.id) # Search the knowledge base - results = client.search("how to use decorators") - for hit in results.results: + page = client.search("how to handle exceptions", limit=5) + for hit in page.items: print(f" {hit.title}: {hit.score:.2f}") - # Manage tags - client.add_tags(doc.id, ["python", "tutorial"]) - - # List topics - topics = client.list_topics() - for t in topics: - print(f"Topic: {t.name}") - - # Get analytics - stats = client.get_analytics() - print(f"Total documents: {stats.total_documents}") + # Counts, topics and index health + overview = client.overview() + print(f"Total documents: {overview.stats.total_documents}") - # Ask a question (RAG) + # Ask a question (the server needs an LLM: see llm.provider) answer = client.ask("What is the recommended error handling pattern?") print(f"Answer: {answer.answer}") diff --git a/sdk/python/src/pylibscope/__init__.py b/sdk/python/src/pylibscope/__init__.py index e1076723..33ca7618 100644 --- a/sdk/python/src/pylibscope/__init__.py +++ b/sdk/python/src/pylibscope/__init__.py @@ -6,39 +6,49 @@ LibscopeError, NotFoundError, ServerError, + TaskFailedError, ValidationError, ) from pylibscope.models import ( - Analytics, AskResult, + AskSource, Document, + DocumentView, Graph, GraphEdge, GraphNode, + Overview, + Page, SearchHit, - SearchResult, - SyncResult, + Stats, + Tag, + Task, Topic, ) __all__ = [ "LibscopeClient", "AsyncLibscopeClient", - "Document", - "SearchResult", - "SearchHit", - "Topic", - "Analytics", "AskResult", + "AskSource", + "Document", + "DocumentView", "Graph", - "GraphNode", "GraphEdge", - "SyncResult", + "GraphNode", + "Overview", + "Page", + "SearchHit", + "Stats", + "Tag", + "Task", + "Topic", "LibscopeError", "LibscopeConnectionError", "NotFoundError", "ValidationError", "ServerError", + "TaskFailedError", ] __version__ = "0.1.0" diff --git a/sdk/python/src/pylibscope/client.py b/sdk/python/src/pylibscope/client.py index cf44401d..1c1ca155 100644 --- a/sdk/python/src/pylibscope/client.py +++ b/sdk/python/src/pylibscope/client.py @@ -1,51 +1,99 @@ -"""Synchronous and asynchronous clients for the libscope REST API.""" +"""Synchronous and asynchronous clients for the libscope REST API (``/api/v1``). + +Slow operations (adding documents, syncing connections) run on the server as background +tasks: those methods return a :class:`Task`. Call ``wait_for_task(task.id)`` to wait for it. +""" from __future__ import annotations -from typing import Any, Dict, List, Optional +import asyncio +import time +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional +from urllib.parse import quote import httpx -from pylibscope.connectors import build_connector_config from pylibscope.exceptions import ( LibscopeConnectionError, LibscopeError, NotFoundError, ServerError, + TaskFailedError, ValidationError, ) from pylibscope.models import ( - Analytics, AskResult, Document, + DocumentView, Graph, - GraphEdge, - GraphNode, + Overview, + Page, SearchHit, - SearchResult, - SyncResult, + Tag, + Task, Topic, ) _API = "/api/v1" -_PATH_DOCUMENTS = "/documents" -_PATH_TOPICS = "/topics" +DEFAULT_BASE_URL = "http://localhost:3378" + + +@dataclass(frozen=True) +class _Call: + """One REST request and how to read its ``data``.""" + + method: str + path: str + parse: Callable[[Any], Any] + params: Optional[Dict[str, Any]] = None + json: Optional[Dict[str, Any]] = None + + +def _defined(**values: Any) -> Dict[str, Any]: + """Drop None values (the server applies its defaults).""" + return {k: v for k, v in values.items() if v is not None} + + +def _filters( + topic: Optional[str], + library: Optional[str], + version: Optional[str], + source_type: Optional[str], + tags: Optional[List[str]], +) -> Dict[str, Any]: + return _defined( + topic=topic, library=library, version=version, sourceType=source_type, tags=tags + ) + + +def _query(params: Dict[str, Any]) -> Dict[str, Any]: + """Query-string form: booleans as true/false; lists are sent as repeated keys.""" + return {k: (str(v).lower() if isinstance(v, bool) else v) for k, v in params.items()} + + +def _doc(document_id: str, rest: str = "") -> str: + return f"/documents/{quote(document_id, safe='')}{rest}" + + +def _started(data: Any) -> Task: + return Task(id=data["taskId"], operation=data.get("operation"), status=data["status"]) + + +def _items(model: Any) -> Callable[[Any], Any]: + return lambda data: [model.model_validate(i) for i in data.get("items", [])] def _raise_for_error(response: httpx.Response) -> None: - """Translate HTTP error responses into SDK exceptions.""" + """Translate an error response into an SDK exception.""" if response.status_code < 400: return - try: - body = response.json() - except Exception: - body = {} - - error = body.get("error", {}) + error = response.json().get("error", {}) + except ValueError: + error = {} code = error.get("code", "UNKNOWN") message = error.get("message", response.text) - if response.status_code == 404: raise NotFoundError(message) if response.status_code == 400: @@ -55,70 +103,109 @@ def _raise_for_error(response: httpx.Response) -> None: raise LibscopeError(message, code=code) -def _extract_data(response: httpx.Response) -> Any: - """Extract the ``data`` envelope from a JSON response.""" - body = response.json() - return body.get("data", body) +class _Calls: + """Request builders shared by both clients.""" + @staticmethod + def health() -> _Call: + return _Call("GET", "/health", lambda d: d) -# --------------------------------------------------------------------------- -# Helpers for normalising API responses into SDK models -# --------------------------------------------------------------------------- + @staticmethod + def overview() -> _Call: + return _Call("GET", "/overview", Overview.model_validate) -def _parse_document(raw: Any) -> Document: - if isinstance(raw, dict): - return Document( - id=raw.get("id", ""), - title=raw.get("title", ""), - url=raw.get("url"), - topic=raw.get("topic"), - topic_id=raw.get("topicId"), - tags=raw.get("tags", []), - content_hash=raw.get("contentHash"), - source_type=raw.get("sourceType"), - created_at=raw.get("createdAt"), - updated_at=raw.get("updatedAt"), - ) - return Document.model_validate(raw) - - -def _parse_search_result(data: Any) -> SearchResult: - results = [] - raw_results = data.get("results", []) if isinstance(data, dict) else [] - for r in raw_results: - results.append( - SearchHit( - document_id=r.get("documentId"), - title=r.get("title"), - content=r.get("content"), - score=r.get("score", 0.0), - ) + @staticmethod + def search(query: str, limit: int, offset: int, min_rating: Optional[float], **f: Any) -> _Call: + params = _defined(query=query, limit=limit, offset=offset, minRating=min_rating) + return _Call( + "GET", "/search", Page[SearchHit].model_validate, params=_query({**params, **f}) ) - total = data.get("totalCount", len(results)) if isinstance(data, dict) else len(results) - return SearchResult(results=results, total_count=total) + @staticmethod + def ask(question: str, top_k: Optional[int], min_rating: Optional[float], **f: Any) -> _Call: + body = _defined(question=question, topK=top_k, minRating=min_rating) + return _Call("POST", "/ask", AskResult.model_validate, json={**body, **f}) -def _parse_topic(raw: Any) -> Topic: - if isinstance(raw, dict): - return Topic( - id=raw.get("id", ""), - name=raw.get("name", ""), - parent_id=raw.get("parentId"), - document_count=raw.get("documentCount"), - ) - return Topic.model_validate(raw) + @staticmethod + def add(body: Dict[str, Any]) -> _Call: + return _Call("POST", "/documents", _started, json=body) + @staticmethod + def get_document(document_id: str) -> _Call: + return _Call("GET", _doc(document_id), DocumentView.model_validate) -def _parse_analytics(raw: Any) -> Analytics: - if isinstance(raw, dict): - return Analytics( - total_documents=raw.get("totalDocuments", 0), - total_chunks=raw.get("totalChunks", 0), - total_topics=raw.get("totalTopics", 0), - total_tags=raw.get("totalTags", 0), - database_size_bytes=raw.get("databaseSizeBytes", 0), - ) - return Analytics.model_validate(raw) + @staticmethod + def list_documents(limit: int, offset: int, **f: Any) -> _Call: + params = _query({"limit": limit, "offset": offset, **f}) + return _Call("GET", "/documents", Page[Document].model_validate, params=params) + + @staticmethod + def delete_document(document_id: str) -> _Call: + return _Call("DELETE", _doc(document_id), lambda d: None) + + @staticmethod + def list_topics() -> _Call: + return _Call("GET", "/topics", _items(Topic)) + + @staticmethod + def create_topic(name: str, parent: Optional[str], description: Optional[str]) -> _Call: + body = _defined(name=name, parent=parent, description=description) + return _Call("POST", "/topics", Topic.model_validate, json=body) + + @staticmethod + def add_tags(document_id: str, tags: List[str]) -> _Call: + path = _doc(document_id, "/tags") + return _Call("POST", path, lambda d: list(d["tags"]), json={"tags": tags}) + + @staticmethod + def remove_tags(document_id: str, tags: List[str]) -> _Call: + path = _doc(document_id, "/tags") + return _Call("DELETE", path, lambda d: list(d["tags"]), params={"tags": tags}) + + @staticmethod + def list_tags() -> _Call: + return _Call("GET", "/tags", _items(Tag)) + + @staticmethod + def graph( + topic: Optional[str], tag: Optional[str], threshold: Optional[float], max_nodes: Optional[int] + ) -> _Call: + params = _defined(topic=topic, tag=tag, threshold=threshold, maxNodes=max_nodes) + return _Call("GET", "/graph", Graph.model_validate, params=params) + + @staticmethod + def sync(body: Dict[str, Any]) -> _Call: + return _Call("POST", "/sync", _started, json=body) + + @staticmethod + def get_task(task_id: str) -> _Call: + return _Call("GET", f"/tasks/{quote(task_id, safe='')}", Task.model_validate) + + @staticmethod + def cancel_task(task_id: str) -> _Call: + return _Call("POST", f"/tasks/{quote(task_id, safe='')}/cancel", lambda d: bool(d["cancelRequested"])) + + +def _add_body( + title: Optional[str] = None, + content: Optional[str] = None, + url: Optional[str] = None, + spider: Optional[bool] = None, + max_pages: Optional[int] = None, + **f: Any, +) -> Dict[str, Any]: + body = _defined(title=title, content=content, url=url, spider=spider, maxPages=max_pages) + return {**body, **f} + + +def _finished(task: Task) -> Task: + if task.status != "completed": + raise TaskFailedError(task) + return task + + +def _headers(api_key: Optional[str]) -> Dict[str, str]: + return {"Authorization": f"Bearer {api_key}"} if api_key else {} # --------------------------------------------------------------------------- @@ -127,17 +214,21 @@ def _parse_analytics(raw: Any) -> Analytics: class LibscopeClient: - """Synchronous client for the libscope REST API.""" + """Synchronous client for the libscope REST API. + + ``api_key`` is sent as ``Authorization: Bearer <key>`` (needed when the server sets + ``LIBSCOPE_API_KEY``). + """ def __init__( self, - base_url: str = "http://localhost:3378", + base_url: str = DEFAULT_BASE_URL, timeout: float = 30.0, + api_key: Optional[str] = None, ) -> None: - self._base_url = base_url.rstrip("/") - self._client = httpx.Client(base_url=self._base_url, timeout=timeout) - - # -- context manager ----------------------------------------------------- + self._client = httpx.Client( + base_url=base_url.rstrip("/") + _API, timeout=timeout, headers=_headers(api_key) + ) def __enter__(self) -> "LibscopeClient": return self @@ -149,210 +240,172 @@ def close(self) -> None: """Close the underlying HTTP client.""" self._client.close() - # -- helpers ------------------------------------------------------------- - - def _url(self, path: str) -> str: - return f"{_API}{path}" - - def _request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: + def _run(self, call: _Call) -> Any: try: - resp = self._client.request(method, self._url(path), **kwargs) + resp = self._client.request(call.method, call.path, params=call.params, json=call.json) except httpx.ConnectError as exc: raise LibscopeConnectionError(str(exc)) from exc _raise_for_error(resp) - return resp + return call.parse(resp.json().get("data")) - # -- document operations ------------------------------------------------- + def health(self) -> Dict[str, Any]: + """Liveness check: ``{"status": "ok"}``.""" + return self._run(_Calls.health()) + + def overview(self) -> Overview: + """Counts, topics, installed packs, embedding index and health.""" + return self._run(_Calls.overview()) def search( self, query: str, *, limit: int = 10, + offset: int = 0, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - min_score: Optional[float] = None, - ) -> SearchResult: - """Perform a semantic search across the knowledge base.""" - params: Dict[str, Any] = {"q": query, "limit": limit} - if topic is not None: - params["topic"] = topic - if tags: - params["tag"] = tags[0] - resp = self._request("GET", "/search", params=params) - data = _extract_data(resp) - result = _parse_search_result(data) - if min_score is not None: - result.results = [h for h in result.results if h.score >= min_score] - result.total_count = len(result.results) - return result - - def add_document( + min_rating: Optional[float] = None, + ) -> Page[SearchHit]: + """Search by meaning and keywords.""" + f = _filters(topic, library, version, source_type, tags) + return self._run(_Calls.search(query, limit, offset, min_rating, **f)) + + def ask( self, - url: str, + question: str, *, + top_k: Optional[int] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - ) -> Document: - """Index a document from a URL.""" - payload: Dict[str, Any] = {"url": url} - if topic is not None: - payload["topic"] = topic - if tags: - payload["tags"] = tags - resp = self._request("POST", "/documents/url", json=payload) - return _parse_document(_extract_data(resp)) + min_rating: Optional[float] = None, + ) -> AskResult: + """Answer a question from the knowledge base (needs an LLM on the server).""" + f = _filters(topic, library, version, source_type, tags) + return self._run(_Calls.ask(question, top_k, min_rating, **f)) def add_text( self, title: str, content: str, *, + url: Optional[str] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - ) -> Document: - """Index a document from raw text.""" - payload: Dict[str, Any] = {"title": title, "content": content} - if topic is not None: - payload["topic"] = topic - if tags: - payload["tags"] = tags - resp = self._request("POST", _PATH_DOCUMENTS, json=payload) - return _parse_document(_extract_data(resp)) - - def get_document(self, doc_id: str) -> Document: - """Retrieve a single document by ID.""" - resp = self._request("GET", f"/documents/{doc_id}") - return _parse_document(_extract_data(resp)) + ) -> Task: + """Start adding a document from text. Returns the background task.""" + f = _filters(topic, library, version, source_type, tags) + return self._run(_Calls.add(_add_body(title=title, content=content, url=url, **f))) - def list_documents( + def add_url( self, + url: str, *, + spider: bool = False, + max_pages: Optional[int] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, + tags: Optional[List[str]] = None, + ) -> Task: + """Start adding a web page (``spider=True``: and the pages it links to) or a public + GitHub/GitLab repository. Returns the background task.""" + f = _filters(topic, library, version, source_type, tags) + body = _add_body(url=url, spider=spider or None, max_pages=max_pages, **f) + return self._run(_Calls.add(body)) + + def get_document(self, document_id: str) -> DocumentView: + """Get a document with its content, tags, links and ratings.""" + return self._run(_Calls.get_document(document_id)) + + def list_documents( + self, + *, limit: int = 50, offset: int = 0, - ) -> List[Document]: - """List documents, optionally filtered by topic.""" - params: Dict[str, Any] = {"limit": limit, "offset": offset} - if topic is not None: - params["topic"] = topic - resp = self._request("GET", _PATH_DOCUMENTS, params=params) - data = _extract_data(resp) - if isinstance(data, list): - return [_parse_document(d) for d in data] - return [] - - def delete_document(self, doc_id: str) -> None: - """Delete a document by ID.""" - self._request("DELETE", f"/documents/{doc_id}") - - # -- topic operations ---------------------------------------------------- + topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, + tags: Optional[List[str]] = None, + ) -> Page[Document]: + """List documents, newest first.""" + f = _filters(topic, library, version, source_type, tags) + return self._run(_Calls.list_documents(limit, offset, **f)) + + def delete_document(self, document_id: str) -> None: + """Delete a document.""" + self._run(_Calls.delete_document(document_id)) def list_topics(self) -> List[Topic]: - """List all topics.""" - resp = self._request("GET", _PATH_TOPICS) - data = _extract_data(resp) - if isinstance(data, list): - return [_parse_topic(t) for t in data] - return [] - - def create_topic(self, name: str, *, parent_id: Optional[str] = None) -> Topic: - """Create a new topic.""" - payload: Dict[str, Any] = {"name": name} - if parent_id is not None: - payload["parentId"] = parent_id - resp = self._request("POST", _PATH_TOPICS, json=payload) - return _parse_topic(_extract_data(resp)) - - # -- tag operations ------------------------------------------------------ - - def add_tags(self, doc_id: str, tags: List[str]) -> None: - """Add tags to a document.""" - self._request("POST", f"/documents/{doc_id}/tags", json={"tags": tags}) - - def remove_tags(self, doc_id: str, tags: List[str]) -> None: - """Remove tags from a document. - - Note: The REST API may not support tag removal yet. - This sends a DELETE request to the document tags endpoint. - """ - self._request("DELETE", f"/documents/{doc_id}/tags", json={"tags": tags}) - - def list_tags(self) -> List[str]: - """List all tags in the knowledge base.""" - resp = self._request("GET", "/tags") - data = _extract_data(resp) - if isinstance(data, list): - return [t if isinstance(t, str) else t.get("name", "") for t in data] - return [] - - # -- analytics ----------------------------------------------------------- - - def get_analytics(self) -> Analytics: - """Get knowledge base statistics.""" - resp = self._request("GET", "/stats") - return _parse_analytics(_extract_data(resp)) - - # -- knowledge graph ----------------------------------------------------- - - def get_graph(self, *, min_similarity: float = 0.7) -> Graph: - """Get the knowledge graph. - - Note: This endpoint may not be available in all server versions. - """ - resp = self._request( - "GET", "/graph", params={"min_similarity": min_similarity} - ) - data = _extract_data(resp) - nodes = [GraphNode(**n) for n in data.get("nodes", [])] if isinstance(data, dict) else [] - edges = [GraphEdge(**e) for e in data.get("edges", [])] if isinstance(data, dict) else [] - return Graph(nodes=nodes, edges=edges) + """List topics with their document counts.""" + return self._run(_Calls.list_topics()) - # -- RAG Q&A ------------------------------------------------------------- + def create_topic( + self, name: str, *, parent: Optional[str] = None, description: Optional[str] = None + ) -> Topic: + """Create a topic (``parent``: ID or name of the parent topic).""" + return self._run(_Calls.create_topic(name, parent, description)) - def ask( + def add_tags(self, document_id: str, tags: List[str]) -> List[str]: + """Add tags to a document. Returns the document's tags.""" + return self._run(_Calls.add_tags(document_id, tags)) + + def remove_tags(self, document_id: str, tags: List[str]) -> List[str]: + """Remove tags from a document. Returns the document's remaining tags.""" + return self._run(_Calls.remove_tags(document_id, tags)) + + def list_tags(self) -> List[Tag]: + """List tags with their document counts.""" + return self._run(_Calls.list_tags()) + + def get_graph( self, - question: str, *, topic: Optional[str] = None, - ) -> AskResult: - """Ask a question using RAG-powered Q&A.""" - payload: Dict[str, Any] = {"question": question} - if topic is not None: - payload["topic"] = topic - resp = self._request("POST", "/ask", json=payload) - data = _extract_data(resp) - if isinstance(data, dict): - return AskResult( - answer=data.get("answer", ""), - sources=data.get("sources", []), - ) - return AskResult() - - # -- connector operations ------------------------------------------------ - - def sync_connector(self, connector: str, **config: Any) -> SyncResult: - """Trigger a connector sync. - - Note: This endpoint may not be available in all server versions. - """ - payload = build_connector_config(connector, **config) - resp = self._request("POST", "/connectors/sync", json=payload) - data = _extract_data(resp) - if isinstance(data, dict): - return SyncResult( - connector=data.get("connector", connector), - documents_synced=data.get("documentsSynced", 0), - errors=data.get("errors", []), - ) - return SyncResult(connector=connector) - - # -- health -------------------------------------------------------------- - - def health(self) -> Dict[str, Any]: - """Check server health.""" - resp = self._request("GET", "/health") - return _extract_data(resp) + tag: Optional[str] = None, + threshold: Optional[float] = None, + max_nodes: Optional[int] = None, + ) -> Graph: + """Knowledge graph of documents, topics and tags.""" + return self._run(_Calls.graph(topic, tag, threshold, max_nodes)) + + def sync(self, name: str) -> Task: + """Start syncing a saved connector connection. Returns the background task.""" + return self._run(_Calls.sync({"name": name})) + + def sync_all(self) -> Task: + """Start syncing every saved connection. Returns the background task.""" + return self._run(_Calls.sync({"all": True})) + + def get_task(self, task_id: str) -> Task: + """Status, progress and result of a background task.""" + return self._run(_Calls.get_task(task_id)) + + def cancel_task(self, task_id: str) -> bool: + """Request cancellation. Returns True when the task was still pending or running.""" + return self._run(_Calls.cancel_task(task_id)) + + def wait_for_task(self, task_id: str, *, timeout: float = 300.0, interval: float = 1.0) -> Task: + """Poll a task until it finishes. Raises TaskFailedError when it failed or was + cancelled, and TimeoutError after ``timeout`` seconds.""" + deadline = time.monotonic() + timeout + while True: + task = self.get_task(task_id) + if task.done: + return _finished(task) + if time.monotonic() >= deadline: + raise TimeoutError(f"Task {task_id} did not finish in {timeout} s") + time.sleep(interval) # --------------------------------------------------------------------------- @@ -361,15 +414,17 @@ def health(self) -> Dict[str, Any]: class AsyncLibscopeClient: - """Asynchronous client for the libscope REST API.""" + """Asynchronous client for the libscope REST API. Same methods as LibscopeClient.""" def __init__( self, - base_url: str = "http://localhost:3378", + base_url: str = DEFAULT_BASE_URL, timeout: float = 30.0, + api_key: Optional[str] = None, ) -> None: - self._base_url = base_url.rstrip("/") - self._client = httpx.AsyncClient(base_url=self._base_url, timeout=timeout) + self._client = httpx.AsyncClient( + base_url=base_url.rstrip("/") + _API, timeout=timeout, headers=_headers(api_key) + ) async def __aenter__(self) -> "AsyncLibscopeClient": return self @@ -381,179 +436,151 @@ async def close(self) -> None: """Close the underlying HTTP client.""" await self._client.aclose() - def _url(self, path: str) -> str: - return f"{_API}{path}" - - async def _request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: + async def _run(self, call: _Call) -> Any: try: - resp = await self._client.request(method, self._url(path), **kwargs) + resp = await self._client.request( + call.method, call.path, params=call.params, json=call.json + ) except httpx.ConnectError as exc: raise LibscopeConnectionError(str(exc)) from exc _raise_for_error(resp) - return resp + return call.parse(resp.json().get("data")) + + async def health(self) -> Dict[str, Any]: + return await self._run(_Calls.health()) - # -- document operations ------------------------------------------------- + async def overview(self) -> Overview: + return await self._run(_Calls.overview()) async def search( self, query: str, *, limit: int = 10, + offset: int = 0, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - min_score: Optional[float] = None, - ) -> SearchResult: - params: Dict[str, Any] = {"q": query, "limit": limit} - if topic is not None: - params["topic"] = topic - if tags: - params["tag"] = tags[0] - resp = await self._request("GET", "/search", params=params) - data = _extract_data(resp) - result = _parse_search_result(data) - if min_score is not None: - result.results = [h for h in result.results if h.score >= min_score] - result.total_count = len(result.results) - return result - - async def add_document( + min_rating: Optional[float] = None, + ) -> Page[SearchHit]: + f = _filters(topic, library, version, source_type, tags) + return await self._run(_Calls.search(query, limit, offset, min_rating, **f)) + + async def ask( self, - url: str, + question: str, *, + top_k: Optional[int] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - ) -> Document: - payload: Dict[str, Any] = {"url": url} - if topic is not None: - payload["topic"] = topic - if tags: - payload["tags"] = tags - resp = await self._request("POST", "/documents/url", json=payload) - return _parse_document(_extract_data(resp)) + min_rating: Optional[float] = None, + ) -> AskResult: + f = _filters(topic, library, version, source_type, tags) + return await self._run(_Calls.ask(question, top_k, min_rating, **f)) async def add_text( self, title: str, content: str, *, + url: Optional[str] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - ) -> Document: - payload: Dict[str, Any] = {"title": title, "content": content} - if topic is not None: - payload["topic"] = topic - if tags: - payload["tags"] = tags - resp = await self._request("POST", _PATH_DOCUMENTS, json=payload) - return _parse_document(_extract_data(resp)) - - async def get_document(self, doc_id: str) -> Document: - resp = await self._request("GET", f"/documents/{doc_id}") - return _parse_document(_extract_data(resp)) + ) -> Task: + f = _filters(topic, library, version, source_type, tags) + return await self._run(_Calls.add(_add_body(title=title, content=content, url=url, **f))) - async def list_documents( + async def add_url( self, + url: str, *, + spider: bool = False, + max_pages: Optional[int] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, + tags: Optional[List[str]] = None, + ) -> Task: + f = _filters(topic, library, version, source_type, tags) + body = _add_body(url=url, spider=spider or None, max_pages=max_pages, **f) + return await self._run(_Calls.add(body)) + + async def get_document(self, document_id: str) -> DocumentView: + return await self._run(_Calls.get_document(document_id)) + + async def list_documents( + self, + *, limit: int = 50, offset: int = 0, - ) -> List[Document]: - params: Dict[str, Any] = {"limit": limit, "offset": offset} - if topic is not None: - params["topic"] = topic - resp = await self._request("GET", _PATH_DOCUMENTS, params=params) - data = _extract_data(resp) - if isinstance(data, list): - return [_parse_document(d) for d in data] - return [] - - async def delete_document(self, doc_id: str) -> None: - await self._request("DELETE", f"/documents/{doc_id}") + topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, + tags: Optional[List[str]] = None, + ) -> Page[Document]: + f = _filters(topic, library, version, source_type, tags) + return await self._run(_Calls.list_documents(limit, offset, **f)) - # -- topic operations ---------------------------------------------------- + async def delete_document(self, document_id: str) -> None: + await self._run(_Calls.delete_document(document_id)) async def list_topics(self) -> List[Topic]: - resp = await self._request("GET", _PATH_TOPICS) - data = _extract_data(resp) - if isinstance(data, list): - return [_parse_topic(t) for t in data] - return [] - - async def create_topic(self, name: str, *, parent_id: Optional[str] = None) -> Topic: - payload: Dict[str, Any] = {"name": name} - if parent_id is not None: - payload["parentId"] = parent_id - resp = await self._request("POST", _PATH_TOPICS, json=payload) - return _parse_topic(_extract_data(resp)) + return await self._run(_Calls.list_topics()) - # -- tag operations ------------------------------------------------------ + async def create_topic( + self, name: str, *, parent: Optional[str] = None, description: Optional[str] = None + ) -> Topic: + return await self._run(_Calls.create_topic(name, parent, description)) - async def add_tags(self, doc_id: str, tags: List[str]) -> None: - await self._request("POST", f"/documents/{doc_id}/tags", json={"tags": tags}) + async def add_tags(self, document_id: str, tags: List[str]) -> List[str]: + return await self._run(_Calls.add_tags(document_id, tags)) - async def remove_tags(self, doc_id: str, tags: List[str]) -> None: - await self._request("DELETE", f"/documents/{doc_id}/tags", json={"tags": tags}) + async def remove_tags(self, document_id: str, tags: List[str]) -> List[str]: + return await self._run(_Calls.remove_tags(document_id, tags)) - async def list_tags(self) -> List[str]: - resp = await self._request("GET", "/tags") - data = _extract_data(resp) - if isinstance(data, list): - return [t if isinstance(t, str) else t.get("name", "") for t in data] - return [] + async def list_tags(self) -> List[Tag]: + return await self._run(_Calls.list_tags()) - # -- analytics ----------------------------------------------------------- - - async def get_analytics(self) -> Analytics: - resp = await self._request("GET", "/stats") - return _parse_analytics(_extract_data(resp)) - - # -- knowledge graph ----------------------------------------------------- - - async def get_graph(self, *, min_similarity: float = 0.7) -> Graph: - resp = await self._request( - "GET", "/graph", params={"min_similarity": min_similarity} - ) - data = _extract_data(resp) - nodes = [GraphNode(**n) for n in data.get("nodes", [])] if isinstance(data, dict) else [] - edges = [GraphEdge(**e) for e in data.get("edges", [])] if isinstance(data, dict) else [] - return Graph(nodes=nodes, edges=edges) - - # -- RAG Q&A ------------------------------------------------------------- - - async def ask( + async def get_graph( self, - question: str, *, topic: Optional[str] = None, - ) -> AskResult: - payload: Dict[str, Any] = {"question": question} - if topic is not None: - payload["topic"] = topic - resp = await self._request("POST", "/ask", json=payload) - data = _extract_data(resp) - if isinstance(data, dict): - return AskResult( - answer=data.get("answer", ""), - sources=data.get("sources", []), - ) - return AskResult() - - # -- connector operations ------------------------------------------------ - - async def sync_connector(self, connector: str, **config: Any) -> SyncResult: - payload = build_connector_config(connector, **config) - resp = await self._request("POST", "/connectors/sync", json=payload) - data = _extract_data(resp) - if isinstance(data, dict): - return SyncResult( - connector=data.get("connector", connector), - documents_synced=data.get("documentsSynced", 0), - errors=data.get("errors", []), - ) - return SyncResult(connector=connector) - - # -- health -------------------------------------------------------------- - - async def health(self) -> Dict[str, Any]: - resp = await self._request("GET", "/health") - return _extract_data(resp) + tag: Optional[str] = None, + threshold: Optional[float] = None, + max_nodes: Optional[int] = None, + ) -> Graph: + return await self._run(_Calls.graph(topic, tag, threshold, max_nodes)) + + async def sync(self, name: str) -> Task: + return await self._run(_Calls.sync({"name": name})) + + async def sync_all(self) -> Task: + return await self._run(_Calls.sync({"all": True})) + + async def get_task(self, task_id: str) -> Task: + return await self._run(_Calls.get_task(task_id)) + + async def cancel_task(self, task_id: str) -> bool: + return await self._run(_Calls.cancel_task(task_id)) + + async def wait_for_task( + self, task_id: str, *, timeout: float = 300.0, interval: float = 1.0 + ) -> Task: + deadline = time.monotonic() + timeout + while True: + task = await self.get_task(task_id) + if task.done: + return _finished(task) + if time.monotonic() >= deadline: + raise TimeoutError(f"Task {task_id} did not finish in {timeout} s") + await asyncio.sleep(interval) diff --git a/sdk/python/src/pylibscope/connectors.py b/sdk/python/src/pylibscope/connectors.py deleted file mode 100644 index 14e2c909..00000000 --- a/sdk/python/src/pylibscope/connectors.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Connector management helpers for libscope SDK.""" - -from __future__ import annotations - -from typing import Any, Dict - - -def build_connector_config(connector: str, **kwargs: Any) -> Dict[str, Any]: - """Build a connector configuration payload. - - Args: - connector: Connector name (e.g., "obsidian", "onenote"). - **kwargs: Connector-specific configuration options. - - Returns: - A dict suitable for posting to the connector sync endpoint. - """ - return {"connector": connector, "config": kwargs} diff --git a/sdk/python/src/pylibscope/exceptions.py b/sdk/python/src/pylibscope/exceptions.py index c9368b2e..25cdd26d 100644 --- a/sdk/python/src/pylibscope/exceptions.py +++ b/sdk/python/src/pylibscope/exceptions.py @@ -1,5 +1,12 @@ """Custom exceptions for the libscope Python SDK.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pylibscope.models import Task + class LibscopeError(Exception): """Base exception for all libscope SDK errors.""" @@ -35,3 +42,12 @@ class ServerError(LibscopeError): def __init__(self, message: str = "Internal server error") -> None: super().__init__(message, code="INTERNAL_ERROR") + + +class TaskFailedError(LibscopeError): + """Raised by ``wait_for_task`` when a background task failed or was cancelled.""" + + def __init__(self, task: Task) -> None: + self.task = task + reason = task.error or task.status + super().__init__(f"Task {task.id} ({task.operation}) {reason}", code="TASK_FAILED") diff --git a/sdk/python/src/pylibscope/models.py b/sdk/python/src/pylibscope/models.py index 9f128b2a..aee2979a 100644 --- a/sdk/python/src/pylibscope/models.py +++ b/sdk/python/src/pylibscope/models.py @@ -1,95 +1,180 @@ -"""Pydantic models for libscope API types.""" +"""Pydantic models for libscope REST API results. + +Field names are snake_case; responses (camelCase) are parsed with aliases. +""" from __future__ import annotations -from typing import Any, List, Optional +import json +from typing import Any, Dict, Generic, List, Optional, TypeVar -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, field_validator +from pydantic.alias_generators import to_camel +T = TypeVar("T") -class Document(BaseModel): - """A document stored in the knowledge base.""" - id: str +class _Model(BaseModel): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + +class Page(_Model, Generic[T]): + """One page of a list result.""" + + items: List[T] = [] + total: int = 0 + limit: int = 0 + offset: int = 0 + + +class Document(_Model): + """A document without its content (list results).""" + + document_id: str title: str - url: Optional[str] = None - topic: Optional[str] = None + source_type: Optional[str] = None + library: Optional[str] = None + version: Optional[str] = None topic_id: Optional[str] = None - tags: List[str] = [] + url: Optional[str] = None content_hash: Optional[str] = None - source_type: Optional[str] = None + submitted_by: Optional[str] = None + content_length: Optional[int] = None created_at: Optional[str] = None updated_at: Optional[str] = None -class SearchHit(BaseModel): - """A single search result with relevance score.""" +class DocumentView(_Model): + """A document with (a page of) its content, tags, links and rating summary.""" - document_id: Optional[str] = None - title: Optional[str] = None - content: Optional[str] = None - score: float = 0.0 + document: Document + content: str = "" + content_length: int = 0 + offset: int = 0 + next_offset: Optional[int] = None + tags: List[str] = [] + links: Dict[str, Any] = {} + ratings: Dict[str, Any] = {} -class SearchResult(BaseModel): - """Response from a search query.""" +class SearchHit(_Model): + """A matching chunk.""" - results: List[SearchHit] = [] - total_count: int = 0 + document_id: str + chunk_id: str + title: str + content: str = "" + source_type: Optional[str] = None + library: Optional[str] = None + version: Optional[str] = None + topic_id: Optional[str] = None + url: Optional[str] = None + score: float = 0.0 + avg_rating: Optional[float] = None -class Topic(BaseModel): - """A topic/category for organizing documents.""" +class Topic(_Model): + """A topic for organizing documents.""" id: str name: str + description: Optional[str] = None parent_id: Optional[str] = None document_count: Optional[int] = None -class Analytics(BaseModel): - """Knowledge base statistics.""" +class Tag(_Model): + """A tag with its document count.""" + + id: str + name: str + document_count: int = 0 + + +class Stats(_Model): + """Counts in an overview.""" total_documents: int = 0 total_chunks: int = 0 total_topics: int = 0 - total_tags: int = 0 database_size_bytes: int = 0 -class AskResult(BaseModel): - """Response from a RAG question-answering query.""" +class Overview(_Model): + """Counts, topics, installed packs, embedding index and health.""" + + stats: Stats = Stats() + topics: List[Topic] = [] + packs: List[Dict[str, Any]] = [] + index: Dict[str, Any] = {} + health: Dict[str, Any] = {} + - answer: str = "" - sources: List[Any] = [] +class AskSource(_Model): + """A chunk an answer is based on.""" + document_id: str + title: str + chunk: str = "" + score: float = 0.0 + + +class AskResult(_Model): + """An answer (mode "answer"), or the context to answer from (mode "context").""" + + mode: str = "answer" + answer: Optional[str] = None + context_prompt: Optional[str] = None + sources: List[AskSource] = [] + model: Optional[str] = None + tokens_used: Optional[int] = None -class GraphNode(BaseModel): - """A node in the knowledge graph.""" + +class GraphNode(_Model): + """A node in the knowledge graph (document, topic or tag).""" id: str label: str type: Optional[str] = None + metadata: Dict[str, Any] = {} -class GraphEdge(BaseModel): +class GraphEdge(_Model): """An edge in the knowledge graph.""" source: str target: str + type: Optional[str] = None weight: float = 0.0 -class Graph(BaseModel): - """Knowledge graph representation.""" +class Graph(_Model): + """Knowledge graph.""" nodes: List[GraphNode] = [] edges: List[GraphEdge] = [] -class SyncResult(BaseModel): - """Result from a connector sync operation.""" +class Task(_Model): + """A background task. ``result`` is the operation result once completed.""" - connector: str - documents_synced: int = 0 - errors: List[str] = [] + id: str + operation: Optional[str] = None + status: str + progress: Optional[Dict[str, Any]] = None + result: Any = None + error: Optional[str] = None + created_at: Optional[str] = None + started_at: Optional[str] = None + completed_at: Optional[str] = None + + @field_validator("result", mode="before") + @classmethod + def _decode_result(cls, value: Any) -> Any: + # The server sends the operation result as a JSON string. + return json.loads(value) if isinstance(value, str) else value + + @property + def done(self) -> bool: + """True when the task completed, failed or was cancelled.""" + return self.status in ("completed", "failed", "cancelled") diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 9ca5002d..aa45fd99 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -1,4 +1,6 @@ -"""Tests for libscope sync and async clients.""" +"""Tests for the libscope sync and async clients (HTTP mocked with respx).""" + +import json import httpx import pytest @@ -7,368 +9,307 @@ from pylibscope.client import AsyncLibscopeClient, LibscopeClient from pylibscope.exceptions import ( LibscopeConnectionError, + LibscopeError, NotFoundError, ServerError, + TaskFailedError, ValidationError, ) -from pylibscope.models import Analytics, AskResult, Document, SearchResult, Topic BASE = "http://localhost:3378" API = f"{BASE}/api/v1" +HIT = { + "documentId": "d1", + "chunkId": "c1", + "title": "Doc 1", + "content": "hello", + "sourceType": "manual", + "score": 0.9, +} +DOC = {"documentId": "d1", "title": "Doc 1", "sourceType": "manual", "contentLength": 5} +STARTED = {"taskId": "t1", "operation": "add", "status": "running"} +ADD_RESULT = {"kind": "content", "documents": [{"documentId": "d1", "title": "Doc 1"}]} -# --------------------------------------------------------------------------- -# Synchronous client tests -# --------------------------------------------------------------------------- +def ok(data, status=200): + return httpx.Response(status, json={"data": data, "meta": {"took": 1}}) -class TestLibscopeClientSearch: - @respx.mock - def test_search(self): - respx.get(f"{API}/search").mock( - return_value=httpx.Response( - 200, - json={ - "data": { - "results": [ - {"documentId": "d1", "title": "Doc 1", "content": "hello", "score": 0.9} - ], - "totalCount": 1, - }, - "meta": {"took": 10}, - }, - ) - ) - with LibscopeClient() as client: - result = client.search("hello") - assert isinstance(result, SearchResult) - assert result.total_count == 1 - assert result.results[0].score == 0.9 +def task(status, **extra): + return {"id": "t1", "operation": "add", "status": status, **extra} + + +def body(route): + return json.loads(route.calls[0].request.content) + + +class TestSearch: @respx.mock - def test_search_with_filters(self): + def test_search_sends_operation_names_and_parses_page(self): route = respx.get(f"{API}/search").mock( - return_value=httpx.Response(200, json={"data": {"results": [], "totalCount": 0}}) + return_value=ok({"items": [HIT], "total": 1, "limit": 5, "offset": 0}) ) with LibscopeClient() as client: - client.search("test", topic="python", tags=["tutorial"], limit=5) - assert "topic=python" in str(route.calls[0].request.url) - assert "tag=tutorial" in str(route.calls[0].request.url) - assert "limit=5" in str(route.calls[0].request.url) - - @respx.mock - def test_search_min_score_filter(self): - respx.get(f"{API}/search").mock( - return_value=httpx.Response( - 200, - json={ - "data": { - "results": [ - {"documentId": "d1", "title": "High", "content": "", "score": 0.95}, - {"documentId": "d2", "title": "Low", "content": "", "score": 0.3}, - ], - "totalCount": 2, - } - }, + page = client.search( + "hello", limit=5, topic="py", source_type="library", tags=["a", "b"], min_rating=3 ) - ) - with LibscopeClient() as client: - result = client.search("test", min_score=0.5) - assert len(result.results) == 1 - assert result.results[0].document_id == "d1" - - -class TestLibscopeClientDocuments: + params = route.calls[0].request.url.params + assert params["query"] == "hello" + assert params["limit"] == "5" + assert params["sourceType"] == "library" + assert params.get_list("tags") == ["a", "b"] + assert params["minRating"] == "3" + assert page.total == 1 + assert page.items[0].document_id == "d1" + assert page.items[0].chunk_id == "c1" + + +class TestDocuments: @respx.mock - def test_add_document_url(self): - respx.post(f"{API}/documents/url").mock( - return_value=httpx.Response( - 201, - json={"data": {"id": "d1", "title": "Python Tutorial", "url": "https://example.com"}}, - ) - ) + def test_add_text_starts_a_task(self): + route = respx.post(f"{API}/documents").mock(return_value=ok(STARTED, 202)) with LibscopeClient() as client: - doc = client.add_document("https://example.com") - assert isinstance(doc, Document) - assert doc.id == "d1" + started = client.add_text("T", "C", topic="py", tags=["x"]) + assert body(route) == {"title": "T", "content": "C", "topic": "py", "tags": ["x"]} + assert (started.id, started.status, started.operation) == ("t1", "running", "add") @respx.mock - def test_add_text(self): - respx.post(f"{API}/documents").mock( - return_value=httpx.Response( - 201, json={"data": {"id": "d2", "title": "My Doc"}} - ) - ) + def test_add_url_with_spider(self): + route = respx.post(f"{API}/documents").mock(return_value=ok(STARTED, 202)) with LibscopeClient() as client: - doc = client.add_text("My Doc", "Some content") - assert doc.title == "My Doc" + client.add_url("https://example.com", spider=True, max_pages=5) + client.add_url("https://example.com/page") + assert body(route) == {"url": "https://example.com", "spider": True, "maxPages": 5} + assert json.loads(route.calls[1].request.content) == {"url": "https://example.com/page"} @respx.mock def test_get_document(self): - respx.get(f"{API}/documents/d1").mock( - return_value=httpx.Response(200, json={"data": {"id": "d1", "title": "Test"}}) + respx.get(f"{API}/documents/d%2F1").mock( + return_value=ok({"document": DOC, "content": "hello", "tags": ["a"], "nextOffset": None}) ) with LibscopeClient() as client: - doc = client.get_document("d1") - assert doc.id == "d1" + view = client.get_document("d/1") + assert view.document.title == "Doc 1" + assert view.content == "hello" + assert view.tags == ["a"] @respx.mock def test_list_documents(self): - respx.get(f"{API}/documents").mock( - return_value=httpx.Response( - 200, - json={"data": [{"id": "d1", "title": "A"}, {"id": "d2", "title": "B"}]}, - ) + route = respx.get(f"{API}/documents").mock( + return_value=ok({"items": [DOC], "total": 1, "limit": 10, "offset": 0}) ) with LibscopeClient() as client: - docs = client.list_documents() - assert len(docs) == 2 + page = client.list_documents(limit=10, library="react") + assert route.calls[0].request.url.params["library"] == "react" + assert page.items[0].content_length == 5 @respx.mock def test_delete_document(self): - respx.delete(f"{API}/documents/d1").mock( - return_value=httpx.Response(200, json={"data": {"deleted": True}}) + route = respx.delete(f"{API}/documents/d1").mock( + return_value=ok({"documentId": "d1", "deleted": True}) ) with LibscopeClient() as client: - client.delete_document("d1") # should not raise + assert client.delete_document("d1") is None + assert route.called -class TestLibscopeClientTopics: +class TestTopicsTagsGraph: @respx.mock - def test_list_topics(self): + def test_topics(self): respx.get(f"{API}/topics").mock( - return_value=httpx.Response( - 200, - json={"data": [{"id": "t1", "name": "Python"}, {"id": "t2", "name": "Rust"}]}, - ) + return_value=ok({"items": [{"id": "py", "name": "Python", "documentCount": 2}]}) ) + create = respx.post(f"{API}/topics").mock(return_value=ok({"id": "web", "name": "Web"})) with LibscopeClient() as client: topics = client.list_topics() - assert len(topics) == 2 - assert all(isinstance(t, Topic) for t in topics) + created = client.create_topic("Web", parent="py") + assert topics[0].document_count == 2 + assert body(create) == {"name": "Web", "parent": "py"} + assert created.id == "web" @respx.mock - def test_create_topic(self): - respx.post(f"{API}/topics").mock( - return_value=httpx.Response(201, json={"data": {"id": "t3", "name": "Go"}}) + def test_tags(self): + add = respx.post(f"{API}/documents/d1/tags").mock( + return_value=ok({"documentId": "d1", "tags": ["a", "b"]}) ) - with LibscopeClient() as client: - topic = client.create_topic("Go") - assert topic.name == "Go" - - -class TestLibscopeClientTags: - @respx.mock - def test_add_tags(self): - respx.post(f"{API}/documents/d1/tags").mock( - return_value=httpx.Response(200, json={"data": ["alpha", "beta"]}) + remove = respx.delete(f"{API}/documents/d1/tags").mock( + return_value=ok({"documentId": "d1", "removed": ["b"], "tags": ["a"]}) + ) + respx.get(f"{API}/tags").mock( + return_value=ok({"items": [{"id": "1", "name": "a", "documentCount": 1}]}) ) with LibscopeClient() as client: - client.add_tags("d1", ["alpha", "beta"]) + assert client.add_tags("d1", ["b"]) == ["a", "b"] + assert client.remove_tags("d1", ["b"]) == ["a"] + assert client.list_tags()[0].name == "a" + assert body(add) == {"tags": ["b"]} + assert remove.calls[0].request.url.params.get_list("tags") == ["b"] @respx.mock - def test_list_tags(self): - respx.get(f"{API}/tags").mock( - return_value=httpx.Response(200, json={"data": ["python", "tutorial"]}) + def test_graph(self): + route = respx.get(f"{API}/graph").mock( + return_value=ok( + { + "nodes": [{"id": "d1", "label": "Doc", "type": "document"}], + "edges": [{"source": "d1", "target": "t", "type": "has_tag", "weight": 1}], + } + ) ) with LibscopeClient() as client: - tags = client.list_tags() - assert tags == ["python", "tutorial"] + graph = client.get_graph(threshold=0.5) + assert route.calls[0].request.url.params["threshold"] == "0.5" + assert graph.nodes[0].type == "document" + assert graph.edges[0].weight == 1 -class TestLibscopeClientAnalytics: +class TestOverviewAsk: @respx.mock - def test_get_analytics(self): - respx.get(f"{API}/stats").mock( - return_value=httpx.Response( - 200, - json={ - "data": { - "totalDocuments": 42, - "totalChunks": 100, - "totalTopics": 3, - "totalTags": 7, - "databaseSizeBytes": 2048, - } - }, + def test_overview(self): + respx.get(f"{API}/overview").mock( + return_value=ok( + { + "stats": {"totalDocuments": 3, "totalChunks": 9, "databaseSizeBytes": 4096}, + "topics": [], + "packs": [], + "health": {"database": "ok"}, + } ) ) with LibscopeClient() as client: - stats = client.get_analytics() - assert isinstance(stats, Analytics) - assert stats.total_documents == 42 - + overview = client.overview() + assert overview.stats.total_documents == 3 + assert overview.health["database"] == "ok" -class TestLibscopeClientAsk: @respx.mock def test_ask(self): - respx.post(f"{API}/ask").mock( - return_value=httpx.Response( - 200, - json={"data": {"answer": "Use decorators.", "sources": [{"id": "d1"}]}}, + route = respx.post(f"{API}/ask").mock( + return_value=ok( + { + "mode": "answer", + "answer": "42", + "model": "m", + "sources": [{"documentId": "d1", "title": "Doc", "chunk": "x", "score": 0.5}], + } ) ) with LibscopeClient() as client: - result = client.ask("How to use decorators?") - assert isinstance(result, AskResult) - assert result.answer == "Use decorators." - + result = client.ask("why?", top_k=3, source_type="manual") + assert body(route) == {"question": "why?", "topK": 3, "sourceType": "manual"} + assert result.answer == "42" + assert result.sources[0].document_id == "d1" -class TestLibscopeClientHealth: @respx.mock def test_health(self): - respx.get(f"{API}/health").mock( - return_value=httpx.Response( - 200, json={"data": {"status": "ok", "docCount": 10}} - ) - ) + respx.get(f"{API}/health").mock(return_value=ok({"status": "ok"})) with LibscopeClient() as client: - h = client.health() - assert h["status"] == "ok" + assert client.health() == {"status": "ok"} -class TestLibscopeClientErrors: +class TestTasks: @respx.mock - def test_not_found(self): - respx.get(f"{API}/documents/missing").mock( - return_value=httpx.Response( - 404, json={"error": {"code": "NOT_FOUND", "message": "Document not found"}} - ) + def test_wait_for_task_returns_the_parsed_result(self): + respx.get(f"{API}/tasks/t1").mock( + side_effect=[ + ok(task("running", progress={"current": 1, "total": 2})), + ok(task("completed", result=json.dumps(ADD_RESULT))), + ] ) with LibscopeClient() as client: - with pytest.raises(NotFoundError): - client.get_document("missing") + done = client.wait_for_task("t1", interval=0) + assert done.status == "completed" + assert done.result["documents"][0]["documentId"] == "d1" @respx.mock - def test_validation_error(self): - respx.post(f"{API}/documents").mock( - return_value=httpx.Response( - 400, - json={"error": {"code": "VALIDATION_ERROR", "message": "title is required"}}, - ) - ) + def test_wait_for_task_raises_when_the_task_failed(self): + respx.get(f"{API}/tasks/t1").mock(return_value=ok(task("failed", error="boom"))) with LibscopeClient() as client: - with pytest.raises(ValidationError): - client.add_text("", "") + with pytest.raises(TaskFailedError, match="boom") as exc: + client.wait_for_task("t1", interval=0) + assert exc.value.task.status == "failed" @respx.mock - def test_server_error(self): - respx.get(f"{API}/stats").mock( - return_value=httpx.Response( - 500, json={"error": {"code": "INTERNAL_ERROR", "message": "oops"}} - ) - ) + def test_wait_for_task_times_out(self): + respx.get(f"{API}/tasks/t1").mock(return_value=ok(task("running"))) with LibscopeClient() as client: - with pytest.raises(ServerError): - client.get_analytics() - - def test_connection_refused(self): - client = LibscopeClient(base_url="http://localhost:19999") - with pytest.raises(LibscopeConnectionError): - client.search("hello") - client.close() + with pytest.raises(TimeoutError): + client.wait_for_task("t1", timeout=0, interval=0) - -class TestLibscopeClientContextManager: @respx.mock - def test_context_manager(self): - respx.get(f"{API}/health").mock( - return_value=httpx.Response(200, json={"data": {"status": "ok"}}) + def test_sync_and_cancel(self): + sync = respx.post(f"{API}/sync").mock(return_value=ok({**STARTED, "operation": "sync"}, 202)) + respx.post(f"{API}/tasks/t1/cancel").mock( + return_value=ok({"taskId": "t1", "cancelRequested": True, "status": "cancelled"}) ) with LibscopeClient() as client: - h = client.health() - assert h["status"] == "ok" - - -# --------------------------------------------------------------------------- -# Async client tests -# --------------------------------------------------------------------------- - - -class TestAsyncLibscopeClient: - @pytest.mark.asyncio + assert client.sync("notes").operation == "sync" + client.sync_all() + assert client.cancel_task("t1") is True + assert body(sync) == {"name": "notes"} + assert json.loads(sync.calls[1].request.content) == {"all": True} + + +class TestErrors: + @pytest.mark.parametrize( + "status,error", + [(404, NotFoundError), (400, ValidationError), (500, ServerError)], + ) @respx.mock - async def test_search(self): - respx.get(f"{API}/search").mock( - return_value=httpx.Response( - 200, - json={ - "data": { - "results": [ - {"documentId": "d1", "title": "Doc", "content": "hi", "score": 0.8} - ], - "totalCount": 1, - } - }, - ) + def test_status_codes(self, status, error): + respx.get(f"{API}/documents/x").mock( + return_value=httpx.Response(status, json={"error": {"code": "C", "message": "msg"}}) ) - async with AsyncLibscopeClient() as client: - result = await client.search("hi") - assert result.total_count == 1 + with LibscopeClient() as client: + with pytest.raises(error, match="msg"): + client.get_document("x") - @pytest.mark.asyncio @respx.mock - async def test_add_text(self): - respx.post(f"{API}/documents").mock( - return_value=httpx.Response(201, json={"data": {"id": "d1", "title": "Test"}}) + def test_other_status_keeps_the_server_code(self): + respx.get(f"{API}/health").mock( + return_value=httpx.Response(401, json={"error": {"code": "UNAUTHORIZED", "message": "no"}}) ) - async with AsyncLibscopeClient() as client: - doc = await client.add_text("Test", "content") - assert doc.id == "d1" + with LibscopeClient() as client: + with pytest.raises(LibscopeError) as exc: + client.health() + assert exc.value.code == "UNAUTHORIZED" - @pytest.mark.asyncio @respx.mock - async def test_list_documents(self): - respx.get(f"{API}/documents").mock( - return_value=httpx.Response( - 200, json={"data": [{"id": "d1", "title": "A"}]} - ) - ) - async with AsyncLibscopeClient() as client: - docs = await client.list_documents() - assert len(docs) == 1 + def test_connection_refused(self): + respx.get(f"{API}/health").mock(side_effect=httpx.ConnectError("refused")) + with LibscopeClient() as client: + with pytest.raises(LibscopeConnectionError): + client.health() - @pytest.mark.asyncio @respx.mock - async def test_error_handling(self): - respx.get(f"{API}/documents/bad").mock( - return_value=httpx.Response( - 404, json={"error": {"code": "NOT_FOUND", "message": "not found"}} - ) - ) - async with AsyncLibscopeClient() as client: - with pytest.raises(NotFoundError): - await client.get_document("bad") + def test_api_key_is_sent_as_bearer_token(self): + route = respx.get("http://server:9000/api/v1/health").mock(return_value=ok({"status": "ok"})) + with LibscopeClient("http://server:9000/", api_key="secret") as client: + client.health() + assert route.calls[0].request.headers["Authorization"] == "Bearer secret" - @pytest.mark.asyncio - @respx.mock - async def test_get_analytics(self): - respx.get(f"{API}/stats").mock( - return_value=httpx.Response( - 200, - json={"data": {"totalDocuments": 5, "totalChunks": 20, "totalTopics": 1, "totalTags": 3, "databaseSizeBytes": 512}}, - ) - ) - async with AsyncLibscopeClient() as client: - stats = await client.get_analytics() - assert stats.total_documents == 5 +class TestAsyncClient: @pytest.mark.asyncio @respx.mock - async def test_ask(self): - respx.post(f"{API}/ask").mock( - return_value=httpx.Response(200, json={"data": {"answer": "yes", "sources": []}}) + async def test_add_wait_search(self): + respx.post(f"{API}/documents").mock(return_value=ok(STARTED, 202)) + respx.get(f"{API}/tasks/t1").mock( + return_value=ok(task("completed", result=json.dumps(ADD_RESULT))) ) + respx.get(f"{API}/search").mock(return_value=ok({"items": [HIT], "total": 1})) async with AsyncLibscopeClient() as client: - result = await client.ask("is it?") - assert result.answer == "yes" + started = await client.add_text("T", "C") + done = await client.wait_for_task(started.id, interval=0) + page = await client.search("hello") + assert done.result["kind"] == "content" + assert page.items[0].title == "Doc 1" @pytest.mark.asyncio @respx.mock - async def test_context_manager(self): - respx.get(f"{API}/health").mock( - return_value=httpx.Response(200, json={"data": {"status": "ok"}}) + async def test_errors(self): + respx.get(f"{API}/documents/x").mock( + return_value=httpx.Response(404, json={"error": {"code": "N", "message": "gone"}}) ) async with AsyncLibscopeClient() as client: - h = await client.health() - assert h["status"] == "ok" + with pytest.raises(NotFoundError): + await client.get_document("x") diff --git a/sdk/python/tests/test_models.py b/sdk/python/tests/test_models.py index 1f472bcd..89344979 100644 --- a/sdk/python/tests/test_models.py +++ b/sdk/python/tests/test_models.py @@ -1,105 +1,48 @@ """Tests for libscope Pydantic models.""" -from pylibscope.models import ( - Analytics, - AskResult, - Document, - Graph, - GraphEdge, - GraphNode, - SearchHit, - SearchResult, - SyncResult, - Topic, -) +import json +from pylibscope.models import AskResult, Document, Page, SearchHit, Task, Topic -class TestDocument: - def test_minimal(self): - doc = Document(id="1", title="Hello") - assert doc.id == "1" - assert doc.title == "Hello" - assert doc.url is None - assert doc.tags == [] - def test_full(self): - doc = Document( - id="abc", - title="Test", - url="https://example.com", - topic="python", - topic_id="t1", - tags=["a", "b"], - content_hash="sha256:...", - source_type="manual", - created_at="2024-01-01", - updated_at="2024-01-02", - ) - assert doc.url == "https://example.com" - assert doc.tags == ["a", "b"] - assert doc.source_type == "manual" +def test_models_read_camel_case_and_accept_snake_case(): + doc = Document.model_validate({"documentId": "d1", "title": "T", "sourceType": "manual"}) + assert (doc.document_id, doc.source_type) == ("d1", "manual") + assert Document(document_id="d2", title="U").document_id == "d2" -class TestSearchResult: - def test_empty(self): - sr = SearchResult() - assert sr.results == [] - assert sr.total_count == 0 +def test_page_of_hits(): + page = Page[SearchHit].model_validate( + { + "items": [{"documentId": "d", "chunkId": "c", "title": "T", "score": 0.4}], + "total": 7, + "limit": 1, + "offset": 2, + } + ) + assert page.items[0].score == 0.4 + assert (page.total, page.limit, page.offset) == (7, 1, 2) - def test_with_hits(self): - hit = SearchHit(document_id="1", title="Doc", content="text", score=0.95) - sr = SearchResult(results=[hit], total_count=1) - assert len(sr.results) == 1 - assert sr.results[0].score == 0.95 - assert sr.results[0].document_id == "1" +def test_task_decodes_its_json_result(): + task = Task.model_validate( + {"id": "t", "status": "completed", "result": json.dumps({"items": [1]})} + ) + assert task.result == {"items": [1]} + assert task.done + assert not Task(id="t", status="running").done -class TestTopic: - def test_basic(self): - t = Topic(id="t1", name="Python") - assert t.name == "Python" - assert t.parent_id is None - def test_with_parent(self): - t = Topic(id="t2", name="Flask", parent_id="t1") - assert t.parent_id == "t1" +def test_ask_result_in_context_mode(): + result = AskResult.model_validate( + {"mode": "context", "contextPrompt": "ctx", "sources": [], "unknown": 1} + ) + assert result.mode == "context" + assert result.context_prompt == "ctx" + assert result.answer is None -class TestAnalytics: - def test_defaults(self): - a = Analytics() - assert a.total_documents == 0 - assert a.database_size_bytes == 0 - - def test_values(self): - a = Analytics(total_documents=42, total_chunks=100, total_topics=5, total_tags=10, database_size_bytes=1024) - assert a.total_documents == 42 - - -class TestAskResult: - def test_basic(self): - r = AskResult(answer="42", sources=[{"id": "1"}]) - assert r.answer == "42" - assert len(r.sources) == 1 - - -class TestGraph: - def test_empty(self): - g = Graph() - assert g.nodes == [] - assert g.edges == [] - - def test_with_data(self): - g = Graph( - nodes=[GraphNode(id="1", label="Doc1")], - edges=[GraphEdge(source="1", target="2", weight=0.8)], - ) - assert len(g.nodes) == 1 - assert g.edges[0].weight == 0.8 - - -class TestSyncResult: - def test_basic(self): - s = SyncResult(connector="obsidian", documents_synced=5) - assert s.connector == "obsidian" - assert s.errors == [] +def test_topic_defaults(): + topic = Topic.model_validate({"id": "a", "name": "A"}) + assert topic.parent_id is None + assert topic.document_count is None From 26396b2e0c2e10c5779189ae2b0fa408d023b2a1 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:47:53 +0000 Subject: [PATCH 56/83] feat(packs)!: one pack registry system (git registries) over operations Remove the URL pack registry (DEFAULT_REGISTRY_URL, listAvailablePacks, loadPackFromRegistry, registryUrl options). Pack names now resolve only from the configured git registries, in core (findRegistryPack/resolveRegistryPack): name or name@version, an optional registry name, a ValidationError naming the registries when several have the pack, NotFoundError when none has it. - install-pack: pack = registry name[@version] or a local .json/.json.gz file (CLI and SDK only; refused over MCP and REST); optional registry. - list-packs: available lists the packs in the synced registry indexes (no network), optional registry. - New operations: list-registries, add-registry, remove-registry, sync-registries, search-registries, create-registry, publish-pack, unpublish-pack. REST: GET /registries and GET /registries/search only. SDK: scope.registries.*. - CLI pack/registry commands call the operations; --registry <name> is the only registry flag (removed --from-registry, pack install --pack-version and -y, registry add --priority/--sync-interval, -r). - Registry config drops priority and syncInterval (auto-sync never ran); file:/// registry URLs accepted; missing git is a ConfigError; registry paths follow homedir() at call time. --- src/LibScope.ts | 21 + src/cli/commands/pack.ts | 125 ++--- src/cli/commands/registry.ts | 515 +++++------------- src/core/operations/index.ts | 3 + src/core/operations/packs.ts | 61 ++- src/core/operations/registries.ts | 191 +++++++ src/core/packs.ts | 121 +--- src/mcp/format.ts | 24 +- src/registry/config.ts | 61 ++- src/registry/git.ts | 16 +- src/registry/publish.ts | 15 +- src/registry/resolve.ts | 240 +++----- src/registry/search.ts | 79 ++- src/registry/sync.ts | 173 ++---- src/registry/types.ts | 74 +-- .../registry/registry-conflict.test.ts | 230 ++------ .../registry/registry-lifecycle.test.ts | 6 +- .../registry/registry-offline.test.ts | 17 +- .../registry/registry-publish.test.ts | 2 +- tests/unit/api-routes.test.ts | 9 + tests/unit/cli-commands.test.ts | 53 +- tests/unit/cli-version-options.test.ts | 79 +-- tests/unit/config-save.test.ts | 2 - tests/unit/libscope.test.ts | 3 + tests/unit/mcp-format.test.ts | 15 +- tests/unit/mcp-server.test.ts | 43 +- tests/unit/operations/registries.test.ts | 154 ++++++ tests/unit/packs.test.ts | 37 -- tests/unit/registry/config.test.ts | 39 +- tests/unit/registry/conflict.test.ts | 449 --------------- tests/unit/registry/git.test.ts | 16 +- tests/unit/registry/publish.test.ts | 2 - tests/unit/registry/resolve.test.ts | 235 ++++++++ tests/unit/registry/search.test.ts | 34 +- tests/unit/registry/stale-cache.test.ts | 109 ---- tests/unit/registry/sync.test.ts | 160 ++---- tests/unit/registry/types.test.ts | 25 +- 37 files changed, 1346 insertions(+), 2092 deletions(-) create mode 100644 src/core/operations/registries.ts create mode 100644 tests/unit/operations/registries.test.ts delete mode 100644 tests/unit/registry/conflict.test.ts create mode 100644 tests/unit/registry/resolve.test.ts delete mode 100644 tests/unit/registry/stale-cache.test.ts diff --git a/src/LibScope.ts b/src/LibScope.ts index 7c51ce50..90fc3ca0 100644 --- a/src/LibScope.ts +++ b/src/LibScope.ts @@ -55,6 +55,16 @@ import { listPacksOperation, removePackOperation, } from "./core/operations/packs.js"; +import { + addRegistryOperation, + createRegistryOperation, + listRegistriesOperation, + publishPackOperation, + removeRegistryOperation, + searchRegistriesOperation, + syncRegistriesOperation, + unpublishPackOperation, +} from "./core/operations/registries.js"; import { askOperation, searchOperation, toRagOptions } from "./core/operations/search.js"; import { deleteSavedSearchOperation, @@ -181,6 +191,16 @@ const NAMESPACES = { list: listPacksOperation, create: createPackOperation, }, + registries: { + list: listRegistriesOperation, + add: addRegistryOperation, + remove: removeRegistryOperation, + sync: syncRegistriesOperation, + search: searchRegistriesOperation, + create: createRegistryOperation, + publish: publishPackOperation, + unpublish: unpublishPackOperation, + }, connectors: { list: lazy( "list-connections", @@ -284,6 +304,7 @@ export class LibScope { readonly links = this.bind(NAMESPACES.links); readonly searches = this.bind(NAMESPACES.searches); readonly packs = this.bind(NAMESPACES.packs); + readonly registries = this.bind(NAMESPACES.registries); readonly connectors = this.bind(NAMESPACES.connectors); readonly tasks = this.bind(NAMESPACES.tasks); readonly admin = this.bind(NAMESPACES.admin); diff --git a/src/cli/commands/pack.ts b/src/cli/commands/pack.ts index eaf0fe66..fda29a8e 100644 --- a/src/cli/commands/pack.ts +++ b/src/cli/commands/pack.ts @@ -1,91 +1,24 @@ /** `libscope pack install|remove|list|create`, over the pack operations. */ import type { Command } from "commander"; -import { createInterface } from "node:readline/promises"; import { createPackOperation, installPackOperation, listPacksOperation, removePackOperation, } from "../../core/operations/index.js"; -import type { InstallResult } from "../../core/packs.js"; -import { loadRegistries } from "../../registry/config.js"; -import { - parsePackSpecifier, - resolvePackFromRegistries, - verifyResolvedPackChecksum, - type ResolvedPack, -} from "../../registry/resolve.js"; -import { ValidationError } from "../../errors.js"; +import type { InstallResult, InstalledPack } from "../../core/packs.js"; +import type { RegistryPack } from "../../registry/types.js"; import { confirmOrCancel } from "../confirm.js"; import { defined, splitList, toNumber } from "../options.js"; import { call, plural, print, printList, run } from "../run.js"; interface InstallFlags { registry?: string; - fromRegistry?: string; - packVersion?: string; - yes?: boolean; batchSize?: number; resumeFrom?: number; concurrency?: number; } -/** Ask which registry to use when a pack is in several. */ -async function chooseRegistry( - packName: string, - sources: Array<{ registryName: string; version: string; priority: number }>, -): Promise<string> { - console.log(`Pack "${packName}" is in several registries:`); - sources.forEach((s, i) => - console.log(` [${i + 1}] ${s.registryName} (v${s.version}, priority ${s.priority})`), - ); - const rl = createInterface({ input: process.stdin, output: process.stdout }); - try { - const answer = (await rl.question("Select registry [1]: ")).trim() || "1"; - const chosen = sources[Number.parseInt(answer, 10) - 1]; - if (!chosen) throw new ValidationError(`Invalid selection: ${answer}`); - return chosen.registryName; - } finally { - rl.close(); - } -} - -/** - * Find `nameOrPath` in the git registries. Returns the resolved pack (checksum verified), or - * undefined for a local file, when no git registry is configured, or when none has the pack. - */ -async function resolveFromGitRegistries( - nameOrPath: string, - flags: InstallFlags, -): Promise<ResolvedPack | undefined> { - const isFile = nameOrPath.endsWith(".json") || nameOrPath.endsWith(".json.gz"); - if (isFile || loadRegistries().length === 0) return undefined; - - const { name, version: specVersion } = parsePackSpecifier(nameOrPath); - const version = flags.packVersion ?? specVersion; - const resolve = ( - registryName: string | undefined, - ): ReturnType<typeof resolvePackFromRegistries> => - resolvePackFromRegistries(name, { - version, - registryName, - conflictResolution: registryName - ? { strategy: "explicit", registryName } - : flags.yes - ? { strategy: "priority" } - : undefined, - }); - - let result = resolve(flags.fromRegistry); - for (const w of result.warnings) console.error(`⚠ ${w}`); - if (result.conflict && !result.resolved) { - result = resolve(await chooseRegistry(name, result.conflict.sources)); - } - if (!result.resolved) return undefined; - await verifyResolvedPackChecksum(result.resolved); - return result.resolved; -} - function printInstall(result: InstallResult): void { if (result.alreadyInstalled) { console.log(`Pack "${result.packName}" is already installed.`); @@ -97,6 +30,25 @@ function printInstall(result: InstallResult): void { ); } +/** Print registry warnings (e.g. a registry that was never synced) to stderr. */ +export function printWarnings(warnings: readonly string[] | undefined): void { + for (const warning of warnings ?? []) console.error(`⚠ ${warning}`); +} + +function printPacks(result: { + items: Array<InstalledPack | RegistryPack>; + warnings?: string[] | undefined; +}): void { + printWarnings(result.warnings); + printList(result.items, "No packs found.", (p) => + console.log( + "installedAt" in p + ? `${p.name} v${p.version} — ${p.description ?? ""} (${plural(p.docCount, "document")})` + : `${p.name} v${p.latestVersion} — ${p.description} (registry ${p.registry})`, + ), + ); +} + interface CreateFlags { name: string; from?: string[]; @@ -135,22 +87,18 @@ export function register(program: Command): void { .description("Install, remove, list and create knowledge packs"); pack - .command("install <nameOrPath>") + .command("install <pack>") .description( - "Install a pack from a git registry, the pack registry URL, or a local .json/.json.gz file (name@version works)", + "Install a pack from the configured registries (name or name@version) or a local .json/.json.gz file", ) - .option("--registry <url>", "Pack registry URL (URL registries)") - .option("--from-registry <name>", "Use this git registry") - .option("--pack-version <semver>", "Version to install (git registries)") - .option("-y, --yes", "Do not prompt: with several registries, use the highest priority") + .option("--registry <name>", "Look only in this registry") .option("--batch-size <n>", "Documents embedded per batch (default 10)", toNumber) .option("--resume-from <n>", "Skip the first N documents (resume a partial install)", toNumber) .option("--concurrency <n>", "Batches embedded in parallel (default 4)", toNumber) - .action(async (nameOrPath: string, flags: InstallFlags) => { - const resolved = await resolveFromGitRegistries(nameOrPath, flags); + .action(async (name: string, flags: InstallFlags) => { const input = defined({ - pack: resolved?.dataPath ?? nameOrPath, - registryUrl: resolved ? undefined : flags.registry, + pack: name, + registry: flags.registry, batchSize: flags.batchSize, resumeFrom: flags.resumeFrom, concurrency: flags.concurrency, @@ -170,21 +118,12 @@ export function register(program: Command): void { pack .command("list") - .description("List installed packs, or packs in the registry with --available") - .option("--available", "List packs available in the registry") - .option("--registry <url>", "Pack registry URL") + .description("List installed packs, or the packs in the configured registries (--available)") + .option("--available", "List the packs in the configured registries") + .option("--registry <name>", "With --available: only this registry") .action(async (flags: { available?: boolean; registry?: string }) => { - const input = defined({ available: flags.available, registryUrl: flags.registry }); - await run(listPacksOperation, input, (r) => - printList<{ name: string; version: string; description: string | null; docCount: number }>( - r.items, - flags.available ? "No packs available." : "No packs installed.", - (p) => - console.log( - `${p.name} v${p.version} — ${p.description ?? ""} (${plural(p.docCount, "document")})`, - ), - ), - ); + const input = defined({ available: flags.available, registry: flags.registry }); + await run(listPacksOperation, input, printPacks); }); pack diff --git a/src/cli/commands/registry.ts b/src/cli/commands/registry.ts index f49ddfb4..7f8f68c6 100644 --- a/src/cli/commands/registry.ts +++ b/src/cli/commands/registry.ts @@ -1,52 +1,20 @@ -/** - * CLI commands for managing pack registries. - * Registered as `libscope registry <subcommand>`. - */ - +/** `libscope registry add|remove|list|sync|search|create|publish|unpublish`, over the registry operations. */ import type { Command } from "commander"; -import { rmSync } from "node:fs"; -import { resolve as pathResolve } from "node:path"; -import { - loadRegistries, - addRegistry, - removeRegistry, - validateRegistryName, - validateGitUrl, -} from "../../registry/config.js"; import { - cloneRegistry, - readIndex, - createRegistryRepo, - checkGitAvailable, -} from "../../registry/git.js"; -import { getRegistryCacheDir } from "../../registry/types.js"; -import { syncRegistryByName, syncAllRegistries } from "../../registry/sync.js"; -import { searchRegistries } from "../../registry/search.js"; -import { publishPack, publishPackToBranch, unpublishPack } from "../../registry/publish.js"; -import { parsePackSpecifier } from "../../registry/resolve.js"; + addRegistryOperation, + createRegistryOperation, + listRegistriesOperation, + publishPackOperation, + removeRegistryOperation, + searchRegistriesOperation, + syncRegistriesOperation, + unpublishPackOperation, +} from "../../core/operations/index.js"; +import type { RegistrySearchResult, RegistrySyncStatus } from "../../registry/types.js"; import { confirmOrCancel } from "../confirm.js"; - -/** Derive a short name from a git URL (e.g. "github.com/org/repo" → "repo"). */ -function deriveNameFromUrl(url: string): string { - // Try URL parsing first for https:// and ssh:// URLs - if (url.startsWith("https://") || url.startsWith("ssh://")) { - try { - const parsed = new URL(url); - const segments = parsed.pathname.split("/").filter(Boolean); - const last = segments[segments.length - 1] ?? ""; - const derived = last.replace(/\.git$/, ""); - if (derived) return derived; - } catch { - // fall through to SCP regex - } - } - - // Fall back to SCP-style SSH format: git@github.com:org/repo.git - const scpMatch = url.match(/:([^/]+\/)*([^/.]+?)(?:\.git)?$/); - if (scpMatch?.[2]) return scpMatch[2]; - - return "registry"; -} +import { defined } from "../options.js"; +import { printList, run } from "../run.js"; +import { printWarnings } from "./pack.js"; /** Truncate a string to a max length, adding "..." if truncated. */ function truncate(str: string, maxLen: number): string { @@ -60,383 +28,174 @@ function padColumns(cols: string[]): string { return cols.map((col, i) => col.padEnd(widths[i] ?? 16)).join(" "); } -/** Sync a single named registry and print the result. */ -async function syncSingleRegistry(name: string): Promise<void> { - const status = await syncRegistryByName(name); - if (status.status === "error") { - console.error(`Error: ${status.error}`); - process.exit(1); - return; - } - if (status.status === "offline") { - console.warn(`Warning: ${status.error}`); - console.warn( - `Registry "${name}" is unreachable. Using cached index from ${status.lastSyncedAt ?? "unknown"}.`, - ); - return; - } - const cacheDir = getRegistryCacheDir(name); - const index = readIndex(cacheDir); - console.log(`Registry "${name}" synced: ${index.length} pack(s) available.`); -} - -/** Sync all registries and print per-registry results. */ -async function syncAllRegistriesAction(): Promise<void> { - const results = await syncAllRegistries(); - if (results.length === 0) { - console.log("No registries configured."); - return; - } - for (const status of results) { - if (status.status === "success") { - const cacheDir = getRegistryCacheDir(status.registryName); - const index = readIndex(cacheDir); - console.log(` ${status.registryName}: synced (${index.length} packs)`); - } else if (status.status === "offline") { - console.warn(` ${status.registryName}: offline (using cached data)`); - } else { - console.error(` ${status.registryName}: error — ${status.error}`); - } - } -} - -/** Format an error for console output. */ -function formatError(err: unknown): string { - return err instanceof Error ? err.message : String(err); -} - -/** Require git to be available, exiting if not. */ -async function requireGit(): Promise<void> { - if (!(await checkGitAvailable())) { - console.error("Error: git is not installed or not in PATH."); - process.exit(1); - } +function packCount(packs: number | null): string { + return packs === null ? "not synced" : `${packs} pack${packs === 1 ? "" : "s"}`; } -/** Derive or validate the registry name from URL/option. Returns the name or exits. */ -function resolveRegistryName(url: string, explicitName?: string): string { - if (explicitName) return explicitName; - - const derived = deriveNameFromUrl(url); - try { - validateRegistryName(derived); - } catch { +/** One line per sync result; a registry that could not be synced sets exit code 1. */ +function printSyncStatus(status: RegistrySyncStatus): void { + if (status.status === "success") { + console.log(`✓ ${status.registry}: synced (${packCount(status.packs)})`); + } else if (status.status === "offline") { console.error( - `Error: Could not derive a valid registry name from URL (got "${derived}"). ` + - "Please provide a name using --name.", + `⚠ ${status.registry}: unreachable, using the local copy from ${status.lastSyncedAt ?? "unknown"} (${status.error ?? ""})`, ); - process.exit(1); + } else { + console.error(`✗ ${status.registry}: ${status.error ?? "sync failed"}`); + process.exitCode = 1; } - return derived; } -/** Parse and validate a non-negative integer option. Exits on invalid input. */ -function parseNonNegativeInt(value: string, optionName: string): number { - const n = Number.parseInt(value, 10); - if (Number.isNaN(n) || n < 0) { - console.error(`Error: "${optionName}" must be a non-negative integer.`); - process.exit(1); +function printSearchResults(result: { items: RegistrySearchResult[]; warnings: string[] }): void { + printWarnings(result.warnings); + if (result.items.length === 0) { + console.log("No packs found."); + return; } - return n; -} - -/** Perform the initial clone/sync after adding a registry. */ -async function initialSync(name: string, url: string): Promise<void> { - const cacheDir = getRegistryCacheDir(name); - console.log(`Cloning registry to ${cacheDir}...`); - try { - await cloneRegistry(url, cacheDir); - const index = readIndex(cacheDir); - console.log(`Synced: ${index.length} pack(s) available.`); - } catch (syncErr) { - console.warn( - `Warning: initial sync failed (${formatError(syncErr)}). ` + - 'You can retry with "libscope registry sync".', + const header = padColumns(["Pack", "Description", "Tags", "Version", "Registry"]); + console.log(header); + console.log("-".repeat(header.length)); + for (const r of result.items) { + const tags = r.tags.length > 0 ? r.tags.join(", ") : "-"; + console.log( + padColumns([ + r.name, + truncate(r.description, 40), + truncate(tags, 20), + r.latestVersion, + r.registry, + ]), ); } } -/** Register all `registry` subcommands on the given Commander program. */ export function register(program: Command): void { - const registryCmd = program + const registry = program .command("registry") - .description("Manage pack registries (git-backed)"); + .description("Manage pack registries (git repositories of packs)"); - // --- registry add --- - registryCmd + registry .command("add <url>") - .description("Add a git-backed pack registry") - .option("-n, --name <alias>", "Short name for this registry") - .option("--priority <n>", "Priority for conflict resolution (lower wins, default: 10)", "10") - .option("--sync-interval <seconds>", "Auto-sync interval in seconds (0 = manual)", "0") - .option("--no-sync", "Skip the initial sync after adding") - .action( - async ( - url: string, - opts: { - name?: string; - priority: string; - syncInterval: string; - sync: boolean; + .description("Add a git repository as a pack registry and clone it") + .option("-n, --name <name>", "Registry name (default: from the URL)") + .option("--no-sync", "Do not clone the registry now") + .action(async (url: string, flags: { name?: string; sync: boolean }) => { + await run( + addRegistryOperation, + defined({ url, name: flags.name, sync: flags.sync }), + (result) => { + console.log(`✓ Registry "${result.name}" added (${result.url})`); + if (result.sync) printSyncStatus(result.sync); }, - ) => { - await requireGit(); - - const name = resolveRegistryName(url, opts.name); - const priority = parseNonNegativeInt(opts.priority, "--priority"); - const syncInterval = parseNonNegativeInt(opts.syncInterval, "--sync-interval"); - - try { - validateRegistryName(name); - validateGitUrl(url); - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - return; - } - - try { - addRegistry({ name, url, syncInterval, priority, lastSyncedAt: null }); - console.log(`Registry "${name}" added (${url}).`); - - if (opts.sync !== false) { - await initialSync(name, url); - } - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - } - }, - ); + ); + }); - // --- registry remove --- - registryCmd + registry .command("remove <name>") - .description("Remove a registry and delete its local cache") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (name: string, opts: { yes?: boolean }) => { - if ( - !(await confirmOrCancel( - `Remove registry "${name}" and its local cache? This cannot be undone.`, - opts.yes, - )) - ) { - return; - } - - try { - removeRegistry(name); - - // Delete local cache - const cacheDir = getRegistryCacheDir(name); - try { - rmSync(cacheDir, { recursive: true, force: true }); - } catch (rmErr) { - console.warn(`Warning: could not remove cache directory (${formatError(rmErr)})`); - } - - console.log(`Registry "${name}" removed.`); - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - } + .description("Remove a registry and delete its local clone") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (name: string, flags: { yes?: boolean }) => { + const question = `Remove registry "${name}" and its local clone?`; + if (!(await confirmOrCancel(question, flags.yes))) return; + await run(removeRegistryOperation, { name }, () => + console.log(`✓ Registry "${name}" removed`), + ); }); - // --- registry list --- - registryCmd + registry .command("list") - .description("List all configured registries") - .action(() => { - const registries = loadRegistries(); - if (registries.length === 0) { - console.log("No registries configured. Use 'libscope registry add <url>' to add one."); - return; - } - - console.log("Configured registries:\n"); - for (const reg of registries) { - const syncInfo = reg.lastSyncedAt ? `last synced ${reg.lastSyncedAt}` : "never synced"; - - // Try to read index to get pack count - let packCount = "?"; - try { - const cacheDir = getRegistryCacheDir(reg.name); - const index = readIndex(cacheDir); - packCount = String(index.length); - } catch { - // Cache doesn't exist yet — packCount remains "?" - } - - console.log( - ` ${reg.name} — ${reg.url} (priority: ${reg.priority}, ${packCount} packs, ${syncInfo})`, - ); - } + .description("List the configured registries") + .action(async () => { + await run(listRegistriesOperation, {}, (result) => + printList( + result.items, + "No registries configured. Add one with: libscope registry add <url>", + (r) => + console.log( + `${r.name} — ${r.url} (${packCount(r.packs)}, last synced ${r.lastSyncedAt ?? "never"})`, + ), + ), + ); }); - // --- registry sync --- - registryCmd + registry .command("sync [name]") - .description("Sync one or all registries (git fetch + fast-forward)") - .action(async (name?: string) => { - await requireGit(); - - if (name) { - await syncSingleRegistry(name); - } else { - await syncAllRegistriesAction(); - } + .description("Fetch the latest packs of one or all registries") + .action(async (name: string | undefined) => { + await run(syncRegistriesOperation, defined({ name }), (result) => + printList(result.items, "No registries configured.", printSyncStatus), + ); }); - // --- registry search --- - registryCmd + registry .command("search <query>") - .description("Search for packs across all configured registries") - .option("-r, --registry <name>", "Search only in a specific registry") - .action((query: string, opts: { registry?: string }) => { - const { results, warnings } = searchRegistries(query, { - registryName: opts.registry, - }); - - for (const w of warnings) { - console.warn(`Warning: ${w}`); - } - - if (results.length === 0) { - console.log(`No packs found matching "${query}".`); - return; - } - - console.log(`Found ${results.length} pack(s) matching "${query}":\n`); - - // Column header - const header = padColumns(["Pack", "Description", "Tags", "Version", "Registry"]); - console.log(header); - console.log("-".repeat(header.length)); - - for (const r of results) { - const tags = r.pack.tags.length > 0 ? r.pack.tags.join(", ") : "-"; - console.log( - padColumns([ - r.pack.name, - truncate(r.pack.description, 40), - truncate(tags, 20), - r.pack.latestVersion, - r.registryName, - ]), - ); - } + .description("Search the packs in the configured registries") + .option("--registry <name>", "Search only this registry") + .action(async (query: string, flags: { registry?: string }) => { + await run( + searchRegistriesOperation, + defined({ query, registry: flags.registry }), + printSearchResults, + ); }); - // --- registry create --- - registryCmd + registry .command("create <path>") - .description("Initialize a new registry git repo with canonical folder structure") - .action(async (rawPath: string) => { - await requireGit(); - - const resolved = pathResolve(rawPath); - try { - await createRegistryRepo(resolved); - console.log(`Registry repo initialized at ${resolved}`); - console.log("Push to a git remote, then add it with 'libscope registry add <url>'."); - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - } + .description("Create an empty registry git repository") + .action(async (path: string) => { + await run(createRegistryOperation, { path }, (result) => { + console.log(`✓ Registry repository created at ${result.path}`); + console.log("Push it to a git remote, then add it with: libscope registry add <url>"); + }); }); - // --- registry publish --- - registryCmd - .command("publish <packFile>") - .description("Publish a pack file to a registry") - .requiredOption("-r, --registry <name>", "Target registry name") - .option("--pack-version <semver>", "Version to publish as (default: auto-bump patch)") + registry + .command("publish <file>") + .description("Publish a pack file to a registry (commit and push)") + .requiredOption("--registry <name>", "Target registry") + .option("--pack-version <semver>", "Version to publish (default: next patch version)") .option("-m, --message <msg>", "Git commit message") - .option("--submit", "Push to a feature branch instead of main (for PR workflow)") + .option("--submit", "Push to a feature branch for a pull request instead of the main branch") .action( async ( - packFile: string, - opts: { registry: string; packVersion?: string; message?: string; submit?: boolean }, + file: string, + flags: { registry: string; packVersion?: string; message?: string; submit?: boolean }, ) => { - await requireGit(); - - const resolved = pathResolve(packFile); - - try { - if (opts.submit) { - const result = await publishPackToBranch({ - registryName: opts.registry, - packFilePath: resolved, - version: opts.packVersion, - commitMessage: opts.message, - }); + const input = defined({ + file, + registry: flags.registry, + version: flags.packVersion, + message: flags.message, + submit: flags.submit, + }); + await run(publishPackOperation, input, (result) => { + if (result.branch) { console.log( - `Pack "${result.packName}@${result.version}" pushed to branch "${result.branch}".`, + `✓ Pack "${result.packName}@${result.version}" pushed to branch "${result.branch}". Open a pull request to merge it.`, ); - console.log("Create a pull request to merge it into the registry."); } else { - const result = await publishPack({ - registryName: opts.registry, - packFilePath: resolved, - version: opts.packVersion, - commitMessage: opts.message, - }); console.log( - `Pack "${result.packName}@${result.version}" published to "${result.registryName}" (checksum: ${result.checksum.slice(0, 12)}...).`, + `✓ Pack "${result.packName}@${result.version}" published to "${result.registryName}" (checksum ${result.checksum.slice(0, 12)}...)`, ); } - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - } + }); }, ); - // --- registry unpublish --- - registryCmd - .command("unpublish <packName>") - .description("Remove a pack version from a registry (accepts name@version)") - .requiredOption("-r, --registry <name>", "Target registry name") - .option("--pack-version <semver>", "Version to unpublish (or use name@version)") + registry + .command("unpublish <pack>") + .description("Remove one version of a pack from a registry (<name>@<version>)") + .requiredOption("--registry <name>", "Target registry") .option("-m, --message <msg>", "Git commit message") - .option("-y, --yes", "Skip confirmation prompt") - .action( - async ( - packSpec: string, - opts: { registry: string; packVersion?: string; message?: string; yes?: boolean }, - ) => { - const { name: packName, version: specVersion } = parsePackSpecifier(packSpec); - const version = opts.packVersion ?? specVersion; - if (!version) { - console.error( - "Error: specify a version with <name>@<version> or --pack-version <semver>.", - ); - process.exit(1); - } - - await requireGit(); - - if ( - !(await confirmOrCancel( - `Unpublish "${packName}@${version}" from "${opts.registry}"? This cannot be undone.`, - opts.yes, - )) - ) { - return; - } - - try { - await unpublishPack({ - registryName: opts.registry, - packName, - version, - commitMessage: opts.message, - }); - console.log(`Pack "${packName}@${version}" unpublished from "${opts.registry}".`); - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - } - }, - ); + .option("-y, --yes", "Do not ask for confirmation") + .action(async (pack: string, flags: { registry: string; message?: string; yes?: boolean }) => { + const question = `Unpublish "${pack}" from "${flags.registry}"? This cannot be undone.`; + if (!(await confirmOrCancel(question, flags.yes))) return; + const input = defined({ pack, registry: flags.registry, message: flags.message }); + await run(unpublishPackOperation, input, (result) => + console.log( + `✓ Pack "${result.pack}@${result.version}" unpublished from "${result.registry}"`, + ), + ); + }); } diff --git a/src/core/operations/index.ts b/src/core/operations/index.ts index 8dfe55a9..761c097c 100644 --- a/src/core/operations/index.ts +++ b/src/core/operations/index.ts @@ -5,6 +5,7 @@ import { documentOperations } from "./documents.js"; import { graphOperations } from "./graph.js"; import { linkOperations } from "./links.js"; import { packOperations } from "./packs.js"; +import { registryOperations } from "./registries.js"; import { savedSearchOperations } from "./searches.js"; import { searchOperations } from "./search.js"; import { tagOperations } from "./tags.js"; @@ -21,6 +22,7 @@ export * from "./documents.js"; export * from "./graph.js"; export * from "./links.js"; export * from "./packs.js"; +export * from "./registries.js"; export * from "./searches.js"; export * from "./search.js"; export * from "./tags.js"; @@ -38,6 +40,7 @@ export const OPERATIONS: readonly Operation[] = [ ...topicOperations, ...savedSearchOperations, ...packOperations, + ...registryOperations, ...connectorOperations, ...adminOperations, ...analyticsOperations, diff --git a/src/core/operations/packs.ts b/src/core/operations/packs.ts index f040410b..2c377a09 100644 --- a/src/core/operations/packs.ts +++ b/src/core/operations/packs.ts @@ -1,10 +1,11 @@ import { z } from "zod"; import { ValidationError } from "../../errors.js"; +import { resolveRegistryPack } from "../../registry/resolve.js"; +import { listRegistryPacks } from "../../registry/search.js"; import { createPack, createPackFromSource, installPack, - listAvailablePacks, listInstalledPacks, removePack, type KnowledgePack, @@ -14,26 +15,34 @@ import * as s from "./schemas.js"; import { defineOperation, type OperationContext } from "./types.js"; const pack = z.string().min(1).describe("Pack name"); -const registryUrl = z.url().optional().describe("Registry URL (default: the public pack registry)"); +/** A configured pack registry (`libscope registry add`). */ +export const registryName = z.string().min(1).describe("Registry name"); function isPackFile(nameOrPath: string): boolean { return nameOrPath.endsWith(".json") || nameOrPath.endsWith(".json.gz"); } -/** REST callers must not read or write files on the server's disk. */ -function assertLocalFilesAllowed(ctx: OperationContext, what: string): void { - if (ctx.surface === "api") { - throw new ValidationError(`${what} is not available over the REST API`); +/** Files on the server's disk are read or written only from the CLI and the Node.js API. */ +export function assertLocalFilesAllowed(ctx: OperationContext, what: string): void { + if (ctx.surface === "api" || ctx.surface === "mcp") { + throw new ValidationError(`${what} is only available from the CLI and the Node.js API`); } } export const installPackOperation = defineOperation({ name: "install-pack", group: "packs", - summary: "Install a knowledge pack from the registry or a local .json/.json.gz file", + summary: "Install a knowledge pack from a registry (name or name@version) or a local file", + description: + "Packs are looked up in the local copies of the configured registries (run sync-registries to update them). Local .json/.json.gz files are accepted from the CLI and the Node.js API only.", input: z.object({ - pack: z.string().min(1).describe("Pack name from the registry, or a local .json/.json.gz file"), - registryUrl, + pack: z + .string() + .min(1) + .describe("Pack name or name@version from a registry, or a local .json/.json.gz file"), + registry: registryName + .optional() + .describe("Look only in this registry (needed when several registries have the pack)"), batchSize: z .number() .int() @@ -57,10 +66,17 @@ export const installPackOperation = defineOperation({ }), annotations: { longRunning: true, idempotent: true }, http: { method: "POST", path: "/packs" }, - handler(ctx, input) { - if (isPackFile(input.pack)) assertLocalFilesAllowed(ctx, "Installing a pack from a file"); - return installPack(ctx.db, ctx.provider, input.pack, { - registryUrl: input.registryUrl, + async handler(ctx, input) { + let file = input.pack; + if (isPackFile(input.pack)) { + assertLocalFilesAllowed(ctx, "Installing a pack from a file"); + if (input.registry !== undefined) { + throw new ValidationError("registry applies to pack names, not to pack files"); + } + } else { + file = (await resolveRegistryPack(input.pack, input.registry)).dataPath; + } + return installPack(ctx.db, ctx.provider, file, { batchSize: input.batchSize, concurrency: input.concurrency, resumeFrom: input.resumeFrom, @@ -86,15 +102,24 @@ export const removePackOperation = defineOperation({ export const listPacksOperation = defineOperation({ name: "list-packs", group: "packs", - summary: "List installed packs, or packs available in the registry", + summary: "List installed packs, or the packs available in the configured registries", input: z.object({ - available: z.boolean().default(false).describe("List registry packs instead of installed ones"), - registryUrl, + available: z + .boolean() + .default(false) + .describe("List the packs in the configured registries instead of installed ones"), + registry: registryName.optional().describe("With available: only this registry"), }), annotations: { readOnly: true }, http: { method: "GET", path: "/packs" }, - async handler(ctx, input) { - if (input.available) return { items: await listAvailablePacks(input.registryUrl) }; + handler(ctx, input) { + if (input.available) { + const { packs, warnings } = listRegistryPacks(input.registry); + return { items: packs, warnings }; + } + if (input.registry !== undefined) { + throw new ValidationError("registry applies only with available: true"); + } return { items: listInstalledPacks(ctx.db) }; }, }); diff --git a/src/core/operations/registries.ts b/src/core/operations/registries.ts new file mode 100644 index 00000000..e5d5a42e --- /dev/null +++ b/src/core/operations/registries.ts @@ -0,0 +1,191 @@ +/** + * Pack registries: git repositories of packs, cloned under ~/.libscope/registries/<name>/. + * Only list-registries and search-registries are read-only and exposed over REST; the others + * change local clones and config (CLI and Node.js API). + */ +import { rmSync } from "node:fs"; +import { resolve as pathResolve } from "node:path"; +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { + addRegistry, + deriveRegistryName, + loadRegistries, + removeRegistry, + requireRegistry, +} from "../../registry/config.js"; +import { createRegistryRepo } from "../../registry/git.js"; +import { publishPack, publishPackToBranch, unpublishPack } from "../../registry/publish.js"; +import { parsePackSpecifier } from "../../registry/resolve.js"; +import { searchRegistries } from "../../registry/search.js"; +import { localPackCount, syncAllRegistries, syncRegistry } from "../../registry/sync.js"; +import { getRegistryCacheDir, type PublishResult } from "../../registry/types.js"; +import { assertLocalFilesAllowed, registryName } from "./packs.js"; +import { defineOperation } from "./types.js"; + +const message = z.string().min(1).optional().describe("Git commit message"); + +export const listRegistriesOperation = defineOperation({ + name: "list-registries", + group: "packs", + summary: "List the configured pack registries", + input: z.object({}), + annotations: { readOnly: true }, + http: { method: "GET", path: "/registries" }, + handler() { + const items = loadRegistries().map((entry) => ({ + name: entry.name, + url: entry.url, + lastSyncedAt: entry.lastSyncedAt, + packs: localPackCount(entry.name), + })); + return { items }; + }, +}); + +export const addRegistryOperation = defineOperation({ + name: "add-registry", + group: "packs", + summary: "Add a git repository as a pack registry, and clone it", + input: z.object({ + url: z + .string() + .min(1) + .describe("Git URL: https://, ssh://, git@host:path, or file:/// for a local repository"), + name: registryName.optional().describe("Registry name (default: from the URL)"), + sync: z.boolean().default(true).describe("Clone the registry now"), + }), + async handler(_ctx, input) { + const name = input.name ?? deriveRegistryName(input.url); + const entry = addRegistry({ name, url: input.url, lastSyncedAt: null }); + return { name, url: entry.url, sync: input.sync ? await syncRegistry(entry) : null }; + }, +}); + +export const removeRegistryOperation = defineOperation({ + name: "remove-registry", + group: "packs", + summary: "Remove a pack registry and delete its local clone", + input: z.object({ name: registryName }), + annotations: { destructive: true }, + handler(_ctx, input) { + removeRegistry(input.name); + rmSync(getRegistryCacheDir(input.name), { recursive: true, force: true }); + return { name: input.name, removed: true }; + }, +}); + +export const syncRegistriesOperation = defineOperation({ + name: "sync-registries", + group: "packs", + summary: "Fetch the latest packs of one or all registries", + description: + "A registry that cannot be reached keeps its local copy (status offline); one that was never cloned reports status error.", + input: z.object({ + name: registryName.optional().describe("Sync only this registry (default: all)"), + }), + annotations: { idempotent: true }, + async handler(_ctx, input) { + const items = + input.name === undefined + ? await syncAllRegistries() + : [await syncRegistry(requireRegistry(input.name))]; + return { items }; + }, +}); + +export const searchRegistriesOperation = defineOperation({ + name: "search-registries", + group: "packs", + summary: "Search the packs in the configured registries by name, description, tags and author", + input: z.object({ + query: z.string().min(1).describe("Search text"), + registry: registryName.optional().describe("Search only this registry"), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/registries/search" }, + handler(_ctx, input) { + const { results, warnings } = searchRegistries(input.query, { registryName: input.registry }); + return { items: results, warnings }; + }, +}); + +export const createRegistryOperation = defineOperation({ + name: "create-registry", + group: "packs", + summary: "Create an empty registry git repository (push it, then add it with add-registry)", + input: z.object({ path: z.string().min(1).describe("Directory to create") }), + async handler(ctx, input) { + assertLocalFilesAllowed(ctx, "Creating a registry"); + const path = pathResolve(input.path); + await createRegistryRepo(path); + return { path }; + }, +}); + +export const publishPackOperation = defineOperation({ + name: "publish-pack", + group: "packs", + summary: "Publish a pack file to a registry (commit and push)", + input: z.object({ + file: z.string().min(1).describe("Pack file (.json or .json.gz)"), + registry: registryName, + version: z + .string() + .min(1) + .optional() + .describe("Version to publish (default: the next patch version, or the pack's version)"), + message, + submit: z + .boolean() + .default(false) + .describe( + "Push to a feature/add-<pack> branch for a pull request instead of the main branch", + ), + }), + async handler(ctx, input): Promise<PublishResult & { branch: string | null }> { + assertLocalFilesAllowed(ctx, "Publishing a pack"); + const options = { + registryName: input.registry, + packFilePath: pathResolve(input.file), + version: input.version, + commitMessage: input.message, + }; + if (input.submit) return publishPackToBranch(options); + return { ...(await publishPack(options)), branch: null }; + }, +}); + +export const unpublishPackOperation = defineOperation({ + name: "unpublish-pack", + group: "packs", + summary: "Remove one version of a pack from a registry (commit and push)", + input: z.object({ + pack: z.string().min(1).describe("name@version to remove"), + registry: registryName, + message, + }), + annotations: { destructive: true }, + async handler(_ctx, input) { + const { name, version } = parsePackSpecifier(input.pack); + if (!version) throw new ValidationError("Give the version to unpublish as name@version"); + await unpublishPack({ + registryName: input.registry, + packName: name, + version, + commitMessage: input.message, + }); + return { registry: input.registry, pack: name, version, removed: true }; + }, +}); + +export const registryOperations = [ + listRegistriesOperation, + addRegistryOperation, + removeRegistryOperation, + syncRegistriesOperation, + searchRegistriesOperation, + createRegistryOperation, + publishPackOperation, + unpublishPackOperation, +] as const; diff --git a/src/core/packs.ts b/src/core/packs.ts index 3c5f5dc9..61b5cc7e 100644 --- a/src/core/packs.ts +++ b/src/core/packs.ts @@ -12,7 +12,7 @@ import { } from "node:path"; import { gzipSync, gunzipSync } from "node:zlib"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { ValidationError, FetchError, NotFoundError } from "../errors.js"; +import { ValidationError, NotFoundError } from "../errors.js"; import { getLogger } from "../logger.js"; import { buildEmbeddingText, @@ -46,13 +46,6 @@ export interface KnowledgePack { }; } -export interface PackInfo { - name: string; - version: string; - description: string; - docCount: number; -} - export interface InstalledPack { name: string; version: string; @@ -69,7 +62,6 @@ export interface InstallResult { } export interface InstallOptions { - registryUrl?: string | undefined; /** Number of documents to embed and insert per batch. Default: 10. */ batchSize?: number | undefined; /** Skip the first N documents (for resuming a partial install). Default: 0. */ @@ -115,8 +107,6 @@ export interface CreatePackFromSourceOptions { onProgress?: ((info: { file: string; index: number; total: number }) => void) | undefined; } -const DEFAULT_REGISTRY_URL = "https://raw.githubusercontent.com/libscope/packs/main/registry.json"; - /** Gzip magic number: first two bytes of a gzip stream. */ const GZIP_MAGIC = Buffer.from([0x1f, 0x8b]); @@ -144,31 +134,6 @@ export function readPackFile(filePath: string): string { return raw.toString("utf-8"); } -/** Validate that a registry URL uses https and is not a private IP. */ -function validateRegistryUrl(url: string): void { - let parsed: URL; - try { - parsed = new URL(url); - } catch { - throw new ValidationError("Invalid registry URL"); - } - if (parsed.protocol !== "https:") { - throw new ValidationError("Registry URL must use https"); - } - const host = parsed.hostname; - if ( - host === "localhost" || - host.startsWith("127.") || - host.startsWith("10.") || - host.startsWith("192.168.") || - host === "0.0.0.0" || - host === "::1" || - host.startsWith("169.254.") - ) { - throw new ValidationError("Registry URL must not point to a private/internal address"); - } -} - /** Validate that a value is a non-empty string, throwing with the given message if not. */ function requireString(value: unknown, message: string, allowEmpty = false): void { if (typeof value !== "string") throw new ValidationError(message); @@ -230,49 +195,10 @@ function validatePack(data: unknown): KnowledgePack { return data as KnowledgePack; } -/** List available packs from a remote registry. */ -export async function listAvailablePacks(registryUrl?: string): Promise<PackInfo[]> { - const url = registryUrl ?? DEFAULT_REGISTRY_URL; - const log = getLogger(); - - validateRegistryUrl(url); - - try { - const response = await fetch(url, { signal: AbortSignal.timeout(30_000) }); - if (!response.ok) { - throw new FetchError(`Registry returned ${response.status}: ${response.statusText}`); - } - const data: unknown = await response.json(); - if (!Array.isArray(data)) { - throw new ValidationError("Registry response is not an array"); - } - - return (data as Array<Record<string, unknown>>).map((entry) => { - const name = entry["name"]; - const version = entry["version"]; - const description = entry["description"]; - const docCount = entry["docCount"] ?? entry["doc_count"]; - return { - name: typeof name === "string" ? name : "", - version: typeof version === "string" ? version : "", - description: typeof description === "string" ? description : "", - docCount: typeof docCount === "number" ? docCount : 0, - }; - }); - } catch (err) { - log.error({ err, url }, "Failed to fetch pack registry"); - if (err instanceof ValidationError) throw err; - if (err instanceof FetchError) throw err; - throw new FetchError( - `Failed to fetch pack registry: ${err instanceof Error ? err.message : String(err)}`, - ); - } -} - /** Load a pack from a local file path. */ -function loadPackFromFile(packNameOrPath: string): KnowledgePack { - const resolved = pathResolve(packNameOrPath); - if (!pathIsAbsolute(packNameOrPath) && !resolved.startsWith(process.cwd())) { +function loadPackFromFile(packPath: string): KnowledgePack { + const resolved = pathResolve(packPath); + if (!pathIsAbsolute(packPath) && !resolved.startsWith(process.cwd())) { throw new ValidationError("Pack file path must be within the current working directory"); } try { @@ -282,31 +208,7 @@ function loadPackFromFile(packNameOrPath: string): KnowledgePack { } catch (err) { if (err instanceof ValidationError) throw err; throw new ValidationError( - `Failed to read pack file "${packNameOrPath}": ${err instanceof Error ? err.message : String(err)}`, - ); - } -} - -/** Load a pack from a remote registry. */ -async function loadPackFromRegistry( - packNameOrPath: string, - registryUrl: string, -): Promise<KnowledgePack> { - validateRegistryUrl(registryUrl); - const baseUrl = registryUrl.replace(/\/[^/]+$/, ""); - const packUrl = `${baseUrl}/${packNameOrPath}.json`; - try { - const response = await fetch(packUrl, { signal: AbortSignal.timeout(30_000) }); - if (!response.ok) { - throw new FetchError(`Pack fetch returned ${response.status}: ${response.statusText}`); - } - const data: unknown = await response.json(); - return validatePack(data); - } catch (err) { - if (err instanceof ValidationError) throw err; - if (err instanceof FetchError) throw err; - throw new FetchError( - `Failed to fetch pack "${packNameOrPath}": ${err instanceof Error ? err.message : String(err)}`, + `Failed to read pack file "${packPath}": ${err instanceof Error ? err.message : String(err)}`, ); } } @@ -464,19 +366,18 @@ function runBatchesConcurrently( }); } -/** Install a pack from a local JSON file path or registry name. */ +/** + * Install a pack from a .json or .json.gz file (gzip is detected by magic bytes). Packs in + * registries are resolved to a file with `resolveRegistryPack` first. + */ export async function installPack( db: Database.Database, provider: EmbeddingProvider, - packNameOrPath: string, + packPath: string, options?: InstallOptions, ): Promise<InstallResult> { const log = getLogger(); - - const isLocalFile = packNameOrPath.endsWith(".json") || packNameOrPath.endsWith(".json.gz"); - const pack = isLocalFile - ? loadPackFromFile(packNameOrPath) - : await loadPackFromRegistry(packNameOrPath, options?.registryUrl ?? DEFAULT_REGISTRY_URL); + const pack = loadPackFromFile(packPath); // Check if already installed const existing = db.prepare("SELECT name FROM packs WHERE name = ?").get(pack.name) as diff --git a/src/mcp/format.ts b/src/mcp/format.ts index beabda81..140e8712 100644 --- a/src/mcp/format.ts +++ b/src/mcp/format.ts @@ -7,7 +7,8 @@ import type { DocumentLinks } from "../core/links.js"; import type { DocumentView } from "../core/document-view.js"; import type { DocumentSummary, ListResult } from "../core/operations/index.js"; import type { Overview } from "../core/overview.js"; -import type { InstallResult, InstalledPack, PackInfo } from "../core/packs.js"; +import type { InstallResult, InstalledPack } from "../core/packs.js"; +import type { RegistryPack } from "../registry/types.js"; import type { AnswerResult, RagSource } from "../core/rag.js"; import type { ReindexResult } from "../core/reindex.js"; import type { Rating } from "../core/ratings.js"; @@ -342,15 +343,18 @@ export function formatInstallPack(result: InstallResult): string { } /** Output of `list-packs`. */ -export function formatPackList(result: { items: Array<InstalledPack | PackInfo> }): string { - if (result.items.length === 0) return "No packs found."; - return result.items - .map((p) => - "installedAt" in p - ? formatInstalledPack(p) - : `- ${p.name} v${p.version} (${p.docCount} docs): ${p.description}`, - ) - .join("\n"); +export function formatPackList(result: { + items: Array<InstalledPack | RegistryPack>; + warnings?: string[] | undefined; +}): string { + const lines = result.items.map((p) => + "installedAt" in p + ? formatInstalledPack(p) + : `- ${p.name} v${p.latestVersion} (registry ${p.registry}): ${p.description}`, + ); + if (lines.length === 0) lines.push("No packs found."); + for (const warning of result.warnings ?? []) lines.push(`Warning: ${warning}`); + return lines.join("\n"); } /** Output of `reindex-documents`. */ diff --git a/src/registry/config.ts b/src/registry/config.ts index 4b656346..1840dfa1 100644 --- a/src/registry/config.ts +++ b/src/registry/config.ts @@ -4,7 +4,7 @@ */ import { readRawUserConfig, writeRawUserConfig } from "../config.js"; -import { ValidationError } from "../errors.js"; +import { NotFoundError, ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; import type { RegistryEntry } from "./types.js"; @@ -33,7 +33,10 @@ export function validateRegistryName(name: string): void { } } -/** Validate a git URL (https, ssh://, or SCP-style). Returns the normalized (trimmed, no trailing slash) URL. */ +/** + * Validate a git URL: https://, ssh://, SCP-style git@host:path, or file:/// for a repository on + * a local or shared disk. Returns the normalized (trimmed, no trailing slash) URL. + */ export function validateGitUrl(url: string): string { // Trim whitespace and trailing slashes const normalized = url.trim().replace(/\/+$/, ""); @@ -46,19 +49,34 @@ export function validateGitUrl(url: string): string { ); } - // Accept https:// URLs, ssh:// URLs, and SCP-style git@host:path URLs const isHttps = normalized.startsWith("https://"); const isSshProtocol = normalized.startsWith("ssh://"); const isScp = /^git@[\w.-]+:/.test(normalized); - if (!isHttps && !isSshProtocol && !isScp) { + const isFile = normalized.startsWith("file:///"); + if (!isHttps && !isSshProtocol && !isScp && !isFile) { throw new ValidationError( - "Registry URL must use https://, ssh://, or SSH (git@host:path) format", + "Registry URL must use https://, ssh://, SSH (git@host:path) or file:/// format", ); } return normalized; } +/** Derive a registry name from a git URL ("https://github.com/org/packs.git" -> "packs"). */ +export function deriveRegistryName(url: string): string { + const trimmed = url.trim().replace(/\/+$/, ""); + const last = trimmed.slice(Math.max(trimmed.lastIndexOf("/"), trimmed.lastIndexOf(":")) + 1); + const name = last.endsWith(".git") ? last.slice(0, -4) : last; + try { + validateRegistryName(name); + } catch { + throw new ValidationError( + `Could not derive a registry name from "${url}" (got "${name}"). Give a name.`, + ); + } + return name; +} + /** Load all registry entries from config. */ export function loadRegistries(): RegistryEntry[] { const config = readRawUserConfig(); @@ -79,13 +97,23 @@ export function getRegistry(name: string): RegistryEntry | undefined { return loadRegistries().find((r) => r.name === name); } -/** Add a new registry entry. Throws if name already exists. */ -export function addRegistry(entry: RegistryEntry): void { +/** The registry called `name`; NotFoundError when none is configured. */ +export function requireRegistry(name: string): RegistryEntry { + const entry = getRegistry(name); + if (!entry) { + throw new NotFoundError( + `Registry "${name}" not found. Add it with: libscope registry add <url> --name ${name}`, + ); + } + return entry; +} + +/** Add a new registry entry. Throws if the name already exists. Returns the stored entry. */ +export function addRegistry(entry: RegistryEntry): RegistryEntry { const log = getLogger(); validateRegistryName(entry.name); // Normalize and validate URL; use the normalized form going forward - const normalizedUrl = validateGitUrl(entry.url); - const normalizedEntry = { ...entry, url: normalizedUrl }; + const normalizedEntry = { ...entry, url: validateGitUrl(entry.url) }; const registries = loadRegistries(); if (registries.some((r) => r.name === normalizedEntry.name)) { @@ -98,19 +126,14 @@ export function addRegistry(entry: RegistryEntry): void { { registry: normalizedEntry.name, url: sanitizeUrl(normalizedEntry.url) }, "Registry added to config", ); + return normalizedEntry; } -/** Remove a registry entry by name. Throws if not found. */ +/** Remove a registry entry by name. NotFoundError when it is not configured. */ export function removeRegistry(name: string): void { - const log = getLogger(); - const registries = loadRegistries(); - const index = registries.findIndex((r) => r.name === name); - if (index === -1) { - throw new ValidationError(`Registry "${name}" not found`); - } - registries.splice(index, 1); - saveRegistries(registries); - log.info({ registry: name }, "Registry removed from config"); + requireRegistry(name); + saveRegistries(loadRegistries().filter((r) => r.name !== name)); + getLogger().info({ registry: name }, "Registry removed from config"); } /** Update the lastSyncedAt timestamp for a registry. */ diff --git a/src/registry/git.ts b/src/registry/git.ts index fe8443c5..b6cff0dd 100644 --- a/src/registry/git.ts +++ b/src/registry/git.ts @@ -8,7 +8,7 @@ import { promisify } from "node:util"; import { existsSync, readFileSync, writeFileSync, mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { getLogger } from "../logger.js"; -import { FetchError, ValidationError } from "../errors.js"; +import { ConfigError, FetchError, ValidationError } from "../errors.js"; import type { PackSummary } from "./types.js"; import { INDEX_FILE, PACKS_DIR } from "./types.js"; @@ -35,6 +35,10 @@ export async function git( const { stdout } = await execFile("git", args, { cwd, timeout }); return stdout.trim(); } catch (err) { + // ENOENT with an existing (or no) working directory: the git binary itself is missing. + if ((err as NodeJS.ErrnoException).code === "ENOENT" && (!cwd || existsSync(cwd))) { + throw new ConfigError("git is not installed or not in PATH", err); + } const message = err instanceof Error ? err.message : String(err); log.error({ args, cwd, err: message }, "Git command failed"); @@ -224,16 +228,6 @@ export async function createRegistryRepo(path: string): Promise<void> { log.info({ path }, "Registry repo initialized"); } -/** Check if git is available on the system. */ -export async function checkGitAvailable(): Promise<boolean> { - try { - await execFile("git", ["--version"]); - return true; - } catch { - return false; - } -} - /** Add, commit, and push changes in a registry repo. */ export async function commitAndPush(repoPath: string, message: string): Promise<void> { await git(["add", "."], { cwd: repoPath }); diff --git a/src/registry/publish.ts b/src/registry/publish.ts index 88e14322..8c36fbc1 100644 --- a/src/registry/publish.ts +++ b/src/registry/publish.ts @@ -5,7 +5,7 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync, copyFileSync, rmSync } from "node:fs"; import { join } from "node:path"; import { getLogger } from "../logger.js"; -import { ValidationError } from "../errors.js"; +import { NotFoundError, ValidationError } from "../errors.js"; import type { PublishOptions, PublishResult, @@ -21,7 +21,7 @@ import { INDEX_FILE, getRegistryCacheDir, } from "./types.js"; -import { getRegistry } from "./config.js"; +import { requireRegistry } from "./config.js"; import { commitAndPush, fetchRegistry, git, clearIndexCache } from "./git.js"; import { computeChecksum, writeChecksumFile } from "./checksum.js"; import { readPackFile, type KnowledgePack } from "../core/packs.js"; @@ -109,10 +109,7 @@ function readPackJson(filePath: string): KnowledgePack { /** Validate that the registry exists and has a local cache; return the cache directory. */ function validateRegistryCache(registryName: string): string { - const entry = getRegistry(registryName); - if (!entry) { - throw new ValidationError(`Registry "${registryName}" not found. Add it first.`); - } + requireRegistry(registryName); const cacheDir = getRegistryCacheDir(registryName); if (!existsSync(cacheDir)) { throw new ValidationError( @@ -183,7 +180,7 @@ function resolveVersionAndManifest( if (manifest.versions.some((v) => v.version === version)) { throw new ValidationError( `Version ${version} of "${pack.name}" already exists in "${registryName}". ` + - "Use --pack-version to specify a different version.", + "Publish it with a different version.", ); } @@ -386,13 +383,13 @@ export async function unpublishPack(options: UnpublishOptions): Promise<void> { const manifestPath = join(packDir, PACK_MANIFEST_FILE); if (!existsSync(manifestPath)) { - throw new ValidationError(`Pack "${packName}" not found in registry "${registryName}".`); + throw new NotFoundError(`Pack "${packName}" not found in registry "${registryName}".`); } const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as PackManifest; const versionIdx = manifest.versions.findIndex((v) => v.version === version); if (versionIdx === -1) { - throw new ValidationError( + throw new NotFoundError( `Version ${version} of "${packName}" not found in registry "${registryName}".`, ); } diff --git a/src/registry/resolve.ts b/src/registry/resolve.ts index c757ed14..fa7e96b6 100644 --- a/src/registry/resolve.ts +++ b/src/registry/resolve.ts @@ -1,22 +1,17 @@ /** - * Registry pack resolution: find and resolve a pack from configured registries. + * Registry pack resolution: turn "name" or "name@version" into a pack file in the local copy + * of one configured registry. No network access; run `libscope registry sync` to update. */ import { existsSync, readFileSync } from "node:fs"; import { getLogger } from "../logger.js"; -import { ValidationError } from "../errors.js"; -import type { - RegistryEntry, - PackSummary, - PackManifest, - RegistryConflict, - ConflictResolution, -} from "./types.js"; -import { getRegistryCacheDir, getPackManifestPath, getPackDataPath } from "./types.js"; +import { NotFoundError, ValidationError } from "../errors.js"; +import type { PackManifest } from "./types.js"; +import { getPackManifestPath, getPackDataPath } from "./types.js"; import { loadRegistries } from "./config.js"; -import { readIndex } from "./git.js"; import { verifyChecksum } from "./checksum.js"; import { validatePathSegment } from "./publish.js"; +import { listRegistryPacks } from "./search.js"; /** Parse a pack specifier like "name@1.2.0" into name and optional version. */ export function parsePackSpecifier(specifier: string): { name: string; version?: string } { @@ -30,62 +25,77 @@ export function parsePackSpecifier(specifier: string): { name: string; version?: return { name: specifier }; } -/** Result of resolving a pack from registries. */ +/** A pack found in a registry. */ export interface ResolvedPack { - registryName: string; - registryUrl: string; - packName: string; + registry: string; + name: string; version: string; - /** Path to the pack data file in the local cache. */ + /** Path to the pack data file in the local copy of the registry. */ dataPath: string; } /** - * Find all registries that have a pack with the given name. + * Find "name" or "name@version" in the configured registries (only `registryName` when given). + * Without a version, the registry's latest version is used. + * + * @throws NotFoundError when no registry (or version) has the pack. + * @throws ValidationError when several registries have it and no registry was named. */ -export function findPackInRegistries(packName: string): { - matches: Array<{ entry: RegistryEntry; pack: PackSummary }>; - warnings: string[]; -} { - const warnings: string[] = []; - const matches: Array<{ entry: RegistryEntry; pack: PackSummary }> = []; - - const registries = loadRegistries(); - for (const entry of registries) { - const cacheDir = getRegistryCacheDir(entry.name); - if (!existsSync(cacheDir)) { - warnings.push( - `Registry "${entry.name}" has never been synced — skipping. Run: libscope registry sync ${entry.name}`, - ); - continue; - } +export function findRegistryPack(specifier: string, registryName?: string): ResolvedPack { + const { name, version } = parsePackSpecifier(specifier); + validatePathSegment(name, "pack name"); + if (version !== undefined) validatePathSegment(version, "version"); + + if (registryName === undefined && loadRegistries().length === 0) { + throw new NotFoundError( + `Pack "${name}" not found: no pack registries are configured. Add one with: libscope registry add <url>`, + ); + } + const { packs, warnings } = listRegistryPacks(registryName); + const matches = packs.filter((p) => p.name === name); + const [match] = matches; + if (!match) { + const where = registryName === undefined ? "any registry" : `registry "${registryName}"`; + throw new NotFoundError([`Pack "${name}" not found in ${where}.`, ...warnings].join(" ")); + } + if (matches.length > 1) { + throw new ValidationError( + `Pack "${name}" is in several registries (${matches.map((m) => m.registry).join(", ")}). ` + + "Choose one with the registry option (CLI: --registry <name>).", + ); + } - try { - const index = readIndex(cacheDir); - const found = index.find((p) => p.name === packName); - if (found) { - matches.push({ entry, pack: found }); - } - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - warnings.push(`Failed to read index for "${entry.name}": ${msg}`); - } + const chosen = version ?? match.latestVersion; + // latestVersion comes from the registry's index.json: validate it like user input. + validatePathSegment(chosen, "version"); + const dataPath = getPackDataPath(match.registry, name, chosen); + if (!existsSync(dataPath)) { + throw new NotFoundError( + `Pack "${name}@${chosen}" not found in registry "${match.registry}". ` + + `Run: libscope registry sync ${match.registry}`, + ); } + return { registry: match.registry, name, version: chosen, dataPath }; +} - return { matches, warnings }; +/** Find a pack in the registries and verify its checksum (see findRegistryPack). */ +export async function resolveRegistryPack( + specifier: string, + registryName?: string, +): Promise<ResolvedPack> { + const resolved = findRegistryPack(specifier, registryName); + await verifyResolvedPackChecksum(resolved); + return resolved; } -/** - * Read a pack manifest from the local cache. - */ +/** Read a pack manifest from the local cache. */ export function readPackManifest(registryName: string, packName: string): PackManifest | null { const manifestPath = getPackManifestPath(registryName, packName); if (!existsSync(manifestPath)) return null; try { return JSON.parse(readFileSync(manifestPath, "utf-8")) as PackManifest; } catch (err) { - const log = getLogger(); - log.warn( + getLogger().warn( { registryName, packName, err: err instanceof Error ? err.message : String(err) }, "Failed to parse pack manifest", ); @@ -94,147 +104,33 @@ export function readPackManifest(registryName: string, packName: string): PackMa } /** - * Resolve a pack from registries, handling version selection and conflicts. - * - * @param packName - Pack name (no version suffix) - * @param options - Resolution options - * @returns Resolved pack info, or null if not found - */ -export function resolvePackFromRegistries( - packName: string, - options?: { - version?: string | undefined; - registryName?: string | undefined; - conflictResolution?: ConflictResolution | undefined; - }, -): { resolved: ResolvedPack | null; conflict?: RegistryConflict; warnings: string[] } { - const log = getLogger(); - - // Validate pack name and version to prevent path traversal via malicious index.json - validatePathSegment(packName, "pack name"); - if (options?.version) { - validatePathSegment(options.version, "version"); - } - - const { matches, warnings } = findPackInRegistries(packName); - - if (matches.length === 0) { - return { resolved: null, warnings }; - } - - // Filter to specific registry if requested - let candidates = matches; - if (options?.registryName) { - candidates = matches.filter((m) => m.entry.name === options.registryName); - if (candidates.length === 0) { - warnings.push(`Pack "${packName}" not found in registry "${options.registryName}".`); - return { resolved: null, warnings }; - } - } - - // Handle conflict: multiple registries have this pack - if (candidates.length > 1) { - const conflict: RegistryConflict = { - packName, - sources: candidates.map((c) => ({ - registryName: c.entry.name, - registryUrl: c.entry.url, - version: c.pack.latestVersion, - priority: c.entry.priority, - })), - }; - - const resolution = options?.conflictResolution ?? { strategy: "priority" }; - - if (resolution.strategy === "priority") { - // Sort by priority (lower wins), pick first - candidates.sort((a, b) => a.entry.priority - b.entry.priority); - candidates = [candidates[0]!]; - log.info( - { packName, registry: candidates[0]!.entry.name }, - "Resolved pack conflict by priority", - ); - } else if (resolution.strategy === "explicit") { - const explicit = candidates.find((c) => c.entry.name === resolution.registryName); - if (!explicit) { - return { resolved: null, conflict, warnings }; - } - candidates = [explicit]; - } else { - // interactive — caller must handle the conflict - return { resolved: null, conflict, warnings }; - } - } - - const match = candidates[0]!; - const version = options?.version ?? match.pack.latestVersion; - - // Try to find the pack data file - const dataPath = getPackDataPath(match.entry.name, packName, version); - if (!existsSync(dataPath)) { - warnings.push( - `Pack "${packName}@${version}" not found in local cache for registry "${match.entry.name}". ` + - "Try syncing first: libscope registry sync", - ); - return { resolved: null, warnings }; - } - - return { - resolved: { - registryName: match.entry.name, - registryUrl: match.entry.url, - packName, - version, - dataPath, - }, - warnings, - }; -} - -/** - * Verify the checksum of a resolved pack's data file against the expected value - * stored in the pack manifest. Throws a ValidationError if the checksum does not - * match, indicating the file may have been tampered with or corrupted. - * - * Call this immediately before installing a registry-resolved pack. + * Verify the checksum of a resolved pack's data file against the value in the pack manifest. + * Throws a ValidationError if the checksum does not match (tampered or corrupted file) or the + * manifest has no checksum for the version. */ export async function verifyResolvedPackChecksum(resolved: ResolvedPack): Promise<void> { const log = getLogger(); - const manifest = readPackManifest(resolved.registryName, resolved.packName); - + const where = { registry: resolved.registry, pack: resolved.name, version: resolved.version }; + const manifest = readPackManifest(resolved.registry, resolved.name); if (!manifest) { - log.warn( - { registryName: resolved.registryName, packName: resolved.packName }, - "No pack manifest found — skipping checksum verification", - ); + log.warn(where, "No pack manifest found — skipping checksum verification"); return; } const versionEntry = manifest.versions.find((v) => v.version === resolved.version); if (!versionEntry) { - log.warn( - { - registryName: resolved.registryName, - packName: resolved.packName, - version: resolved.version, - }, - "Version entry not found in manifest — skipping checksum verification", - ); + log.warn(where, "Version entry not found in manifest — skipping checksum verification"); return; } if (!versionEntry.checksum) { throw new ValidationError( - `Pack "${resolved.packName}@${resolved.version}" in registry "${resolved.registryName}" ` + + `Pack "${resolved.name}@${resolved.version}" in registry "${resolved.registry}" ` + "has no checksum recorded. The registry may be corrupted or from an older format.", ); } // verifyChecksum throws ValidationError on mismatch await verifyChecksum(resolved.dataPath, versionEntry.checksum); - - log.info( - { registryName: resolved.registryName, packName: resolved.packName, version: resolved.version }, - "Pack checksum verified before installation", - ); + log.info(where, "Pack checksum verified before installation"); } diff --git a/src/registry/search.ts b/src/registry/search.ts index 80531b85..b5e27e9b 100644 --- a/src/registry/search.ts +++ b/src/registry/search.ts @@ -1,14 +1,20 @@ /** - * Registry search: find packs across all configured registries. + * List and search the packs of the configured registries, from their local (synced) indexes. */ import { existsSync } from "node:fs"; import { getLogger } from "../logger.js"; -import type { RegistryEntry, PackSummary, RegistrySearchResult } from "./types.js"; +import type { RegistryEntry, PackSummary, RegistryPack, RegistrySearchResult } from "./types.js"; import { getRegistryCacheDir } from "./types.js"; -import { loadRegistries } from "./config.js"; +import { loadRegistries, requireRegistry } from "./config.js"; import { readIndex } from "./git.js"; +/** Packs in registries plus warnings for registries that could not be read. */ +export interface RegistryPackList { + packs: RegistryPack[]; + warnings: string[]; +} + /** * Compute a relevance score for a pack against a query. * Higher = better match. Returns 0 for no match. @@ -50,43 +56,39 @@ function scoreMatch(pack: PackSummary, query: string): number { return score; } -/** Resolve which registries to search, returning them or adding a warning if not found. */ -function resolveRegistries( - registryName: string | undefined, - warnings: string[], -): RegistryEntry[] | null { - if (!registryName) return loadRegistries(); - const all = loadRegistries(); - const entry = all.find((r) => r.name === registryName); - if (!entry) { - warnings.push(`Registry "${registryName}" not found.`); - return null; - } - return [entry]; -} - /** Read packs from a single registry, appending warnings on failure. */ -function readRegistryPacks(entry: RegistryEntry, warnings: string[]): PackSummary[] | null { +function readRegistryPacks(entry: RegistryEntry, warnings: string[]): PackSummary[] { const cacheDir = getRegistryCacheDir(entry.name); if (!existsSync(cacheDir)) { warnings.push( `Registry "${entry.name}" has never been synced. Run: libscope registry sync ${entry.name}`, ); - return null; + return []; } try { return readIndex(cacheDir); } catch (err) { const msg = err instanceof Error ? err.message : String(err); warnings.push(`Failed to read index for "${entry.name}": ${msg}`); - getLogger().warn( - { registry: entry.name, err: msg }, - "Failed to read registry index during search", - ); - return null; + getLogger().warn({ registry: entry.name, err: msg }, "Failed to read registry index"); + return []; } } +/** + * Every pack in the local indexes of all registries, or of `registryName` only + * (NotFoundError when that registry is not configured). No network access. + */ +export function listRegistryPacks(registryName?: string): RegistryPackList { + const registries = + registryName === undefined ? loadRegistries() : [requireRegistry(registryName)]; + const warnings: string[] = []; + const packs = registries.flatMap((entry) => + readRegistryPacks(entry, warnings).map((pack) => ({ ...pack, registry: entry.name })), + ); + return { packs, warnings }; +} + /** * Search for packs across all (or a specific) registry. * Returns results sorted by relevance score (highest first). @@ -95,29 +97,12 @@ export function searchRegistries( query: string, options?: { registryName?: string | undefined }, ): { results: RegistrySearchResult[]; warnings: string[] } { - const warnings: string[] = []; - const results: RegistrySearchResult[] = []; - - const registries = resolveRegistries(options?.registryName, warnings); - if (!registries) return { results, warnings }; - - for (const entry of registries) { - const packs = readRegistryPacks(entry, warnings); - if (!packs) continue; - - for (const pack of packs) { - const score = scoreMatch(pack, query); - if (score > 0) { - results.push({ registryName: entry.name, pack, score }); - } - } - } + const { packs, warnings } = listRegistryPacks(options?.registryName); + const results = packs + .map((pack) => ({ ...pack, score: scoreMatch(pack, query) })) + .filter((r) => r.score > 0); // Sort by score descending, then by name - results.sort((a, b) => { - if (b.score !== a.score) return b.score - a.score; - return a.pack.name.localeCompare(b.pack.name); - }); - + results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name)); return { results, warnings }; } diff --git a/src/registry/sync.ts b/src/registry/sync.ts index 01a361fd..f989883b 100644 --- a/src/registry/sync.ts +++ b/src/registry/sync.ts @@ -1,11 +1,13 @@ /** * Registry sync engine: keeps local caches up to date and handles offline gracefully. + * Syncs run only when asked for (`libscope registry sync`, `registry add`); reading + * registries never touches the network. */ import { existsSync, writeFileSync, readFileSync, unlinkSync, mkdirSync } from "node:fs"; import { join } from "node:path"; import { getLogger } from "../logger.js"; -import type { RegistryEntry, PackSummary, RegistrySyncStatus } from "./types.js"; +import type { RegistryEntry, RegistrySyncStatus } from "./types.js"; import { getRegistryCacheDir } from "./types.js"; import { loadRegistries, updateRegistrySyncTime } from "./config.js"; import { cloneRegistry, fetchRegistry, readIndex, clearIndexCache } from "./git.js"; @@ -67,87 +69,79 @@ function releaseSyncLock(lockPath: string): void { } } +/** Number of packs in a registry's local index, or null when it has no readable index. */ +export function localPackCount(registryName: string): number | null { + const cacheDir = getRegistryCacheDir(registryName); + if (!existsSync(cacheDir)) return null; + try { + return readIndex(cacheDir).length; + } catch { + return null; + } +} + +/** True when the registry has a cloned local copy. */ +function hasClone(cacheDir: string): boolean { + return existsSync(join(cacheDir, ".git")); +} + /** * Sync a single registry: clone if missing, fetch if already cached. - * Returns the sync status. On failure, falls back to cached data with a warning. - * If another process is already syncing this registry, returns status "error" with - * a descriptive message so the caller can fall back to cached data. + * Never throws: on failure the status is "offline" (the cached copy is still used) or + * "error" (no cached copy, or another process is syncing this registry). */ export async function syncRegistry(entry: RegistryEntry): Promise<RegistrySyncStatus> { const log = getLogger(); const cacheDir = getRegistryCacheDir(entry.name); + const status = ( + state: RegistrySyncStatus["status"], + lastSyncedAt: string | null, + error?: string, + ): RegistrySyncStatus => ({ + registry: entry.name, + status: state, + lastSyncedAt, + ...(error === undefined ? {} : { error }), + packs: localPackCount(entry.name), + }); - const result: RegistrySyncStatus = { - registryName: entry.name, - status: "syncing", - lastSyncedAt: entry.lastSyncedAt, - }; - - // Acquire lock before touching git state const lockPath = acquireSyncLock(cacheDir); if (lockPath === null) { - // Another live process holds the lock — skip this sync - result.status = "error"; - result.error = `Registry "${entry.name}" sync is already in progress by another process. Try again shortly.`; - return result; + return status( + "error", + entry.lastSyncedAt, + `Registry "${entry.name}" sync is already in progress by another process. Try again shortly.`, + ); } try { - if (existsSync(cacheDir) && existsSync(join(cacheDir, ".git"))) { + if (hasClone(cacheDir)) { await fetchRegistry(cacheDir); } else { await cloneRegistry(entry.url, cacheDir); } - // Invalidate cached index so next readIndex() picks up fresh data clearIndexCache(cacheDir); - updateRegistrySyncTime(entry.name); - result.status = "success"; - result.lastSyncedAt = new Date().toISOString(); log.info({ registry: entry.name }, "Registry synced successfully"); + return status("success", new Date().toISOString()); } catch (err) { const message = err instanceof Error ? err.message : String(err); - - if (existsSync(cacheDir) && existsSync(join(cacheDir, ".git"))) { - // We have a cached version — fall back to it - result.status = "offline"; - result.error = message; + if (hasClone(cacheDir)) { log.warn( { registry: entry.name, err: message }, `Registry "${entry.name}" is unreachable. Using cached index from ${entry.lastSyncedAt ?? "unknown"}.`, ); - } else { - // No cache at all - result.status = "error"; - result.error = message; - log.error( - { registry: entry.name, err: message }, - `Registry "${entry.name}" has never been synced and is unreachable.`, - ); + return status("offline", entry.lastSyncedAt, message); } + log.error( + { registry: entry.name, err: message }, + `Registry "${entry.name}" has never been synced and is unreachable.`, + ); + return status("error", entry.lastSyncedAt, message); } finally { releaseSyncLock(lockPath); } - - return result; -} - -/** - * Sync a named registry. Throws if registry not found. - */ -export async function syncRegistryByName(name: string): Promise<RegistrySyncStatus> { - const registries = loadRegistries(); - const entry = registries.find((r) => r.name === name); - if (!entry) { - return { - registryName: name, - status: "error", - lastSyncedAt: null, - error: `Registry "${name}" not found. Run 'libscope registry add <url>' first.`, - }; - } - return syncRegistry(entry); } /** Maximum number of concurrent git fetch operations. */ @@ -173,81 +167,10 @@ async function runConcurrent<T>(tasks: Array<() => Promise<T>>, concurrency: num return results; } -/** - * Sync all configured registries concurrently. Returns status for each. - */ +/** Sync all configured registries concurrently. Returns status for each. */ export async function syncAllRegistries(): Promise<RegistrySyncStatus[]> { - const registries = loadRegistries(); - if (registries.length === 0) return []; - - return runConcurrent( - registries.map((entry) => () => syncRegistry(entry)), - SYNC_CONCURRENCY, - ); -} - -/** - * Check if a registry is stale (syncInterval > 0 and time since last sync exceeds interval). - */ -export function isRegistryStale(entry: RegistryEntry): boolean { - if (entry.syncInterval <= 0) return false; - if (!entry.lastSyncedAt) return true; - - const lastSync = new Date(entry.lastSyncedAt).getTime(); - const now = Date.now(); - const intervalMs = entry.syncInterval * 1000; - return now - lastSync > intervalMs; -} - -/** - * Sync all stale registries concurrently. Intended for non-blocking startup check. - * Returns status array; errors are logged but not thrown. - */ -export async function syncStaleRegistries(): Promise<RegistrySyncStatus[]> { - const registries = loadRegistries(); - const stale = registries.filter(isRegistryStale); - if (stale.length === 0) return []; - - const log = getLogger(); - log.debug({ count: stale.length }, "Syncing stale registries concurrently"); - return runConcurrent( - stale.map((entry) => () => syncRegistry(entry)), + loadRegistries().map((entry) => () => syncRegistry(entry)), SYNC_CONCURRENCY, ); } - -/** - * Read the cached index for a registry. - * If the cache is stale, syncs first. On sync failure, uses cached data. - * Returns null with an error message if no cache exists and sync fails. - */ -export async function getRegistryIndex( - entry: RegistryEntry, -): Promise<{ packs: PackSummary[]; warning?: string }> { - const cacheDir = getRegistryCacheDir(entry.name); - - // Auto-sync if stale - if (isRegistryStale(entry) || !existsSync(cacheDir)) { - const status = await syncRegistry(entry); - if (status.status === "error") { - return { - packs: [], - warning: - status.error ?? - `Registry "${entry.name}" has never been synced and is unreachable. Run: libscope registry sync when online.`, - }; - } - if (status.status === "offline") { - const packs = readIndex(cacheDir); - return { - packs, - warning: `Registry "${entry.name}" is unreachable. Using cached index from ${entry.lastSyncedAt ?? "unknown"}.`, - }; - } - } - - // Read from cache - const packs = readIndex(cacheDir); - return { packs }; -} diff --git a/src/registry/types.ts b/src/registry/types.ts index 35cf95e8..ddf1cef5 100644 --- a/src/registry/types.ts +++ b/src/registry/types.ts @@ -21,8 +21,10 @@ import { homedir } from "node:os"; // Folder structure constants // --------------------------------------------------------------------------- -/** Root directory for all registry caches. */ -export const REGISTRIES_DIR = join(homedir(), ".libscope", "registries"); +/** Root directory for all registry caches (~/.libscope/registries, read at call time). */ +export function getRegistriesDir(): string { + return join(homedir(), ".libscope", "registries"); +} /** Name of the top-level index file in each registry. */ export const INDEX_FILE = "index.json"; @@ -44,21 +46,12 @@ export const CHECKSUM_FILE = "checksum.sha256"; export interface RegistryEntry { /** User-chosen short name (e.g. "official", "team-internal"). */ name: string; - /** Git remote URL (https or ssh). */ + /** Git remote URL (https://, ssh://, git@host:path or file://). */ url: string; - /** How often to auto-sync, in seconds. 0 = manual only. */ - syncInterval: number; - /** Priority for conflict resolution — lower wins. */ - priority: number; /** ISO-8601 timestamp of last successful sync, or null if never synced. */ lastSyncedAt: string | null; } -/** Shape of the "registries" key in ~/.libscope/config.json. */ -export interface RegistryConfigBlock { - registries: RegistryEntry[]; -} - // --------------------------------------------------------------------------- // Registry index (index.json at repo/cache root) // --------------------------------------------------------------------------- @@ -114,51 +107,32 @@ export interface PackManifest { } // --------------------------------------------------------------------------- -// Search results +// Packs listed from registries // --------------------------------------------------------------------------- -/** A pack search result, combining summary info with registry source. */ -export interface RegistrySearchResult { - /** Which registry this result came from. */ - registryName: string; - /** Pack summary from that registry's index. */ - pack: PackSummary; - /** Relevance score (higher = better match). */ - score: number; +/** A pack from a registry index, with the registry it came from. */ +export interface RegistryPack extends PackSummary { + registry: string; } -// --------------------------------------------------------------------------- -// Conflict resolution -// --------------------------------------------------------------------------- - -/** When multiple registries offer the same pack, the user must choose. */ -export interface RegistryConflict { - packName: string; - /** One entry per registry that has this pack. */ - sources: Array<{ - registryName: string; - registryUrl: string; - version: string; - priority: number; - }>; +/** A pack search result: the pack and its relevance score (higher = better match). */ +export interface RegistrySearchResult extends RegistryPack { + score: number; } -/** Resolution strategy for pack conflicts. */ -export type ConflictResolution = - | { strategy: "priority" } - | { strategy: "interactive" } - | { strategy: "explicit"; registryName: string }; - // --------------------------------------------------------------------------- // Sync state // --------------------------------------------------------------------------- -/** Status of a registry sync operation. */ +/** Result of syncing one registry. */ export interface RegistrySyncStatus { - registryName: string; - status: "syncing" | "success" | "error" | "offline"; + registry: string; + /** "offline": the fetch failed and the cached copy is used; "error": no usable copy. */ + status: "success" | "error" | "offline"; lastSyncedAt: string | null; error?: string; + /** Packs in the local index after the sync, or null when there is no readable index. */ + packs: number | null; } // --------------------------------------------------------------------------- @@ -199,30 +173,30 @@ export interface UnpublishOptions { /** Get the local cache directory for a named registry. */ export function getRegistryCacheDir(registryName: string): string { - return join(REGISTRIES_DIR, registryName); + return join(getRegistriesDir(), registryName); } /** Get the path to a registry's local index.json. */ export function getRegistryIndexPath(registryName: string): string { - return join(REGISTRIES_DIR, registryName, INDEX_FILE); + return join(getRegistriesDir(), registryName, INDEX_FILE); } /** Get the path to a pack's manifest within a registry cache. */ export function getPackManifestPath(registryName: string, packName: string): string { - return join(REGISTRIES_DIR, registryName, PACKS_DIR, packName, PACK_MANIFEST_FILE); + return join(getRegistriesDir(), registryName, PACKS_DIR, packName, PACK_MANIFEST_FILE); } /** Get the directory for a specific pack version within a registry cache. */ export function getPackVersionDir(registryName: string, packName: string, version: string): string { - return join(REGISTRIES_DIR, registryName, PACKS_DIR, packName, version); + return join(getRegistriesDir(), registryName, PACKS_DIR, packName, version); } /** Get the path to the pack data file for a specific version. */ export function getPackDataPath(registryName: string, packName: string, version: string): string { - return join(REGISTRIES_DIR, registryName, PACKS_DIR, packName, version, `${packName}.json`); + return join(getRegistriesDir(), registryName, PACKS_DIR, packName, version, `${packName}.json`); } /** Get the path to the checksum file for a specific pack version. */ export function getChecksumPath(registryName: string, packName: string, version: string): string { - return join(REGISTRIES_DIR, registryName, PACKS_DIR, packName, version, CHECKSUM_FILE); + return join(getRegistriesDir(), registryName, PACKS_DIR, packName, version, CHECKSUM_FILE); } diff --git a/tests/integration/registry/registry-conflict.test.ts b/tests/integration/registry/registry-conflict.test.ts index f682cb16..af89ad24 100644 --- a/tests/integration/registry/registry-conflict.test.ts +++ b/tests/integration/registry/registry-conflict.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { randomUUID, createHash } from "node:crypto"; import { execSync } from "node:child_process"; import { initLogger } from "../../../src/logger.js"; +import { NotFoundError } from "../../../src/errors.js"; import type { RegistryEntry, PackSummary } from "../../../src/registry/types.js"; const gitEnv = { @@ -29,10 +30,13 @@ vi.mock("node:os", async (importOriginal) => { const { loadRegistries, saveRegistries, getRegistry } = await import("../../../src/registry/config.js"); const { syncRegistry } = await import("../../../src/registry/sync.js"); -const { resolvePackFromRegistries } = await import("../../../src/registry/resolve.js"); +const { findRegistryPack } = await import("../../../src/registry/resolve.js"); +const { installPackOperation, listPacksOperation, runOperation } = + await import("../../../src/core/operations/index.js"); +const { makeContext } = await import("../../unit/operations/helpers.js"); -function makeEntry(name: string, url: string, priority = 1): RegistryEntry { - return { name, url, syncInterval: 3600, priority, lastSyncedAt: null }; +function makeEntry(name: string, url: string): RegistryEntry { + return { name, url, lastSyncedAt: null }; } function addTestRegistry(entry: RegistryEntry): void { @@ -112,183 +116,69 @@ describe("integration: registry conflict resolution", () => { rmSync(tempDir, { recursive: true, force: true }); }); - it("should detect conflict when two registries have the same pack name", async () => { - const sharedPack: PackSummary = { - name: "shared-pack", - description: "Shared", + function pack(name: string, author = "author"): PackSummary { + return { + name, + description: `The ${name} pack`, tags: [], latestVersion: "1.0.0", - author: "author", + author, updatedAt: "2026-01-01", }; + } - const repo1 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo2 = createBareRepoWithPacks(tempDir, [{ ...sharedPack, author: "other-author" }]); - - addTestRegistry(makeEntry("reg1", repo1, 2)); - addTestRegistry(makeEntry("reg2", repo2, 1)); - - await syncRegistry(getRegistry("reg1")!); - await syncRegistry(getRegistry("reg2")!); - - // With interactive resolution, should get conflict back - const { resolved, conflict } = resolvePackFromRegistries("shared-pack", { - conflictResolution: { strategy: "interactive" }, - }); - - expect(resolved).toBeNull(); - expect(conflict).toBeDefined(); - expect(conflict!.sources).toHaveLength(2); - expect(conflict!.sources.map((s) => s.registryName).sort((a, b) => a.localeCompare(b))).toEqual( - ["reg1", "reg2"], - ); - }); - - it("should resolve conflict with explicit --registry flag", async () => { - const sharedPack: PackSummary = { - name: "shared-pack", - description: "Shared", - tags: [], - latestVersion: "1.0.0", - author: "author-1", - updatedAt: "2026-01-01", - }; - - const repo1 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo2 = createBareRepoWithPacks(tempDir, [{ ...sharedPack, author: "author-2" }]); - - addTestRegistry(makeEntry("reg1", repo1)); - addTestRegistry(makeEntry("reg2", repo2)); - await syncRegistry(getRegistry("reg1")!); - await syncRegistry(getRegistry("reg2")!); - - const { resolved } = resolvePackFromRegistries("shared-pack", { - registryName: "reg1", - }); - - expect(resolved).not.toBeNull(); - expect(resolved!.registryName).toBe("reg1"); - }); - - it("should resolve conflict by priority (lower wins)", async () => { - const sharedPack: PackSummary = { - name: "priority-pack", - description: "Priority test", - tags: [], - latestVersion: "1.0.0", - author: "author", - updatedAt: "2026-01-01", - }; - - const repo1 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo2 = createBareRepoWithPacks(tempDir, [sharedPack]); - - addTestRegistry(makeEntry("high-priority", repo1, 10)); - addTestRegistry(makeEntry("low-priority", repo2, 1)); - await syncRegistry(getRegistry("high-priority")!); - await syncRegistry(getRegistry("low-priority")!); - - const { resolved } = resolvePackFromRegistries("priority-pack", { - conflictResolution: { strategy: "priority" }, - }); - - expect(resolved).not.toBeNull(); - expect(resolved!.registryName).toBe("low-priority"); - }); - - it("should not conflict when packs have different names", async () => { - const repo1 = createBareRepoWithPacks(tempDir, [ - { - name: "pack-a", - description: "Pack A", - tags: [], - latestVersion: "1.0.0", - author: "a", - updatedAt: "2026-01-01", - }, - ]); - const repo2 = createBareRepoWithPacks(tempDir, [ - { - name: "pack-b", - description: "Pack B", - tags: [], - latestVersion: "1.0.0", - author: "b", - updatedAt: "2026-01-01", - }, - ]); - - addTestRegistry(makeEntry("no-conflict-1", repo1)); - addTestRegistry(makeEntry("no-conflict-2", repo2)); - await syncRegistry(getRegistry("no-conflict-1")!); - await syncRegistry(getRegistry("no-conflict-2")!); - - const { resolved: r1, conflict: c1 } = resolvePackFromRegistries("pack-a"); - expect(r1).not.toBeNull(); - expect(c1).toBeUndefined(); + async function addSynced(name: string, packs: PackSummary[]): Promise<void> { + addTestRegistry(makeEntry(name, createBareRepoWithPacks(tempDir, packs))); + expect((await syncRegistry(getRegistry(name)!)).status).toBe("success"); + } - const { resolved: r2, conflict: c2 } = resolvePackFromRegistries("pack-b"); - expect(r2).not.toBeNull(); - expect(c2).toBeUndefined(); + it("refuses an ambiguous pack name and installs from the named registry", async () => { + await addSynced("reg1", [pack("shared-pack")]); + await addSynced("reg2", [pack("shared-pack", "other-author")]); + const t = makeContext(); + try { + await expect( + runOperation(installPackOperation, t.ctx, { pack: "shared-pack" }), + ).rejects.toThrow(/several registries \(reg1, reg2\)/); + + const result = await runOperation(installPackOperation, t.ctx, { + pack: "shared-pack@1.0.0", + registry: "reg2", + }); + expect(result).toMatchObject({ packName: "shared-pack", documentsInstalled: 1 }); + const row = t.db.prepare("SELECT content FROM documents").get() as { content: string }; + expect(row.content).toContain("other-author"); + } finally { + t.db.close(); + } }); - it("should handle conflict with three registries", async () => { - const sharedPack: PackSummary = { - name: "triple-pack", - description: "Three-way conflict", - tags: [], - latestVersion: "1.0.0", - author: "author", - updatedAt: "2026-01-01", - }; - - const repo1 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo2 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo3 = createBareRepoWithPacks(tempDir, [sharedPack]); - - addTestRegistry(makeEntry("triple-1", repo1)); - addTestRegistry(makeEntry("triple-2", repo2)); - addTestRegistry(makeEntry("triple-3", repo3)); - await syncRegistry(getRegistry("triple-1")!); - await syncRegistry(getRegistry("triple-2")!); - await syncRegistry(getRegistry("triple-3")!); - - const { conflict } = resolvePackFromRegistries("triple-pack", { - conflictResolution: { strategy: "interactive" }, - }); - - expect(conflict).toBeDefined(); - expect(conflict!.sources).toHaveLength(3); + it("resolves packs with different names without a registry", async () => { + await addSynced("one", [pack("alpha")]); + await addSynced("two", [pack("beta")]); + expect(findRegistryPack("alpha").registry).toBe("one"); + expect(findRegistryPack("beta").registry).toBe("two"); + expect(() => findRegistryPack("gamma")).toThrow(NotFoundError); }); - it("should list all conflicting registries in conflict object", async () => { - const sharedPack: PackSummary = { - name: "info-pack", - description: "Info test", - tags: [], - latestVersion: "1.0.0", - author: "author", - updatedAt: "2026-01-01", - }; - - const repo1 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo2 = createBareRepoWithPacks(tempDir, [sharedPack]); - - addTestRegistry(makeEntry("info-reg1", repo1, 2)); - addTestRegistry(makeEntry("info-reg2", repo2, 1)); - await syncRegistry(getRegistry("info-reg1")!); - await syncRegistry(getRegistry("info-reg2")!); - - const { conflict } = resolvePackFromRegistries("info-pack", { - conflictResolution: { strategy: "interactive" }, - }); - - expect(conflict!.packName).toBe("info-pack"); - for (const source of conflict!.sources) { - expect(source.registryName).toBeTruthy(); - expect(source.registryUrl).toBeTruthy(); - expect(source.version).toBe("1.0.0"); - expect(typeof source.priority).toBe("number"); + it("lists the packs available in the synced registries", async () => { + await addSynced("one", [pack("alpha")]); + await addSynced("two", [pack("beta"), pack("alpha")]); + const t = makeContext(); + try { + const all = await runOperation(listPacksOperation, t.ctx, { available: true }); + expect(all.items.map((p) => ("registry" in p ? `${p.registry}/${p.name}` : p.name))).toEqual([ + "one/alpha", + "two/beta", + "two/alpha", + ]); + const two = await runOperation(listPacksOperation, t.ctx, { + available: true, + registry: "two", + }); + expect(two.items).toHaveLength(2); + } finally { + t.db.close(); } }); }); diff --git a/tests/integration/registry/registry-lifecycle.test.ts b/tests/integration/registry/registry-lifecycle.test.ts index f510723b..24ea4485 100644 --- a/tests/integration/registry/registry-lifecycle.test.ts +++ b/tests/integration/registry/registry-lifecycle.test.ts @@ -37,8 +37,6 @@ function makeEntry(name: string, url: string): RegistryEntry { return { name, url, - syncInterval: 3600, - priority: 1, lastSyncedAt: null, }; } @@ -188,7 +186,7 @@ describe("integration: registry lifecycle", () => { const { results } = searchRegistries("react"); expect(results.length).toBeGreaterThan(0); - expect(results[0]!.pack.name).toBe("react-docs"); + expect(results[0]!.name).toBe("react-docs"); }); it("should remove a registry and clean up cache", async () => { @@ -285,7 +283,7 @@ describe("integration: registry lifecycle", () => { // Search across both const { results } = searchRegistries("pack"); expect(results.length).toBe(2); - expect(results.map((r) => r.pack.name).sort((a, b) => a.localeCompare(b))).toEqual([ + expect(results.map((r) => r.name).sort((a, b) => a.localeCompare(b))).toEqual([ "pack-from-reg1", "pack-from-reg2", ]); diff --git a/tests/integration/registry/registry-offline.test.ts b/tests/integration/registry/registry-offline.test.ts index 30889e93..8ad55cf1 100644 --- a/tests/integration/registry/registry-offline.test.ts +++ b/tests/integration/registry/registry-offline.test.ts @@ -28,12 +28,12 @@ vi.mock("node:os", async (importOriginal) => { const { loadRegistries, saveRegistries, getRegistry } = await import("../../../src/registry/config.js"); -const { syncRegistry, getRegistryIndex } = await import("../../../src/registry/sync.js"); -const { searchRegistries } = await import("../../../src/registry/search.js"); +const { syncRegistry } = await import("../../../src/registry/sync.js"); +const { searchRegistries, listRegistryPacks } = await import("../../../src/registry/search.js"); const { getRegistryCacheDir } = await import("../../../src/registry/types.js"); function makeEntry(name: string, url: string): RegistryEntry { - return { name, url, syncInterval: 3600, priority: 1, lastSyncedAt: null }; + return { name, url, lastSyncedAt: null }; } function addTestRegistry(entry: RegistryEntry): void { @@ -120,7 +120,7 @@ describe("integration: registry offline / unreachable remote", () => { const entry = getRegistry("named-error")!; const status = await syncRegistry(entry); - expect(status.registryName).toBe("named-error"); + expect(status.registry).toBe("named-error"); expect(status.status).toBe("error"); }); @@ -147,7 +147,7 @@ describe("integration: registry offline / unreachable remote", () => { // Search should still return results from cache const { results } = searchRegistries("stale"); expect(results.length).toBeGreaterThan(0); - expect(results[0]!.pack.name).toBe("stale-searchable"); + expect(results[0]!.name).toBe("stale-searchable"); }); it("should fail clearly when no cache exists and remote is unreachable", async () => { @@ -162,13 +162,12 @@ describe("integration: registry offline / unreachable remote", () => { expect(existsSync(getRegistryCacheDir("no-cache"))).toBe(false); }); - it("should return error with getRegistryIndex when never synced and unreachable", async () => { + it("warns about a registry that was never synced when listing packs", () => { addTestRegistry(makeEntry("never-synced", join(tempDir, "ghost.git"))); - const entry = getRegistry("never-synced")!; - const { packs, warning } = await getRegistryIndex(entry); + const { packs, warnings } = listRegistryPacks(); expect(packs).toEqual([]); - expect(warning).toBeTruthy(); + expect(warnings[0]).toContain("never been synced"); }); }); diff --git a/tests/integration/registry/registry-publish.test.ts b/tests/integration/registry/registry-publish.test.ts index 45baf361..64b98b5c 100644 --- a/tests/integration/registry/registry-publish.test.ts +++ b/tests/integration/registry/registry-publish.test.ts @@ -34,7 +34,7 @@ const { verifyChecksum } = await import("../../../src/registry/checksum.js"); const { getRegistryCacheDir } = await import("../../../src/registry/types.js"); function makeEntry(name: string, url: string): RegistryEntry { - return { name, url, syncInterval: 3600, priority: 1, lastSyncedAt: null }; + return { name, url, lastSyncedAt: null }; } function addTestRegistry(entry: RegistryEntry): void { diff --git a/tests/unit/api-routes.test.ts b/tests/unit/api-routes.test.ts index e14ee44b..c775211b 100644 --- a/tests/unit/api-routes.test.ts +++ b/tests/unit/api-routes.test.ts @@ -211,6 +211,15 @@ const CASES: Record<string, RouteCases> = { "GET /api/v1/packs": { ok: () => ({ path: `${V}/packs` }), bad: () => ({ path: `${V}/packs`, query: { available: "maybe" } }), + missing: () => ({ path: `${V}/packs`, query: { available: "true", registry: "nope" } }), + }, + "GET /api/v1/registries": { + ok: () => ({ path: `${V}/registries` }), + }, + "GET /api/v1/registries/search": { + ok: () => ({ path: `${V}/registries/search`, query: { query: "react" } }), + bad: () => ({ path: `${V}/registries/search` }), + missing: () => ({ path: `${V}/registries/search`, query: { query: "x", registry: "nope" } }), }, "GET /api/v1/connections": { ok: () => ({ path: `${V}/connections` }) }, diff --git a/tests/unit/cli-commands.test.ts b/tests/unit/cli-commands.test.ts index 849793a7..c4e312aa 100644 --- a/tests/unit/cli-commands.test.ts +++ b/tests/unit/cli-commands.test.ts @@ -227,11 +227,52 @@ const SINGLE: Array<[string[], string, Record<string, unknown>]> = [ { pack: "./p.json", batchSize: 5, resumeFrom: 2, concurrency: 3 }, ], [["pack", "remove", "p", "-y"], "remove-pack", { pack: "p" }], + [ + ["pack", "install", "react-docs@1.2.0", "--registry", "official"], + "install-pack", + { pack: "react-docs@1.2.0", registry: "official" }, + ], [["pack", "list"], "list-packs", {}], [ - ["pack", "list", "--available", "--registry", "https://r.example"], + ["pack", "list", "--available", "--registry", "official"], "list-packs", - { available: true, registryUrl: "https://r.example" }, + { available: true, registry: "official" }, + ], + [ + ["registry", "add", "https://g.example/org/packs.git", "--name", "team", "--no-sync"], + "add-registry", + { url: "https://g.example/org/packs.git", name: "team", sync: false }, + ], + [["registry", "remove", "team", "-y"], "remove-registry", { name: "team" }], + [["registry", "list"], "list-registries", {}], + [["registry", "sync"], "sync-registries", {}], + [["registry", "sync", "team"], "sync-registries", { name: "team" }], + [ + ["registry", "search", "react", "--registry", "team"], + "search-registries", + { query: "react", registry: "team" }, + ], + [["registry", "create", "./reg"], "create-registry", { path: "./reg" }], + [ + [ + "registry", + "publish", + "p.json.gz", + "--registry", + "team", + "--pack-version", + "2.0.0", + "-m", + "msg", + "--submit", + ], + "publish-pack", + { file: "p.json.gz", registry: "team", version: "2.0.0", message: "msg", submit: true }, + ], + [ + ["registry", "unpublish", "p@1.0.0", "--registry", "team", "-y"], + "unpublish-pack", + { pack: "p@1.0.0", registry: "team" }, ], [ ["pack", "create", "--name", "p", "--from", "./src", "--exclude", "*.min.js"], @@ -393,7 +434,7 @@ function commandPath(argv: string[]): string { } describe("CLI command set", () => { - /** Commands not covered above: they do not call operations (files, servers, registries). */ + /** Commands not covered above: they do not call operations (config files, servers). */ const NOT_OPERATIONS = new Set([ "serve", "doctor", @@ -408,7 +449,7 @@ describe("CLI command set", () => { "workspace delete", ]); - it("covers every command with a contract test (registry commands are tested elsewhere)", () => { + it("covers every command with a contract test", () => { const tested = new Set([ ...SINGLE.map(([argv]) => commandPath(argv)), "docs delete", @@ -418,9 +459,7 @@ describe("CLI command set", () => { "admin stats", "connect", ]); - const missing = leafCommands(program).filter( - (c) => !c.startsWith("registry ") && !NOT_OPERATIONS.has(c) && !tested.has(c), - ); + const missing = leafCommands(program).filter((c) => !NOT_OPERATIONS.has(c) && !tested.has(c)); expect(missing).toEqual([]); }); diff --git a/tests/unit/cli-version-options.test.ts b/tests/unit/cli-version-options.test.ts index c16e2962..de71ae65 100644 --- a/tests/unit/cli-version-options.test.ts +++ b/tests/unit/cli-version-options.test.ts @@ -1,21 +1,8 @@ -import { describe, it, expect, vi, beforeAll, afterEach } from "vitest"; +import { describe, it, expect, vi, beforeAll } from "vitest"; import { Command, CommanderError } from "commander"; import { readFileSync } from "node:fs"; -vi.mock("../../src/registry/git.js", async (importOriginal) => { - const original = await importOriginal<typeof import("../../src/registry/git.js")>(); - return { ...original, checkGitAvailable: vi.fn().mockResolvedValue(true) }; -}); - -vi.mock("../../src/registry/publish.js", () => ({ - publishPack: vi.fn(), - publishPackToBranch: vi.fn(), - unpublishPack: vi.fn().mockResolvedValue(undefined), -})); - import { program } from "../../src/cli/index.js"; -import { register as registerRegistryCommands } from "../../src/cli/commands/registry.js"; -import { unpublishPack } from "../../src/registry/publish.js"; const pkgVersion = ( JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf-8")) as { @@ -82,78 +69,16 @@ describe("CLI subcommand version flags", () => { }); it.each([ - [["pack", "install"], ["react-docs"]], [ ["pack", "create"], ["--name", "react-docs"], ], [ ["registry", "publish"], - ["pack.json", "-r", "main"], - ], - [ - ["registry", "unpublish"], - ["react-docs", "-r", "main"], + ["pack.json", "--registry", "main"], ], ])("%j accepts --pack-version", async (path, rest) => { const opts = await parseWithStubbedAction(path, [...rest, "--pack-version", "3.0.0"]); expect(opts.packVersion).toBe("3.0.0"); }); }); - -describe("registry unpublish version resolution", () => { - function buildProgram(): Command { - const root = new Command(); - root.exitOverride(); - registerRegistryCommands(root); - return root; - } - - afterEach(() => { - vi.restoreAllMocks(); - vi.mocked(unpublishPack).mockClear(); - }); - - function stubExit(): void { - vi.spyOn(console, "log").mockImplementation(() => {}); - vi.spyOn(process, "exit").mockImplementation((code) => { - throw new Error(`process.exit(${String(code)})`); - }); - } - - it("accepts name@version", async () => { - stubExit(); - await buildProgram().parseAsync( - ["registry", "unpublish", "react-docs@1.2.0", "-r", "main", "-y"], - { - from: "user", - }, - ); - expect(unpublishPack).toHaveBeenCalledWith( - expect.objectContaining({ registryName: "main", packName: "react-docs", version: "1.2.0" }), - ); - }); - - it("prefers --pack-version over the name@version suffix", async () => { - stubExit(); - await buildProgram().parseAsync( - ["registry", "unpublish", "react-docs@1.2.0", "-r", "main", "-y", "--pack-version", "1.3.0"], - { from: "user" }, - ); - expect(unpublishPack).toHaveBeenCalledWith( - expect.objectContaining({ packName: "react-docs", version: "1.3.0" }), - ); - }); - - it("exits with an error when no version is given", async () => { - stubExit(); - const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - await expect( - buildProgram().parseAsync(["registry", "unpublish", "react-docs", "-r", "main", "-y"], { - from: "user", - }), - ).rejects.toThrow("process.exit(1)"); - expect(errSpy).toHaveBeenCalledWith(expect.stringContaining("--pack-version")); - expect(unpublishPack).not.toHaveBeenCalled(); - }); -}); diff --git a/tests/unit/config-save.test.ts b/tests/unit/config-save.test.ts index 631fd175..66bd5fdc 100644 --- a/tests/unit/config-save.test.ts +++ b/tests/unit/config-save.test.ts @@ -68,8 +68,6 @@ function writeExisting(data: unknown, path = configPath()): void { const REGISTRY = { name: "team", url: "https://example.com/registry.git", - syncInterval: 3600, - priority: 1, lastSyncedAt: null, }; diff --git a/tests/unit/libscope.test.ts b/tests/unit/libscope.test.ts index 2a8321ee..b28cdb6a 100644 --- a/tests/unit/libscope.test.ts +++ b/tests/unit/libscope.test.ts @@ -174,6 +174,9 @@ describe("LibScope", () => { it("packs, tasks, webhooks, analytics", async () => { expect((await scope.packs.list()).items).toEqual([]); + await expect( + scope.registries.search({ query: "x", registry: "no-such-registry-for-tests" }), + ).rejects.toBeInstanceOf(NotFoundError); expect(Array.isArray((await scope.tasks.list()).items)).toBe(true); await expect(scope.tasks.get({ taskId: "missing" })).rejects.toBeInstanceOf(NotFoundError); expect((await scope.webhooks.list()).items).toEqual([]); diff --git a/tests/unit/mcp-format.test.ts b/tests/unit/mcp-format.test.ts index 77789d4f..a9b794d8 100644 --- a/tests/unit/mcp-format.test.ts +++ b/tests/unit/mcp-format.test.ts @@ -268,9 +268,18 @@ describe("MCP output formatting", () => { }); it("pack list formats registry packs", () => { - expect( - formatPackList({ items: [{ name: "p", version: "1", description: "desc", docCount: 3 }] }), - ).toBe("- p v1 (3 docs): desc"); + const pack = { + name: "p", + description: "desc", + tags: [], + latestVersion: "1", + author: "a", + updatedAt: "2026-01-01", + registry: "team", + }; + expect(formatPackList({ items: [pack], warnings: ['Registry "x" was never synced'] })).toBe( + '- p v1 (registry team): desc\nWarning: Registry "x" was never synced', + ); }); it("reindex reports failures and a rebuilt index", () => { diff --git a/tests/unit/mcp-server.test.ts b/tests/unit/mcp-server.test.ts index 4b7fe74c..2560a244 100644 --- a/tests/unit/mcp-server.test.ts +++ b/tests/unit/mcp-server.test.ts @@ -25,6 +25,8 @@ const { createMcpServer, parseToolsets, buildInstructions } = await import("../../src/mcp/server.js"); const { saveConnectorSettings } = await import("../../src/connectors/saved-config.js"); const { taskRegistry } = await import("../../src/core/tasks.js"); +const { saveRegistries } = await import("../../src/registry/config.js"); +const { getPackDataPath, getRegistryCacheDir } = await import("../../src/registry/types.js"); initLogger("silent"); @@ -541,29 +543,46 @@ describe("MCP server: tool calls", () => { await callError(client, "sync", { name: "bad name!" }); }); - it("install-pack and list-packs (admin): install from a file and list", async () => { - const path = join(tempHome, "demo.json"); + it("install-pack and list-packs (admin): install from a registry and list", async () => { + // A synced registry "team" holding pack "demo" (local copy only, no git needed). + saveRegistries([{ name: "team", url: "https://example.com/team.git", lastSyncedAt: null }]); + const dataPath = getPackDataPath("team", "demo", "1.0.0"); + mkdirSync(join(dataPath, ".."), { recursive: true }); + const pack = { + name: "demo", + version: "1.0.0", + description: "Test pack", + documents: [{ title: "Pack doc", content: "Content from a pack", source: "test" }], + metadata: { author: "tests", license: "MIT", createdAt: new Date().toISOString() }, + }; + writeFileSync(dataPath, JSON.stringify(pack)); + const summary = { name: "demo", description: "Test pack", tags: [], latestVersion: "1.0.0" }; writeFileSync( - path, - JSON.stringify({ - name: "demo", - version: "1.0.0", - description: "Test pack", - documents: [{ title: "Pack doc", content: "Content from a pack", source: "test" }], - metadata: { author: "tests", license: "MIT", createdAt: new Date().toISOString() }, - }), + join(getRegistryCacheDir("team"), "index.json"), + JSON.stringify([{ ...summary, author: "tests", updatedAt: "2026-01-01" }]), ); + expect(await callOk(client, "list-packs")).toBe("No packs found."); - expect(await callOk(client, "install-pack", { pack: path })).toBe( + expect(await callOk(client, "list-packs", { available: true })).toBe( + "- demo v1.0.0 (registry team): Test pack", + ); + expect(await callOk(client, "install-pack", { pack: "demo" })).toBe( "Installed pack demo: 1 documents.", ); - expect(await callOk(client, "install-pack", { pack: path })).toBe( + expect(await callOk(client, "install-pack", { pack: "demo@1.0.0", registry: "team" })).toBe( "Pack demo is already installed.", ); expect(await callOk(client, "list-packs")).toMatch(/^- demo v1\.0\.0 \(1 docs, installed /); expect(await callOk(client, "overview")).toContain("Packs:\n- demo v1.0.0"); + // Local files are not read over MCP. + const file = join(tempHome, "demo.json"); + writeFileSync(file, JSON.stringify(pack)); + expect(await callError(client, "install-pack", { pack: file })).toContain( + "only available from the CLI", + ); await callError(client, "install-pack", {}); + await callError(client, "install-pack", { pack: "missing" }); await callError(client, "list-packs", { available: "yes" }); }); diff --git a/tests/unit/operations/registries.test.ts b/tests/unit/operations/registries.test.ts new file mode 100644 index 00000000..90041a0a --- /dev/null +++ b/tests/unit/operations/registries.test.ts @@ -0,0 +1,154 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { gzipSync } from "node:zlib"; +import { execFileSync } from "node:child_process"; +import { NotFoundError, ValidationError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; + +// Registries live under ~/.libscope/registries: point homedir() at a temp directory. +let tempHome = tmpdir(); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const ops = await import("../../../src/core/operations/index.js"); +const { getRegistryCacheDir } = await import("../../../src/registry/types.js"); +const { makeContext, run, expectValidationError } = await import("./helpers.js"); +type TestContext = ReturnType<typeof makeContext>; + +initLogger("silent"); + +describe("registry operations", () => { + let t: TestContext; + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "libscope-registry-ops-")); + tempHome = join(dir, "home"); + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + function writeGzPack(name: string): string { + const path = join(dir, `${name}.json.gz`); + const pack = { + name, + version: "1.0.0", + description: `The ${name} pack`, + documents: [{ title: "Pack doc", content: "Content from a registry pack", source: "t" }], + metadata: { author: "tests", license: "MIT", createdAt: "2026-01-01T00:00:00.000Z" }, + }; + writeFileSync(path, gzipSync(JSON.stringify(pack))); + return path; + } + + /** create-registry, then a bare clone to push to; returns its file:/// URL. */ + async function makeRemote(): Promise<string> { + const created = await run(ops.createRegistryOperation, t.ctx, { path: join(dir, "src") }); + const bare = join(dir, "packs.git"); + execFileSync("git", ["clone", "--quiet", "--bare", created.path, bare]); + return `file://${bare}`; + } + + it("adds, publishes, lists, searches, installs, unpublishes and removes", async () => { + const url = await makeRemote(); + const added = await run(ops.addRegistryOperation, t.ctx, { url }); + expect(added).toMatchObject({ name: "packs", sync: { status: "success", packs: 0 } }); + + const published = await run(ops.publishPackOperation, t.ctx, { + file: writeGzPack("demo"), + registry: "packs", + }); + expect(published).toMatchObject({ packName: "demo", version: "1.0.0", branch: null }); + + const registries = await run(ops.listRegistriesOperation, t.ctx, {}); + expect(registries.items).toEqual([ + expect.objectContaining({ name: "packs", url, packs: 1 }) as unknown, + ]); + + const found = await run(ops.searchRegistriesOperation, t.ctx, { query: "demo" }); + expect(found.items.map((p) => [p.registry, p.name])).toEqual([["packs", "demo"]]); + + const available = await run(ops.listPacksOperation, t.ctx, { available: true }); + expect(available).toMatchObject({ items: [{ name: "demo", registry: "packs" }] }); + + const installed = await run(ops.installPackOperation, t.ctx, { pack: "demo@1.0.0" }); + expect(installed).toMatchObject({ packName: "demo", documentsInstalled: 1 }); + + await expectValidationError( + run(ops.unpublishPackOperation, t.ctx, { pack: "demo", registry: "packs" }), + ); + expect( + await run(ops.unpublishPackOperation, t.ctx, { pack: "demo@1.0.0", registry: "packs" }), + ).toMatchObject({ removed: true }); + expect((await run(ops.searchRegistriesOperation, t.ctx, { query: "demo" })).items).toEqual([]); + + await run(ops.removeRegistryOperation, t.ctx, { name: "packs" }); + expect(existsSync(getRegistryCacheDir("packs"))).toBe(false); + expect((await run(ops.listRegistriesOperation, t.ctx, {})).items).toEqual([]); + }); + + it("syncs one or all registries and reports registries that cannot be reached", async () => { + const url = await makeRemote(); + await run(ops.addRegistryOperation, t.ctx, { url, name: "first", sync: false }); + await run(ops.addRegistryOperation, t.ctx, { + url: `file://${join(dir, "missing.git")}`, + name: "broken", + sync: false, + }); + expect((await run(ops.listPacksOperation, t.ctx, { available: true })).warnings).toHaveLength( + 2, + ); + + const one = await run(ops.syncRegistriesOperation, t.ctx, { name: "first" }); + expect(one.items).toMatchObject([{ registry: "first", status: "success", packs: 0 }]); + const all = await run(ops.syncRegistriesOperation, t.ctx, {}); + expect(all.items.map((s) => s.status)).toEqual(["success", "error"]); + }); + + it("throws NotFoundError for unknown registries and packs", async () => { + await expect(run(ops.installPackOperation, t.ctx, { pack: "demo" })).rejects.toBeInstanceOf( + NotFoundError, + ); + for (const [op, input] of [ + [ops.syncRegistriesOperation, { name: "nope" }], + [ops.searchRegistriesOperation, { query: "x", registry: "nope" }], + [ops.removeRegistryOperation, { name: "nope" }], + [ops.listPacksOperation, { available: true, registry: "nope" }], + [ops.installPackOperation, { pack: "demo", registry: "nope" }], + ] as const) { + await expect(run(op, t.ctx, input), op.name).rejects.toBeInstanceOf(NotFoundError); + } + }); + + it("validates input", async () => { + await expectValidationError(run(ops.listPacksOperation, t.ctx, { registry: "packs" })); + await expectValidationError( + run(ops.installPackOperation, t.ctx, { pack: "a.json", registry: "packs" }), + ); + await expectValidationError(run(ops.addRegistryOperation, t.ctx, { url: "/srv/x.git" })); + await expectValidationError(run(ops.searchRegistriesOperation, t.ctx, { query: "" })); + }); + + it("reads and writes local files only from the CLI and the Node.js API", async () => { + for (const surface of ["api", "mcp"] as const) { + const other = makeContext({ db: t.db, surface }); + await expect( + run(ops.installPackOperation, other.ctx, { pack: writeGzPack("x") }), + ).rejects.toBeInstanceOf(ValidationError); + await expect( + run(ops.publishPackOperation, other.ctx, { file: "x.json", registry: "packs" }), + ).rejects.toThrow(/only available from the CLI/); + await expect( + run(ops.createRegistryOperation, other.ctx, { path: join(dir, "r") }), + ).rejects.toThrow(/only available from the CLI/); + } + }); +}); diff --git a/tests/unit/packs.test.ts b/tests/unit/packs.test.ts index f07fd9d4..6a501ffc 100644 --- a/tests/unit/packs.test.ts +++ b/tests/unit/packs.test.ts @@ -11,7 +11,6 @@ import { removePack, listInstalledPacks, createPack, - listAvailablePacks, createPackFromSource, } from "../../src/core/packs.js"; import type { KnowledgePack } from "../../src/core/packs.js"; @@ -496,42 +495,6 @@ describe("knowledge packs", () => { /must be within the current working directory/, ); }); - - it("should reject http registry URLs", async () => { - await expect( - installPack(db, provider, "some-pack", { - registryUrl: "http://evil.com/registry.json", - }), - ).rejects.toThrow(/must use https/); - }); - - it("should reject private IP registry URLs", async () => { - await expect( - installPack(db, provider, "some-pack", { - registryUrl: "https://127.0.0.1/registry.json", - }), - ).rejects.toThrow(/private/); - }); - - it("should reject localhost registry URLs", async () => { - await expect( - installPack(db, provider, "some-pack", { - registryUrl: "https://localhost/registry.json", - }), - ).rejects.toThrow(/private/); - }); - - it("should reject http registry URL in listAvailablePacks", async () => { - await expect(listAvailablePacks("http://evil.com/registry.json")).rejects.toThrow( - /must use https/, - ); - }); - - it("should reject private IP in listAvailablePacks", async () => { - await expect(listAvailablePacks("https://192.168.1.1/registry.json")).rejects.toThrow( - /private/, - ); - }); }); describe("createPackFromSource", () => { diff --git a/tests/unit/registry/config.test.ts b/tests/unit/registry/config.test.ts index 98e89022..5e0d8743 100644 --- a/tests/unit/registry/config.test.ts +++ b/tests/unit/registry/config.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { initLogger } from "../../../src/logger.js"; +import { NotFoundError } from "../../../src/errors.js"; import type { RegistryEntry } from "../../../src/registry/types.js"; // Mock homedir before importing registry config module @@ -28,14 +29,14 @@ const { validateRegistryName, validateGitUrl, sanitizeUrl, + requireRegistry, + deriveRegistryName, } = await import("../../../src/registry/config.js"); function makeEntry(overrides: Partial<RegistryEntry> = {}): RegistryEntry { return { name: "test-reg", url: "https://github.com/org/registry.git", - syncInterval: 3600, - priority: 1, lastSyncedAt: null, ...overrides, }; @@ -110,6 +111,11 @@ describe("registry config", () => { expect(() => validateGitUrl("not-a-url")).toThrow(); }); + it("accepts file:/// URLs and rejects plain local paths", () => { + expect(validateGitUrl("file:///srv/registry.git/")).toBe("file:///srv/registry.git"); + expect(() => validateGitUrl("/srv/registry.git")).toThrow(); + }); + // Security/robustness fixes: whitespace trimming and trailing slash normalisation it("should trim leading and trailing whitespace", () => { const result = validateGitUrl(" https://github.com/org/repo.git "); @@ -284,8 +290,33 @@ describe("registry config", () => { expect(loadRegistries()).toHaveLength(0); }); - it("should throw when removing non-existent registry", () => { - expect(() => removeRegistry("nonexistent")).toThrow(/not found/); + it("should throw NotFoundError when removing non-existent registry", () => { + expect(() => removeRegistry("nonexistent")).toThrow(NotFoundError); + }); + }); + + describe("requireRegistry", () => { + it("returns the entry, or throws NotFoundError", () => { + addRegistry(makeEntry({ name: "here" })); + expect(requireRegistry("here").name).toBe("here"); + expect(() => requireRegistry("gone")).toThrow(NotFoundError); + }); + }); + + describe("deriveRegistryName", () => { + it.each([ + ["https://github.com/org/packs.git", "packs"], + ["https://github.com/org/packs/", "packs"], + ["git@github.com:org/team-packs.git", "team-packs"], + ["git@host:packs.git", "packs"], + ["ssh://git@host:7999/proj/my_packs.git", "my_packs"], + ["file:///srv/registries/local.git", "local"], + ])("%s -> %s", (url, name) => { + expect(deriveRegistryName(url)).toBe(name); + }); + + it("throws ValidationError when no valid name can be derived", () => { + expect(() => deriveRegistryName("https://example.com/x.git")).toThrow(/Give a name/); }); }); diff --git a/tests/unit/registry/conflict.test.ts b/tests/unit/registry/conflict.test.ts deleted file mode 100644 index 2838bf95..00000000 --- a/tests/unit/registry/conflict.test.ts +++ /dev/null @@ -1,449 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdirSync, writeFileSync, rmSync } from "node:fs"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; -import { randomUUID, createHash } from "node:crypto"; -import { initLogger } from "../../../src/logger.js"; -import type { RegistryEntry, PackSummary } from "../../../src/registry/types.js"; - -let tempHome: string = join(tmpdir(), `libscope-conflict-test-${process.pid}`); -mkdirSync(tempHome, { recursive: true }); - -vi.mock("node:os", async (importOriginal) => { - const orig = await importOriginal<typeof import("node:os")>(); - return { - ...orig, - homedir: () => tempHome, - }; -}); - -const { - findPackInRegistries, - resolvePackFromRegistries, - parsePackSpecifier, - verifyResolvedPackChecksum, -} = await import("../../../src/registry/resolve.js"); -const { saveRegistries } = await import("../../../src/registry/config.js"); -const { getRegistryCacheDir, getPackDataPath, getPackManifestPath } = - await import("../../../src/registry/types.js"); -const { clearIndexCache } = await import("../../../src/registry/git.js"); - -function makeEntry(name: string, overrides: Partial<RegistryEntry> = {}): RegistryEntry { - return { - name, - url: "https://github.com/org/registry.git", - syncInterval: 3600, - priority: 1, - lastSyncedAt: new Date().toISOString(), - ...overrides, - }; -} - -function makePack(name: string, overrides: Partial<PackSummary> = {}): PackSummary { - return { - name, - description: `The ${name} pack`, - tags: [], - latestVersion: "1.0.0", - author: "author", - updatedAt: "2026-01-01T00:00:00.000Z", - ...overrides, - }; -} - -function setupRegistry(regName: string, packs: PackSummary[]): void { - const cacheDir = getRegistryCacheDir(regName); - mkdirSync(cacheDir, { recursive: true }); - writeFileSync(join(cacheDir, "index.json"), JSON.stringify(packs), "utf-8"); -} - -function setupPackDataFile(regName: string, packName: string, version: string): void { - const dataPath = getPackDataPath(regName, packName, version); - mkdirSync(join(dataPath, ".."), { recursive: true }); - writeFileSync( - dataPath, - JSON.stringify({ - name: packName, - version, - description: "test", - documents: [], - metadata: { author: "test", license: "MIT", createdAt: "2026-01-01" }, - }), - "utf-8", - ); -} - -/** - * Set up a pack data file with a matching manifest that includes a real checksum. - * Returns the checksum that was written. - */ -function setupPackDataFileWithManifest( - regName: string, - packName: string, - version: string, - content?: string, -): string { - const dataPath = getPackDataPath(regName, packName, version); - mkdirSync(join(dataPath, ".."), { recursive: true }); - - const packContent = - content ?? - JSON.stringify({ - name: packName, - version, - description: "test", - documents: [], - metadata: { author: "test", license: "MIT", createdAt: "2026-01-01" }, - }); - writeFileSync(dataPath, packContent, "utf-8"); - - const checksum = createHash("sha256").update(packContent, "utf-8").digest("hex"); - - const manifestPath = getPackManifestPath(regName, packName); - mkdirSync(join(manifestPath, ".."), { recursive: true }); - writeFileSync( - manifestPath, - JSON.stringify({ - name: packName, - description: "test", - tags: [], - author: "test", - license: "MIT", - versions: [ - { - version, - publishedAt: "2026-01-01T00:00:00.000Z", - checksumPath: `${version}/checksum.sha256`, - checksum, - docCount: 0, - }, - ], - }), - "utf-8", - ); - - return checksum; -} - -describe("registry conflict resolution", () => { - beforeEach(() => { - initLogger("silent"); - clearIndexCache(); - tempHome = join(tmpdir(), `libscope-conflict-${randomUUID()}`); - mkdirSync(tempHome, { recursive: true }); - }); - - afterEach(() => { - rmSync(tempHome, { recursive: true, force: true }); - }); - - describe("parsePackSpecifier", () => { - it("should parse 'name' as name only", () => { - expect(parsePackSpecifier("react-docs")).toEqual({ name: "react-docs" }); - }); - - it("should parse 'name@version' into name and version", () => { - expect(parsePackSpecifier("react-docs@1.2.0")).toEqual({ - name: "react-docs", - version: "1.2.0", - }); - }); - - it("should handle scoped-like names with @ at the start", () => { - // Last @ is the version delimiter - expect(parsePackSpecifier("@org/pack@2.0.0")).toEqual({ - name: "@org/pack", - version: "2.0.0", - }); - }); - - it("should return just name when no @ after first character", () => { - expect(parsePackSpecifier("simple-pack")).toEqual({ name: "simple-pack" }); - }); - }); - - describe("findPackInRegistries", () => { - it("should return empty matches when no registries configured", () => { - const { matches } = findPackInRegistries("anything"); - expect(matches).toEqual([]); - }); - - it("should find pack in single registry", () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("react-docs")]); - - const { matches } = findPackInRegistries("react-docs"); - expect(matches).toHaveLength(1); - expect(matches[0]!.entry.name).toBe("reg1"); - expect(matches[0]!.pack.name).toBe("react-docs"); - }); - - it("should find pack in multiple registries", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - - const { matches } = findPackInRegistries("shared-pack"); - expect(matches).toHaveLength(2); - }); - - it("should warn for unsynced registries", () => { - saveRegistries([makeEntry("unsynced")]); - // No cache dir created - - const { matches, warnings } = findPackInRegistries("anything"); - expect(matches).toEqual([]); - expect(warnings).toHaveLength(1); - expect(warnings[0]).toContain("never been synced"); - }); - - it("should return empty matches when pack not found", () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("other-pack")]); - - const { matches } = findPackInRegistries("nonexistent"); - expect(matches).toEqual([]); - }); - }); - - describe("resolvePackFromRegistries", () => { - it("should resolve pack from single registry", () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("test-pack")]); - setupPackDataFile("reg1", "test-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("test-pack"); - expect(resolved).not.toBeNull(); - expect(resolved!.registryName).toBe("reg1"); - expect(resolved!.packName).toBe("test-pack"); - expect(resolved!.version).toBe("1.0.0"); - }); - - it("should return null when pack not found anywhere", () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("other-pack")]); - - const { resolved } = resolvePackFromRegistries("nonexistent"); - expect(resolved).toBeNull(); - }); - - it("should detect conflict when pack exists in multiple registries", () => { - saveRegistries([makeEntry("reg1", { priority: 2 }), makeEntry("reg2", { priority: 1 })]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - setupPackDataFile("reg2", "shared-pack", "1.0.0"); - - // Default resolution is "priority" — reg2 has lower priority (wins) - const { resolved } = resolvePackFromRegistries("shared-pack"); - expect(resolved).not.toBeNull(); - expect(resolved!.registryName).toBe("reg2"); - }); - - it("should resolve conflict by priority (lower wins)", () => { - saveRegistries([makeEntry("reg-a", { priority: 10 }), makeEntry("reg-b", { priority: 1 })]); - setupRegistry("reg-a", [makePack("shared-pack")]); - setupRegistry("reg-b", [makePack("shared-pack")]); - setupPackDataFile("reg-b", "shared-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("shared-pack", { - conflictResolution: { strategy: "priority" }, - }); - expect(resolved!.registryName).toBe("reg-b"); - }); - - it("should resolve conflict with explicit registry", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - setupPackDataFile("reg1", "shared-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("shared-pack", { - conflictResolution: { strategy: "explicit", registryName: "reg1" }, - }); - expect(resolved!.registryName).toBe("reg1"); - }); - - it("should return conflict for interactive strategy without resolving", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - - const { resolved, conflict } = resolvePackFromRegistries("shared-pack", { - conflictResolution: { strategy: "interactive" }, - }); - expect(resolved).toBeNull(); - expect(conflict).toBeDefined(); - expect(conflict!.sources).toHaveLength(2); - expect(conflict!.packName).toBe("shared-pack"); - }); - - it("should return null when explicit registry doesn't have the pack", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - - const { resolved } = resolvePackFromRegistries("shared-pack", { - registryName: "reg3", - }); - expect(resolved).toBeNull(); - }); - - it("should filter to specified registryName option", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - setupPackDataFile("reg1", "shared-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("shared-pack", { - registryName: "reg1", - }); - expect(resolved!.registryName).toBe("reg1"); - }); - - it("should use specified version", () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("test-pack", { latestVersion: "2.0.0" })]); - setupPackDataFile("reg1", "test-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("test-pack", { version: "1.0.0" }); - expect(resolved!.version).toBe("1.0.0"); - }); - - it("should include all candidate registries in conflict", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2"), makeEntry("reg3")]); - setupRegistry("reg1", [makePack("shared")]); - setupRegistry("reg2", [makePack("shared")]); - setupRegistry("reg3", [makePack("shared")]); - - const { conflict } = resolvePackFromRegistries("shared", { - conflictResolution: { strategy: "interactive" }, - }); - expect(conflict!.sources).toHaveLength(3); - }); - - it("should not conflict when packs have different names", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("pack-a")]); - setupRegistry("reg2", [makePack("pack-b")]); - setupPackDataFile("reg1", "pack-a", "1.0.0"); - - const { resolved, conflict } = resolvePackFromRegistries("pack-a"); - expect(resolved).not.toBeNull(); - expect(conflict).toBeUndefined(); - }); - }); - - describe("verifyResolvedPackChecksum", () => { - it("should pass when file matches recorded checksum", async () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("test-pack")]); - setupPackDataFileWithManifest("reg1", "test-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("test-pack"); - expect(resolved).not.toBeNull(); - - await expect(verifyResolvedPackChecksum(resolved!)).resolves.toBeUndefined(); - }); - - it("should throw when file has been tampered with", async () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("test-pack")]); - setupPackDataFileWithManifest("reg1", "test-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("test-pack"); - expect(resolved).not.toBeNull(); - - // Tamper with the pack data file after checksum was recorded - writeFileSync(resolved!.dataPath, '{"tampered":true}', "utf-8"); - - await expect(verifyResolvedPackChecksum(resolved!)).rejects.toThrow( - /Checksum verification failed/, - ); - }); - - it("should skip verification and warn when no manifest exists", async () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("no-manifest-pack")]); - setupPackDataFile("reg1", "no-manifest-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("no-manifest-pack"); - expect(resolved).not.toBeNull(); - - // Should not throw — manifest is missing, logs a warning and skips - await expect(verifyResolvedPackChecksum(resolved!)).resolves.toBeUndefined(); - }); - - it("should throw when manifest has no checksum for the version", async () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("zero-checksum-pack")]); - setupPackDataFile("reg1", "zero-checksum-pack", "1.0.0"); - - // Write a manifest with an empty checksum - const manifestPath = getPackManifestPath("reg1", "zero-checksum-pack"); - mkdirSync(join(manifestPath, ".."), { recursive: true }); - writeFileSync( - manifestPath, - JSON.stringify({ - name: "zero-checksum-pack", - description: "test", - tags: [], - author: "test", - license: "MIT", - versions: [ - { - version: "1.0.0", - publishedAt: "2026-01-01T00:00:00.000Z", - checksumPath: "1.0.0/checksum.sha256", - checksum: "", - docCount: 0, - }, - ], - }), - "utf-8", - ); - - const { resolved } = resolvePackFromRegistries("zero-checksum-pack"); - expect(resolved).not.toBeNull(); - - await expect(verifyResolvedPackChecksum(resolved!)).rejects.toThrow( - /has no checksum recorded/, - ); - }); - - it("should skip verification when version entry not found in manifest", async () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("version-mismatch-pack")]); - setupPackDataFile("reg1", "version-mismatch-pack", "1.0.0"); - - // Manifest only records version 2.0.0, not 1.0.0 - const manifestPath = getPackManifestPath("reg1", "version-mismatch-pack"); - mkdirSync(join(manifestPath, ".."), { recursive: true }); - writeFileSync( - manifestPath, - JSON.stringify({ - name: "version-mismatch-pack", - description: "test", - tags: [], - author: "test", - license: "MIT", - versions: [ - { - version: "2.0.0", - publishedAt: "2026-01-01T00:00:00.000Z", - checksumPath: "2.0.0/checksum.sha256", - checksum: "abc123", - docCount: 0, - }, - ], - }), - "utf-8", - ); - - const { resolved } = resolvePackFromRegistries("version-mismatch-pack"); - expect(resolved).not.toBeNull(); - - // Should not throw — version entry missing, logs a warning and skips - await expect(verifyResolvedPackChecksum(resolved!)).resolves.toBeUndefined(); - }); - }); -}); diff --git a/tests/unit/registry/git.test.ts b/tests/unit/registry/git.test.ts index 035b5bf7..e62fe69f 100644 --- a/tests/unit/registry/git.test.ts +++ b/tests/unit/registry/git.test.ts @@ -16,12 +16,13 @@ const gitEnv = { import { readIndex, createRegistryRepo, - checkGitAvailable, + git, cloneRegistry, fetchRegistry, commitAndPush, clearIndexCache, } from "../../../src/registry/git.js"; +import { ConfigError } from "../../../src/errors.js"; describe("registry git helpers", () => { let tempDir: string; @@ -35,10 +36,15 @@ describe("registry git helpers", () => { rmSync(tempDir, { recursive: true, force: true }); }); - describe("checkGitAvailable", () => { - it("should return true when git is available", async () => { - const result = await checkGitAvailable(); - expect(result).toBe(true); + describe("git", () => { + it("reports a missing git binary as a ConfigError", async () => { + const savedPath = process.env["PATH"]; + process.env["PATH"] = tempDir; + try { + await expect(git(["--version"])).rejects.toBeInstanceOf(ConfigError); + } finally { + process.env["PATH"] = savedPath; + } }); }); diff --git a/tests/unit/registry/publish.test.ts b/tests/unit/registry/publish.test.ts index aeff7240..91b316b7 100644 --- a/tests/unit/registry/publish.test.ts +++ b/tests/unit/registry/publish.test.ts @@ -64,8 +64,6 @@ function makeEntry( return { name, url, - syncInterval: 3600, - priority: 1, lastSyncedAt: null, ...overrides, }; diff --git a/tests/unit/registry/resolve.test.ts b/tests/unit/registry/resolve.test.ts new file mode 100644 index 00000000..3476bc03 --- /dev/null +++ b/tests/unit/registry/resolve.test.ts @@ -0,0 +1,235 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { gzipSync } from "node:zlib"; +import { randomUUID, createHash } from "node:crypto"; +import { initLogger } from "../../../src/logger.js"; +import { NotFoundError, ValidationError } from "../../../src/errors.js"; +import type { RegistryEntry, PackSummary } from "../../../src/registry/types.js"; + +let tempHome: string = join(tmpdir(), `libscope-resolve-test-${process.pid}`); + +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const { findRegistryPack, resolveRegistryPack, parsePackSpecifier, verifyResolvedPackChecksum } = + await import("../../../src/registry/resolve.js"); +const { saveRegistries } = await import("../../../src/registry/config.js"); +const { getRegistryCacheDir, getPackDataPath, getPackManifestPath } = + await import("../../../src/registry/types.js"); +const { clearIndexCache } = await import("../../../src/registry/git.js"); + +function makeEntry(name: string): RegistryEntry { + return { name, url: "https://github.com/org/registry.git", lastSyncedAt: null }; +} + +function makePack(name: string, overrides: Partial<PackSummary> = {}): PackSummary { + return { + name, + description: `The ${name} pack`, + tags: [], + latestVersion: "1.0.0", + author: "author", + updatedAt: "2026-01-01T00:00:00.000Z", + ...overrides, + }; +} + +function setupRegistry(regName: string, packs: PackSummary[]): void { + const cacheDir = getRegistryCacheDir(regName); + mkdirSync(cacheDir, { recursive: true }); + writeFileSync(join(cacheDir, "index.json"), JSON.stringify(packs), "utf-8"); +} + +function packJson(name: string, version: string): string { + return JSON.stringify({ + name, + version, + description: "test", + documents: [], + metadata: { author: "test", license: "MIT", createdAt: "2026-01-01" }, + }); +} + +/** Write the pack data file and, unless `checksum` is null, a manifest recording it. */ +function setupPackData( + regName: string, + packName: string, + version: string, + options: { data?: Buffer | string; checksum?: string | null; manifestVersion?: string } = {}, +): string { + const dataPath = getPackDataPath(regName, packName, version); + mkdirSync(join(dataPath, ".."), { recursive: true }); + const data = options.data ?? packJson(packName, version); + writeFileSync(dataPath, data); + if (options.checksum === null) return dataPath; + + const manifestPath = getPackManifestPath(regName, packName); + mkdirSync(join(manifestPath, ".."), { recursive: true }); + const checksum = options.checksum ?? createHash("sha256").update(data).digest("hex"); + const entryVersion = options.manifestVersion ?? version; + writeFileSync( + manifestPath, + JSON.stringify({ + name: packName, + description: "test", + tags: [], + author: "test", + license: "MIT", + versions: [ + { + version: entryVersion, + publishedAt: "2026-01-01T00:00:00.000Z", + checksumPath: `${entryVersion}/checksum.sha256`, + checksum, + docCount: 0, + }, + ], + }), + ); + return dataPath; +} + +describe("registry pack resolution", () => { + beforeEach(() => { + initLogger("silent"); + clearIndexCache(); + tempHome = join(tmpdir(), `libscope-resolve-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + }); + + afterEach(() => { + rmSync(tempHome, { recursive: true, force: true }); + }); + + describe("parsePackSpecifier", () => { + it("parses 'name' and 'name@version'", () => { + expect(parsePackSpecifier("react-docs")).toEqual({ name: "react-docs" }); + expect(parsePackSpecifier("react-docs@1.2.0")).toEqual({ + name: "react-docs", + version: "1.2.0", + }); + }); + + it("does not treat a leading @ as a version separator", () => { + expect(parsePackSpecifier("@scope")).toEqual({ name: "@scope" }); + }); + }); + + describe("findRegistryPack", () => { + it("finds the latest version in the single registry that has the pack", () => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("react", { latestVersion: "2.0.0" })]); + const dataPath = setupPackData("reg1", "react", "2.0.0"); + + expect(findRegistryPack("react")).toEqual({ + registry: "reg1", + name: "react", + version: "2.0.0", + dataPath, + }); + }); + + it("installs the version given as name@version", () => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("react", { latestVersion: "2.0.0" })]); + setupPackData("reg1", "react", "2.0.0"); + setupPackData("reg1", "react", "1.0.0"); + + expect(findRegistryPack("react@1.0.0").version).toBe("1.0.0"); + }); + + it("throws ValidationError naming every registry when several have the pack", () => { + saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); + setupRegistry("reg1", [makePack("react")]); + setupRegistry("reg2", [makePack("react")]); + + expect(() => findRegistryPack("react")).toThrow(ValidationError); + expect(() => findRegistryPack("react")).toThrow(/several registries \(reg1, reg2\)/); + }); + + it("uses the named registry when several have the pack", () => { + saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); + setupRegistry("reg1", [makePack("react")]); + setupRegistry("reg2", [makePack("react")]); + setupPackData("reg2", "react", "1.0.0"); + + expect(findRegistryPack("react", "reg2").registry).toBe("reg2"); + }); + + it("throws NotFoundError when no registry has the pack", () => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("other")]); + + expect(() => findRegistryPack("react")).toThrow(NotFoundError); + expect(() => findRegistryPack("react", "reg1")).toThrow(/not found in registry "reg1"/); + }); + + it("throws NotFoundError when no registries are configured", () => { + expect(() => findRegistryPack("react")).toThrow(/no pack registries are configured/); + }); + + it("throws NotFoundError for an unknown registry name", () => { + saveRegistries([makeEntry("reg1")]); + expect(() => findRegistryPack("react", "nope")).toThrow(NotFoundError); + }); + + it("throws NotFoundError when the version is not in the local copy", () => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("react")]); + setupPackData("reg1", "react", "1.0.0"); + + expect(() => findRegistryPack("react@9.9.9")).toThrow(NotFoundError); + }); + + it("mentions registries that were never synced", () => { + saveRegistries([makeEntry("reg1"), makeEntry("unsynced")]); + setupRegistry("reg1", []); + + expect(() => findRegistryPack("react")).toThrow(/"unsynced" has never been synced/); + }); + + it("rejects path traversal in names and versions", () => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("react", { latestVersion: "../../etc" })]); + + expect(() => findRegistryPack("../react")).toThrow(ValidationError); + expect(() => findRegistryPack("react@../x")).toThrow(ValidationError); + expect(() => findRegistryPack("react")).toThrow(ValidationError); + }); + }); + + describe("resolveRegistryPack (checksums)", () => { + beforeEach(() => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("p")]); + }); + + it("resolves when the file matches the recorded checksum, also for gzip data", async () => { + setupPackData("reg1", "p", "1.0.0", { data: gzipSync(packJson("p", "1.0.0")) }); + await expect(resolveRegistryPack("p")).resolves.toMatchObject({ name: "p" }); + }); + + it("throws when the file was changed after publishing", async () => { + const dataPath = setupPackData("reg1", "p", "1.0.0"); + writeFileSync(dataPath, '{"tampered":true}'); + await expect(resolveRegistryPack("p")).rejects.toThrow(/Checksum verification failed/); + }); + + it("throws when the manifest has no checksum for the version", async () => { + setupPackData("reg1", "p", "1.0.0", { checksum: "" }); + await expect(resolveRegistryPack("p")).rejects.toThrow(/has no checksum recorded/); + }); + + it("skips verification without a manifest or a manifest entry for the version", async () => { + setupPackData("reg1", "p", "1.0.0", { checksum: null }); + await expect(resolveRegistryPack("p")).resolves.toMatchObject({ version: "1.0.0" }); + + setupPackData("reg1", "p", "1.0.0", { manifestVersion: "2.0.0", checksum: "abc" }); + await expect(verifyResolvedPackChecksum(findRegistryPack("p"))).resolves.toBeUndefined(); + }); + }); +}); diff --git a/tests/unit/registry/search.test.ts b/tests/unit/registry/search.test.ts index d89da2a1..42c6507b 100644 --- a/tests/unit/registry/search.test.ts +++ b/tests/unit/registry/search.test.ts @@ -4,9 +4,10 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { initLogger } from "../../../src/logger.js"; +import { NotFoundError } from "../../../src/errors.js"; import type { RegistryEntry, PackSummary } from "../../../src/registry/types.js"; -// Mock homedir before importing any registry modules — REGISTRIES_DIR is module-level +// Registry paths follow homedir(): point it at a temp directory. let tempHome: string = join(tmpdir(), `libscope-search-test-${process.pid}`); mkdirSync(tempHome, { recursive: true }); @@ -19,7 +20,7 @@ vi.mock("node:os", async (importOriginal) => { }); // Import AFTER mock is set up — getRegistryCacheDir picks up mocked homedir -const { searchRegistries } = await import("../../../src/registry/search.js"); +const { searchRegistries, listRegistryPacks } = await import("../../../src/registry/search.js"); const { saveRegistries } = await import("../../../src/registry/config.js"); const { getRegistryCacheDir } = await import("../../../src/registry/types.js"); const { clearIndexCache } = await import("../../../src/registry/git.js"); @@ -28,8 +29,6 @@ function makeEntry(name: string, overrides: Partial<RegistryEntry> = {}): Regist return { name, url: "https://github.com/org/registry.git", - syncInterval: 3600, - priority: 1, lastSyncedAt: new Date().toISOString(), ...overrides, }; @@ -88,7 +87,7 @@ describe("registry search", () => { const { results } = searchRegistries("react-docs"); expect(results).toHaveLength(1); - expect(results[0]!.pack.name).toBe("react-docs"); + expect(results[0]!.name).toBe("react-docs"); // 100 (exact name) + 20 (description contains "react-docs" via default desc) expect(results[0]!.score).toBeGreaterThanOrEqual(100); }); @@ -144,7 +143,7 @@ describe("registry search", () => { const { results } = searchRegistries("docs"); expect(results).toHaveLength(2); - expect(results.map((r) => r.pack.name).sort((a, b) => a.localeCompare(b))).toEqual([ + expect(results.map((r) => r.name).sort((a, b) => a.localeCompare(b))).toEqual([ "react-docs", "vue-docs", ]); @@ -160,7 +159,7 @@ describe("registry search", () => { const { results } = searchRegistries("react"); // "react" has exact name match (100) + more → higher score // "react-docs" has partial name match (50) + less - expect(results[0]!.pack.name).toBe("react"); + expect(results[0]!.name).toBe("react"); expect(results[0]!.score).toBeGreaterThan(results[1]!.score); }); @@ -179,14 +178,21 @@ describe("registry search", () => { const { results } = searchRegistries("docs", { registryName: "reg1" }); expect(results).toHaveLength(1); - expect(results[0]!.registryName).toBe("reg1"); + expect(results[0]!.registry).toBe("reg1"); }); - it("should warn when specified registryName does not exist", () => { - const { results, warnings } = searchRegistries("test", { registryName: "nonexistent" }); - expect(results).toEqual([]); - expect(warnings).toHaveLength(1); - expect(warnings[0]).toContain("not found"); + it("throws NotFoundError when the named registry does not exist", () => { + expect(() => searchRegistries("test", { registryName: "nonexistent" })).toThrow(NotFoundError); + }); + + it("lists every pack of every registry with listRegistryPacks", () => { + saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); + setupRegistry("reg1", [makePack("a"), makePack("b")]); + setupRegistry("reg2", [makePack("c")]); + const { packs, warnings } = listRegistryPacks(); + expect(packs.map((p) => `${p.registry}/${p.name}`)).toEqual(["reg1/a", "reg1/b", "reg2/c"]); + expect(warnings).toEqual([]); + expect(listRegistryPacks("reg2").packs).toHaveLength(1); }); it("should include registryName in results", () => { @@ -194,7 +200,7 @@ describe("registry search", () => { setupRegistry("my-registry", [makePack("test-pack")]); const { results } = searchRegistries("test"); - expect(results[0]!.registryName).toBe("my-registry"); + expect(results[0]!.registry).toBe("my-registry"); }); it("should handle corrupted index.json gracefully", () => { diff --git a/tests/unit/registry/stale-cache.test.ts b/tests/unit/registry/stale-cache.test.ts deleted file mode 100644 index 834215ef..00000000 --- a/tests/unit/registry/stale-cache.test.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import type { RegistryEntry } from "../../../src/registry/types.js"; -import { isRegistryStale } from "../../../src/registry/sync.js"; - -function makeEntry(overrides: Partial<RegistryEntry> = {}): RegistryEntry { - return { - name: "test-reg", - url: "https://github.com/org/registry.git", - syncInterval: 3600, // 1 hour - priority: 1, - lastSyncedAt: null, - ...overrides, - }; -} - -describe("registry stale-cache detection", () => { - beforeEach(() => { - vi.useFakeTimers(); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - describe("isRegistryStale", () => { - it("should return true when lastSyncedAt is null (never synced)", () => { - const entry = makeEntry({ lastSyncedAt: null }); - expect(isRegistryStale(entry)).toBe(true); - }); - - it("should return false when syncInterval is 0 (manual only)", () => { - const entry = makeEntry({ syncInterval: 0, lastSyncedAt: null }); - expect(isRegistryStale(entry)).toBe(false); - }); - - it("should return false when syncInterval is negative", () => { - const entry = makeEntry({ syncInterval: -1, lastSyncedAt: null }); - expect(isRegistryStale(entry)).toBe(false); - }); - - it("should return true when last sync was longer ago than syncInterval", () => { - const now = new Date("2026-03-11T12:00:00.000Z"); - vi.setSystemTime(now); - - // Last synced 2 hours ago, interval is 1 hour - const entry = makeEntry({ - syncInterval: 3600, - lastSyncedAt: "2026-03-11T10:00:00.000Z", - }); - expect(isRegistryStale(entry)).toBe(true); - }); - - it("should return false when last sync was within syncInterval", () => { - const now = new Date("2026-03-11T12:00:00.000Z"); - vi.setSystemTime(now); - - // Last synced 30 minutes ago, interval is 1 hour - const entry = makeEntry({ - syncInterval: 3600, - lastSyncedAt: "2026-03-11T11:30:00.000Z", - }); - expect(isRegistryStale(entry)).toBe(false); - }); - - it("should return true at exactly the boundary (1 ms past interval)", () => { - // syncInterval = 60 seconds = 60000ms - const entry = makeEntry({ syncInterval: 60 }); - const baseTime = new Date("2026-03-11T12:00:00.000Z"); - entry.lastSyncedAt = baseTime.toISOString(); - - // Set time to 60001ms later (1ms past the boundary) - vi.setSystemTime(new Date(baseTime.getTime() + 60001)); - expect(isRegistryStale(entry)).toBe(true); - }); - - it("should return false at exactly the boundary (exactly syncInterval)", () => { - const entry = makeEntry({ syncInterval: 60 }); - const baseTime = new Date("2026-03-11T12:00:00.000Z"); - entry.lastSyncedAt = baseTime.toISOString(); - - // Set time to exactly 60000ms later - vi.setSystemTime(new Date(baseTime.getTime() + 60000)); - expect(isRegistryStale(entry)).toBe(false); - }); - - it("should handle very short syncInterval (1 second)", () => { - const entry = makeEntry({ syncInterval: 1 }); - const baseTime = new Date("2026-03-11T12:00:00.000Z"); - entry.lastSyncedAt = baseTime.toISOString(); - - vi.setSystemTime(new Date(baseTime.getTime() + 2000)); - expect(isRegistryStale(entry)).toBe(true); - }); - - it("should handle very large syncInterval (24 hours)", () => { - const entry = makeEntry({ syncInterval: 86400 }); - const now = new Date("2026-03-11T12:00:00.000Z"); - vi.setSystemTime(now); - - // Synced 12 hours ago — still fresh - entry.lastSyncedAt = "2026-03-11T00:00:00.000Z"; - expect(isRegistryStale(entry)).toBe(false); - - // Synced 25 hours ago — stale - entry.lastSyncedAt = "2026-03-10T11:00:00.000Z"; - expect(isRegistryStale(entry)).toBe(true); - }); - }); -}); diff --git a/tests/unit/registry/sync.test.ts b/tests/unit/registry/sync.test.ts index 0d4edc50..2b9cd97d 100644 --- a/tests/unit/registry/sync.test.ts +++ b/tests/unit/registry/sync.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdirSync, writeFileSync, rmSync, mkdtempSync, existsSync } from "node:fs"; +import { mkdirSync, writeFileSync, rmSync, mkdtempSync, existsSync, renameSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; @@ -18,13 +18,8 @@ vi.mock("node:os", async (importOriginal) => { }; }); -const { - syncRegistry, - syncAllRegistries, - syncStaleRegistries, - syncRegistryByName, - getRegistryIndex, -} = await import("../../../src/registry/sync.js"); +const { syncRegistry, syncAllRegistries, localPackCount } = + await import("../../../src/registry/sync.js"); const { loadRegistries, saveRegistries } = await import("../../../src/registry/config.js"); function makeEntry( @@ -32,16 +27,18 @@ function makeEntry( url: string, overrides: Partial<RegistryEntry> = {}, ): RegistryEntry { - return { - name, - url, - syncInterval: 3600, - priority: 1, - lastSyncedAt: null, - ...overrides, - }; + return { name, url, lastSyncedAt: null, ...overrides }; } +const PACK: PackSummary = { + name: "test-pack", + description: "Test", + tags: [], + latestVersion: "1.0.0", + author: "a", + updatedAt: "2026-01-01", +}; + function addTestRegistry(entry: RegistryEntry): void { const registries = loadRegistries(); registries.push(entry); @@ -80,18 +77,33 @@ describe("registry sync functions", () => { rmSync(tempDir, { recursive: true, force: true }); }); - describe("syncRegistryByName", () => { - it("should return error for non-existent registry name", async () => { - const status = await syncRegistryByName("nonexistent"); - expect(status.status).toBe("error"); - expect(status.error).toContain("not found"); + describe("syncRegistry", () => { + it("clones the registry and reports its pack count", async () => { + const bareRepo = createBareRepo(tempDir, [PACK]); + addTestRegistry(makeEntry("by-name", bareRepo)); + expect(localPackCount("by-name")).toBeNull(); + + const status = await syncRegistry(makeEntry("by-name", bareRepo)); + expect(status).toMatchObject({ registry: "by-name", status: "success", packs: 1 }); + expect(loadRegistries()[0]!.lastSyncedAt).not.toBeNull(); }); - it("should sync an existing registry by name", async () => { - const bareRepo = createBareRepo(tempDir); - addTestRegistry(makeEntry("by-name", bareRepo)); - const status = await syncRegistryByName("by-name"); - expect(status.status).toBe("success"); + it("keeps the local copy when the remote is unreachable (offline)", async () => { + const repo = createBareRepo(tempDir, [PACK]); + const entry = makeEntry("offline-idx", repo); + addTestRegistry(entry); + await syncRegistry(entry); + + renameSync(repo, repo + ".broken"); + const status = await syncRegistry(entry); + expect(status).toMatchObject({ status: "offline", packs: 1 }); + expect(status.error).toBeTruthy(); + }); + + it("reports error when the registry was never cloned and cannot be reached", async () => { + const entry = makeEntry("never", join(tempDir, "missing.git")); + addTestRegistry(entry); + expect(await syncRegistry(entry)).toMatchObject({ status: "error", packs: null }); }); }); @@ -113,102 +125,6 @@ describe("registry sync functions", () => { }); }); - describe("syncStaleRegistries", () => { - it("should return empty array when no registries are stale", async () => { - const repo = createBareRepo(tempDir); - addTestRegistry( - makeEntry("fresh", repo, { - syncInterval: 99999, - lastSyncedAt: new Date().toISOString(), - }), - ); - const results = await syncStaleRegistries(); - expect(results).toEqual([]); - }); - - it("should sync registries that are stale", async () => { - const repo = createBareRepo(tempDir); - addTestRegistry( - makeEntry("stale-one", repo, { - syncInterval: 1, - lastSyncedAt: "2020-01-01T00:00:00.000Z", // very old - }), - ); - const results = await syncStaleRegistries(); - expect(results).toHaveLength(1); - expect(results[0]!.status).toBe("success"); - }); - - it("should return empty when all registries have syncInterval=0 (manual)", async () => { - const repo = createBareRepo(tempDir); - addTestRegistry(makeEntry("manual", repo, { syncInterval: 0 })); - const results = await syncStaleRegistries(); - expect(results).toEqual([]); - }); - }); - - describe("getRegistryIndex", () => { - it("should return packs from a synced registry", async () => { - const packs: PackSummary[] = [ - { - name: "test-pack", - description: "Test", - tags: [], - latestVersion: "1.0.0", - author: "a", - updatedAt: "2026-01-01", - }, - ]; - const repo = createBareRepo(tempDir, packs); - addTestRegistry(makeEntry("idx-test", repo)); - await syncRegistry(makeEntry("idx-test", repo)); - - const entry = makeEntry("idx-test", repo, { - syncInterval: 0, // manual, won't auto-sync - lastSyncedAt: new Date().toISOString(), - }); - const { packs: result, warning } = await getRegistryIndex(entry); - expect(result).toHaveLength(1); - expect(result[0]!.name).toBe("test-pack"); - expect(warning).toBeUndefined(); - }); - - it("should return warning when remote unreachable and has stale cache", async () => { - const packs: PackSummary[] = [ - { - name: "offline-pack", - description: "Offline", - tags: [], - latestVersion: "1.0.0", - author: "a", - updatedAt: "2026-01-01", - }, - ]; - const repo = createBareRepo(tempDir, packs); - const entry = makeEntry("offline-idx", repo, { - syncInterval: 1, - lastSyncedAt: "2020-01-01T00:00:00.000Z", - }); - addTestRegistry(entry); - - // Sync once to populate cache - await syncRegistry(entry); - - // Break the remote - const { renameSync } = await import("node:fs"); - renameSync(repo, repo + ".broken"); - - // getRegistryIndex should fall back to cache with warning - const staleEntry = makeEntry("offline-idx", repo, { - syncInterval: 1, - lastSyncedAt: "2020-01-01T00:00:00.000Z", - }); - const { packs: result, warning } = await getRegistryIndex(staleEntry); - expect(result).toHaveLength(1); - expect(warning).toContain("unreachable"); - }); - }); - // Robustness fix: sync locking prevents concurrent syncs describe("sync locking", () => { it("should create a lock file during sync and remove it when done", async () => { diff --git a/tests/unit/registry/types.test.ts b/tests/unit/registry/types.test.ts index e5e0c6f9..8a081a37 100644 --- a/tests/unit/registry/types.test.ts +++ b/tests/unit/registry/types.test.ts @@ -15,8 +15,6 @@ import type { RegistryEntry, PackSummary, PackManifest, - RegistryConfigBlock, - ConflictResolution, RegistrySyncStatus, } from "../../../src/registry/types.js"; @@ -73,12 +71,9 @@ describe("registry types — type shape validation", () => { const entry: RegistryEntry = { name: "official", url: "https://github.com/org/registry.git", - syncInterval: 3600, - priority: 1, lastSyncedAt: null, }; expect(entry.name).toBe("official"); - expect(entry.syncInterval).toBe(3600); expect(entry.lastSyncedAt).toBeNull(); }); @@ -116,25 +111,9 @@ describe("registry types — type shape validation", () => { expect(manifest.versions[0]!.docCount).toBe(5); }); - it("RegistryConfigBlock should wrap registries array", () => { - const block: RegistryConfigBlock = { - registries: [], - }; - expect(block.registries).toEqual([]); - }); - - it("ConflictResolution should support 'priority', 'interactive', and 'explicit' strategies", () => { - const byPriority: ConflictResolution = { strategy: "priority" }; - const interactive: ConflictResolution = { strategy: "interactive" }; - const explicit: ConflictResolution = { strategy: "explicit", registryName: "official" }; - expect(byPriority.strategy).toBe("priority"); - expect(interactive.strategy).toBe("interactive"); - expect(explicit.strategy).toBe("explicit"); - }); - it("RegistrySyncStatus should support all status values", () => { - const statuses: RegistrySyncStatus["status"][] = ["syncing", "success", "error", "offline"]; - expect(statuses).toHaveLength(4); + const statuses: RegistrySyncStatus["status"][] = ["success", "error", "offline"]; + expect(statuses).toHaveLength(3); }); }); From 7e700fd0eb1f514deaa716f6b7bdb5db056db45c Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:48:05 +0000 Subject: [PATCH 57/83] docs(packs): one registry system; migration table for packs and registries --- README.md | 12 +- docs/guide/architecture.md | 2 +- docs/guide/knowledge-packs.md | 21 ++-- docs/guide/pack-registries.md | 56 +++++----- docs/migration-v2.md | 56 ++++++++++ docs/reference/cli.md | 87 ++++++++------- docs/reference/mcp-tools.md | 18 +-- docs/reference/registry.md | 205 +++++++++++++++++----------------- docs/reference/rest-api.md | 14 ++- 9 files changed, 269 insertions(+), 202 deletions(-) diff --git a/README.md b/README.md index ec5038ff..c5342a4e 100644 --- a/README.md +++ b/README.md @@ -467,13 +467,13 @@ libscope pack install react-docs@1.2.0 # specific version libscope registry create ./my-registry # Publish a pack file to your registry -libscope registry publish ./my-pack.json -r my-registry --pack-version 1.0.0 +libscope registry publish ./my-pack.json --registry my-registry --pack-version 1.0.0 # Submit a pack to someone else's registry (creates a feature branch) -libscope registry publish ./my-pack.json -r community --submit +libscope registry publish ./my-pack.json --registry community --submit ``` -Authentication is delegated to git — SSH keys and HTTPS credential helpers work automatically. Registries cache locally and support offline index lookups. See the [Pack Registries guide](/guide/pack-registries) for full details. +Authentication is delegated to git — SSH keys and HTTPS credential helpers work automatically. Search and install read the local copies of the registries; `libscope registry sync` updates them. See the [Pack Registries guide](/guide/pack-registries) for full details. There's also a web dashboard at `http://localhost:3377` when you run `libscope serve`, with search, document browsing, topic navigation, and a knowledge graph visualization at `/graph`. @@ -565,10 +565,10 @@ There's also a web dashboard at `http://localhost:3377` when you run `libscope s | `libscope registry remove <name>` | Unregister a registry | | `libscope registry list` | List configured registries | | `libscope registry sync [<name>]` | Sync one or all registries | -| `libscope registry search <query> [-r <name>]` | Search registry pack indexes | +| `libscope registry search <query> [--registry <name>]` | Search registry pack indexes | | `libscope registry create <path>` | Initialize a new registry repo | -| `libscope registry publish <file> -r <name>` | Publish a pack file to a registry | -| `libscope registry unpublish <pack> -r <name>` | Remove a pack version from a registry | +| `libscope registry publish <file> --registry <name>` | Publish a pack file to a registry | +| `libscope registry unpublish <name@version> --registry <name>` | Remove a pack version from a registry | **Utilities** diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 8f2e11db..77ea0cb8 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -116,7 +116,7 @@ src/ │ ├── types.ts # RegistryEntry, PackSummary, PackManifest │ ├── config.ts # registry list in ~/.libscope/config.json │ ├── git.ts # git clone/pull/commit/push -│ ├── sync.ts # registry syncing with auto-sync intervals +│ ├── sync.ts # registry syncing (on request only) │ ├── search.ts # pack search across registries │ ├── resolve.ts # pack resolution (version conflicts) │ ├── publish.ts # publishing packs to registries diff --git a/docs/guide/knowledge-packs.md b/docs/guide/knowledge-packs.md index edb97644..5306dad3 100644 --- a/docs/guide/knowledge-packs.md +++ b/docs/guide/knowledge-packs.md @@ -1,24 +1,27 @@ # Knowledge Packs -Knowledge packs are portable collections of documents you can share, install, and version. Think of them like npm packages, but for documentation. +Knowledge packs are portable collections of documents you can share, install, and version. Think of them like npm packages, but for documentation. A pack is a `.json` file, or a gzip-compressed `.json.gz` file. ## Installing Packs ```bash # From a local file libscope pack install ./react-docs.json +libscope pack install ./react-docs.json.gz -# From a registry -libscope pack install react-docs --registry https://registry.example.com +# From a pack registry (see Pack Registries) +libscope pack install react-docs +libscope pack install react-docs@1.2.0 +libscope pack install react-docs --registry official # List installed packs libscope pack list -# List available packs from a registry +# List the packs available in your registries libscope pack list --available ``` -When you install a pack, its documents get indexed into your knowledge base just like any other content. They show up in search results and can be queried via RAG. +Pack names are looked up in the git [pack registries](/guide/pack-registries) you added with `libscope registry add`. When you install a pack, its documents get indexed into your knowledge base just like any other content. They show up in search results and can be queried via RAG. ## Creating Packs @@ -33,7 +36,7 @@ libscope pack create \ --author "team" ``` -This creates a JSON file containing the documents, their metadata, and topic assignments. Share it with your team, commit it to a repo, or publish it to a registry. +This creates a JSON file containing the documents, their metadata, and topic assignments. Share it with your team, commit it to a repo, or [publish it to a registry](/guide/pack-registries#publishing-packs). ## Removing Packs @@ -45,9 +48,9 @@ This removes the pack's documents from your knowledge base. ## MCP Usage -Packs are also available via MCP tools: +Packs are also available in the MCP admin toolset (see [MCP tools](/reference/mcp-tools)): -- `install-pack` — install from registry or file path -- `list-packs` — list installed or available packs +- `install-pack` — install a pack by name (or `name@version`) from the configured registries. Local pack files can be installed only with the CLI or the Node.js API. +- `list-packs` — list installed packs, or with `available: true` the packs in the configured registries Your AI assistant can install packs directly when it needs documentation for a specific library. diff --git a/docs/guide/pack-registries.md b/docs/guide/pack-registries.md index 13b58ee4..74a7cbed 100644 --- a/docs/guide/pack-registries.md +++ b/docs/guide/pack-registries.md @@ -7,16 +7,16 @@ Authentication is handled entirely by git. If you have SSH keys or an HTTPS cred ## Adding a Registry ```bash -# Add a public registry +# Add a public registry (the name "libscope-registry" comes from the URL) libscope registry add https://github.com/org/libscope-registry.git -# Add with a custom alias and priority -libscope registry add git@github.com:team/internal-packs.git --name team-packs --priority 5 +# Add with a custom name +libscope registry add git@github.com:team/internal-packs.git --name team-packs -# Add with auto-sync every 24 hours -libscope registry add https://github.com/org/registry.git --sync-interval 86400 +# Add a registry on a local or shared disk +libscope registry add file:///srv/libscope-packs.git -# Add without cloning immediately +# Add without cloning now libscope registry add https://github.com/org/registry.git --no-sync # List configured registries @@ -26,7 +26,7 @@ libscope registry list libscope registry remove team-packs ``` -On first add, libscope clones the registry's index locally to `~/.libscope/registries/<name>/`. Subsequent syncs fetch only changes. +When you add a registry, libscope clones it to `~/.libscope/registries/<name>/`. Later syncs fetch only changes. ## Searching Registries @@ -35,33 +35,36 @@ On first add, libscope clones the registry's index locally to `~/.libscope/regis libscope registry search "react" # Search a specific registry -libscope registry search "react" -r official +libscope registry search "react" --registry official ``` Results show the pack name, description, tags, latest version, and which registry it came from. ## Installing Packs from a Registry -The existing `pack install` command now resolves packs from your configured registries: +`pack install` finds pack names in your configured registries: ```bash # Install the latest version libscope pack install react-docs # Install a specific version -libscope pack install react-docs --pack-version 1.2.0 -# or libscope pack install react-docs@1.2.0 -# Install from a specific registry (skips conflict resolution) +# Install from a specific registry libscope pack install react-docs --registry official + +# List the packs in all registries +libscope pack list --available ``` -If multiple registries contain a pack with the same name, libscope resolves the conflict by priority (lower `priority` value wins). You can override this with `--registry <name>`. +If more than one registry has a pack with the same name, the install fails and the error names the registries. Run it again with `--registry <name>`. + +Before it installs a pack, libscope verifies the pack file against the checksum in the registry. ### Offline Behavior -If a registry is unreachable during install, libscope falls back to the cached index with a warning. If the registry has never been synced, it tells you to run `libscope registry sync` when online. +Searching, listing and installing read only the local copies of the registries. They do not use the network. A registry that was never synced is skipped with a warning that tells you to run `libscope registry sync`. ## Syncing Registries @@ -73,7 +76,7 @@ libscope registry sync libscope registry sync official ``` -Registries also auto-sync when the local cache is older than the configured `syncInterval` (in seconds). This happens automatically before pack installs when the cache is stale. +Registries are synced only when you run `registry sync` (or `registry add`). If a registry cannot be reached, libscope keeps its local copy and reports it as offline. ## Creating Your Own Registry @@ -89,18 +92,16 @@ This creates a git repo with the correct folder structure (`index.json`, `packs/ ```bash # Publish a pack file to a registry you own -libscope registry publish ./my-pack.json -r my-registry --pack-version 1.0.0 +libscope registry publish ./my-pack.json --registry my-registry --pack-version 1.0.0 -# Auto-bump patch version (from latest in registry) -libscope registry publish ./my-pack.json -r my-registry +# Publish the next patch version (from the latest in the registry) +libscope registry publish ./my-pack.json.gz --registry my-registry # Submit a pack to someone else's registry (creates a feature branch) -libscope registry publish ./my-pack.json -r community --submit +libscope registry publish ./my-pack.json --registry community --submit # Unpublish a specific version -libscope registry unpublish my-pack -r my-registry --pack-version 1.0.0 -# or -libscope registry unpublish my-pack@1.0.0 -r my-registry +libscope registry unpublish my-pack@1.0.0 --registry my-registry ``` Publishing assembles the pack into the registry's folder structure, generates a SHA-256 checksum, updates `index.json` and `pack.json`, and commits + pushes. The `--submit` flag pushes to a `feature/add-<pack-name>` branch instead — you then create a pull request manually. @@ -111,13 +112,12 @@ Every published pack version includes a `checksum.sha256` file. On install, libs ## Versioning -Pack versions follow [semver](https://semver.org/) (e.g. `1.0.0`, `1.2.3`). When you publish without `--pack-version`, the patch version is auto-bumped from the latest. Old versions are preserved in the registry. `pack install` defaults to the latest version unless you specify one. - -## MCP Usage +Pack versions follow [semver](https://semver.org/) (e.g. `1.0.0`, `1.2.3`). When you publish without `--pack-version`, the patch version is auto-bumped from the latest. Old versions are preserved in the registry. `pack install` defaults to the latest version unless you specify one with `name@version`. -Your AI assistant can also work with registries through MCP: +## Other Interfaces -- `install-pack` — install from a registry by name -- `list-packs --available` — browse packs available in registries +- **Node.js API:** `scope.registries.list()`, `.add()`, `.remove()`, `.sync()`, `.search()`, `.create()`, `.publish()`, `.unpublish()`, and `scope.packs.install({ pack, registry })`. +- **REST API:** `GET /api/v1/registries`, `GET /api/v1/registries/search`, `GET /api/v1/packs?available=true` and `POST /api/v1/packs` (read-only access to registries; add, sync and publish with the CLI or the Node.js API). +- **MCP (admin toolset):** `install-pack` installs from a registry by name, and `list-packs` with `available: true` browses the packs in the registries. See the [Registry Reference](/reference/registry) for complete schema details, configuration format, and all CLI flags. diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 14b36656..629ff931 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -382,3 +382,59 @@ New in 2.0: `doctor`, `docs rate`, `topics delete`, `admin prune`, `connect docs | `connect <type>` always started from scratch. | `connect <type>` creates or updates a saved connection (only the given settings change) and syncs it. | | `import-backup` did not ask for confirmation. | `admin restore` asks; add `-y` to skip the question. | | Import progress was printed on stdout. | Progress is one line on stderr, only on a terminal. | + +## Packs and registries + +LibScope 1.x had two pack registry systems: a URL registry (one JSON file, by default `https://raw.githubusercontent.com/libscope/packs/main/registry.json`) and git registries (`libscope registry add`). LibScope 2.0 keeps only the git registries. `--registry <name>` (`registry` in the SDK, REST and MCP) always means the name of a configured git registry. See the [Pack Registries guide](guide/pack-registries.md). + +### Removed: the URL registry + +| 1.x | 2.0 | +| ---------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| `pack install <name> --registry <url>` | `registry add <git-url>`, then `pack install <name>` | +| `pack list --available --registry <url>` | `pack list --available [--registry <name>]` (packs in the configured git registries) | +| `pack install <name>` without git registries fetched `<name>.json` from the URL registry | Fails with "no pack registries are configured"; add a registry first | +| `install-pack` and `list-packs` input `registryUrl` (MCP, REST, SDK) | `registry` (a registry name) | +| `listAvailablePacks(registryUrl)`, `PackInfo` (`src/core/packs.ts`) | `listRegistryPacks(registryName?)` (`src/registry/search.ts`), `RegistryPack` | +| `installPack(db, provider, nameOrPath, { registryUrl })` | `installPack(db, provider, packFile)`; resolve names with `resolveRegistryPack(spec, registry?)` | + +### Removed and renamed options + +| 1.x option | 2.0 | +| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| `pack install --from-registry <name>` | `pack install --registry <name>` | +| `pack install --pack-version <v>` | `pack install <name>@<v>` | +| `pack install -y` (with several registries, use the highest priority) | Removed. A pack in several registries is an error that names them; use `--registry <name>` | +| Interactive registry choice in `pack install` | Removed (same error) | +| `registry add --priority <n>`, config field `registries[].priority` | Removed (no priority order; the field is ignored) | +| `registry add --sync-interval <s>`, config field `registries[].syncInterval` | Removed. It never triggered a sync; run `registry sync` | +| `registry search -r <name>`, `registry publish -r <name>`, `registry unpublish -r <name>` | `--registry <name>` | +| `registry unpublish <pack> --pack-version <v>` | `registry unpublish <pack>@<v>` | + +### Behavior changes + +| 1.x | 2.0 | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| MCP `install-pack` accepted a local `.json`/`.json.gz` path. | Local pack files are installed only from the CLI and the SDK (MCP and REST refuse them). MCP and REST install by name. | +| A pack in several registries was resolved by priority or a prompt. | It is a `ValidationError` that names the registries. | +| An unknown registry, pack or version was a `ValidationError` or `null`. | It is a `NotFoundError` (404 over REST). | +| `registry add` accepted only `https://`, `ssh://` and `git@host:path`. | `file:///` URLs (a repository on a local or shared disk) are also accepted. | +| `registry` commands exited with their own messages. | They call the registry operations and print errors like every other command. | +| `pack list --available` read the URL registry over the network. | It reads the local copies of the git registries. Only `registry add`, `sync`, `publish` and `unpublish` use the network. | + +### New operations + +`list-registries`, `add-registry`, `remove-registry`, `sync-registries`, `search-registries`, `create-registry`, `publish-pack` and `unpublish-pack`. In the SDK they are `scope.registries.list()`, `.add()`, `.remove()`, `.sync()`, `.search()`, `.create()`, `.publish()` and `.unpublish()`. Over REST, only the read-only ones have routes: `GET /api/v1/registries` and `GET /api/v1/registries/search`. They are not MCP tools. + +### Library code + +| 1.x | 2.0 | +| ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| `findPackInRegistries`, `resolvePackFromRegistries` (`src/registry/resolve.ts`) | `findRegistryPack(spec, registry?)`, `resolveRegistryPack(spec, registry?)` | +| `RegistryConflict`, `ConflictResolution`, `RegistryConfigBlock` types | Removed | +| `syncRegistryByName`, `syncStaleRegistries`, `getRegistryIndex`, `isRegistryStale` (`src/registry/sync.ts`) | `syncRegistry(requireRegistry(name))`, `syncAllRegistries()`, `listRegistryPacks()` | +| `checkGitAvailable` (`src/registry/git.ts`) | Removed: a missing git binary is a `ConfigError` from any git call | +| `REGISTRIES_DIR` constant | `getRegistriesDir()` (follows the current home directory) | +| `RegistrySearchResult` `{ registryName, pack, score }` | `{ ...pack, registry, score }` | +| `RegistrySyncStatus.registryName`, status `"syncing"` | `registry`; statuses `success`, `offline`, `error`; new field `packs` | +| `ResolvedPack` `{ registryName, registryUrl, packName, version, dataPath }` | `{ registry, name, version, dataPath }` | diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a2d61ba8..247ddd84 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -245,81 +245,82 @@ Running `connect` again for a saved connection changes only the settings you giv ## Knowledge packs -| Command | Description | -| -------------------------------------------- | -------------------------------------------------------------------------------- | -| `pack install <nameOrPath>` | Install from a git registry, the pack registry URL, or a `.json`/`.json.gz` file | -| `pack remove <name> [-y]` | Remove a pack and its documents | -| `pack list [--available] [--registry <url>]` | Installed packs, or packs in the registry | -| `pack create --name <name> [options]` | Create a pack from indexed documents, or from files and URLs (`--from`) | +| Command | Description | +| --------------------------------------------- | --------------------------------------------------------------------------------------------- | +| `pack install <pack> [--registry <name>]` | Install `name` or `name@version` from the configured registries, or a `.json`/`.json.gz` file | +| `pack remove <name> [-y]` | Remove a pack and its documents | +| `pack list [--available] [--registry <name>]` | Installed packs, or the packs in the configured registries | +| `pack create --name <name> [options]` | Create a pack from indexed documents, or from files and URLs (`--from`) | ```bash -libscope pack install react-docs # latest from any configured git registry +libscope pack install react-docs # latest version libscope pack install react-docs@1.2.0 -libscope pack install react-docs --from-registry official +libscope pack install react-docs --registry official # when several registries have react-docs libscope pack install ./react-docs.json.gz libscope pack create --name team-docs --topic engineering libscope pack create --name react-docs --from ./react/docs --exclude "*.min.js" ``` -`pack install` options: `--registry <url>` (URL registry), `--from-registry <name>` and `--pack-version <semver>` (git registries), `-y` (with several registries, use the highest priority instead of asking), `--batch-size <n>`, `--concurrency <n>`, `--resume-from <n>`. +`pack install` options: `--registry <name>`, `--batch-size <n>`, `--concurrency <n>`, `--resume-from <n>`. Pack names are looked up in the local copies of the registries; run `libscope registry sync` to update them. When more than one registry has the pack, the command fails and names the registries: choose one with `--registry`. `pack create` options: `--from <sources...>`, `--topic`, `--pack-version`, `--description`, `--author`, `--license`, `--output <path>` (default `<name>.json`, or `<name>.json.gz` with `--from`), `--extensions <exts>`, `--exclude <globs...>`, `--no-recursive`. ## Pack registries -| Command | Description | -| ----------------------------------------------------------------- | ------------------------------------------------------- | -| `libscope registry add <url> [-n <alias>]` | Register a git repo as a pack registry | -| `libscope registry remove <name> [-y]` | Unregister a registry | -| `libscope registry list` | List configured registries | -| `libscope registry sync [<name>]` | Sync one or all registries | -| `libscope registry search <query> [-r <name>]` | Search registry pack indexes | -| `libscope registry create <path>` | Initialize a new registry repo | -| `libscope registry publish <file> -r <name>` | Publish a pack file to a registry | -| `libscope registry unpublish <pack> -r <name> --pack-version <v>` | Remove a pack version from a registry (or `<pack>@<v>`) | +A registry is a git repository of packs. `--registry <name>` always means the name of a configured registry. + +| Command | Description | +| -------------------------------------------------------------- | ----------------------------------------------- | +| `libscope registry add <url> [-n <name>] [--no-sync]` | Add a git repository as a registry and clone it | +| `libscope registry remove <name> [-y]` | Remove a registry and its local clone | +| `libscope registry list` | List the configured registries | +| `libscope registry sync [<name>]` | Fetch the latest packs of one or all registries | +| `libscope registry search <query> [--registry <name>]` | Search the packs in the registries | +| `libscope registry create <path>` | Create an empty registry repository | +| `libscope registry publish <file> --registry <name>` | Publish a pack file to a registry | +| `libscope registry unpublish <name@version> --registry <name>` | Remove one version of a pack from a registry | ### `libscope registry add` ```bash libscope registry add https://github.com/org/registry.git -libscope registry add git@github.com:team/packs.git --name team --priority 5 -libscope registry add https://github.com/org/registry.git --sync-interval 86400 --no-sync +libscope registry add git@github.com:team/packs.git --name team +libscope registry add file:///srv/libscope-packs.git --no-sync ``` -| Option | Description | -| --------------------------- | --------------------------------------------------------- | -| `-n, --name <alias>` | Short name for this registry (default: inferred from URL) | -| `--priority <n>` | Conflict resolution priority — lower wins (default: 10) | -| `--sync-interval <seconds>` | Auto-sync interval in seconds, 0 = manual (default: 0) | -| `--no-sync` | Skip initial sync after adding | +| Option | Description | +| ------------------- | ------------------------------------------------- | +| `-n, --name <name>` | Registry name (default: the last part of the URL) | +| `--no-sync` | Do not clone the registry now | + +URLs can use `https://`, `ssh://`, `git@host:path` or `file:///`. ### `libscope registry publish` ```bash -libscope registry publish ./my-pack.json -r my-registry --pack-version 1.0.0 -libscope registry publish ./my-pack.json -r my-registry # auto-bump patch version -libscope registry publish ./my-pack.json -r community --submit # push to a feature branch +libscope registry publish ./my-pack.json --registry my-registry --pack-version 1.0.0 +libscope registry publish ./my-pack.json.gz --registry my-registry # next patch version +libscope registry publish ./my-pack.json --registry community --submit # push to a feature branch ``` -| Option | Description | -| ------------------------- | ------------------------------------------------ | -| `-r, --registry <name>` | Target registry (required) | -| `--pack-version <semver>` | Version to publish as (default: auto-bump patch) | -| `-m, --message <msg>` | Git commit message | -| `--submit` | Push to a feature branch instead of main | +| Option | Description | +| ------------------------- | ----------------------------------------------------------------------- | +| `--registry <name>` | Target registry (required) | +| `--pack-version <semver>` | Version to publish (default: next patch version, or the pack's version) | +| `-m, --message <msg>` | Git commit message | +| `--submit` | Push to a feature branch instead of the main branch | ### `libscope registry unpublish` ```bash -libscope registry unpublish my-pack -r my-registry --pack-version 1.0.0 +libscope registry unpublish my-pack@1.0.0 --registry my-registry ``` -| Option | Description | -| ------------------------- | --------------------------------------------------------------- | -| `-r, --registry <name>` | Target registry (required) | -| `--pack-version <semver>` | Version to remove (required unless given as `<pack>@<version>`) | -| `-m, --message <msg>` | Git commit message | -| `-y, --yes` | Skip confirmation prompt | +| Option | Description | +| --------------------- | -------------------------- | +| `--registry <name>` | Target registry (required) | +| `-m, --message <msg>` | Git commit message | +| `-y, --yes` | Skip confirmation prompt | `libscope registry create ./my-registry` creates a git repo with the registry folder structure. See the [Registry Reference](/reference/registry). diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 235a8fbb..21afb3c1 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -245,28 +245,30 @@ Annotations: not destructive. ### install-pack -Install a knowledge pack from the registry or a local .json/.json.gz file +Install a knowledge pack from a registry (name or name@version) or a local file + +Packs are looked up in the local copies of the configured registries (run sync-registries to update them). Local .json/.json.gz files are accepted from the CLI and the Node.js API only. Annotations: not destructive, idempotent. | Parameter | Type | Required | Description | | ------------- | ------- | -------- | -------------------------------------------------------------------------------------------------------- | -| `pack` | string | yes | Pack name from the registry, or a local .json/.json.gz file | -| `registryUrl` | string | | Registry URL (default: the public pack registry) | +| `pack` | string | yes | Pack name or name@version from a registry, or a local .json/.json.gz file | +| `registry` | string | | Look only in this registry (needed when several registries have the pack) | | `batchSize` | integer | | Documents per batch (default 10) | | `concurrency` | integer | | Batches embedded in parallel (default 4) | | `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"} Default: `false`. | ### list-packs -List installed packs, or packs available in the registry +List installed packs, or the packs available in the configured registries Annotations: read-only. -| Parameter | Type | Required | Description | -| ------------- | ------- | -------- | --------------------------------------------------------------- | -| `available` | boolean | | List registry packs instead of installed ones Default: `false`. | -| `registryUrl` | string | | Registry URL (default: the public pack registry) | +| Parameter | Type | Required | Description | +| ----------- | ------- | -------- | --------------------------------------------------------------------------------------- | +| `available` | boolean | | List the packs in the configured registries instead of installed ones Default: `false`. | +| `registry` | string | | With available: only this registry | ### reindex-documents diff --git a/docs/reference/registry.md b/docs/reference/registry.md index 7310b10e..97584c31 100644 --- a/docs/reference/registry.md +++ b/docs/reference/registry.md @@ -1,81 +1,79 @@ # Registry Reference -Complete reference for the git-based pack registry feature. +Complete reference for git-based pack registries. A registry is a git repository of knowledge packs; `--registry <name>` always means the name of a configured registry. ## CLI Commands ### `libscope registry add` -Register a git repository as a pack registry. +Add a git repository as a pack registry and clone it. ```bash libscope registry add <git-url> [options] ``` -| Option | Description | -| ----------------------------- | -------------------------------------------------------- | -| `<git-url>` | Git clone URL (HTTPS or SSH) | -| `-n, --name <alias>` | Short name for this registry (default: inferred from URL)| -| `--priority <n>` | Priority for conflict resolution — lower wins (default: 10) | -| `--sync-interval <seconds>` | Auto-sync interval in seconds, 0 = manual only (default: 0) | -| `--no-sync` | Skip the initial sync after adding | +| Option | Description | +| ------------------- | ------------------------------------------------------------ | +| `<git-url>` | Git URL: `https://`, `ssh://`, `git@host:path` or `file:///` | +| `-n, --name <name>` | Registry name (default: the last part of the URL) | +| `--no-sync` | Do not clone the registry now | ```bash # Examples libscope registry add https://github.com/org/registry.git -libscope registry add git@github.com:team/packs.git --name team --priority 5 -libscope registry add https://github.com/org/registry.git --sync-interval 86400 +libscope registry add git@github.com:team/packs.git --name team +libscope registry add file:///srv/libscope-packs.git --no-sync ``` ### `libscope registry remove` -Unregister a registry and delete its local cache. +Remove a registry and delete its local clone. ```bash libscope registry remove <name> [-y, --yes] ``` -| Option | Description | -| ------------ | -------------------------- | -| `-y, --yes` | Skip confirmation prompt | +| Option | Description | +| ----------- | ------------------------ | +| `-y, --yes` | Skip confirmation prompt | ### `libscope registry list` -List all configured registries with their sync status. +List the configured registries. ```bash libscope registry list ``` -Output includes: name, URL, priority, pack count, and last synced timestamp. +Output includes: name, URL, pack count, and last synced timestamp. ### `libscope registry sync` -Manually sync one or all registries (git fetch + fast-forward). +Fetch the latest packs of one or all registries (git fetch + reset to the remote branch). This is the only command that updates the local copies. ```bash libscope registry sync [<name>] ``` -Without a name, syncs all registries. With a name, syncs only that registry. +Without a name, syncs all registries. With a name, syncs only that registry. A registry that cannot be reached keeps its local copy (`offline`); one that was never cloned reports an error and the command exits with code 1. ### `libscope registry search` -Search across cached registry indexes. +Search the local copies of the registry indexes. ```bash -libscope registry search <query> [-r, --registry <name>] +libscope registry search <query> [--registry <name>] ``` -| Option | Description | -| --------------------- | ---------------------------------------------- | -| `<query>` | Search term (matches name, description, tags, author) | -| `-r, --registry <name>` | Limit search to a specific registry | +| Option | Description | +| ------------------- | ----------------------------------------------------- | +| `<query>` | Search term (matches name, description, tags, author) | +| `--registry <name>` | Search only this registry | ```bash # Examples libscope registry search "react" -libscope registry search "kubernetes" -r official +libscope registry search "kubernetes" --registry official ``` ### `libscope registry create` @@ -87,34 +85,37 @@ libscope registry create <path> ``` Creates a git repo with: + - `index.json` — empty pack index (JSON array) - `packs/` — directory for pack contents (with `.gitkeep`) - An initial commit ### `libscope registry publish` -Publish a pack file to a registry. +Publish a pack file (`.json` or `.json.gz`) to a registry. ```bash -libscope registry publish <packFile> -r <name> [options] +libscope registry publish <file> --registry <name> [options] ``` -| Option | Description | -| ----------------------- | -------------------------------------------------------- | -| `<packFile>` | Path to the pack `.json` file to publish | -| `-r, --registry <name>` | Target registry (required) | -| `--pack-version <semver>` | Version to publish as (default: auto-bump patch) | -| `-m, --message <msg>` | Git commit message | -| `--submit` | Push to a feature branch instead of main (for PR workflow) | +| Option | Description | +| ------------------------- | ----------------------------------------------------------------------- | +| `<file>` | Path to the pack `.json` or `.json.gz` file to publish | +| `--registry <name>` | Target registry (required) | +| `--pack-version <semver>` | Version to publish (default: next patch version, or the pack's version) | +| `-m, --message <msg>` | Git commit message | +| `--submit` | Push to a feature branch instead of main (for PR workflow) | **Direct publish** (you have write access): + ```bash -libscope registry publish ./react-docs.json -r my-registry --pack-version 1.0.0 +libscope registry publish ./react-docs.json --registry my-registry --pack-version 1.0.0 ``` **Submit for inclusion** (you don't have write access): + ```bash -libscope registry publish ./react-docs.json -r community --submit +libscope registry publish ./react-docs.json --registry community --submit ``` The `--submit` flag creates a `feature/add-<pack-name>` branch and pushes it. You then create a pull request manually. @@ -124,44 +125,58 @@ The `--submit` flag creates a `feature/add-<pack-name>` branch and pushes it. Yo Remove a specific pack version from a registry. ```bash -libscope registry unpublish <packName>[@<semver>] -r <name> [--pack-version <semver>] [options] +libscope registry unpublish <name>@<version> --registry <name> [options] ``` -| Option | Description | -| ----------------------- | --------------------------------- | -| `<packName>` | Name of the pack to unpublish | -| `-r, --registry <name>` | Target registry (required) | -| `--pack-version <semver>` | Version to remove (required unless given as `<packName>@<semver>`) | -| `-m, --message <msg>` | Git commit message | -| `-y, --yes` | Skip confirmation prompt | +| Option | Description | +| --------------------- | ----------------------------- | +| `<name>@<version>` | Pack and version to unpublish | +| `--registry <name>` | Target registry (required) | +| `-m, --message <msg>` | Git commit message | +| `-y, --yes` | Skip confirmation prompt | If the last version of a pack is unpublished, the entire pack is removed from the registry index. -### `libscope pack install` (extended) +### `libscope pack install` -The existing `pack install` command is extended to resolve packs from registries. +`pack install` resolves pack names from the configured registries. ```bash -libscope pack install <name> [--pack-version <semver>] [--registry <name>] +libscope pack install <name>[@<version>] [--registry <name>] ``` -| Option | Description | -| -------------------- | ---------------------------------------------- | -| `--pack-version <semver>` | Install a specific version (default: latest) | -| `--registry <name>` | Install from a specific registry | +| Option | Description | +| ------------------- | -------------------------------------------------------------- | +| `--registry <name>` | Look only in this registry (needed when several have the pack) | ```bash -# Install latest from any registry +# Install latest from the registry that has the pack libscope pack install react-docs # Install specific version libscope pack install react-docs@1.2.0 -libscope pack install react-docs --pack-version 1.2.0 # Install from a specific registry libscope pack install react-docs --registry official ``` +`libscope pack list --available [--registry <name>]` lists the packs in the registries. + +### Other interfaces + +| Operation | Node.js API | REST | MCP (admin toolset) | +| ------------------- | ------------------------------ | ---------------------------------- | ------------------- | +| `list-registries` | `scope.registries.list()` | `GET /api/v1/registries` | — | +| `search-registries` | `scope.registries.search()` | `GET /api/v1/registries/search` | — | +| `add-registry` | `scope.registries.add()` | — | — | +| `remove-registry` | `scope.registries.remove()` | — | — | +| `sync-registries` | `scope.registries.sync()` | — | — | +| `create-registry` | `scope.registries.create()` | — | — | +| `publish-pack` | `scope.registries.publish()` | — | — | +| `unpublish-pack` | `scope.registries.unpublish()` | — | — | +| `install-pack` | `scope.packs.install()` | `POST /api/v1/packs` | `install-pack` | +| `list-packs` | `scope.packs.list()` | `GET /api/v1/packs?available=true` | `list-packs` | + --- ## Registry Folder Structure @@ -209,14 +224,14 @@ A JSON array of pack summaries for fast search without traversing subdirectories ] ``` -| Field | Type | Description | -| ---------------- | ---------- | ---------------------------------------- | -| `name` | `string` | Pack name (unique within the registry) | -| `description` | `string` | One-line description | -| `tags` | `string[]` | Tags/categories for search filtering | -| `latestVersion` | `string` | Latest published semver version | -| `author` | `string` | Author name or handle | -| `updatedAt` | `string` | ISO-8601 timestamp of last publish | +| Field | Type | Description | +| --------------- | ---------- | -------------------------------------- | +| `name` | `string` | Pack name (unique within the registry) | +| `description` | `string` | One-line description | +| `tags` | `string[]` | Tags/categories for search filtering | +| `latestVersion` | `string` | Latest published semver version | +| `author` | `string` | Author name or handle | +| `updatedAt` | `string` | ISO-8601 timestamp of last publish | ## Schema: `pack.json` @@ -248,18 +263,18 @@ Per-pack manifest with full metadata and version history. } ``` -| Field | Type | Description | -| ------------------------ | ---------- | ---------------------------------------------- | -| `name` | `string` | Pack name | -| `description` | `string` | One-line description | -| `tags` | `string[]` | Tags/categories | -| `author` | `string` | Author name or handle | -| `license` | `string` | License identifier (e.g. "MIT") | -| `versions[].version` | `string` | Semver version string | -| `versions[].publishedAt` | `string` | ISO-8601 publish timestamp | -| `versions[].checksumPath`| `string` | Relative path to the checksum file | -| `versions[].checksum` | `string` | SHA-256 checksum hex value | -| `versions[].docCount` | `number` | Number of documents in this version | +| Field | Type | Description | +| ------------------------- | ---------- | ----------------------------------- | +| `name` | `string` | Pack name | +| `description` | `string` | One-line description | +| `tags` | `string[]` | Tags/categories | +| `author` | `string` | Author name or handle | +| `license` | `string` | License identifier (e.g. "MIT") | +| `versions[].version` | `string` | Semver version string | +| `versions[].publishedAt` | `string` | ISO-8601 publish timestamp | +| `versions[].checksumPath` | `string` | Relative path to the checksum file | +| `versions[].checksum` | `string` | SHA-256 checksum hex value | +| `versions[].docCount` | `number` | Number of documents in this version | Versions are ordered newest first. @@ -275,28 +290,22 @@ Registries are stored in `~/.libscope/config.json` under the `registries` key: { "name": "official", "url": "git@github.com:org/libscope-registry.git", - "syncInterval": 86400, - "priority": 10, "lastSyncedAt": "2026-03-10T14:30:00Z" }, { "name": "team", "url": "https://github.com/team/internal-packs.git", - "syncInterval": 0, - "priority": 5, "lastSyncedAt": null } ] } ``` -| Field | Type | Description | Default | -| -------------- | ----------------- | -------------------------------------------------- | ------- | -| `name` | `string` | Local alias for the registry | — | -| `url` | `string` | Git clone URL (HTTPS or SSH) | — | -| `syncInterval` | `number` | Auto-sync interval in seconds (0 = manual only) | `0` | -| `priority` | `number` | Conflict resolution priority — lower wins | `10` | -| `lastSyncedAt` | `string \| null` | ISO-8601 timestamp of last sync, null if never | `null` | +| Field | Type | Description | Default | +| -------------- | ---------------- | -------------------------------------------------------------- | ------- | +| `name` | `string` | Local name for the registry (`/^[a-zA-Z0-9_-]+$/`, 2-64 chars) | — | +| `url` | `string` | Git URL: `https://`, `ssh://`, `git@host:path` or `file:///` | — | +| `lastSyncedAt` | `string \| null` | ISO-8601 timestamp of last sync, null if never | `null` | You can edit this file directly or use `libscope registry add/remove`. @@ -315,15 +324,13 @@ To test access: `git ls-remote <registry-url>`. If that works, libscope will too ## Offline Behavior -Registries cache their index locally at `~/.libscope/registries/<name>/`. - -| Scenario | Behavior | -| ------------------------------------- | ------------------------------------------------------------------------------------------ | -| Registry unreachable, cache exists | Uses cached index with warning: "Registry '\<name\>' is unreachable. Using cached index from \<date\>." | -| Registry unreachable, no cache | Fails with: "Registry '\<name\>' has never been synced and is unreachable." | -| Cache stale, registry reachable | Auto-syncs before proceeding | +Each registry is cloned to `~/.libscope/registries/<name>/`. Searching, listing (`pack list --available`) and installing read only this local copy; they never use the network. The network is used only by `registry add`, `registry sync`, `registry publish` and `registry unpublish`. -Pack content downloads still require network access — only the index lookup can work offline. +| Scenario | Behavior | +| -------------------------------------------------------- | --------------------------------------------------------------------------- | +| `registry sync`, registry unreachable, local copy exists | Status `offline`: the local copy is kept and used | +| `registry sync`, registry unreachable, no local copy | Status `error` and exit code 1 | +| Search, list or install, registry never synced | The registry is skipped with a warning: run `libscope registry sync <name>` | --- @@ -342,17 +349,11 @@ Pack versions follow [semver](https://semver.org/): - Versions must be valid semver strings (e.g. `1.0.0`, `2.3.1`) - `pack install <name>` installs the latest version -- `pack install <name>@1.0.0` or `--pack-version 1.0.0` installs a specific version +- `pack install <name>@1.0.0` installs a specific version - Old versions are preserved in the registry — publishing a new version does not remove previous ones - When publishing without `--pack-version`, the patch version is auto-bumped from the latest - The `latestVersion` in `index.json` always points to the most recently published version -## Conflict Resolution - -When multiple registries contain a pack with the same name: - -- **Priority-based** (default): the registry with the lowest `priority` value wins -- **Explicit**: use `--registry <name>` to specify which registry to use -- **Interactive**: when running in a terminal without `--registry`, libscope prompts you to choose +## Packs in Several Registries -In non-interactive / CI mode, conflicts without `--registry` fail with an actionable error. +When more than one configured registry has a pack with the same name, `pack install <name>` fails with an error that names the registries. Choose one with `--registry <name>` (`registry` in the Node.js, REST and MCP interfaces). diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index b7ed864a..3a7f6b68 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -104,11 +104,15 @@ The parameters of each route are in the OpenAPI document (`GET /openapi.json`). ### Packs -| Method | Path | Operation | Description | -| -------- | --------------------- | -------------- | ------------------------------------------------------------------------------------------------ | -| `POST` | `/api/v1/packs` | `install-pack` | Install a knowledge pack from the registry or a local .json/.json.gz file. Runs as a task (202). | -| `DELETE` | `/api/v1/packs/:pack` | `remove-pack` | Remove an installed pack and its documents. | -| `GET` | `/api/v1/packs` | `list-packs` | List installed packs, or packs available in the registry. | +| Method | Path | Operation | Description | +| -------- | --------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `POST` | `/api/v1/packs` | `install-pack` | Install a knowledge pack from a registry (`pack`: name or name@version; optional `registry`). Runs as a task (202). | +| `DELETE` | `/api/v1/packs/:pack` | `remove-pack` | Remove an installed pack and its documents. | +| `GET` | `/api/v1/packs` | `list-packs` | List installed packs, or with `available=true` the packs in the configured registries (optional `registry`). | +| `GET` | `/api/v1/registries` | `list-registries` | List the configured pack registries. | +| `GET` | `/api/v1/registries/search` | `search-registries` | Search the packs in the configured registries (`query`, optional `registry`). | + +Registries are read from the local copies under `~/.libscope/registries/`. Adding, removing, syncing and publishing to registries is done with the CLI or the SDK. ### Connectors From 60b00b842887593682afaf64d4009a8c872cef6d Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:48:31 +0000 Subject: [PATCH 58/83] fix(docker)!: run the REST API with valid 2.0 commands and keep data in the volume - CMD serve api on 0.0.0.0:3378 (was serve = MCP on stdio); healthcheck uses /api/v1/health (stats was removed); EXPOSE/compose port 3378. - HOME=/data so ~/.libscope (config, workspaces) is in the volume; the unused LIBSCOPE_DATA_DIR is removed. - node:22-bookworm-slim base and sharp rebuilt: the local embedding model did not load on Alpine (onnxruntime-node needs glibc). - .dockerignore excludes sdk/. - Docs: serve --api/--dashboard -> serve api/dashboard; migration notes for the client SDKs, saved search filters and the Docker image. --- .dockerignore | 3 +- Dockerfile | 32 +++++++++++-------- docker-compose.yml | 5 ++- docs/guide/connectors.md | 2 +- docs/guide/dashboard.md | 12 ++++---- docs/migration-v2.md | 63 ++++++++++++++++++++++++++++++++++++++ docs/reference/rest-api.md | 2 +- 7 files changed, 94 insertions(+), 25 deletions(-) diff --git a/.dockerignore b/.dockerignore index c4997d6d..b7b551a8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,5 @@ dist *.md tests coverage -sdk-python -sdk-go +sdk .github diff --git a/Dockerfile b/Dockerfile index a8242a76..e4ed9f77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ +# Debian (glibc) base: the local embedding model (onnxruntime-node) does not run on Alpine (musl). + # Build stage -FROM node:22-alpine AS builder +FROM node:22-bookworm-slim AS builder WORKDIR /app COPY package*.json ./ RUN npm ci @@ -7,21 +9,27 @@ COPY . . RUN npm run build # Production stage -FROM node:22-alpine AS production +FROM node:22-bookworm-slim AS production WORKDIR /app -RUN apk add --no-cache python3 make g++ && \ - addgroup -g 1001 libscope && \ - adduser -u 1001 -G libscope -s /bin/sh -D libscope +RUN groupadd -g 1001 libscope && \ + useradd -u 1001 -g libscope -s /bin/sh -M libscope && \ + mkdir /data && chown libscope:libscope /data COPY package*.json ./ -RUN npm ci --omit=dev --ignore-scripts && \ - npm rebuild better-sqlite3 && \ - apk del python3 make g++ +RUN apt-get update && \ + apt-get install -y --no-install-recommends python3 make g++ && \ + npm ci --omit=dev --ignore-scripts && \ + npm rebuild better-sqlite3 sharp && \ + npm cache clean --force && \ + apt-get purge -y python3 make g++ && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* COPY --from=builder /app/dist ./dist USER libscope -EXPOSE 3420 3421 -ENV LIBSCOPE_DATA_DIR=/data +# Config, secrets, connections and workspace databases live in $HOME/.libscope. +ENV HOME=/data VOLUME ["/data"] +EXPOSE 3378 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s \ - CMD node -e "fetch('http://localhost:3420/api/v1/stats').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))" + CMD node -e "fetch('http://localhost:3378/api/v1/health').then(r => process.exit(r.status < 500 ? 0 : 1)).catch(() => process.exit(1))" ENTRYPOINT ["node", "dist/cli/index.js"] -CMD ["serve"] +CMD ["serve", "api", "--host", "0.0.0.0", "--port", "3378"] diff --git a/docker-compose.yml b/docker-compose.yml index 720cc75e..911be0e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,13 +2,12 @@ services: libscope: build: . ports: - - "3420:3420" # API - - "3421:3421" # Web UI + - "3378:3378" # REST API volumes: - libscope-data:/data environment: - - LIBSCOPE_DATA_DIR=/data - OPENAI_API_KEY=${OPENAI_API_KEY:-} + - LIBSCOPE_API_KEY=${LIBSCOPE_API_KEY:-} restart: unless-stopped # Optional: Ollama for local embeddings diff --git a/docs/guide/connectors.md b/docs/guide/connectors.md index 3bdf9e5c..2188339f 100644 --- a/docs/guide/connectors.md +++ b/docs/guide/connectors.md @@ -24,7 +24,7 @@ Each file also stores `connectorType` and, after `libscope schedule set`, a `sch `--sync` on any connect command re-runs the saved config (`--name` selects which one). Other options on the command line are ignored with `--sync`. After each successful sync, `lastSync` is set to the time the sync started. -To sync a saved config on a schedule, set a cron expression. Schedules run while the API server is running (`libscope serve --api`): +To sync a saved config on a schedule, set a cron expression. Schedules run while the API server is running (`libscope serve api`): ```bash libscope schedule set notion "0 */6 * * *" diff --git a/docs/guide/dashboard.md b/docs/guide/dashboard.md index 06a07dea..b19cac28 100644 --- a/docs/guide/dashboard.md +++ b/docs/guide/dashboard.md @@ -5,17 +5,17 @@ LibScope includes a local web dashboard for browsing, searching, and managing yo ## Starting the Dashboard ```bash -libscope serve --dashboard +libscope serve dashboard ``` This starts an HTTP server at `http://localhost:3377` by default. ```bash # Use a custom port -libscope serve --dashboard --port 8080 +libscope serve dashboard --port 8080 # Bind to all interfaces (for LAN access) -libscope serve --dashboard --host 0.0.0.0 --port 3377 +libscope serve dashboard --host 0.0.0.0 --port 3377 ``` The dashboard page needs no external files. The knowledge graph page (`/graph`) loads d3 from `d3js.org`. @@ -75,7 +75,7 @@ The dashboard pages read their data from a small JSON API on the same server. Ea | `GET /api/search?q&limit&topic` | `search` | | `GET /api/graph?threshold&maxNodes&topic&tag` | `graph` | -These URLs are for the dashboard pages. For your own scripts, use the [REST API](../reference/rest-api.md) (`libscope serve --api`), which has every operation, API-key authentication and an OpenAPI document. +These URLs are for the dashboard pages. For your own scripts, use the [REST API](../reference/rest-api.md) (`libscope serve api`), which has every operation, API-key authentication and an OpenAPI document. ## Rate Limiting @@ -87,10 +87,10 @@ The dashboard, the REST API and the MCP server are separate servers. You can sta ```bash # Dashboard in one terminal -libscope serve --dashboard --port 3377 +libscope serve dashboard --port 3377 # REST API in another -libscope serve --api --port 3378 +libscope serve api --port 3378 # MCP server (or configure it in your AI client) libscope serve diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 14b36656..1f3cba76 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -382,3 +382,66 @@ New in 2.0: `doctor`, `docs rate`, `topics delete`, `admin prune`, `connect docs | `connect <type>` always started from scratch. | `connect <type>` creates or updates a saved connection (only the given settings change) and syncs it. | | `import-backup` did not ask for confirmation. | `admin restore` asks; add `-y` to skip the question. | | Import progress was printed on stdout. | Progress is one line on stderr, only on a terminal. | + +## Client SDKs + +The Python (`pylibscope`) and Go (`sdk/go`) clients now call the 2.0 `/api/v1` routes. Methods that add documents or sync connections start a background task on the server and return a task at once. Wait for it with `wait_for_task` (Python) or `WaitForTask` (Go). Both poll `GET /api/v1/tasks/:taskId`. Both clients accept an API key (`api_key=` / `WithAPIKey`), sent as `Authorization: Bearer <key>`. + +### Python (`pylibscope`) + +| 1.x | 2.0 | +| --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `add_document(url, ...)` returned a `Document` | `add_url(url, *, spider, max_pages, ...)` returns a `Task`. Get the documents from `wait_for_task(task.id).result["documents"]`. | +| `add_text(title, content, ...)` returned a `Document` | `add_text(...)` returns a `Task` | +| `search(query, *, limit, topic, tags, min_score)` | `search(query, *, limit, offset, topic, library, version, source_type, tags, min_rating)` returns `Page[SearchHit]`. `min_score` is removed. Every tag in `tags` is sent (1.x sent only the first). | +| `SearchResult` (`results`, `total_count`) | `Page[SearchHit]` (`items`, `total`, `limit`, `offset`) | +| `get_document(doc_id)` returned a `Document` | `get_document(document_id)` returns a `DocumentView` (`document`, `content`, `tags`, `links`, `ratings`) | +| `list_documents(...)` returned `List[Document]` | `list_documents(...)` returns `Page[Document]` | +| `Document.id` | `Document.document_id` | +| `get_analytics()` and `Analytics` | `overview()` and `Overview` (`overview().stats.total_documents`) | +| `create_topic(name, *, parent_id)` | `create_topic(name, *, parent, description)` | +| `add_tags` and `remove_tags` returned `None` | Both return the document's tags. `remove_tags` now removes the tags (1.x sent a body that the server ignored). | +| `list_tags()` returned `List[str]` | `list_tags()` returns `List[Tag]` (`name`, `document_count`) | +| `get_graph(*, min_similarity)` | `get_graph(*, topic, tag, threshold, max_nodes)` | +| `ask(question, *, topic)` returned `AskResult(answer, sources)` | `ask(question, *, top_k, min_rating, topic, library, version, source_type, tags)` returns `AskResult` (`mode`, `answer` or `context_prompt`, `sources`) | +| `sync_connector(connector, **config)` and `SyncResult` | `sync(name)` and `sync_all()` return a `Task`. Save connections with `libscope connect <type>`. | +| `pylibscope.connectors.build_connector_config` | Removed | +| — | New: `get_task`, `cancel_task`, `wait_for_task`, `TaskFailedError`, `Tag`, `Overview`, `DocumentView`, `Page` | + +### Go (`github.com/RobertLD/libscope/sdk/go`) + +| 1.x | 2.0 | +| ---------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `AddDocument(ctx, url, opts...)` returned `*Document` | `AddURL(ctx, url, params...)` returns `*Task`. Call `WaitForTask`, then `Task.DecodeResult` into an `IngestResult`. | +| `AddText(ctx, title, content, opts...)` returned `*Document` | `AddText(ctx, title, content, params...)` returns `*Task` | +| `Search(...)` returned `*SearchResult` (`Results`, `Total`, `Query`) | `Search(...)` returns `*Page[SearchHit]` (`Items`, `Total`, `Limit`, `Offset`) | +| `SearchHit.Document`, `SearchHit.ChunkText` | `SearchHit.DocumentID`, `ChunkID`, `Title`, `Content` and the other document fields | +| `GetDocument` returned `*Document` | `GetDocument` returns `*DocumentView` | +| `ListDocuments` returned `[]Document` | `ListDocuments` returns `*Page[Document]` | +| `Document.ID`, `Document.Source` | `Document.DocumentID`, `Document.SourceType` | +| `GetStats(ctx)` and `Stats{TotalDocuments, DatabaseSize}` | `Overview(ctx)`. Counts are in `Overview.Stats` (`TotalDocuments`, `TotalChunks`, `TotalTopics`, `DatabaseSizeBytes`). | +| `HealthStatus.DocCount`, `HealthStatus.DBSize` | Removed. `Health` returns only `Status`. | +| `AddTagsToDocument(ctx, docID, tags)` returned `[]Tag` | `AddTags(ctx, id, tags)` returns the document's tags (`[]string`). New: `RemoveTags`. | +| Option types `SearchOption`, `AddDocumentOption`, `AddTextOption`, `ListOption`, `CreateTopicOption` | One `Param` type for all methods | +| `WithDocumentTopic`, `WithTextTopic`, `WithListTopic` | `WithTopic` | +| `WithDocumentTags`, `WithTextTags` | `WithTags` | +| `WithListLimit` | `WithLimit` | +| `WithTextSource` | `WithSourceType` | +| `WithParentID` | `WithParent` | +| `WithMinScore` (filtered on the client) | Removed. `WithMinRating` filters by document rating on the server. | +| `WithTags(...)` sent only the first tag | `WithTags(...)` sends every tag | +| — | New: `Ask`, `GetGraph`, `Sync`, `SyncAll`, `GetTask`, `CancelTask`, `WaitForTask`, `WithAPIKey`, `TaskError`. `errors.Is(err, ErrNotFound)` now works. | + +## Saved searches + +Saved search filters use the field names of the `search` operation: `sourceType` (was `source`), `topic`, `library`, `version`, `tags`, `minRating` and `limit`. Database migration 19 (schema version 19) renames `source` to `sourceType` in existing saved searches. A saved search runs the `search` operation with its stored filters. That operation ignores stored keys that it does not accept, for example `dateFrom` and `dateTo` (the 1.x REST API could store them). + +## Docker image + +| 1.x | 2.0 | +| -------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | +| `CMD ["serve"]` started the MCP server on stdio | `CMD ["serve", "api", "--host", "0.0.0.0", "--port", "3378"]` starts the REST API | +| Exposed ports 3420 and 3421 | Exposes port 3378 (REST API). docker-compose maps `3378:3378` and passes `LIBSCOPE_API_KEY`. | +| `LIBSCOPE_DATA_DIR=/data` (LibScope did not read it, so data was not stored in the volume) | `HOME=/data`. Config, secrets, connections and workspace databases are in `/data/.libscope`. | +| The health check called `GET /api/v1/stats` | The health check calls `GET /api/v1/health` | +| `node:22-alpine` base image: the local embedding model failed to load (onnxruntime-node needs glibc), so adding documents failed with the default config | `node:22-bookworm-slim` base image | diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index b7ed864a..242b80c4 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -3,7 +3,7 @@ Start the REST API server: ```bash -libscope serve --api --port 3378 +libscope serve api --port 3378 ``` Every route under `/api/v1` runs one LibScope operation: the same operation, with the same parameter names, defaults and validation, as the CLI, the MCP server and the SDK. The OpenAPI 3.1 document at `GET /openapi.json` is generated from the operations, so it always matches the server. From e88456beb5bbfc73013f23be41b47b734bb2cfa8 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:49:27 +0000 Subject: [PATCH 59/83] docs(migration): list-packs takes a registry name, not registryUrl --- docs/migration-v2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 629ff931..541cd64e 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -183,7 +183,7 @@ The MCP server now has 11 core tools and an optional admin toolset. Old tool nam | `get-related` | `search` with `relatedTo` | `chunkId` -> `relatedTo` (a chunk ID or a document ID). `minScore` and `includeLinkedDocuments` are removed. Use `version`, `sourceType`, `minRating` as filters. | | `ask-question` | `ask` | Added `version`, `sourceType`, `tags`, `minRating`. | | `health-check`, `list-topics` | `overview` | `list-topics` `parentId` is removed; `overview` lists every topic with its document count. | -| `list-packs` | `overview` (installed packs), or admin `list-packs` | Admin `list-packs` keeps `available` and `registryUrl`. | +| `list-packs` | `overview` (installed packs), or admin `list-packs` | Admin `list-packs` keeps `available`; `registryUrl` is replaced by `registry` (a git registry name). | | `get-document` | `get-document` | Added `offset` and `maxLength` for paging. The output includes tags, links (with `linkId`) and ratings. | | `get-document-links` | `get-document` | Links are part of the document output. | | `list-documents` | `list-documents` | Added `version`, `tags`, `offset`. `limit` max 1000. The output shows the total. | From c9a6d9fa17ab30ca9478f1beb9f945d74f94630d Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 18:59:33 +0000 Subject: [PATCH 60/83] docs: generate the CLI, MCP, config and REST references from code scripts/gen-docs.mjs fills the generated blocks in docs/reference from the built dist: the commander program, the MCP tools/list of createMcpServer (core and admin toolsets), getConfigKeyTable() and API_ROUTES. npm run docs:gen rewrites the blocks; npm run docs:check fails when one is out of date and runs in the CI build job. --- .github/workflows/ci.yml | 2 + docs/reference/cli.md | 788 ++++++++++++++++++++++++-------- docs/reference/configuration.md | 86 ++-- docs/reference/mcp-tools.md | 179 ++++---- docs/reference/rest-api.md | 170 +++---- package.json | 2 + scripts/gen-docs.mjs | 398 ++++++++++++++++ 7 files changed, 1200 insertions(+), 425 deletions(-) create mode 100644 scripts/gen-docs.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7eef0349..8fee8f0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,3 +72,5 @@ jobs: test -f dist/mcp/server.js test -f dist/cli/index.js test -f dist/core/index.js + - name: Check generated docs (run `npm run docs:gen` to update) + run: npm run docs:check diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 247ddd84..b52879bd 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -2,6 +2,8 @@ Every command is a thin wrapper over one LibScope operation, so the CLI, the MCP server, the REST API and the SDK use the same parameters and defaults. Upgrading from 1.x? See [Migrating to LibScope 2.0](/migration-v2#cli). +The usage lines and option tables on this page are generated from the CLI code (`npm run docs:gen`). `libscope <command> --help` prints the same text. + | Command | What it does | | ------------------------------------------------------------- | -------------------------------------------------------------- | | [`add`](#libscope-add) | Add files, directories, URLs (optionally crawled) or repos | @@ -23,12 +25,19 @@ Every command is a thin wrapper over one LibScope operation, so the CLI, the MCP These go before or after the command name. -| Option | Description | -| --------------------- | --------------------------------------------------------------- | -| `--json` | Print the operation result as JSON (for scripts) | -| `-v, --verbose` | Debug logging, and the stack trace when a command fails | -| `--log-level <level>` | `debug`, `info`, `warn`, `error` or `silent` (default `silent`) | -| `--workspace <name>` | Use this workspace instead of the active one | +<!-- generated:start cli:global --> +Usage: `libscope [options] <command>` + +| Option | Description | +| --------------------- | ------------------------------------------------------------------------------- | +| `-V, --version` | output the version number | +| `--json` | Print results as JSON | +| `-v, --verbose` | Debug logging, and stack traces on errors | +| `--log-level <level>` | Log level (default silent). One of: `debug`, `info`, `warn`, `error`, `silent`. | +| `--workspace <name>` | Use this workspace instead of the active one | + +`-h, --help` shows the help of the program or of any command. +<!-- generated:end cli:global --> Every result shows document IDs (and chunk IDs for search hits), so you can pass them to the next command. @@ -51,10 +60,6 @@ Commands that select documents (`search`, `ask`, `docs list`, `searches save`, ` ## `libscope add` -```bash -libscope add <sources...> [options] -``` - Each source can be a file, a directory, a URL or a GitHub/GitLab repository URL. LibScope detects which one it is. A local file's absolute path is stored as its URL, so adding a changed file again replaces the old version. ```bash @@ -67,40 +72,42 @@ libscope add https://github.com/org/repo --branch main --path docs libscope add ./docs --dry-run # list what would be added ``` -| Option | Applies to | Description | -| ------------------------ | ---------- | ---------------------------------------------------------------------------------------------------- | -| `--topic <topic>` | all | Topic ID or name (the topic must exist) | -| `--library <name>` | all | Library name | -| `--lib-version <ver>` | all | Library version | -| `--source-type <type>` | all | Default: `library` with `--library`, `topic` with `--topic`, else `manual` (repositories: `library`) | -| `--tags <a,b>` | all | Tags to add to every new document | -| `--title <title>` | file, URL | Title (default: file name or page title) | -| `--format <ext>` | files | Read files as this format, e.g. `.pdf` | -| `--dedup <mode>` | all | `skip`, `warn` or `force` | -| `--expires <time>` | all | ISO 8601 time after which `admin prune` deletes the documents | -| `--include <globs>` | directory | Only files matching these globs (comma-separated) | -| `--exclude <globs>` | directory | Skip files matching these globs (comma-separated) | -| `--watch` | directory | Keep running and re-index files when they change | -| `--spider` | URL | Also crawl linked pages | -| `--max-pages <n>` | crawl | Page limit (default 25, max 200) | -| `--max-depth <n>` | crawl | Link depth (default 2, max 5) | -| `--no-same-domain` | crawl | Also follow links to other domains | -| `--path-prefix <path>` | crawl | Only follow links under this path | -| `--exclude-urls <globs>` | crawl | Skip URLs matching these globs (comma-separated) | -| `--branch <name>` | repository | Branch (default: from the URL, else `main`) | -| `--path <dirs>` | repository | Only these subdirectories (comma-separated) | -| `--extensions <exts>` | repository | File extensions (default `.md,.mdx,.txt,.rst`) | -| `--token <token>` | repository | Access token for a private repository | -| `--dry-run` | all | List what would be added without adding it | +<!-- generated:start cli:add --> +Usage: `libscope add <sources...>` + +Add files, directories, URLs or GitHub/GitLab repository URLs to the knowledge base. + +| Option | Description | +| ------------------------- | ------------------------------------------------------------- | +| `--topic <topic>` | Topic (ID or name) | +| `--library <name>` | Library name | +| `--lib-version <version>` | Library version | +| `--source-type <type>` | Source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Tags to add (comma-separated) | +| `--title <title>` | Title (one file or URL; default: detected) | +| `--format <ext>` | Read files as this format, e.g. .pdf | +| `--dedup <mode>` | Duplicate handling: skip, warn or force | +| `--expires <time>` | ISO 8601 time after which `admin prune` deletes the documents | +| `--include <globs>` | Directory: only files matching these globs (comma-separated) | +| `--exclude <globs>` | Directory: skip files matching these globs (comma-separated) | +| `--watch` | Directory: keep running and re-index files when they change | +| `--spider` | URL: also crawl linked pages | +| `--max-pages <n>` | Crawl: page limit (default 25, max 200) | +| `--max-depth <n>` | Crawl: link depth (default 2, max 5) | +| `--no-same-domain` | Crawl: also follow links to other domains | +| `--path-prefix <path>` | Crawl: only follow links under this path | +| `--exclude-urls <globs>` | Crawl: skip URLs matching these globs (comma-separated) | +| `--branch <name>` | Repository: branch (default: from the URL, else main) | +| `--path <dirs>` | Repository: only these subdirectories (comma-separated) | +| `--extensions <exts>` | Repository: file extensions (default .md,.mdx,.txt,.rst) | +| `--token <token>` | Repository: access token for a private repository | +| `--dry-run` | List what would be added without adding it | +<!-- generated:end cli:add --> With `--json`, `add` prints one result per source: `{ kind, documents: [{ documentId, title, chunkCount, source }], errors, skipped, planned?, crawl? }`. ## `libscope search` -```bash -libscope search [query] [options] -``` - ```bash libscope search "authentication best practices" --library my-lib -n 10 libscope search "deploy process" --context 1 # show neighbouring chunks @@ -108,15 +115,25 @@ libscope search --related <documentId|chunkId> # content similar to a document libscope search # interactive search (on a terminal) ``` -Takes the [document filters](#document-filters) and: - -| Option | Description | -| ------------------- | ------------------------------------------------------ | -| `--related <id>` | Find content similar to this document or chunk ID | -| `--offset <n>` | Results to skip (paging) | -| `--min-rating <n>` | Only documents with at least this average rating (1-5) | -| `--max-per-doc <n>` | At most this many chunks per document | -| `--context <n>` | Neighbouring chunks to show around each result (0-2) | +<!-- generated:start cli:search --> +Usage: `libscope search [query]` + +Search by meaning and keywords. With no query on a terminal, start interactive search. + +| Option | Description | +| ------------------------- | --------------------------------------------------------------- | +| `--related <id>` | Find content similar to this document or chunk ID instead | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--lib-version <version>` | Only this library version | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `-n, --limit <n>` | Maximum results | +| `--offset <n>` | Results to skip (paging) | +| `--min-rating <n>` | Only documents rated at least this (1-5) | +| `--max-per-doc <n>` | At most this many chunks per document | +| `--context <n>` | Neighbouring chunks to show around each result (0-2) | +<!-- generated:end cli:search --> With no query and no `--related`, on a terminal, `search` starts interactive mode: type a query, see results, repeat. Type `quit`, `exit` or press Ctrl+D to leave. The filters given on the command line apply to every query. @@ -127,30 +144,135 @@ libscope ask "How do I configure OAuth2?" --library my-lib -n 8 libscope ask "What changed in v2?" --model gpt-4o-mini ``` -Takes the [document filters](#document-filters) (`-n` is the number of chunks used as context, default 5), `--min-rating <n>` and `--model <model>` (LLM model for this question). +<!-- generated:start cli:ask --> +Usage: `libscope ask <question>` + +Answer a question from the knowledge base with the configured LLM. + +| Option | Description | +| ------------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--lib-version <version>` | Only this library version | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `-n, --limit <n>` | Chunks to use as context (default 5) | +| `--min-rating <n>` | Only documents rated at least this (1-5) | +| `--model <model>` | LLM model for this question (default: llm.model) | +<!-- generated:end cli:ask --> `ask` needs an LLM. With `llm.provider` set to `auto` (the default), LibScope uses OpenAI when an OpenAI API key is set, else Anthropic when an Anthropic key is set, else Ollama when `llm.url` is set or the embedding provider is Ollama. Without an LLM, `ask` fails with a hint; `libscope doctor` shows what is resolved. With `llm.provider passthrough`, `ask` prints the retrieved context instead of an answer. ## `libscope docs` -| Command | Description | -| -------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -| `docs list [filters] [--offset <n>]` | List documents, newest first | -| `docs show <documentId> [--offset <n>] [--max-length <n>]` | Show a document with its tags, links and rating summary | -| `docs update <documentId> [options]` | Change title, content, metadata or tags | -| `docs delete <documentId> [-y]` | Delete a document with its chunks, links and ratings | -| `docs history <documentId>` | List saved versions | -| `docs rollback <documentId> <version>` | Restore a saved version (the current state is saved first) | -| `docs rate <documentId> <1-5> [--feedback <text>] [--correction <text>] [--chunk <chunkId>]` | Rate a document or one chunk | -| `docs link <documentId> <targetDocumentId> --type <type> [--label <text>]` | Link two documents (`see_also`, `prerequisite`, `supersedes`, `related`, `references`) | -| `docs unlink <linkId>` | Delete a link | -| `docs links [documentId] [--type <type>]` | Links of one document (both directions), or every link | -| `docs prereqs <documentId>` | Documents to read first, following prerequisite links | -| `docs tag <documentId> <tags...>` | Add tags (space- or comma-separated) | -| `docs untag <documentId> <tags...>` | Remove tags | -| `docs suggest-tags <documentId> [-n <n>]` | Suggest tags from the content | - -`docs update` options: `--title`, `--content <text>`, `--content-file <path>`, `--library`, `--lib-version`, `--url`, `--topic <topic>` and `--tags <a,b>` (replaces all tags). Changing the content re-chunks and re-embeds the document. +<!-- generated:start cli:docs --> +### `libscope docs list` + +List documents, newest first. + +| Option | Description | +| ------------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--lib-version <version>` | Only this library version | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `-n, --limit <n>` | Maximum results | +| `--offset <n>` | Documents to skip (paging) | + +### `libscope docs show <documentId>` + +Show a document with its tags, links and ratings. + +| Option | Description | +| ------------------ | -------------------------------------------- | +| `--offset <n>` | Start the content at this character | +| `--max-length <n>` | Show at most this many characters of content | + +### `libscope docs history <documentId>` + +List saved versions of a document. + +### `libscope docs update <documentId>` + +Change a document's title, content, metadata or tags. + +| Option | Description | +| ------------------------- | --------------------------------------------- | +| `--title <title>` | New title | +| `--content <text>` | New content (re-chunked and re-embedded) | +| `--content-file <path>` | Read the new content from a file | +| `--library <name>` | New library name | +| `--lib-version <version>` | New library version | +| `--url <url>` | New URL | +| `--topic <topic>` | New topic (ID or name) | +| `--tags <tags>` | Replace the tags with these (comma-separated) | + +### `libscope docs delete <documentId>` + +Delete a document with its chunks, links and ratings. + +| Option | Description | +| ----------- | --------------------------- | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope docs rollback <documentId> <version>` + +Restore a saved version (the current state is saved first). + +### `libscope docs rate <documentId> <rating>` + +Rate a document from 1 (poor) to 5 (excellent). + +| Option | Description | +| --------------------- | ------------------------------ | +| `--feedback <text>` | What is good or wrong | +| `--correction <text>` | Suggested replacement text | +| `--chunk <chunkId>` | Rate one chunk of the document | + +### `libscope docs link <documentId> <targetDocumentId>` + +Link one document to another. + +| Option | Description | +| ---------------- | ----------------------------------------------------------------------------- | +| `--type <type>` | Link type: see_also, prerequisite, supersedes, related, references. Required. | +| `--label <text>` | Short description of the relationship | + +### `libscope docs unlink <linkId>` + +Delete a link (link IDs are shown by `docs links`). + +### `libscope docs links [documentId]` + +List the links of a document, or every link. + +| Option | Description | +| --------------- | ---------------------------------------------------------------------------- | +| `--type <type>` | Only this link type: see_also, prerequisite, supersedes, related, references | + +### `libscope docs prereqs <documentId>` + +List the documents to read first (following prerequisite links). + +### `libscope docs tag <documentId> <tags...>` + +Add tags to a document (space- or comma-separated). + +### `libscope docs untag <documentId> <tags...>` + +Remove tags from a document. + +### `libscope docs suggest-tags <documentId>` + +Suggest tags from a document's content. + +| Option | Description | +| ----------------- | ------------------- | +| `-n, --limit <n>` | Maximum suggestions | +<!-- generated:end cli:docs --> + +`docs update --tags` replaces all tags. Changing the content re-chunks and re-embeds the document. ```bash libscope docs update <documentId> --title "New Title" --tags guide,v2 @@ -159,23 +281,74 @@ libscope docs update <documentId> --content-file ./updated.md ## Topics and tags -| Command | Description | -| ---------------------------------------------------------------- | ------------------------------------------------ | -| `topics list [--parent <topic>]` | Topics with document counts | -| `topics create <name> [--description <text>] [--parent <topic>]` | Create a topic | -| `topics delete <topic> [--delete-documents] [-y]` | Delete a topic (documents are kept unless asked) | -| `tags list` | Tags with document counts | +<!-- generated:start cli:topics,tags --> +### `libscope topics list` + +List topics with their document counts. + +| Option | Description | +| ------------------ | ------------------------------------------------ | +| `--parent <topic>` | Only direct subtopics of this topic (ID or name) | + +### `libscope topics create <name>` + +Create a topic. + +| Option | Description | +| ---------------------- | ------------------------- | +| `--description <text>` | What the topic covers | +| `--parent <topic>` | Parent topic (ID or name) | + +### `libscope topics delete <topic>` + +Delete a topic (its documents are kept without a topic). + +| Option | Description | +| -------------------- | --------------------------------- | +| `--delete-documents` | Also delete the topic's documents | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope tags list` + +List all tags with their document counts. +<!-- generated:end cli:topics,tags --> `<topic>` is a topic ID or name everywhere. ## `libscope searches` -| Command | Description | -| ----------------------------------------------------------- | ---------------------------------- | -| `searches save <name> <query> [filters] [--min-rating <n>]` | Save a query with its filters | -| `searches list [-n <n>] [--offset <n>]` | List saved searches | -| `searches run <name>` | Run a saved search (name or ID) | -| `searches delete <name>` | Delete a saved search (name or ID) | +<!-- generated:start cli:searches --> +### `libscope searches save <name> <query>` + +Save a query and its filters. + +| Option | Description | +| ------------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--lib-version <version>` | Only this library version | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `-n, --limit <n>` | Maximum results | +| `--min-rating <n>` | Only documents rated at least this (1-5) | + +### `libscope searches list` + +List saved searches. + +| Option | Description | +| ----------------- | ------------------------ | +| `-n, --limit <n>` | Maximum results | +| `--offset <n>` | Results to skip (paging) | + +### `libscope searches run <search>` + +Run a saved search (name or ID). + +### `libscope searches delete <search>` + +Delete a saved search (name or ID). +<!-- generated:end cli:searches --> ## `libscope bulk` @@ -187,28 +360,60 @@ libscope bulk retag --topic guides --add important,v2 --remove draft -y libscope bulk move --library react --since 2024-01-01 --to frontend -y ``` -| Option | Description | -| ------------------------------------------------- | ------------------------------------------------- | -| `--topic`, `--library`, `--source-type`, `--tags` | Filters ([document filters](#document-filters)) | -| `--since <date>` | Created on or after (ISO 8601) | -| `--before <date>` | Created on or before (ISO 8601) | -| `--add <tags>` | `retag`: tags to add (comma-separated) | -| `--remove <tags>` | `retag`: tags to remove (comma-separated) | -| `--to <topic>` | `move`: destination topic (required) | -| `--dry-run` | List the matching documents without changing them | -| `-y, --yes` | Do not ask for confirmation | +<!-- generated:start cli:bulk --> +### `libscope bulk delete` + +Delete matching documents. + +| Option | Description | +| ---------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `--since <date>` | Only documents created on or after (ISO 8601) | +| `--before <date>` | Only documents created on or before (ISO 8601) | +| `--dry-run` | List the matching documents without changing them | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope bulk retag` + +Add and/or remove tags on matching documents. + +| Option | Description | +| ---------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `--since <date>` | Only documents created on or after (ISO 8601) | +| `--before <date>` | Only documents created on or before (ISO 8601) | +| `--dry-run` | List the matching documents without changing them | +| `-y, --yes` | Do not ask for confirmation | +| `--add <tags>` | Tags to add (comma-separated) | +| `--remove <tags>` | Tags to remove (comma-separated) | + +### `libscope bulk move` + +Move matching documents to another topic. + +| Option | Description | +| ---------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `--since <date>` | Only documents created on or after (ISO 8601) | +| `--before <date>` | Only documents created on or before (ISO 8601) | +| `--dry-run` | List the matching documents without changing them | +| `-y, --yes` | Do not ask for confirmation | +| `--to <topic>` | Destination topic (ID or name). Required. | +<!-- generated:end cli:bulk --> ## Connectors A connection is a saved connector configuration in `~/.libscope/connectors/<name>.json` (mode `0600`, credentials included). -| Command | Description | -| ----------------------------------------------------------- | ------------------------------------------------------------- | -| `connect <type> [source] [options]` | Save a connection and sync it | -| `sync [name]` / `sync --all` | Sync one or every saved connection with its saved settings | -| `disconnect <name> [--keep-documents] [--type <type>] [-y]` | Delete the connection's documents and its saved settings | -| `connections` | List connections with schedule, last sync and last run status | - Connector types: `notion`, `slack`, `confluence`, `obsidian`, `onenote`, `docs` (a documentation site). `[source]` is the vault path for `obsidian` and the URL for `confluence` and `docs`. ```bash @@ -223,35 +428,59 @@ libscope connect notion --name work --schedule "0 */6 * * *" # second Notion co Running `connect` again for a saved connection changes only the settings you give and keeps the rest. For example, `libscope connect notion --schedule off --no-sync` removes the schedule. -| Option | Types | Description | -| --------------------------------------------- | ----------------------------------- | ----------------------------------------------------------------------- | -| `--name <name>` | all | Connection name (default: the type) | -| `--schedule <cron>` | all | Sync on this schedule while `libscope serve api` runs; `off` removes it | -| `--no-sync` | all | Save without syncing now | -| `--token <token>` | notion, slack, confluence, onenote | API token (onenote: an access token that cannot be refreshed) | -| `--exclude <list>` | slack, notion, confluence, obsidian | Channels, page IDs, space keys or file globs to skip | -| `--channels <list>` | slack | Channel names or IDs (default `all`) | -| `--thread-mode <mode>` | slack | `aggregate` (default) or `separate` | -| `--email <email>` | confluence | User email (Cloud) | -| `--server` | confluence | Server/Data Center instead of Cloud | -| `--spaces <keys>` | confluence | Space keys (default `all`) | -| `--topic-mapping <mode>` | obsidian | `folder` (default) or `frontmatter` | -| `--notebook <name>` | onenote | One notebook (default: all) | -| `--client-id <id>` | onenote | Azure app client ID for device-code sign-in | -| `--tenant-id <id>` | onenote | Azure tenant ID (default `common`) | -| `--site-type <type>` | docs | `auto`, `sphinx`, `vitepress`, `doxygen` or `generic` | -| `--library`, `--lib-version` | docs | Library name and version for the pages | -| `--max-pages`, `--max-depth`, `--path-prefix` | docs | Crawl limits (default 500 pages, depth 10) | +<!-- generated:start cli:connect,sync,disconnect,connections --> +### `libscope connect <type> [source]` + +Save a connection and sync it. Types: notion, slack, confluence, obsidian, onenote, docs. [source] is the vault path (obsidian) or the URL (confluence, docs). Running it again for a saved connection changes only the given settings. + +| Option | Description | +| ------------------------- | ----------------------------------------------------------------------------------- | +| `--name <name>` | Connection name (default: the type) | +| `--schedule <cron>` | Sync on this cron schedule while `libscope serve api` runs ("off" removes it) | +| `--no-sync` | Save the connection without syncing now | +| `--token <token>` | notion, slack, confluence: API token; onenote: access token | +| `--exclude <list>` | Comma-separated: slack channels, notion page IDs, confluence spaces, obsidian globs | +| `--channels <list>` | slack: channel names or IDs (default all) | +| `--thread-mode <mode>` | slack: aggregate (one document per thread) or separate | +| `--email <email>` | confluence: user email (Cloud) | +| `--server` | confluence: Server/Data Center instead of Cloud | +| `--spaces <keys>` | confluence: space keys (default all) | +| `--topic-mapping <mode>` | obsidian: topics from folder (default) or frontmatter | +| `--notebook <name>` | onenote: one notebook (default all) | +| `--client-id <id>` | onenote: Azure app client ID for device sign-in | +| `--tenant-id <id>` | onenote: Azure tenant ID (default common) | +| `--site-type <type>` | docs: auto, sphinx, vitepress, doxygen or generic | +| `--library <name>` | docs: library name for the pages | +| `--lib-version <version>` | docs: library version for the pages | +| `--max-pages <n>` | docs: page limit (default 500) | +| `--max-depth <n>` | docs: link depth (default 10) | +| `--path-prefix <path>` | docs: only pages under this path | + +### `libscope sync [name]` + +Sync a saved connection with its saved settings, or all with --all. + +| Option | Description | +| ------- | --------------------------- | +| `--all` | Sync every saved connection | + +### `libscope disconnect <name>` + +Delete a connection's documents and its saved settings (including credentials). + +| Option | Description | +| ------------------ | ------------------------------------------------------ | +| `--type <type>` | Connector type, when no saved connection has this name | +| `--keep-documents` | Only delete the saved settings | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope connections` + +List saved connections with their schedule and last sync. +<!-- generated:end cli:connect,sync,disconnect,connections --> ## Knowledge packs -| Command | Description | -| --------------------------------------------- | --------------------------------------------------------------------------------------------- | -| `pack install <pack> [--registry <name>]` | Install `name` or `name@version` from the configured registries, or a `.json`/`.json.gz` file | -| `pack remove <name> [-y]` | Remove a pack and its documents | -| `pack list [--available] [--registry <name>]` | Installed packs, or the packs in the configured registries | -| `pack create --name <name> [options]` | Create a pack from indexed documents, or from files and URLs (`--from`) | - ```bash libscope pack install react-docs # latest version libscope pack install react-docs@1.2.0 @@ -261,68 +490,131 @@ libscope pack create --name team-docs --topic engineering libscope pack create --name react-docs --from ./react/docs --exclude "*.min.js" ``` -`pack install` options: `--registry <name>`, `--batch-size <n>`, `--concurrency <n>`, `--resume-from <n>`. Pack names are looked up in the local copies of the registries; run `libscope registry sync` to update them. When more than one registry has the pack, the command fails and names the registries: choose one with `--registry`. +Pack names are looked up in the local copies of the registries; run `libscope registry sync` to update them. When more than one registry has the pack, the command fails and names the registries: choose one with `--registry`. -`pack create` options: `--from <sources...>`, `--topic`, `--pack-version`, `--description`, `--author`, `--license`, `--output <path>` (default `<name>.json`, or `<name>.json.gz` with `--from`), `--extensions <exts>`, `--exclude <globs...>`, `--no-recursive`. +<!-- generated:start cli:pack --> +### `libscope pack install <pack>` -## Pack registries +Install a pack from the configured registries (name or name@version) or a local .json/.json.gz file. -A registry is a git repository of packs. `--registry <name>` always means the name of a configured registry. +| Option | Description | +| ------------------- | ----------------------------------------------------- | +| `--registry <name>` | Look only in this registry | +| `--batch-size <n>` | Documents embedded per batch (default 10) | +| `--resume-from <n>` | Skip the first N documents (resume a partial install) | +| `--concurrency <n>` | Batches embedded in parallel (default 4) | -| Command | Description | -| -------------------------------------------------------------- | ----------------------------------------------- | -| `libscope registry add <url> [-n <name>] [--no-sync]` | Add a git repository as a registry and clone it | -| `libscope registry remove <name> [-y]` | Remove a registry and its local clone | -| `libscope registry list` | List the configured registries | -| `libscope registry sync [<name>]` | Fetch the latest packs of one or all registries | -| `libscope registry search <query> [--registry <name>]` | Search the packs in the registries | -| `libscope registry create <path>` | Create an empty registry repository | -| `libscope registry publish <file> --registry <name>` | Publish a pack file to a registry | -| `libscope registry unpublish <name@version> --registry <name>` | Remove one version of a pack from a registry | +### `libscope pack remove <name>` -### `libscope registry add` +Remove an installed pack and its documents. -```bash -libscope registry add https://github.com/org/registry.git -libscope registry add git@github.com:team/packs.git --name team -libscope registry add file:///srv/libscope-packs.git --no-sync -``` +| Option | Description | +| ----------- | --------------------------- | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope pack list` -| Option | Description | -| ------------------- | ------------------------------------------------- | -| `-n, --name <name>` | Registry name (default: the last part of the URL) | -| `--no-sync` | Do not clone the registry now | +List installed packs, or the packs in the configured registries (--available). -URLs can use `https://`, `ssh://`, `git@host:path` or `file:///`. +| Option | Description | +| ------------------- | ------------------------------------------- | +| `--available` | List the packs in the configured registries | +| `--registry <name>` | With --available: only this registry | -### `libscope registry publish` +### `libscope pack create` + +Create a pack from indexed documents, or from files, folders and URLs (--from). + +| Option | Description | +| -------------------------- | ---------------------------------------------------------------------------- | +| `--name <name>` | Pack name. Required. | +| `--from <sources...>` | Files, folders or URLs to build the pack from | +| `--topic <topic>` | Only this topic's documents (without --from) | +| `--pack-version <version>` | Pack version (default 1.0.0) | +| `--description <text>` | Pack description | +| `--author <name>` | Pack author | +| `--license <license>` | Pack license | +| `--output <path>` | Output file (default <name>.json, or <name>.json.gz with --from) | +| `--extensions <exts>` | With --from: file extensions to include (comma-separated) | +| `--exclude <globs...>` | With --from: globs to skip | +| `--no-recursive` | With --from: do not walk subdirectories | +<!-- generated:end cli:pack --> + +## Pack registries + +A registry is a git repository of packs. `--registry <name>` always means the name of a configured registry. ```bash +libscope registry add https://github.com/org/registry.git +libscope registry add git@github.com:team/packs.git --name team +libscope registry add file:///srv/libscope-packs.git --no-sync libscope registry publish ./my-pack.json --registry my-registry --pack-version 1.0.0 libscope registry publish ./my-pack.json.gz --registry my-registry # next patch version libscope registry publish ./my-pack.json --registry community --submit # push to a feature branch +libscope registry unpublish my-pack@1.0.0 --registry my-registry ``` -| Option | Description | -| ------------------------- | ----------------------------------------------------------------------- | -| `--registry <name>` | Target registry (required) | -| `--pack-version <semver>` | Version to publish (default: next patch version, or the pack's version) | -| `-m, --message <msg>` | Git commit message | -| `--submit` | Push to a feature branch instead of the main branch | +Registry URLs can use `https://`, `ssh://`, `git@host:path` or `file:///`. `libscope registry create ./my-registry` creates a git repository with the registry folder structure. See the [Registry Reference](/reference/registry). -### `libscope registry unpublish` +<!-- generated:start cli:registry --> +### `libscope registry add <url>` -```bash -libscope registry unpublish my-pack@1.0.0 --registry my-registry -``` +Add a git repository as a pack registry and clone it. + +| Option | Description | +| ------------------- | ------------------------------------- | +| `-n, --name <name>` | Registry name (default: from the URL) | +| `--no-sync` | Do not clone the registry now | + +### `libscope registry remove <name>` + +Remove a registry and delete its local clone. + +| Option | Description | +| ----------- | --------------------------- | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope registry list` + +List the configured registries. -| Option | Description | -| --------------------- | -------------------------- | -| `--registry <name>` | Target registry (required) | -| `-m, --message <msg>` | Git commit message | -| `-y, --yes` | Skip confirmation prompt | +### `libscope registry sync [name]` -`libscope registry create ./my-registry` creates a git repo with the registry folder structure. See the [Registry Reference](/reference/registry). +Fetch the latest packs of one or all registries. + +### `libscope registry search <query>` + +Search the packs in the configured registries. + +| Option | Description | +| ------------------- | ------------------------- | +| `--registry <name>` | Search only this registry | + +### `libscope registry create <path>` + +Create an empty registry git repository. + +### `libscope registry publish <file>` + +Publish a pack file to a registry (commit and push). + +| Option | Description | +| ------------------------- | ---------------------------------------------------------------------- | +| `--registry <name>` | Target registry. Required. | +| `--pack-version <semver>` | Version to publish (default: next patch version) | +| `-m, --message <msg>` | Git commit message | +| `--submit` | Push to a feature branch for a pull request instead of the main branch | + +### `libscope registry unpublish <pack>` + +Remove one version of a pack from a registry (<name>@<version>). + +| Option | Description | +| --------------------- | --------------------------- | +| `--registry <name>` | Target registry. Required. | +| `-m, --message <msg>` | Git commit message | +| `-y, --yes` | Do not ask for confirmation | +<!-- generated:end cli:registry --> ## `libscope serve` @@ -332,58 +624,148 @@ libscope serve api # REST API on http://localhost:3378 libscope serve dashboard --port 8080 # web dashboard (default port 3377) ``` -| Option | Description | -| --------------- | --------------------------------------- | -| `--port <n>` | Port (`api` and `dashboard`) | -| `--host <host>` | Host to listen on (default `localhost`) | +<!-- generated:start cli:serve --> +Usage: `libscope serve [mode]` + +Start the MCP server on stdio (mcp, default), the REST API (api, port 3378) or the web dashboard (dashboard, port 3377). + +| Option | Description | +| --------------- | ------------------------------------- | +| `--port <n>` | Port (api and dashboard) | +| `--host <host>` | Host to listen on (default localhost) | +<!-- generated:end cli:serve --> Connection schedules (`connect --schedule`) run while `serve api` runs. ## `libscope config` -| Command | Description | -| -------------------------- | ----------------------------------------- | -| `config show` | Effective configuration (API keys masked) | -| `config get <key>` | One effective value (API keys masked) | -| `config set <key> <value>` | Set a value in `~/.libscope/config.json` | -| `config unset <key>` | Remove a value from the user config file | -| `config path` | Print the user config file path | +<!-- generated:start cli:config --> +### `libscope config show` + +Show the effective configuration (API keys are masked). + +### `libscope config get <key>` + +Print the effective value of a key (API keys are masked). + +### `libscope config set <key> <value>` + +Set a key in the user config file. Keys: embedding.provider, embedding.model, embedding.url, embedding.dimensions, llm.provider, llm.model, llm.url, openai.apiKey, anthropic.apiKey, database.path, indexing.maxDocumentSize, indexing.allowPrivateUrls, indexing.allowSelfSignedCerts, logging.level, mcp.toolsets. API keys are written to ~/.libscope/secrets.json (mode 0600), never to config.json. + +### `libscope config unset <key>` -Keys: `embedding.provider`, `embedding.model`, `embedding.url`, `embedding.dimensions`, `llm.provider`, `llm.model`, `llm.url`, `openai.apiKey`, `anthropic.apiKey`, `database.path`, `indexing.maxDocumentSize`, `indexing.allowPrivateUrls`, `indexing.allowSelfSignedCerts`, `logging.level`. `openai.apiKey` and `anthropic.apiKey` are written to `~/.libscope/secrets.json` (mode `0600`), never to `config.json`; the `LIBSCOPE_OPENAI_API_KEY` / `OPENAI_API_KEY` and `LIBSCOPE_ANTHROPIC_API_KEY` / `ANTHROPIC_API_KEY` environment variables take precedence. See the [configuration reference](/reference/configuration). +Remove a key from the user config file (or secrets file for API keys). + +### `libscope config path` + +Print the path of the user config file. +<!-- generated:end cli:config --> + +`openai.apiKey` and `anthropic.apiKey` are written to `~/.libscope/secrets.json` (mode `0600`), never to `config.json`; the `LIBSCOPE_OPENAI_API_KEY` / `OPENAI_API_KEY` and `LIBSCOPE_ANTHROPIC_API_KEY` / `ANTHROPIC_API_KEY` environment variables take precedence. See the [configuration reference](/reference/configuration) for every key. ## Workspaces -| Command | Description | -| ------------------------------ | ------------------------------------------ | -| `workspace create <name>` | Create a workspace | -| `workspace list` | List workspaces (`*` marks the active one) | -| `workspace use <name>` | Make a workspace the active one | -| `workspace delete <name> [-y]` | Delete a workspace and its database | +A workspace is a separate knowledge base with its own database (`~/.libscope/workspaces/<name>/libscope.db`). The global `--workspace <name>` option and the `LIBSCOPE_WORKSPACE` environment variable select one for a single command. + +<!-- generated:start cli:workspace --> +### `libscope workspace create <name>` + +Create a workspace. + +### `libscope workspace list` + +List workspaces. + +### `libscope workspace use <name>` + +Make a workspace the active one. + +### `libscope workspace delete <name>` + +Delete a workspace and its database. + +| Option | Description | +| ----------- | --------------------------- | +| `-y, --yes` | Do not ask for confirmation | +<!-- generated:end cli:workspace --> ## `libscope webhooks` -| Command | Description | -| ------------------------------------------------------ | ------------------------------------------ | -| `webhooks create <url> --events <list> [--secret <s>]` | Register a webhook | -| `webhooks list` | List webhooks (secrets are never shown) | -| `webhooks delete <webhookId>` | Delete a webhook | -| `webhooks test <webhookId>` | Send a test event and show the HTTP status | +<!-- generated:start cli:webhooks --> +### `libscope webhooks list` + +List webhooks (secrets are never shown). + +### `libscope webhooks create <url>` + +Register a webhook. + +| Option | Description | +| ------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `--events <events>` | Comma-separated: document.created, document.updated, document.deleted, document.rated, search.executed. Required. | +| `--secret <secret>` | HMAC-SHA256 signing secret (needs LIBSCOPE_SECRET_KEY) | + +### `libscope webhooks delete <webhookId>` + +Delete a webhook. + +### `libscope webhooks test <webhookId>` + +Send a test event and show the HTTP status. +<!-- generated:end cli:webhooks --> Events: `document.created`, `document.updated`, `document.deleted`, `document.rated`, `search.executed`. A secret signs each POST with HMAC-SHA256; storing it needs `LIBSCOPE_SECRET_KEY`. ## `libscope admin` -| Command | Description | -| --------------------------------------------------------------------- | --------------------------------------------------------------------------- | -| `admin reindex` | Re-embed chunks with the configured embedding model | -| `admin reindex --rebuild` | Recreate the vector index for a new model or vector size, then re-embed all | -| `admin dedupe [--threshold <0-1>] [--strategy exact\|semantic\|both]` | Find duplicate and near-duplicate documents | -| `admin backup <file>` | Write the whole knowledge base to a JSON file | -| `admin restore <file> [-y]` | Import a file written by `admin backup` | -| `admin stats [--days <n>]` | Counts, index, most returned and stale documents, search analytics | -| `admin prune` | Delete documents whose expiry time (`add --expires`) has passed | +<!-- generated:start cli:admin --> +### `libscope admin reindex` + +Re-embed chunks with the configured embedding model. + +| Option | Description | +| ------------------------ | --------------------------------------------------------------------- | +| `--rebuild` | Recreate the vector index for the configured model, then re-embed all | +| `--doc <documentIds...>` | Only these documents | +| `--since <date>` | Only documents created on or after (ISO 8601) | +| `--before <date>` | Only documents created on or before (ISO 8601) | +| `--batch-size <n>` | Chunks per embedding call (default 50) | + +### `libscope admin dedupe` + +Find duplicate and near-duplicate documents. + +| Option | Description | +| ----------------------- | --------------------------------------- | +| `--threshold <n>` | Similarity threshold 0-1 (default 0.95) | +| `--strategy <strategy>` | exact, semantic or both (default) | + +### `libscope admin prune` + +Delete documents whose expiry time has passed. + +### `libscope admin backup <file>` + +Write the whole knowledge base to a JSON file. -`admin reindex` also takes `--doc <documentIds...>`, `--since <date>`, `--before <date>` and `--batch-size <n>` (not with `--rebuild`). +### `libscope admin restore <file>` + +Import a file written by `admin backup`. + +| Option | Description | +| ----------- | --------------------------- | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope admin stats` + +Counts, index, most returned and stale documents, and search analytics. + +| Option | Description | +| ------------ | ------------------------------------------------------- | +| `--days <n>` | Look-back days for stale documents and search analytics | +<!-- generated:end cli:admin --> + +`admin reindex --rebuild` recreates the vector index for a new embedding model or vector size, then re-embeds every chunk. The other `admin reindex` filters do not apply with `--rebuild`. ## `libscope doctor` @@ -392,4 +774,14 @@ libscope doctor # report only libscope doctor --fix # also create the database and vector index ``` +<!-- generated:start cli:doctor --> +Usage: `libscope doctor` + +Check the setup: config, workspace, database, embedding model, index and LLM. + +| Option | Description | +| ------- | -------------------------------------------------- | +| `--fix` | Create what is missing (database and vector index) | +<!-- generated:end cli:doctor --> + `doctor` shows the config and secrets files, the active workspace and database path, the configured embedding model and the model the vector index was built with, and which LLM `ask` will use. It then lists checks with a fix for each warning or error, for example `libscope admin reindex --rebuild` when the index was built with a different embedding model. It exits with code 1 when a check fails. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 4c560e5d..e548f85b 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -18,63 +18,35 @@ A value in a config file that fails validation is ignored with a warning, and th ## All Config Keys -Every key `section.field` has the environment variable `LIBSCOPE_<SECTION>_<FIELD>` (for example `embedding.model` → `LIBSCOPE_EMBEDDING_MODEL`). - -### Embedding - -| Key | Type | Default | Env var | Description | -| ---------------------- | --------------------------- | ------------------------------------------------------------ | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `embedding.provider` | `local`, `ollama`, `openai` | `local` | `LIBSCOPE_EMBEDDING_PROVIDER` | Embedding provider. A config file can also name a provider registered in code with `registerProvider()`. | -| `embedding.model` | string | ollama: `nomic-embed-text`; openai: `text-embedding-3-small` | `LIBSCOPE_EMBEDDING_MODEL` | Embedding model. The local provider always uses `Xenova/all-MiniLM-L6-v2` and ignores this key. | -| `embedding.url` | string | `http://localhost:11434` | `LIBSCOPE_EMBEDDING_URL` | Server URL of the embedding provider. Used by the `ollama` provider. | -| `embedding.dimensions` | integer (1–10000) | size of the model | `LIBSCOPE_EMBEDDING_DIMENSIONS` | Vector size of the Ollama or OpenAI model. Needed only for models whose size LibScope does not know. For `text-embedding-3-*`, it also requests shorter vectors. After you change it, run `libscope reindex --rebuild`. | - -### LLM (for RAG) - -| Key | Type | Default | Env var | Description | -| -------------- | ------------------------------------------------------ | --------------------------------------------------------------------------------- | ----------------------- | ----------------------------------------------------------------------- | -| `llm.provider` | `auto`, `openai`, `anthropic`, `ollama`, `passthrough` | `auto` | `LIBSCOPE_LLM_PROVIDER` | LLM for `ask`. See [Automatic LLM selection](#automatic-llm-selection). | -| `llm.model` | string | openai: `gpt-4o-mini`; anthropic: `claude-3-5-haiku-20241022`; ollama: `llama3.2` | `LIBSCOPE_LLM_MODEL` | LLM model. | -| `llm.url` | string | `embedding.url`, then `http://localhost:11434` | `LIBSCOPE_LLM_URL` | Server URL of the LLM. Used by the `ollama` provider. | - -### API keys (secrets) - -| Key | Type | Default | Env vars (first set wins) | Description | -| ------------------ | ------ | ------- | ------------------------------------------------- | ------------------------------------ | -| `openai.apiKey` | string | — | `LIBSCOPE_OPENAI_API_KEY`, `OPENAI_API_KEY` | OpenAI API key (embeddings and LLM). | -| `anthropic.apiKey` | string | — | `LIBSCOPE_ANTHROPIC_API_KEY`, `ANTHROPIC_API_KEY` | Anthropic API key (LLM). | - -Each API key is read in this order: `LIBSCOPE_<PROVIDER>_API_KEY`, then `<PROVIDER>_API_KEY`, then `~/.libscope/secrets.json`. `libscope config set openai.apiKey <key>` writes the key to `secrets.json`, never to `config.json`. `libscope config show` and `libscope config get` mask keys (for example `sk-…abcd`). - -### Database - -| Key | Type | Default | Env var | Description | -| --------------- | ------ | -------------------------------------------------------- | ------------------------ | ---------------------------------------------------------------------- | -| `database.path` | string | unset (`~/.libscope/workspaces/<workspace>/libscope.db`) | `LIBSCOPE_DATABASE_PATH` | SQLite file to use instead of the workspace database. `~` is expanded. | - -### Logging - -| Key | Type | Default | Env var | Description | -| --------------- | ------------------------------------------ | ------- | ------------------------ | ----------- | -| `logging.level` | `debug`, `info`, `warn`, `error`, `silent` | `info` | `LIBSCOPE_LOGGING_LEVEL` | Log level. | - -### Indexing - -| Key | Type | Default | Env var | Description | -| ------------------------------- | ------- | ----------- | ------------------------------------------- | ----------------------------------------------------- | -| `indexing.maxDocumentSize` | integer | `104857600` | `LIBSCOPE_INDEXING_MAX_DOCUMENT_SIZE` | Largest document to index, in bytes (100 MB). | -| `indexing.allowPrivateUrls` | boolean | `false` | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS` | Allow fetching from private or internal IP addresses. | -| `indexing.allowSelfSignedCerts` | boolean | `false` | `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS` | Accept self-signed or untrusted TLS certificates. | - -Booleans accept `true`, `false`, `1`, and `0` in environment variables and in `config set`. - -### MCP - -| Key | Type | Default | Env var | Description | -| -------------- | ------------------------------ | ------- | ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `mcp.toolsets` | list of `core`, `admin`, `all` | none | `LIBSCOPE_MCP_TOOLSETS` | Optional MCP toolsets. `admin` adds `sync`, `install-pack`, `list-packs` and `reindex-documents`. `all` enables every optional toolset. The core tools are always on. | - -A list key accepts a JSON array in a config file, or a comma-separated list in a config file, an environment variable, and `config set` (for example `libscope config set mcp.toolsets admin`). +Every key `section.field` has the environment variable `LIBSCOPE_<SECTION>_<FIELD>` (for example `embedding.model` → `LIBSCOPE_EMBEDDING_MODEL`). This table is generated from the schema (`npm run docs:gen`). + +<!-- generated:start config:keys --> +| Key | Type | Default | Environment variables | Secret | Description | +| ------------------------------- | -------------------------------------------------------------- | --------------------------------------------------------------------------- | ------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `embedding.provider` | `local` \| `ollama` \| `openai` | local | `LIBSCOPE_EMBEDDING_PROVIDER` | | Embedding provider. A config file may also name a provider registered in code. | +| `embedding.model` | string | ollama: nomic-embed-text; openai: text-embedding-3-small | `LIBSCOPE_EMBEDDING_MODEL` | | Embedding model. Ignored by the local provider (always Xenova/all-MiniLM-L6-v2). | +| `embedding.url` | string | http://localhost:11434 | `LIBSCOPE_EMBEDDING_URL` | | Server URL of the embedding provider. Used by the ollama provider. | +| `embedding.dimensions` | integer | size of the model | `LIBSCOPE_EMBEDDING_DIMENSIONS` | | Vector size, for models whose size is not built in. | +| `llm.provider` | `auto` \| `openai` \| `anthropic` \| `ollama` \| `passthrough` | auto | `LIBSCOPE_LLM_PROVIDER` | | LLM for "ask". "auto": passthrough under MCP, else openai if an OpenAI key is set, else anthropic if an Anthropic key is set, else ollama if llm.url is set or embedding.provider is ollama, else none. | +| `llm.model` | string | openai: gpt-4o-mini; anthropic: claude-3-5-haiku-20241022; ollama: llama3.2 | `LIBSCOPE_LLM_MODEL` | | LLM model. | +| `llm.url` | string | embedding.url, then http://localhost:11434 | `LIBSCOPE_LLM_URL` | | Server URL of the LLM. Used by the ollama provider. | +| `openai.apiKey` | string | — | `LIBSCOPE_OPENAI_API_KEY`, `OPENAI_API_KEY` | yes | OpenAI API key (embeddings and LLM). | +| `anthropic.apiKey` | string | — | `LIBSCOPE_ANTHROPIC_API_KEY`, `ANTHROPIC_API_KEY` | yes | Anthropic API key (LLM). | +| `database.path` | string | ~/.libscope/workspaces/<workspace>/libscope.db | `LIBSCOPE_DATABASE_PATH` | | SQLite database file. A leading ~ is expanded. Unset: the workspace database. | +| `indexing.maxDocumentSize` | integer | 104857600 | `LIBSCOPE_INDEXING_MAX_DOCUMENT_SIZE` | | Largest document to index, in bytes. | +| `indexing.allowPrivateUrls` | boolean | false | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS` | | Allow fetching URLs on private or local networks. | +| `indexing.allowSelfSignedCerts` | boolean | false | `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS` | | Accept self-signed TLS certificates when fetching. | +| `logging.level` | `debug` \| `info` \| `warn` \| `error` \| `silent` | info | `LIBSCOPE_LOGGING_LEVEL` | | Log level. | +| `mcp.toolsets` | list of `core` \| `admin` \| `all` | none | `LIBSCOPE_MCP_TOOLSETS` | | Optional MCP toolsets (comma list). admin adds sync, install-pack, list-packs and reindex-documents; all enables every optional toolset. The core tools are always on. | +<!-- generated:end config:keys --> + +Notes: + +- `embedding.dimensions` is needed only for Ollama or OpenAI models whose vector size LibScope does not know. For `text-embedding-3-*`, it also requests shorter vectors. After you change the embedding provider, model or dimensions, run `libscope admin reindex --rebuild`. +- `llm.provider`: see [Automatic LLM selection](#automatic-llm-selection). +- Each API key is read in this order: `LIBSCOPE_<PROVIDER>_API_KEY`, then `<PROVIDER>_API_KEY`, then `~/.libscope/secrets.json`. `libscope config set openai.apiKey <key>` writes the key to `secrets.json`, never to `config.json`. `libscope config show` and `libscope config get` mask keys (for example `sk-…abcd`). +- Booleans accept `true`, `false`, `1` and `0` in environment variables and in `config set`. +- A list key (`mcp.toolsets`) accepts a JSON array in a config file, or a comma-separated list in a config file, an environment variable, and `config set` (for example `libscope config set mcp.toolsets admin`). ## Automatic LLM selection diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index b561c1a5..7e97c2ed 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -56,43 +56,48 @@ The `mcp.toolsets` config key enables optional toolsets (`libscope config set mc ## Core tools +The tool descriptions and parameter tables below are generated from the server code (`npm run docs:gen`). + +<!-- generated:start mcp:core --> ### search Search the knowledge base by meaning and keywords (query), or find content similar to a document or chunk (relatedTo). Results carry documentId and chunkId. Annotations: read-only. -| Parameter | Type | Required | Description | -| ---------------------- | ----------------------------------------------------- | -------- | ------------------------------------------------------------------------- | -| `query` | string | | What to search for | -| `relatedTo` | string | | Document or chunk ID: return similar content instead of running a query | -| `topic` | string | | Topic ID or name | -| `library` | string | | Library name | -| `version` | string | | Library version | -| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | -| `tags` | string[] | | Only documents carrying all of these tags | -| `minRating` | number | | Minimum average rating | -| `limit` | integer | | Maximum results (default 10, max 100) | -| `offset` | integer | | Results to skip (paging) Default: `0`. | -| `maxChunksPerDocument` | integer | | At most this many chunks per document (default: no limit) | -| `contextChunks` | integer | | Neighbouring chunks to include before and after each result Default: `0`. | +| Parameter | Type | Required | Description | +| ---------------------- | ----------------------------------------------------- | -------- | --------------------------------------------------------------------------- | +| `query` | string | | What to search for | +| `relatedTo` | string | | Document or chunk ID: return similar content instead of running a query | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | +| `tags` | string[] | | Only documents carrying all of these tags | +| `minRating` | number | | Minimum average rating | +| `limit` | integer | | Maximum results (default 10, max 100) | +| `offset` | integer | | Results to skip (paging). Default: `0`. | +| `maxChunksPerDocument` | integer | | At most this many chunks per document (default: no limit) | +| `contextChunks` | integer | | Neighbouring chunks to include before and after each result. Default: `0`. | +| `diversity` | number | | Query search: MMR reranking, 0 = relevance only (default), 1 = most diverse | ### ask -Retrieve the knowledge-base context for a question (passthrough: no LLM is called; answer from the returned context yourself) +Answer a question from the knowledge base with an LLM, or (passthrough) return the context to answer from Annotations: read-only. -| Parameter | Type | Required | Description | -| ------------ | ----------------------------------------------------- | -------- | ------------------------------------------- | -| `question` | string | yes | The question | -| `topic` | string | | Topic ID or name | -| `library` | string | | Library name | -| `version` | string | | Library version | -| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | -| `tags` | string[] | | Only documents carrying all of these tags | -| `minRating` | number | | Minimum average rating | -| `topK` | integer | | Chunks to retrieve as context Default: `5`. | +| Parameter | Type | Required | Description | +| -------------- | ----------------------------------------------------- | -------- | ---------------------------------------------------------------------------- | +| `question` | string | yes | The question | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | +| `tags` | string[] | | Only documents carrying all of these tags | +| `minRating` | number | | Minimum average rating | +| `topK` | integer | | Chunks to retrieve as context. Default: `5`. | +| `systemPrompt` | string | | System prompt for the LLM (default: answer from the context and cite titles) | ### get-document @@ -103,7 +108,7 @@ Annotations: read-only. | Parameter | Type | Required | Description | | ------------ | ------- | -------- | ------------------------------------------------------ | | `documentId` | string | yes | Document ID | -| `offset` | integer | | Character offset into the content Default: `0`. | +| `offset` | integer | | Character offset into the content. Default: `0`. | | `maxLength` | integer | | Maximum characters of content to return (default: all) | ### list-documents @@ -120,7 +125,7 @@ Annotations: read-only. | `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | | `tags` | string[] | | Only documents carrying all of these tags | | `limit` | integer | | Maximum results (default 50, max 1000) | -| `offset` | integer | | Results to skip (paging) Default: `0`. | +| `offset` | integer | | Results to skip (paging). Default: `0`. | ### overview @@ -136,29 +141,29 @@ Add to the knowledge base: inline content (with title), a web page (url), a site Annotations: not destructive. -| Parameter | Type | Required | Description | -| ----------------- | ----------------------------------------------------- | -------- | -------------------------------------------------------------------------------------------------------- | -| `content` | string | | Inline document content (markdown) | -| `title` | string | | Title (required with content; detected for files and URLs) | -| `url` | string | | Source URL. Without content, the URL is fetched; with content, it is stored | -| `topic` | string | | Topic ID or name | -| `library` | string | | Library name | -| `version` | string | | Library version | -| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Source type (default: library if library is set, topic if topic is set, else manual) | -| `tags` | string[] | | Tags to add to every new document | -| `expiresAt` | string | | ISO 8601 time after which the document is pruned | -| `dedup` | `skip` \| `warn` \| `force` | | Duplicate handling: skip returns the existing document, warn indexes anyway, force skips the check | -| `spider` | boolean | | URL: also crawl linked pages Default: `false`. | -| `maxPages` | integer | | Crawl: page limit (default 25, max 200) | -| `maxDepth` | integer | | Crawl: link depth (default 2, max 5) | -| `sameDomain` | boolean | | Crawl: stay on the seed domain (default true) | -| `pathPrefix` | string | | Crawl: only follow links under this path | -| `excludePatterns` | string[] | | Crawl: globs of URLs to skip | -| `branch` | string | | Repository: branch (default: from URL, else main) | -| `paths` | string[] | | Repository: only these subdirectories | -| `extensions` | string[] | | Repository: file extensions (default .md, .mdx, .txt, .rst) | -| `dryRun` | boolean | | List what would be added without adding it Default: `false`. | -| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"} Default: `false`. | +| Parameter | Type | Required | Description | +| ----------------- | ----------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------- | +| `content` | string | | Inline document content (markdown) | +| `title` | string | | Title (required with content; detected for files and URLs) | +| `url` | string (uri) | | Source URL. Without content, the URL is fetched; with content, it is stored | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Source type (default: library if library is set, topic if topic is set, else manual) | +| `tags` | string[] | | Tags to add to every new document | +| `expiresAt` | string (date-time) | | ISO 8601 time after which the document is pruned | +| `dedup` | `skip` \| `warn` \| `force` | | Duplicate handling: skip returns the existing document, warn indexes anyway, force skips the check | +| `spider` | boolean | | URL: also crawl linked pages. Default: `false`. | +| `maxPages` | integer | | Crawl: page limit (default 25, max 200) | +| `maxDepth` | integer | | Crawl: link depth (default 2, max 5) | +| `sameDomain` | boolean | | Crawl: stay on the seed domain (default true) | +| `pathPrefix` | string | | Crawl: only follow links under this path | +| `excludePatterns` | string[] | | Crawl: globs of URLs to skip | +| `branch` | string | | Repository: branch (default: from URL, else main) | +| `paths` | string[] | | Repository: only these subdirectories | +| `extensions` | string[] | | Repository: file extensions (default .md, .mdx, .txt, .rst) | +| `dryRun` | boolean | | List what would be added without adding it. Default: `false`. | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"}. Default: `false`. | ### update-document @@ -166,16 +171,16 @@ Update a document's title, content, metadata or tags (content changes are re-ind Annotations: not destructive, idempotent. -| Parameter | Type | Required | Description | -| ------------ | -------------- | -------- | ---------------------------------------- | -| `documentId` | string | yes | Document ID | -| `title` | string | | New title | -| `content` | string | | New content (re-chunked and re-embedded) | -| `library` | string \| null | | New library (null clears it) | -| `version` | string \| null | | New version (null clears it) | -| `url` | string \| null | | New URL (null clears it) | -| `topic` | string \| null | | New topic ID or name (null clears it) | -| `tags` | string[] | | Replace the document's tags with these | +| Parameter | Type | Required | Description | +| ------------ | -------------------- | -------- | ---------------------------------------- | +| `documentId` | string | yes | Document ID | +| `title` | string | | New title | +| `content` | string | | New content (re-chunked and re-embedded) | +| `library` | string \| null | | New library (null clears it) | +| `version` | string \| null | | New version (null clears it) | +| `url` | string (uri) \| null | | New URL (null clears it) | +| `topic` | string \| null | | New topic ID or name (null clears it) | +| `tags` | string[] | | Replace the document's tags with these | ### delete-document @@ -226,38 +231,39 @@ Annotations: not destructive, idempotent. | --------- | ------------------------------ | -------- | ------------------------------------------------------------------------ | | `action` | `status` \| `cancel` \| `list` | yes | What to do: status, cancel, list | | `taskId` | string | | Task ID returned when a background task started (action: status, cancel) | +<!-- generated:end mcp:core --> ## Admin tools These tools are registered only when `mcp.toolsets` contains `admin` or `all`. +<!-- generated:start mcp:admin --> ### sync Sync one saved connector connection (name) or all of them (all: true) with the settings saved by 'libscope connect' Annotations: not destructive. -| Parameter | Type | Required | Description | -| --------- | ------- | -------- | -------------------------------------------------------------------------------------------------------- | -| `name` | string | | Saved connection name (see `libscope connections`) | -| `all` | boolean | | Sync every saved connection Default: `false`. | -| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"} Default: `false`. | +| Parameter | Type | Required | Description | +| --------- | ------- | -------- | --------------------------------------------------------------------------------------------------------- | +| `name` | string | | Name of a saved connection | +| `all` | boolean | | Sync every saved connection. Default: `false`. | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"}. Default: `false`. | ### install-pack Install a knowledge pack from a registry (name or name@version) or a local file -Packs are looked up in the local copies of the configured registries (run sync-registries to update them). Local .json/.json.gz files are accepted from the CLI and the Node.js API only. - Annotations: not destructive, idempotent. -| Parameter | Type | Required | Description | -| ------------- | ------- | -------- | -------------------------------------------------------------------------------------------------------- | -| `pack` | string | yes | Pack name or name@version from a registry, or a local .json/.json.gz file | -| `registry` | string | | Look only in this registry (needed when several registries have the pack) | -| `batchSize` | integer | | Documents per batch (default 10) | -| `concurrency` | integer | | Batches embedded in parallel (default 4) | -| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"} Default: `false`. | +| Parameter | Type | Required | Description | +| ------------- | ------- | -------- | --------------------------------------------------------------------------------------------------------- | +| `pack` | string | yes | Pack name or name@version from a registry, or a local .json/.json.gz file | +| `registry` | string | | Look only in this registry (needed when several registries have the pack) | +| `batchSize` | integer | | Documents per batch (default 10) | +| `concurrency` | integer | | Batches embedded in parallel (default 4) | +| `resumeFrom` | integer | | Skip the first N documents (resume a partial install) | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"}. Default: `false`. | ### list-packs @@ -265,10 +271,10 @@ List installed packs, or the packs available in the configured registries Annotations: read-only. -| Parameter | Type | Required | Description | -| ----------- | ------- | -------- | --------------------------------------------------------------------------------------- | -| `available` | boolean | | List the packs in the configured registries instead of installed ones Default: `false`. | -| `registry` | string | | With available: only this registry | +| Parameter | Type | Required | Description | +| ----------- | ------- | -------- | ---------------------------------------------------------------------------------------- | +| `available` | boolean | | List the packs in the configured registries instead of installed ones. Default: `false`. | +| `registry` | string | | With available: only this registry | ### reindex-documents @@ -276,11 +282,12 @@ Re-embed chunks with the configured embedding model (rebuild after changing mode Annotations: not destructive, idempotent. -| Parameter | Type | Required | Description | -| ------------- | ----------------------------------- | -------- | -------------------------------------------------------------------------------------------------------- | -| `documentIds` | string[] | | Only these documents | -| `since` | string (ISO 8601 date or date-time) | | Only documents created on or after | -| `before` | string (ISO 8601 date or date-time) | | Only documents created on or before | -| `batchSize` | integer | | Chunks per embedding call (default 50) | -| `rebuild` | boolean | | Drop and recreate the vector table for the configured model, then re-embed everything Default: `false`. | -| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"} Default: `false`. | +| Parameter | Type | Required | Description | +| ------------- | ----------------------------------- | -------- | --------------------------------------------------------------------------------------------------------- | +| `documentIds` | string[] | | Only these documents | +| `since` | string (date) \| string (date-time) | | Only documents created on or after | +| `before` | string (date) \| string (date-time) | | Only documents created on or before | +| `batchSize` | integer | | Chunks per embedding call (default 50) | +| `rebuild` | boolean | | Drop and recreate the vector table for the configured model, then re-embed everything. Default: `false`. | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"}. Default: `false`. | +<!-- generated:end mcp:admin --> diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index 5bafa553..3bada982 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -37,128 +37,130 @@ Every route under `/api/v1` runs one LibScope operation: the same operation, wit ## Endpoints -The parameters of each route are in the OpenAPI document (`GET /openapi.json`). +This list is generated from the routes (`npm run docs:gen`). The parameters of each route are in the OpenAPI document (`GET /openapi.json`). Routes with status `202` start a [background task](#conventions). +<!-- generated:start rest:routes --> ### Meta -| Method | Path | Operation | Description | -| ------ | ---------------- | --------- | ---------------------------------------------------------------------- | -| `GET` | `/openapi.json` | — | This OpenAPI document. | -| `GET` | `/api/v1/health` | — | Liveness check (use GET /api/v1/overview for counts and index health). | +| Method | Path | Operation | Status | Description | +| ------ | ---------------- | --------- | ------ | --------------------------------------------------------------------- | +| `GET` | `/openapi.json` | — | `200` | This OpenAPI document | +| `GET` | `/api/v1/health` | — | `200` | Liveness check (use GET /api/v1/overview for counts and index health) | ### Documents -| Method | Path | Operation | Description | -| -------- | ---------------------------------------- | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `POST` | `/api/v1/documents` | `add` | Add content to the knowledge base: inline content, a URL (optionally crawled), a GitHub/GitLab repository, or (CLI/SDK only) a local file or directory. Runs as a task (202). | -| `GET` | `/api/v1/documents/:documentId` | `get-document` | Get a document with its tags, links and rating summary; long content can be paged. | -| `GET` | `/api/v1/documents` | `list-documents` | List documents (newest first) with optional filters. | -| `PATCH` | `/api/v1/documents/:documentId` | `update-document` | Update a document's title, content, metadata or tags (content changes are re-indexed). | -| `DELETE` | `/api/v1/documents/:documentId` | `delete-document` | Delete a document with its chunks, vectors, tags, links and ratings. | -| `POST` | `/api/v1/documents/:documentId/ratings` | `rate-document` | Rate a document (1-5), optionally with feedback or a suggested correction. | -| `GET` | `/api/v1/documents/:documentId/versions` | `document-history` | List saved versions of a document, newest first. | -| `POST` | `/api/v1/documents/:documentId/rollback` | `rollback-document` | Restore a document to a saved version (the current state is saved first). | +| Method | Path | Operation | Status | Description | +| -------- | ---------------------------------------- | ------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `POST` | `/api/v1/documents` | `add` | `202` (task) | Add content to the knowledge base: inline content, a URL (optionally crawled), a GitHub/GitLab repository, or (CLI/SDK only) a local file or directory | +| `GET` | `/api/v1/documents/:documentId` | `get-document` | `200` | Get a document with its tags, links and rating summary; long content can be paged | +| `GET` | `/api/v1/documents` | `list-documents` | `200` | List documents (newest first) with optional filters | +| `PATCH` | `/api/v1/documents/:documentId` | `update-document` | `200` | Update a document's title, content, metadata or tags (content changes are re-indexed) | +| `DELETE` | `/api/v1/documents/:documentId` | `delete-document` | `200` | Delete a document with its chunks, vectors, tags, links and ratings | +| `POST` | `/api/v1/documents/:documentId/ratings` | `rate-document` | `200` | Rate a document (1-5), optionally with feedback or a suggested correction | +| `GET` | `/api/v1/documents/:documentId/versions` | `document-history` | `200` | List saved versions of a document, newest first | +| `POST` | `/api/v1/documents/:documentId/rollback` | `rollback-document` | `200` | Restore a document to a saved version (the current state is saved first) | ### Search and Q&A -| Method | Path | Operation | Description | -| ------ | ---------------- | --------- | ---------------------------------------------------------------------------------------------------------- | -| `GET` | `/api/v1/search` | `search` | Search the knowledge base by meaning and keywords, or find content related to a document or chunk. | -| `POST` | `/api/v1/ask` | `ask` | Answer a question from the knowledge base with an LLM, or (passthrough) return the context to answer from. | +| Method | Path | Operation | Status | Description | +| ------ | ---------------- | --------- | ------ | --------------------------------------------------------------------------------------------------------- | +| `GET` | `/api/v1/search` | `search` | `200` | Search the knowledge base by meaning and keywords, or find content related to a document or chunk | +| `POST` | `/api/v1/ask` | `ask` | `200` | Answer a question from the knowledge base with an LLM, or (passthrough) return the context to answer from | ### Links and graph -| Method | Path | Operation | Description | -| -------- | --------------------------------------------- | ------------------ | ------------------------------------------------------------------------------------------ | -| `POST` | `/api/v1/documents/:documentId/links` | `link-documents` | Create a typed link from one document to another. | -| `DELETE` | `/api/v1/links/:linkId` | `unlink-documents` | Delete a link between documents. | -| `GET` | `/api/v1/links` | `list-links` | List links of one document (outgoing and incoming) or all links. | -| `GET` | `/api/v1/documents/:documentId/prerequisites` | `prerequisites` | List the documents to read before this one (following prerequisite links). | -| `GET` | `/api/v1/graph` | `graph` | Knowledge graph: documents, topics and tags as nodes; links, tags and similarity as edges. | +| Method | Path | Operation | Status | Description | +| -------- | --------------------------------------------- | ------------------ | ------ | ----------------------------------------------------------------------------------------- | +| `POST` | `/api/v1/documents/:documentId/links` | `link-documents` | `200` | Create a typed link from one document to another | +| `DELETE` | `/api/v1/links/:linkId` | `unlink-documents` | `200` | Delete a link between documents | +| `GET` | `/api/v1/links` | `list-links` | `200` | List links of one document (outgoing and incoming) or all links | +| `GET` | `/api/v1/documents/:documentId/prerequisites` | `prerequisites` | `200` | List the documents to read before this one (following prerequisite links) | +| `GET` | `/api/v1/graph` | `graph` | `200` | Knowledge graph: documents, topics and tags as nodes; links, tags and similarity as edges | ### Tags -| Method | Path | Operation | Description | -| -------- | ---------------------------------------------- | -------------- | ---------------------------------------------------- | -| `POST` | `/api/v1/documents/:documentId/tags` | `add-tags` | Add tags to a document (tags are created as needed). | -| `DELETE` | `/api/v1/documents/:documentId/tags` | `remove-tags` | Remove tags from a document. | -| `GET` | `/api/v1/tags` | `list-tags` | List all tags with their document counts. | -| `GET` | `/api/v1/documents/:documentId/suggested-tags` | `suggest-tags` | Suggest tags for a document from its content. | +| Method | Path | Operation | Status | Description | +| -------- | ---------------------------------------------- | -------------- | ------ | --------------------------------------------------- | +| `POST` | `/api/v1/documents/:documentId/tags` | `add-tags` | `200` | Add tags to a document (tags are created as needed) | +| `DELETE` | `/api/v1/documents/:documentId/tags` | `remove-tags` | `200` | Remove tags from a document | +| `GET` | `/api/v1/tags` | `list-tags` | `200` | List all tags with their document counts | +| `GET` | `/api/v1/documents/:documentId/suggested-tags` | `suggest-tags` | `200` | Suggest tags for a document from its content | ### Topics -| Method | Path | Operation | Description | -| -------- | ----------------------- | -------------- | ---------------------------------------------- | -| `GET` | `/api/v1/topics` | `list-topics` | List topics with their document counts. | -| `POST` | `/api/v1/topics` | `create-topic` | Create a topic. | -| `DELETE` | `/api/v1/topics/:topic` | `delete-topic` | Delete a topic, optionally with its documents. | +| Method | Path | Operation | Status | Description | +| -------- | ----------------------- | -------------- | ------ | --------------------------------------------- | +| `GET` | `/api/v1/topics` | `list-topics` | `200` | List topics with their document counts | +| `POST` | `/api/v1/topics` | `create-topic` | `200` | Create a topic | +| `DELETE` | `/api/v1/topics/:topic` | `delete-topic` | `200` | Delete a topic, optionally with its documents | ### Saved searches -| Method | Path | Operation | Description | -| -------- | ------------------------------ | --------------------- | ------------------------------------------------- | -| `POST` | `/api/v1/searches` | `save-search` | Save a search query and its filters under a name. | -| `GET` | `/api/v1/searches` | `list-saved-searches` | List saved searches. | -| `POST` | `/api/v1/searches/:search/run` | `run-saved-search` | Run a saved search. | -| `DELETE` | `/api/v1/searches/:search` | `delete-saved-search` | Delete a saved search. | +| Method | Path | Operation | Status | Description | +| -------- | ------------------------------ | --------------------- | ------ | ------------------------------------------------ | +| `POST` | `/api/v1/searches` | `save-search` | `200` | Save a search query and its filters under a name | +| `GET` | `/api/v1/searches` | `list-saved-searches` | `200` | List saved searches | +| `POST` | `/api/v1/searches/:search/run` | `run-saved-search` | `200` | Run a saved search | +| `DELETE` | `/api/v1/searches/:search` | `delete-saved-search` | `200` | Delete a saved search | -### Packs +### Packs and registries -| Method | Path | Operation | Description | -| -------- | --------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------- | -| `POST` | `/api/v1/packs` | `install-pack` | Install a knowledge pack from a registry (`pack`: name or name@version; optional `registry`). Runs as a task (202). | -| `DELETE` | `/api/v1/packs/:pack` | `remove-pack` | Remove an installed pack and its documents. | -| `GET` | `/api/v1/packs` | `list-packs` | List installed packs, or with `available=true` the packs in the configured registries (optional `registry`). | -| `GET` | `/api/v1/registries` | `list-registries` | List the configured pack registries. | -| `GET` | `/api/v1/registries/search` | `search-registries` | Search the packs in the configured registries (`query`, optional `registry`). | - -Registries are read from the local copies under `~/.libscope/registries/`. Adding, removing, syncing and publishing to registries is done with the CLI or the SDK. +| Method | Path | Operation | Status | Description | +| -------- | --------------------------- | ------------------- | ------------ | ----------------------------------------------------------------------------------- | +| `POST` | `/api/v1/packs` | `install-pack` | `202` (task) | Install a knowledge pack from a registry (name or name@version) or a local file | +| `DELETE` | `/api/v1/packs/:pack` | `remove-pack` | `200` | Remove an installed pack and its documents | +| `GET` | `/api/v1/packs` | `list-packs` | `200` | List installed packs, or the packs available in the configured registries | +| `GET` | `/api/v1/registries` | `list-registries` | `200` | List the configured pack registries | +| `GET` | `/api/v1/registries/search` | `search-registries` | `200` | Search the packs in the configured registries by name, description, tags and author | ### Connectors -| Method | Path | Operation | Description | -| -------- | --------------------------- | ------------------ | ------------------------------------------------------------------------------------------ | -| `GET` | `/api/v1/connections` | `list-connections` | List saved connector connections with their schedule and last sync. | -| `POST` | `/api/v1/sync` | `sync` | Sync one saved connection, or all of them, using the saved settings. Runs as a task (202). | -| `DELETE` | `/api/v1/connections/:name` | `disconnect` | Remove a connection's documents and its saved settings (including credentials). | +| Method | Path | Operation | Status | Description | +| -------- | --------------------------- | ------------------ | ------------ | ------------------------------------------------------------------------------ | +| `GET` | `/api/v1/connections` | `list-connections` | `200` | List saved connector connections with their schedule and last sync | +| `POST` | `/api/v1/sync` | `sync` | `202` (task) | Sync one saved connection, or all of them, using the saved settings | +| `DELETE` | `/api/v1/connections/:name` | `disconnect` | `200` | Remove a connection's documents and its saved settings (including credentials) | ### Admin and bulk -| Method | Path | Operation | Description | -| ------ | ----------------------------- | --------------- | ---------------------------------------------------------------------------------------------------------- | -| `GET` | `/api/v1/overview` | `overview` | Knowledge base overview: counts, topics, installed packs, embedding model of the index, and health. | -| `POST` | `/api/v1/admin/reindex` | `reindex` | Re-embed chunks with the configured embedding model (rebuild after changing models). Runs as a task (202). | -| `GET` | `/api/v1/admin/duplicates` | `dedupe` | Find groups of duplicate or near-duplicate documents. Runs as a task (202). | -| `POST` | `/api/v1/admin/prune-expired` | `prune-expired` | Delete documents whose expiry time (expiresAt) has passed. | -| `POST` | `/api/v1/bulk/delete` | `bulk-delete` | Delete every document matching the filters (at most 1000 per call). | -| `POST` | `/api/v1/bulk/retag` | `bulk-retag` | Add and/or remove tags on every document matching the filters. | -| `POST` | `/api/v1/bulk/move` | `bulk-move` | Move every document matching the filters to another topic. | +| Method | Path | Operation | Status | Description | +| ------ | ----------------------------- | --------------- | ------------ | -------------------------------------------------------------------------------------------------- | +| `GET` | `/api/v1/overview` | `overview` | `200` | Knowledge base overview: counts, topics, installed packs, embedding model of the index, and health | +| `POST` | `/api/v1/admin/reindex` | `reindex` | `202` (task) | Re-embed chunks with the configured embedding model (rebuild after changing models) | +| `GET` | `/api/v1/admin/duplicates` | `dedupe` | `202` (task) | Find groups of duplicate or near-duplicate documents | +| `POST` | `/api/v1/admin/prune-expired` | `prune-expired` | `200` | Delete documents whose expiry time (expiresAt) has passed | +| `POST` | `/api/v1/bulk/delete` | `bulk-delete` | `200` | Delete every document matching the filters (at most 1000 per call) | +| `POST` | `/api/v1/bulk/retag` | `bulk-retag` | `200` | Add and/or remove tags on every document matching the filters | +| `POST` | `/api/v1/bulk/move` | `bulk-move` | `200` | Move every document matching the filters to another topic | ### Analytics -| Method | Path | Operation | Description | -| ------ | ------------------------------- | ------------------ | --------------------------------------------------------------- | -| `GET` | `/api/v1/analytics/popular` | `popular` | Documents returned most often in search results. | -| `GET` | `/api/v1/analytics/stale` | `stale` | Documents that no search returned in the given number of days. | -| `GET` | `/api/v1/analytics/top-queries` | `top-queries` | Most frequent search queries. | -| `GET` | `/api/v1/analytics/searches` | `search-analytics` | Search volume, top and zero-result queries, and knowledge gaps. | +| Method | Path | Operation | Status | Description | +| ------ | ------------------------------- | ------------------ | ------ | -------------------------------------------------------------- | +| `GET` | `/api/v1/analytics/popular` | `popular` | `200` | Documents returned most often in search results | +| `GET` | `/api/v1/analytics/stale` | `stale` | `200` | Documents that no search returned in the given number of days | +| `GET` | `/api/v1/analytics/top-queries` | `top-queries` | `200` | Most frequent search queries | +| `GET` | `/api/v1/analytics/searches` | `search-analytics` | `200` | Search volume, top and zero-result queries, and knowledge gaps | ### Webhooks -| Method | Path | Operation | Description | -| -------- | ---------------------------------- | ---------------- | ----------------------------------------------------------------------- | -| `POST` | `/api/v1/webhooks` | `create-webhook` | Register a URL to receive a signed POST for document and search events. | -| `GET` | `/api/v1/webhooks` | `list-webhooks` | List registered webhooks (secrets are never shown). | -| `DELETE` | `/api/v1/webhooks/:webhookId` | `delete-webhook` | Delete a webhook. | -| `POST` | `/api/v1/webhooks/:webhookId/test` | `test-webhook` | Send a test event to a webhook and report the HTTP status. | +| Method | Path | Operation | Status | Description | +| -------- | ---------------------------------- | ---------------- | ------ | ---------------------------------------------------------------------- | +| `POST` | `/api/v1/webhooks` | `create-webhook` | `200` | Register a URL to receive a signed POST for document and search events | +| `GET` | `/api/v1/webhooks` | `list-webhooks` | `200` | List registered webhooks (secrets are never shown) | +| `DELETE` | `/api/v1/webhooks/:webhookId` | `delete-webhook` | `200` | Delete a webhook | +| `POST` | `/api/v1/webhooks/:webhookId/test` | `test-webhook` | `200` | Send a test event to a webhook and report the HTTP status | ### Tasks -| Method | Path | Operation | Description | -| ------ | ------------------------------ | ------------- | ------------------------------------------------------------- | -| `GET` | `/api/v1/tasks/:taskId` | `get-task` | Status, progress and result of a background task. | -| `POST` | `/api/v1/tasks/:taskId/cancel` | `cancel-task` | Request cancellation of a pending or running background task. | -| `GET` | `/api/v1/tasks` | `list-tasks` | List background tasks from the last hour, newest first. | +| Method | Path | Operation | Status | Description | +| ------ | ------------------------------ | ------------- | ------ | ------------------------------------------------------------ | +| `GET` | `/api/v1/tasks/:taskId` | `get-task` | `200` | Status, progress and result of a background task | +| `POST` | `/api/v1/tasks/:taskId/cancel` | `cancel-task` | `200` | Request cancellation of a pending or running background task | +| `GET` | `/api/v1/tasks` | `list-tasks` | `200` | List background tasks from the last hour, newest first | +<!-- generated:end rest:routes --> + +Registries are read from the local copies under `~/.libscope/registries/`. Adding, removing, syncing and publishing to registries is done with the CLI or the SDK. ## Examples diff --git a/package.json b/package.json index 7f674244..24522f7c 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,8 @@ "test:coverage": "vitest run --coverage", "prepare": "husky", "serve": "node dist/mcp/main.js", + "docs:gen": "node scripts/gen-docs.mjs", + "docs:check": "node scripts/gen-docs.mjs --check", "docs:dev": "vitepress dev docs", "docs:build": "vitepress build docs", "docs:preview": "vitepress preview docs" diff --git a/scripts/gen-docs.mjs b/scripts/gen-docs.mjs new file mode 100644 index 00000000..2119a33c --- /dev/null +++ b/scripts/gen-docs.mjs @@ -0,0 +1,398 @@ +#!/usr/bin/env node +/** + * Fill the generated blocks of the reference docs from the built code (dist/). + * + * npm run build && npm run docs:gen rewrite the blocks + * npm run build && npm run docs:check exit 1 when a block is out of date + * + * A block is the text between `<!-- generated:start NAME -->` and `<!-- generated:end NAME -->`. + * Everything outside the blocks is hand-written and kept. Block names: + * + * cli:global docs/reference/cli.md global options of the `libscope` program + * cli:<cmd>[,<cmd>...] docs/reference/cli.md these top-level commands and their subcommands + * mcp:core, mcp:admin docs/reference/mcp-tools.md MCP tools with their parameters + * config:keys docs/reference/configuration.md every config key + * rest:routes docs/reference/rest-api.md every REST route + * + * The generator needs no network and no embedding model: it constructs the CLI program without + * running it, and builds the MCP server over an in-memory database with a stub provider. + */ +import { readFileSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { format, resolveConfig } from "prettier"; +import Database from "better-sqlite3"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const fromDist = (path) => import(pathToFileURL(join(ROOT, "dist", path)).href); + +const DOCS = { + cli: "docs/reference/cli.md", + mcp: "docs/reference/mcp-tools.md", + config: "docs/reference/configuration.md", + rest: "docs/reference/rest-api.md", +}; + +// --------------------------------------------------------------------------------------------- +// Markdown helpers + +/** Text for a table cell or a paragraph: no HTML or Vue interpolation, no broken tables. */ +function text(value) { + return String(value ?? "") + .replaceAll(homedir(), "~") + .replaceAll("\n", " ") + .replaceAll("|", "\\|") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll("{{", "{ {"); +} + +/** Inline code; the content keeps `<` and `|` (escaped for tables). */ +function code(value) { + return `\`${String(value).replaceAll(homedir(), "~").replaceAll("|", "\\|")}\``; +} + +/** Text that ends with a period when `withPeriod` (when more text follows in the cell). */ +function sentence(value, withPeriod = true) { + const t = text(value).trim(); + if (t === "") return ""; + return withPeriod && !/[.!?]$/.test(t) ? `${t}.` : t; +} + +function table(headers, rows) { + const line = (cells) => `| ${cells.join(" | ")} |`; + return [line(headers), line(headers.map(() => "---")), ...rows.map(line)].join("\n"); +} + +// --------------------------------------------------------------------------------------------- +// CLI (commander program from dist/cli/index.js; importing it does not run it) + +function commandUsage(cmd, parents) { + const args = cmd.registeredArguments.map((a) => { + const name = `${a.name()}${a.variadic ? "..." : ""}`; + return a.required ? `<${name}>` : `[${name}]`; + }); + return [...parents, cmd.name(), ...args].join(" "); +} + +function optionDescription(opt) { + const extras = []; + if (opt.mandatory) extras.push("Required."); + if (opt.argChoices) extras.push(`One of: ${opt.argChoices.map(code).join(", ")}.`); + if (opt.defaultValue !== undefined && !opt.negate && !/\bdefault\b/i.test(opt.description)) { + extras.push( + `Default: ${code(opt.defaultValueDescription ?? JSON.stringify(opt.defaultValue))}.`, + ); + } + return [sentence(opt.description, extras.length > 0), ...extras].filter(Boolean).join(" "); +} + +function optionsTable(cmd) { + const options = cmd.options.filter((o) => !o.hidden); + if (options.length === 0) return ""; + return table( + ["Option", "Description"], + options.map((o) => [code(o.flags), optionDescription(o)]), + ); +} + +function renderLeafCommand(cmd, parents, heading) { + const parts = []; + const usage = `${code(commandUsage(cmd, parents))}`; + if (heading) parts.push(`${heading} ${usage}`, sentence(cmd.description())); + else parts.push(`Usage: ${usage}`, sentence(cmd.description())); + const options = optionsTable(cmd); + if (options) parts.push(options); + return parts.join("\n\n"); +} + +/** + * A top-level command. A command group renders each subcommand under an h3. A single command + * renders under an h3 when the block covers several commands, else without a heading (the + * hand-written h2 above the block names it). + */ +function renderCommand(cmd, alone) { + const visible = cmd.commands.filter((c) => !c._hidden); + if (visible.length === 0) return renderLeafCommand(cmd, ["libscope"], alone ? "" : "###"); + const parents = ["libscope", cmd.name()]; + return visible.map((sub) => renderLeafCommand(sub, parents, "###")).join("\n\n"); +} + +async function cliBlocks() { + const { program } = await fromDist("cli/index.js"); + const byName = new Map(program.commands.map((c) => [c.name(), c])); + const global = [ + "Usage: `libscope [options] <command>`", + optionsTable(program), + "`-h, --help` shows the help of the program or of any command.", + ].join("\n\n"); + return { + commands: program.commands.filter((c) => !c._hidden).map((c) => c.name()), + render(name) { + if (name === "cli:global") return global; + const names = name.slice("cli:".length).split(","); + return names + .map((n) => { + const cmd = byName.get(n); + if (!cmd) throw new Error(`${name}: the CLI has no command "${n}"`); + return renderCommand(cmd, names.length === 1); + }) + .join("\n\n"); + }, + }; +} + +// --------------------------------------------------------------------------------------------- +// MCP tools (createMcpServer over an in-memory database; tools/list over an in-memory transport) + +function schemaType(s) { + if (!s || typeof s !== "object") return "any"; + if (s.enum) return s.enum.map((v) => code(v)).join(" \\| "); + if (s.anyOf) return s.anyOf.map(schemaType).join(" \\| "); + if (s.type === "array") return `${schemaType(s.items)}[]`; + if (s.type === "string" && s.format) return `string (${s.format})`; + return s.type ?? "any"; +} + +function annotationText(annotations = {}) { + if (annotations.readOnlyHint) return "read-only"; + return [ + annotations.destructiveHint ? "destructive" : "not destructive", + annotations.idempotentHint ? "idempotent" : null, + ] + .filter(Boolean) + .join(", "); +} + +function renderTool(tool) { + const parts = [`### ${tool.name}`, text(tool.description)]; + parts.push(`Annotations: ${annotationText(tool.annotations)}.`); + const props = tool.inputSchema?.properties ?? {}; + const required = new Set(tool.inputSchema?.required ?? []); + const rows = Object.entries(props).map(([key, s]) => { + // Many descriptions already state the default in words. + const statesDefault = /\bdefault\b/i.test(s.description ?? ""); + const def = + s.default !== undefined && !statesDefault + ? ` Default: ${code(JSON.stringify(s.default))}.` + : ""; + const description = [sentence(s.description, def !== ""), def.trim()].filter(Boolean).join(" "); + return [code(key), schemaType(s), required.has(key) ? "yes" : "", description]; + }); + parts.push( + rows.length ? table(["Parameter", "Type", "Required", "Description"], rows) : "No parameters.", + ); + return parts.join("\n\n"); +} + +async function listTools(ctx, createMcpServer, toolsets) { + const mcp = createMcpServer({ ctx, toolsets }); + const client = new Client({ name: "gen-docs", version: "0" }); + const [clientSide, serverSide] = InMemoryTransport.createLinkedPair(); + await Promise.all([mcp.server.connect(serverSide), client.connect(clientSide)]); + const { tools } = await client.listTools(); + await client.close(); + await mcp.close(); + return tools; +} + +async function mcpBlocks() { + const { createMcpServer } = await fromDist("mcp/server.js"); + const { createOperationContext } = await fromDist("core/operations/index.js"); + const { runMigrations } = await fromDist("db/schema.js"); + const { ConfigSchema } = await fromDist("config-schema.js"); + const { initLogger } = await fromDist("logger.js"); + initLogger("silent"); + + const db = new Database(":memory:"); + runMigrations(db); + const config = ConfigSchema.parse( + Object.fromEntries(Object.keys(ConfigSchema.shape).map((section) => [section, {}])), + ); + const provider = { + name: "docs", + dimensions: 4, + embed: async () => [0, 0, 0, 1], + embedBatch: async (texts) => texts.map(() => [0, 0, 0, 1]), + }; + // A stub LLM registers `ask` with its general description (the server registers it only + // when an LLM or passthrough is available). + const llm = { model: "docs", complete: async () => ({ text: "" }) }; + const ctx = createOperationContext({ db, provider, config, surface: "mcp", llm }); + + const core = await listTools(ctx, createMcpServer, []); + const all = await listTools(ctx, createMcpServer, ["admin"]); + db.close(); + const coreNames = new Set(core.map((t) => t.name)); + const admin = all.filter((t) => !coreNames.has(t.name)); + return { + render(name) { + if (name === "mcp:core") return core.map(renderTool).join("\n\n"); + if (name === "mcp:admin") return admin.map(renderTool).join("\n\n"); + throw new Error(`unknown block ${name}`); + }, + }; +} + +// --------------------------------------------------------------------------------------------- +// Config keys (getConfigKeyTable from dist/config-schema.js) + +/** "a | b" -> `a` \| `b`; "list of a | b" -> list of `a` \| `b`; "string" stays. */ +function configType(type) { + const list = type.startsWith("list of "); + const values = list ? type.slice("list of ".length) : type; + if (!list && !values.includes(" | ")) return values; + const choices = values.split(" | ").map(code).join(" \\| "); + return list ? `list of ${choices}` : choices; +} + +async function configBlocks() { + const { getConfigKeyTable } = await fromDist("config-schema.js"); + const rows = getConfigKeyTable().map((row) => [ + code(row.key), + configType(row.type), + row.default === "" ? "—" : text(row.default), + row.env.map(code).join(", "), + row.secret ? "yes" : "", + sentence(row.description, false), + ]); + const keys = table( + ["Key", "Type", "Default", "Environment variables", "Secret", "Description"], + rows, + ); + return { + render(name) { + if (name === "config:keys") return keys; + throw new Error(`unknown block ${name}`); + }, + }; +} + +// --------------------------------------------------------------------------------------------- +// REST routes (API_ROUTES from dist/api/routes.js) + +const ROUTE_GROUPS = [ + ["meta", "Meta"], + ["documents", "Documents"], + ["search", "Search and Q&A"], + ["links", "Links and graph"], + ["tags", "Tags"], + ["topics", "Topics"], + ["searches", "Saved searches"], + ["packs", "Packs and registries"], + ["connectors", "Connectors"], + ["admin", "Admin and bulk"], + ["analytics", "Analytics"], + ["webhooks", "Webhooks"], + ["tasks", "Tasks"], +]; + +async function restBlocks() { + const { API_ROUTES } = await fromDist("api/routes.js"); + const groups = new Map(ROUTE_GROUPS.map(([key]) => [key, []])); + for (const route of API_ROUTES) { + const group = route.operation?.group ?? "meta"; + if (!groups.has(group)) + throw new Error(`REST route group "${group}" has no heading in gen-docs`); + groups.get(group).push(route); + } + const sections = ROUTE_GROUPS.filter(([key]) => groups.get(key).length > 0).map( + ([key, title]) => { + const rows = groups + .get(key) + .map((r) => [ + code(r.method), + code(r.path), + r.operation ? code(r.operation.name) : "—", + r.operation?.annotations?.longRunning ? "`202` (task)" : "`200`", + sentence(r.summary, false), + ]); + return `### ${title}\n\n${table(["Method", "Path", "Operation", "Status", "Description"], rows)}`; + }, + ); + const routes = sections.join("\n\n"); + return { + render(name) { + if (name === "rest:routes") return routes; + throw new Error(`unknown block ${name}`); + }, + }; +} + +// --------------------------------------------------------------------------------------------- +// Blocks in files + +const BLOCK = /<!-- generated:start ([\w:,-]+) -->\n([\s\S]*?)<!-- generated:end \1 -->/g; + +async function formatBlock(content, file) { + const options = (await resolveConfig(file)) ?? {}; + return format(content, { ...options, parser: "markdown", filepath: file }); +} + +async function main() { + const check = process.argv.includes("--check"); + const generators = { + cli: await cliBlocks(), + mcp: await mcpBlocks(), + config: await configBlocks(), + rest: await restBlocks(), + }; + const problems = []; + const required = { + cli: ["cli:global"], + mcp: ["mcp:core", "mcp:admin"], + config: ["config:keys"], + rest: ["rest:routes"], + }; + + for (const [kind, relPath] of Object.entries(DOCS)) { + const file = join(ROOT, relPath); + const original = readFileSync(file, "utf8"); + const seen = new Set(); + let updated = original; + for (const match of original.matchAll(BLOCK)) { + const [whole, name, current] = match; + const prefix = name.split(":")[0]; + if (prefix !== kind) { + problems.push(`${relPath}: block "${name}" does not belong in this file`); + continue; + } + seen.add(name); + const body = await formatBlock(generators[kind].render(name), file); + if (current !== body) { + if (check) problems.push(`${relPath}: block "${name}" is out of date`); + const block = `<!-- generated:start ${name} -->\n${body}<!-- generated:end ${name} -->`; + updated = updated.replace(whole, () => block); + } + } + for (const name of required[kind]) { + if (!seen.has(name)) problems.push(`${relPath}: missing block "${name}"`); + } + if (kind === "cli") { + const covered = new Set( + [...seen].filter((n) => n !== "cli:global").flatMap((n) => n.slice(4).split(",")), + ); + for (const command of generators.cli.commands) { + if (!covered.has(command)) + problems.push(`${relPath}: no block documents "libscope ${command}"`); + } + } + if (!check && updated !== original) { + writeFileSync(file, updated); + console.log(`updated ${relative(ROOT, file)}`); + } + } + + if (problems.length > 0) { + for (const p of problems) console.error(`✗ ${p}`); + if (check) console.error("Run `npm run build && npm run docs:gen` and commit the result."); + process.exitCode = 1; + } else if (check) { + console.log("Generated docs are up to date."); + } +} + +await main(); From b7bc1d20b23442d94141b9049535c0aa02d56ed0 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 19:07:30 +0000 Subject: [PATCH 61/83] docs(reference): fix hand-written parts around the generated blocks Config defaults are rendered as code (a bare URL failed the vitepress dead-link check). The configuration reference no longer lists the connector env vars that 2.0 does not read, and adds LIBSCOPE_GIT_TIMEOUT_MS. Blocks keep the blank lines prettier writes. --- docs/reference/cli.md | 36 ++++++++++++++++++++++++++- docs/reference/configuration.md | 43 ++++++++++++++++----------------- docs/reference/mcp-tools.md | 4 +++ docs/reference/rest-api.md | 2 ++ scripts/gen-docs.mjs | 12 +++++++-- 5 files changed, 72 insertions(+), 25 deletions(-) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index b52879bd..b21c7c38 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -26,6 +26,7 @@ The usage lines and option tables on this page are generated from the CLI code ( These go before or after the command name. <!-- generated:start cli:global --> + Usage: `libscope [options] <command>` | Option | Description | @@ -37,6 +38,7 @@ Usage: `libscope [options] <command>` | `--workspace <name>` | Use this workspace instead of the active one | `-h, --help` shows the help of the program or of any command. + <!-- generated:end cli:global --> Every result shows document IDs (and chunk IDs for search hits), so you can pass them to the next command. @@ -53,7 +55,7 @@ Commands that select documents (`search`, `ask`, `docs list`, `searches save`, ` | ---------------------- | -------------------------------------------------------------- | | `--topic <topic>` | Topic ID or name | | `--library <name>` | Library name | -| `--lib-version <ver>` | Library version (not on `bulk`) | +| `--lib-version <version>` | Library version (not on `bulk`) | | `--source-type <type>` | `library`, `topic`, `manual` or `model-generated` | | `--tags <a,b>` | Only documents with all these tags | | `-n, --limit <n>` | Maximum results (`ask`: chunks used as context; not on `bulk`) | @@ -73,6 +75,7 @@ libscope add ./docs --dry-run # list what would be added ``` <!-- generated:start cli:add --> + Usage: `libscope add <sources...>` Add files, directories, URLs or GitHub/GitLab repository URLs to the knowledge base. @@ -102,6 +105,7 @@ Add files, directories, URLs or GitHub/GitLab repository URLs to the knowledge b | `--extensions <exts>` | Repository: file extensions (default .md,.mdx,.txt,.rst) | | `--token <token>` | Repository: access token for a private repository | | `--dry-run` | List what would be added without adding it | + <!-- generated:end cli:add --> With `--json`, `add` prints one result per source: `{ kind, documents: [{ documentId, title, chunkCount, source }], errors, skipped, planned?, crawl? }`. @@ -116,6 +120,7 @@ libscope search # interactive search (on a term ``` <!-- generated:start cli:search --> + Usage: `libscope search [query]` Search by meaning and keywords. With no query on a terminal, start interactive search. @@ -133,6 +138,7 @@ Search by meaning and keywords. With no query on a terminal, start interactive s | `--min-rating <n>` | Only documents rated at least this (1-5) | | `--max-per-doc <n>` | At most this many chunks per document | | `--context <n>` | Neighbouring chunks to show around each result (0-2) | + <!-- generated:end cli:search --> With no query and no `--related`, on a terminal, `search` starts interactive mode: type a query, see results, repeat. Type `quit`, `exit` or press Ctrl+D to leave. The filters given on the command line apply to every query. @@ -145,6 +151,7 @@ libscope ask "What changed in v2?" --model gpt-4o-mini ``` <!-- generated:start cli:ask --> + Usage: `libscope ask <question>` Answer a question from the knowledge base with the configured LLM. @@ -159,6 +166,7 @@ Answer a question from the knowledge base with the configured LLM. | `-n, --limit <n>` | Chunks to use as context (default 5) | | `--min-rating <n>` | Only documents rated at least this (1-5) | | `--model <model>` | LLM model for this question (default: llm.model) | + <!-- generated:end cli:ask --> `ask` needs an LLM. With `llm.provider` set to `auto` (the default), LibScope uses OpenAI when an OpenAI API key is set, else Anthropic when an Anthropic key is set, else Ollama when `llm.url` is set or the embedding provider is Ollama. Without an LLM, `ask` fails with a hint; `libscope doctor` shows what is resolved. With `llm.provider passthrough`, `ask` prints the retrieved context instead of an answer. @@ -166,6 +174,7 @@ Answer a question from the knowledge base with the configured LLM. ## `libscope docs` <!-- generated:start cli:docs --> + ### `libscope docs list` List documents, newest first. @@ -270,6 +279,7 @@ Suggest tags from a document's content. | Option | Description | | ----------------- | ------------------- | | `-n, --limit <n>` | Maximum suggestions | + <!-- generated:end cli:docs --> `docs update --tags` replaces all tags. Changing the content re-chunks and re-embeds the document. @@ -282,6 +292,7 @@ libscope docs update <documentId> --content-file ./updated.md ## Topics and tags <!-- generated:start cli:topics,tags --> + ### `libscope topics list` List topics with their document counts. @@ -311,6 +322,7 @@ Delete a topic (its documents are kept without a topic). ### `libscope tags list` List all tags with their document counts. + <!-- generated:end cli:topics,tags --> `<topic>` is a topic ID or name everywhere. @@ -318,6 +330,7 @@ List all tags with their document counts. ## `libscope searches` <!-- generated:start cli:searches --> + ### `libscope searches save <name> <query>` Save a query and its filters. @@ -348,6 +361,7 @@ Run a saved search (name or ID). ### `libscope searches delete <search>` Delete a saved search (name or ID). + <!-- generated:end cli:searches --> ## `libscope bulk` @@ -361,6 +375,7 @@ libscope bulk move --library react --since 2024-01-01 --to frontend -y ``` <!-- generated:start cli:bulk --> + ### `libscope bulk delete` Delete matching documents. @@ -408,6 +423,7 @@ Move matching documents to another topic. | `--dry-run` | List the matching documents without changing them | | `-y, --yes` | Do not ask for confirmation | | `--to <topic>` | Destination topic (ID or name). Required. | + <!-- generated:end cli:bulk --> ## Connectors @@ -429,6 +445,7 @@ libscope connect notion --name work --schedule "0 */6 * * *" # second Notion co Running `connect` again for a saved connection changes only the settings you give and keeps the rest. For example, `libscope connect notion --schedule off --no-sync` removes the schedule. <!-- generated:start cli:connect,sync,disconnect,connections --> + ### `libscope connect <type> [source]` Save a connection and sync it. Types: notion, slack, confluence, obsidian, onenote, docs. [source] is the vault path (obsidian) or the URL (confluence, docs). Running it again for a saved connection changes only the given settings. @@ -477,6 +494,7 @@ Delete a connection's documents and its saved settings (including credentials). ### `libscope connections` List saved connections with their schedule and last sync. + <!-- generated:end cli:connect,sync,disconnect,connections --> ## Knowledge packs @@ -493,6 +511,7 @@ libscope pack create --name react-docs --from ./react/docs --exclude "*.min.js" Pack names are looked up in the local copies of the registries; run `libscope registry sync` to update them. When more than one registry has the pack, the command fails and names the registries: choose one with `--registry`. <!-- generated:start cli:pack --> + ### `libscope pack install <pack>` Install a pack from the configured registries (name or name@version) or a local .json/.json.gz file. @@ -538,6 +557,7 @@ Create a pack from indexed documents, or from files, folders and URLs (--from). | `--extensions <exts>` | With --from: file extensions to include (comma-separated) | | `--exclude <globs...>` | With --from: globs to skip | | `--no-recursive` | With --from: do not walk subdirectories | + <!-- generated:end cli:pack --> ## Pack registries @@ -557,6 +577,7 @@ libscope registry unpublish my-pack@1.0.0 --registry my-registry Registry URLs can use `https://`, `ssh://`, `git@host:path` or `file:///`. `libscope registry create ./my-registry` creates a git repository with the registry folder structure. See the [Registry Reference](/reference/registry). <!-- generated:start cli:registry --> + ### `libscope registry add <url>` Add a git repository as a pack registry and clone it. @@ -614,6 +635,7 @@ Remove one version of a pack from a registry (<name>@<version>). | `--registry <name>` | Target registry. Required. | | `-m, --message <msg>` | Git commit message | | `-y, --yes` | Do not ask for confirmation | + <!-- generated:end cli:registry --> ## `libscope serve` @@ -625,6 +647,7 @@ libscope serve dashboard --port 8080 # web dashboard (default port 3377) ``` <!-- generated:start cli:serve --> + Usage: `libscope serve [mode]` Start the MCP server on stdio (mcp, default), the REST API (api, port 3378) or the web dashboard (dashboard, port 3377). @@ -633,6 +656,7 @@ Start the MCP server on stdio (mcp, default), the REST API (api, port 3378) or t | --------------- | ------------------------------------- | | `--port <n>` | Port (api and dashboard) | | `--host <host>` | Host to listen on (default localhost) | + <!-- generated:end cli:serve --> Connection schedules (`connect --schedule`) run while `serve api` runs. @@ -640,6 +664,7 @@ Connection schedules (`connect --schedule`) run while `serve api` runs. ## `libscope config` <!-- generated:start cli:config --> + ### `libscope config show` Show the effective configuration (API keys are masked). @@ -659,6 +684,7 @@ Remove a key from the user config file (or secrets file for API keys). ### `libscope config path` Print the path of the user config file. + <!-- generated:end cli:config --> `openai.apiKey` and `anthropic.apiKey` are written to `~/.libscope/secrets.json` (mode `0600`), never to `config.json`; the `LIBSCOPE_OPENAI_API_KEY` / `OPENAI_API_KEY` and `LIBSCOPE_ANTHROPIC_API_KEY` / `ANTHROPIC_API_KEY` environment variables take precedence. See the [configuration reference](/reference/configuration) for every key. @@ -668,6 +694,7 @@ Print the path of the user config file. A workspace is a separate knowledge base with its own database (`~/.libscope/workspaces/<name>/libscope.db`). The global `--workspace <name>` option and the `LIBSCOPE_WORKSPACE` environment variable select one for a single command. <!-- generated:start cli:workspace --> + ### `libscope workspace create <name>` Create a workspace. @@ -687,11 +714,13 @@ Delete a workspace and its database. | Option | Description | | ----------- | --------------------------- | | `-y, --yes` | Do not ask for confirmation | + <!-- generated:end cli:workspace --> ## `libscope webhooks` <!-- generated:start cli:webhooks --> + ### `libscope webhooks list` List webhooks (secrets are never shown). @@ -712,6 +741,7 @@ Delete a webhook. ### `libscope webhooks test <webhookId>` Send a test event and show the HTTP status. + <!-- generated:end cli:webhooks --> Events: `document.created`, `document.updated`, `document.deleted`, `document.rated`, `search.executed`. A secret signs each POST with HMAC-SHA256; storing it needs `LIBSCOPE_SECRET_KEY`. @@ -719,6 +749,7 @@ Events: `document.created`, `document.updated`, `document.deleted`, `document.ra ## `libscope admin` <!-- generated:start cli:admin --> + ### `libscope admin reindex` Re-embed chunks with the configured embedding model. @@ -763,6 +794,7 @@ Counts, index, most returned and stale documents, and search analytics. | Option | Description | | ------------ | ------------------------------------------------------- | | `--days <n>` | Look-back days for stale documents and search analytics | + <!-- generated:end cli:admin --> `admin reindex --rebuild` recreates the vector index for a new embedding model or vector size, then re-embeds every chunk. The other `admin reindex` filters do not apply with `--rebuild`. @@ -775,6 +807,7 @@ libscope doctor --fix # also create the database and vector index ``` <!-- generated:start cli:doctor --> + Usage: `libscope doctor` Check the setup: config, workspace, database, embedding model, index and LLM. @@ -782,6 +815,7 @@ Check the setup: config, workspace, database, embedding model, index and LLM. | Option | Description | | ------- | -------------------------------------------------- | | `--fix` | Create what is missing (database and vector index) | + <!-- generated:end cli:doctor --> `doctor` shows the config and secrets files, the active workspace and database path, the configured embedding model and the model the vector index was built with, and which LLM `ask` will use. It then lists checks with a fix for each warning or error, for example `libscope admin reindex --rebuild` when the index was built with a different embedding model. It exits with code 1 when a check fails. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index e548f85b..426c0ff5 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -21,23 +21,25 @@ A value in a config file that fails validation is ignored with a warning, and th Every key `section.field` has the environment variable `LIBSCOPE_<SECTION>_<FIELD>` (for example `embedding.model` → `LIBSCOPE_EMBEDDING_MODEL`). This table is generated from the schema (`npm run docs:gen`). <!-- generated:start config:keys --> + | Key | Type | Default | Environment variables | Secret | Description | | ------------------------------- | -------------------------------------------------------------- | --------------------------------------------------------------------------- | ------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `embedding.provider` | `local` \| `ollama` \| `openai` | local | `LIBSCOPE_EMBEDDING_PROVIDER` | | Embedding provider. A config file may also name a provider registered in code. | +| `embedding.provider` | `local` \| `ollama` \| `openai` | `local` | `LIBSCOPE_EMBEDDING_PROVIDER` | | Embedding provider. A config file may also name a provider registered in code. | | `embedding.model` | string | ollama: nomic-embed-text; openai: text-embedding-3-small | `LIBSCOPE_EMBEDDING_MODEL` | | Embedding model. Ignored by the local provider (always Xenova/all-MiniLM-L6-v2). | -| `embedding.url` | string | http://localhost:11434 | `LIBSCOPE_EMBEDDING_URL` | | Server URL of the embedding provider. Used by the ollama provider. | +| `embedding.url` | string | `http://localhost:11434` | `LIBSCOPE_EMBEDDING_URL` | | Server URL of the embedding provider. Used by the ollama provider. | | `embedding.dimensions` | integer | size of the model | `LIBSCOPE_EMBEDDING_DIMENSIONS` | | Vector size, for models whose size is not built in. | -| `llm.provider` | `auto` \| `openai` \| `anthropic` \| `ollama` \| `passthrough` | auto | `LIBSCOPE_LLM_PROVIDER` | | LLM for "ask". "auto": passthrough under MCP, else openai if an OpenAI key is set, else anthropic if an Anthropic key is set, else ollama if llm.url is set or embedding.provider is ollama, else none. | +| `llm.provider` | `auto` \| `openai` \| `anthropic` \| `ollama` \| `passthrough` | `auto` | `LIBSCOPE_LLM_PROVIDER` | | LLM for "ask". "auto": passthrough under MCP, else openai if an OpenAI key is set, else anthropic if an Anthropic key is set, else ollama if llm.url is set or embedding.provider is ollama, else none. | | `llm.model` | string | openai: gpt-4o-mini; anthropic: claude-3-5-haiku-20241022; ollama: llama3.2 | `LIBSCOPE_LLM_MODEL` | | LLM model. | -| `llm.url` | string | embedding.url, then http://localhost:11434 | `LIBSCOPE_LLM_URL` | | Server URL of the LLM. Used by the ollama provider. | +| `llm.url` | string | embedding.url, then `http://localhost:11434` | `LIBSCOPE_LLM_URL` | | Server URL of the LLM. Used by the ollama provider. | | `openai.apiKey` | string | — | `LIBSCOPE_OPENAI_API_KEY`, `OPENAI_API_KEY` | yes | OpenAI API key (embeddings and LLM). | | `anthropic.apiKey` | string | — | `LIBSCOPE_ANTHROPIC_API_KEY`, `ANTHROPIC_API_KEY` | yes | Anthropic API key (LLM). | -| `database.path` | string | ~/.libscope/workspaces/<workspace>/libscope.db | `LIBSCOPE_DATABASE_PATH` | | SQLite database file. A leading ~ is expanded. Unset: the workspace database. | -| `indexing.maxDocumentSize` | integer | 104857600 | `LIBSCOPE_INDEXING_MAX_DOCUMENT_SIZE` | | Largest document to index, in bytes. | -| `indexing.allowPrivateUrls` | boolean | false | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS` | | Allow fetching URLs on private or local networks. | -| `indexing.allowSelfSignedCerts` | boolean | false | `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS` | | Accept self-signed TLS certificates when fetching. | -| `logging.level` | `debug` \| `info` \| `warn` \| `error` \| `silent` | info | `LIBSCOPE_LOGGING_LEVEL` | | Log level. | -| `mcp.toolsets` | list of `core` \| `admin` \| `all` | none | `LIBSCOPE_MCP_TOOLSETS` | | Optional MCP toolsets (comma list). admin adds sync, install-pack, list-packs and reindex-documents; all enables every optional toolset. The core tools are always on. | +| `database.path` | string | `~/.libscope/workspaces/<workspace>/libscope.db` | `LIBSCOPE_DATABASE_PATH` | | SQLite database file. A leading ~ is expanded. Unset: the workspace database. | +| `indexing.maxDocumentSize` | integer | `104857600` | `LIBSCOPE_INDEXING_MAX_DOCUMENT_SIZE` | | Largest document to index, in bytes. | +| `indexing.allowPrivateUrls` | boolean | `false` | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS` | | Allow fetching URLs on private or local networks. | +| `indexing.allowSelfSignedCerts` | boolean | `false` | `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS` | | Accept self-signed TLS certificates when fetching. | +| `logging.level` | `debug` \| `info` \| `warn` \| `error` \| `silent` | `info` | `LIBSCOPE_LOGGING_LEVEL` | | Log level. | +| `mcp.toolsets` | list of `core` \| `admin` \| `all` | `none` | `LIBSCOPE_MCP_TOOLSETS` | | Optional MCP toolsets (comma list). admin adds sync, install-pack, list-packs and reindex-documents; all enables every optional toolset. The core tools are always on. | + <!-- generated:end config:keys --> Notes: @@ -64,18 +66,15 @@ Set `llm.provider` to a specific provider to skip this selection. These variables are not config keys: -| Variable | Description | Default | -| --------------------- | ------------------------------------------------------------------------ | --------- | -| `LIBSCOPE_WORKSPACE` | Active workspace (overrides `libscope workspace use`) | `default` | -| `LIBSCOPE_API_KEY` | REST API key. When set, requests must send `Authorization: Bearer <key>` | — | -| `LIBSCOPE_SECRET_KEY` | Key used to encrypt stored webhook secrets | — | -| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (to stderr) in the CLI | — | -| `ONENOTE_CLIENT_ID` | OneNote app client ID | — | -| `ONENOTE_TENANT_ID` | OneNote tenant ID | `common` | -| `NOTION_TOKEN` | Notion integration token | — | -| `CONFLUENCE_URL` | Confluence base URL | — | -| `CONFLUENCE_EMAIL` | Confluence user email | — | -| `CONFLUENCE_TOKEN` | Confluence API token | — | +| Variable | Description | Default | +| ------------------------- | ----------------------------------------------------------------------------------- | --------- | +| `LIBSCOPE_WORKSPACE` | Active workspace (overrides `libscope workspace use`) | `default` | +| `LIBSCOPE_API_KEY` | REST API key. When set, requests must send `Authorization: Bearer <key>` | — | +| `LIBSCOPE_SECRET_KEY` | Key used to encrypt stored webhook secrets | — | +| `LIBSCOPE_VERBOSE` | `1` turns on debug logs (JSON, on stderr) in the CLI | — | +| `LIBSCOPE_GIT_TIMEOUT_MS` | Timeout of each git call made for pack registries, in milliseconds (at most 300000) | `60000` | + +Connector credentials are not read from environment variables. Give them to `libscope connect <type>` (see [Connectors](/guide/connectors)). LibScope does not load `.env` files. Export variables in your shell, or set them in the `env` block of your MCP client config. diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 7e97c2ed..8cd4eb10 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -59,6 +59,7 @@ The `mcp.toolsets` config key enables optional toolsets (`libscope config set mc The tool descriptions and parameter tables below are generated from the server code (`npm run docs:gen`). <!-- generated:start mcp:core --> + ### search Search the knowledge base by meaning and keywords (query), or find content similar to a document or chunk (relatedTo). Results carry documentId and chunkId. @@ -231,6 +232,7 @@ Annotations: not destructive, idempotent. | --------- | ------------------------------ | -------- | ------------------------------------------------------------------------ | | `action` | `status` \| `cancel` \| `list` | yes | What to do: status, cancel, list | | `taskId` | string | | Task ID returned when a background task started (action: status, cancel) | + <!-- generated:end mcp:core --> ## Admin tools @@ -238,6 +240,7 @@ Annotations: not destructive, idempotent. These tools are registered only when `mcp.toolsets` contains `admin` or `all`. <!-- generated:start mcp:admin --> + ### sync Sync one saved connector connection (name) or all of them (all: true) with the settings saved by 'libscope connect' @@ -290,4 +293,5 @@ Annotations: not destructive, idempotent. | `batchSize` | integer | | Chunks per embedding call (default 50) | | `rebuild` | boolean | | Drop and recreate the vector table for the configured model, then re-embed everything. Default: `false`. | | `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"}. Default: `false`. | + <!-- generated:end mcp:admin --> diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index 3bada982..6edc7e85 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -40,6 +40,7 @@ Every route under `/api/v1` runs one LibScope operation: the same operation, wit This list is generated from the routes (`npm run docs:gen`). The parameters of each route are in the OpenAPI document (`GET /openapi.json`). Routes with status `202` start a [background task](#conventions). <!-- generated:start rest:routes --> + ### Meta | Method | Path | Operation | Status | Description | @@ -158,6 +159,7 @@ This list is generated from the routes (`npm run docs:gen`). The parameters of e | `GET` | `/api/v1/tasks/:taskId` | `get-task` | `200` | Status, progress and result of a background task | | `POST` | `/api/v1/tasks/:taskId/cancel` | `cancel-task` | `200` | Request cancellation of a pending or running background task | | `GET` | `/api/v1/tasks` | `list-tasks` | `200` | List background tasks from the last hour, newest first | + <!-- generated:end rest:routes --> Registries are read from the local copies under `~/.libscope/registries/`. Adding, removing, syncing and publishing to registries is done with the CLI or the SDK. diff --git a/scripts/gen-docs.mjs b/scripts/gen-docs.mjs index 2119a33c..90fb1d02 100644 --- a/scripts/gen-docs.mjs +++ b/scripts/gen-docs.mjs @@ -249,12 +249,19 @@ function configType(type) { return list ? `list of ${choices}` : choices; } +/** A single value in code; in a sentence, only the URLs (a bare URL would become a link). */ +function defaultCell(value) { + if (value === "") return "—"; + if (!/\s/.test(value)) return code(value); + return text(value).replaceAll(/https?:\/\/[^\s,;]+/g, (url) => `\`${url}\``); +} + async function configBlocks() { const { getConfigKeyTable } = await fromDist("config-schema.js"); const rows = getConfigKeyTable().map((row) => [ code(row.key), configType(row.type), - row.default === "" ? "—" : text(row.default), + defaultCell(row.default), row.env.map(code).join(", "), row.secret ? "yes" : "", sentence(row.description, false), @@ -361,7 +368,8 @@ async function main() { continue; } seen.add(name); - const body = await formatBlock(generators[kind].render(name), file); + // Blank lines around the markers, as prettier writes them. + const body = `\n${await formatBlock(generators[kind].render(name), file)}\n`; if (current !== body) { if (check) problems.push(`${relPath}: block "${name}" is out of date`); const block = `<!-- generated:start ${name} -->\n${body}<!-- generated:end ${name} -->`; From 092eb5a712b26fd62ca5e5f1b92ed0646d3da10e Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 19:07:31 +0000 Subject: [PATCH 62/83] docs(migration): verify the 2.0 guide against the code and link its sections Adds an intro list of the sections. Fixes: buildKnowledgeGraph maps to scope.links.graph(); DELETE /api/v1/packs/:pack and the registry routes in the new-route list; new MCP parameters diversity, systemPrompt and resumeFrom; drops the createLlmProvider row (no longer exported). The guide is linked from the site nav and sidebar. --- docs/.vitepress/config.ts | 5 +++++ docs/migration-v2.md | 35 +++++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 350a972f..46d211a7 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -33,6 +33,10 @@ export default defineConfig({ text: "Changelog", link: "/changelog", }, + { + text: "Migrating to 2.0", + link: "/migration-v2", + }, { text: "Contributing", link: "/contributing", @@ -48,6 +52,7 @@ export default defineConfig({ items: [ { text: "Getting Started", link: "/guide/getting-started" }, { text: "Configuration", link: "/guide/configuration" }, + { text: "Migrating to 2.0", link: "/migration-v2" }, ], }, { diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 8ab8942c..a1a9646c 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -2,6 +2,18 @@ LibScope 2.0 removes old names without aliases. This page lists every removed or renamed name and what to use instead. Your indexed data is not changed by the upgrade. +- [Configuration](#configuration): config keys, environment variables, API keys +- [Library API](#library-api): package root exports, the `LibScope` class, `libscope/lite` +- [MCP](#mcp): server startup, renamed, merged and removed tools +- [REST API](#rest-api): routes, request fields, responses, CORS +- [CLI](#cli): commands, options, behavior +- [Packs and registries](#packs-and-registries): the URL registry is removed +- [Client SDKs](#client-sdks): Python and Go clients +- [Saved searches](#saved-searches): filter field names +- [Docker image](#docker-image): command, port, data volume + +The references list the 2.0 names: [CLI](reference/cli.md), [MCP tools](reference/mcp-tools.md), [REST API](reference/rest-api.md) and [configuration](reference/configuration.md). + ## Configuration ### Config keys @@ -41,13 +53,12 @@ These variables do not change: `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_LLM_PROV ### Library (SDK) code -| Old (1.x) | New (2.0) | -| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | -| `LibScopeConfig.embedding.openaiApiKey`, `llm.openaiApiKey` | `LibScopeConfig.openai.apiKey` | -| `LibScopeConfig.llm.anthropicApiKey` | `LibScopeConfig.anthropic.apiKey` | -| `embedding.ollamaUrl`, `ollamaModel`, `openaiModel` | `embedding.url`, `embedding.model` | -| `llm.ollamaUrl` | `llm.url` | -| `createLlmProvider(config)` read `ANTHROPIC_API_KEY` itself | `createLlmProvider(config, { surface })` reads only `config`; `loadConfig()` reads the env vars. | +| Old (1.x) | New (2.0) | +| ----------------------------------------------------------- | ---------------------------------- | +| `LibScopeConfig.embedding.openaiApiKey`, `llm.openaiApiKey` | `LibScopeConfig.openai.apiKey` | +| `LibScopeConfig.llm.anthropicApiKey` | `LibScopeConfig.anthropic.apiKey` | +| `embedding.ollamaUrl`, `ollamaModel`, `openaiModel` | `embedding.url`, `embedding.model` | +| `llm.ollamaUrl` | `llm.url` | ## Library API @@ -111,7 +122,7 @@ Each method below is on a `LibScope` instance (`const scope = LibScope.create()` | `ConnectorScheduler`, `loadScheduleEntries` | CLI `libscope connect <type> --schedule <cron>` | | `createWorkspace`, `deleteWorkspace`, `listWorkspaces`, `getWorkspacePath`, `getWorkspacesDir`, `getActiveWorkspace`, `setActiveWorkspace`, `DEFAULT_WORKSPACE` | `workspace` option of `LibScope.create()`; CLI `libscope workspace` | | `FileWatcher`, `DEFAULT_WATCH_EXTENSIONS` | CLI `libscope add <dir> --watch` | -| `buildKnowledgeGraph`, `detectClusters` | — (the web dashboard shows the graph) | +| `buildKnowledgeGraph`, `detectClusters` | `scope.links.graph()` (`detectClusters` —) | | `startApiServer` | CLI `libscope serve api` | The removed type exports follow their functions. Use `LibScopeInput<"group", "method">` and `LibScopeOutput<"group", "method">` for the input and result of any namespace method. @@ -179,9 +190,9 @@ The MCP server now has 11 core tools and an optional admin toolset. Old tool nam | 1.x tool | 2.0 tool | Parameter changes | | ------------------------------------------------------------------------------------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `search-docs` | `search` | `source` -> `sourceType` (enum). Added `tags`, `relatedTo`. `limit` max 100. `maxChunksPerDocument` minimum is 1. | +| `search-docs` | `search` | `source` -> `sourceType` (enum). Added `tags`, `relatedTo`, `diversity`. `limit` max 100. `maxChunksPerDocument` minimum is 1. | | `get-related` | `search` with `relatedTo` | `chunkId` -> `relatedTo` (a chunk ID or a document ID). `minScore` and `includeLinkedDocuments` are removed. Use `version`, `sourceType`, `minRating` as filters. | -| `ask-question` | `ask` | Added `version`, `sourceType`, `tags`, `minRating`. | +| `ask-question` | `ask` | Added `version`, `sourceType`, `tags`, `minRating`, `systemPrompt`. | | `health-check`, `list-topics` | `overview` | `list-topics` `parentId` is removed; `overview` lists every topic with its document count. | | `list-packs` | `overview` (installed packs), or admin `list-packs` | Admin `list-packs` keeps `available`; `registryUrl` is replaced by `registry` (a git registry name). | | `get-document` | `get-document` | Added `offset` and `maxLength` for paging. The output includes tags, links (with `linkId`) and ratings. | @@ -194,7 +205,7 @@ The MCP server now has 11 core tools and an optional admin toolset. Old tool nam | `get-task` | `task` with `action: "status"` | `taskId` unchanged. An unknown task is an error. | | `cancel-task` | `task` with `action: "cancel"` | `taskId` unchanged. An unknown task is an error. | | — | `task` with `action: "list"` | New. | -| `install-pack` (always on) | `install-pack` (admin toolset) | `nameOrPath` -> `pack`. Added `batchSize`, `concurrency`. | +| `install-pack` (always on) | `install-pack` (admin toolset) | `nameOrPath` -> `pack`. Added `batchSize`, `concurrency`, `resumeFrom`. | | `reindex-documents` (always on) | `reindex-documents` (admin toolset) | Added `rebuild`. | | `sync-slack`, `sync-notion`, `sync-confluence`, `sync-onenote`, `sync-obsidian-vault` | `sync` (admin toolset) | `{ "name": "<saved connection>" }` or `{ "all": true }`. Credentials and connector settings are no longer tool parameters: save them with `libscope connect <type>`. | @@ -234,7 +245,7 @@ Every `/api/v1` route now runs one operation and is generated from it. The [REST These routes did not change their path: `GET /openapi.json`, `GET /api/v1/health`, `GET` and `POST /api/v1/documents`, `GET`, `PATCH` and `DELETE /api/v1/documents/:id`, `POST /api/v1/documents/:id/tags`, `POST /api/v1/documents/:id/links`, `DELETE /api/v1/links/:id`, `POST /api/v1/ask`, `GET` and `POST /api/v1/topics`, `GET /api/v1/tags`, `GET /api/v1/analytics/searches`, the saved-search routes, `POST /api/v1/bulk/{delete,retag,move}` and the webhook routes. -New routes: `DELETE /api/v1/documents/:id/tags`, `POST /api/v1/documents/:id/ratings`, `GET /api/v1/documents/:id/versions`, `POST /api/v1/documents/:id/rollback`, `GET /api/v1/documents/:id/prerequisites`, `GET /api/v1/links`, `GET /api/v1/graph`, `DELETE /api/v1/topics/:topic`, `GET`, `POST` and `DELETE /api/v1/packs`, `GET /api/v1/connections`, `DELETE /api/v1/connections/:name`, `POST /api/v1/sync`, `GET /api/v1/overview`, `POST /api/v1/admin/reindex`, `GET /api/v1/admin/duplicates`, `POST /api/v1/admin/prune-expired`, `GET /api/v1/analytics/{popular,stale,top-queries}`, `GET /api/v1/tasks`, `GET /api/v1/tasks/:taskId` and `POST /api/v1/tasks/:taskId/cancel`. +New routes: `DELETE /api/v1/documents/:id/tags`, `POST /api/v1/documents/:id/ratings`, `GET /api/v1/documents/:id/versions`, `POST /api/v1/documents/:id/rollback`, `GET /api/v1/documents/:id/prerequisites`, `GET /api/v1/links`, `GET /api/v1/graph`, `DELETE /api/v1/topics/:topic`, `GET` and `POST /api/v1/packs`, `DELETE /api/v1/packs/:pack`, `GET /api/v1/registries`, `GET /api/v1/registries/search`, `GET /api/v1/connections`, `DELETE /api/v1/connections/:name`, `POST /api/v1/sync`, `GET /api/v1/overview`, `POST /api/v1/admin/reindex`, `GET /api/v1/admin/duplicates`, `POST /api/v1/admin/prune-expired`, `GET /api/v1/analytics/{popular,stale,top-queries}`, `GET /api/v1/tasks`, `GET /api/v1/tasks/:taskId` and `POST /api/v1/tasks/:taskId/cancel`. ### Request fields From a248043eb7bae0941c6eca0a1008a5b1cfd5340e Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 19:07:32 +0000 Subject: [PATCH 63/83] docs(claude): describe the 2.0 structure, config and generated docs Project structure with src/core/operations as the single operation layer and the CLI, MCP, REST and SDK adapters; how to add a feature; NotFoundError; schema version 19; config schema and 2.0 env vars; docs:gen/docs:check. --- CLAUDE.md | 113 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 73 insertions(+), 40 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 76ceecea..11dd52a0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # libscope — Agent Guidelines -AI-powered knowledge base with MCP integration. TypeScript, Node.js >=20, CommonJS (ES2022 target). +AI-powered knowledge base with MCP integration. TypeScript, Node.js >=20, ES modules (`"type": "module"`, NodeNext, ES2022 target). ## Build & Test @@ -13,9 +13,13 @@ npm run format # Auto-format npm test # Run all tests via vitest npm run test:coverage # Coverage thresholds: statements 75%, branches 74%, functions 75%, lines 75% npm run build # tsc — outputs to dist/ +npm run docs:gen # After build: rewrite the generated blocks in docs/reference/*.md +npm run docs:check # After build: exit 1 if a generated block is out of date (CI runs this) ``` -CI runs on Node 20 and 22. CI checks: lint, format:check, typecheck, test:coverage, build. Build verifies `dist/mcp/server.js`, `dist/cli/index.js`, `dist/core/index.js` exist. +CI runs on Node 20 and 22. CI checks: lint, format:check, typecheck, test:coverage, build, docs:check. Build verifies `dist/mcp/server.js`, `dist/cli/index.js`, `dist/core/index.js` exist. + +When you change a CLI command or option, an operation's input schema or summary, an MCP tool, a REST route or a config key, run `npm run build && npm run docs:gen` and commit the changed `docs/reference/*.md`. The generator is `scripts/gen-docs.mjs`; do not edit text between `<!-- generated:start ... -->` and `<!-- generated:end ... -->` by hand. **Before every push, run this exact sequence locally:** ```bash @@ -67,38 +71,66 @@ When delegating to subagents for mechanical fixes: ## Project Structure +Every feature is one **operation** in `src/core/operations/`. The CLI, MCP server, REST API and SDK are thin adapters over the operations: they parse input, call `runOperation(op, ctx, input)` (or `startOperationTask` for long-running operations), and format the result. Put logic in an operation or a core module, never in a surface. + ``` src/ -├── cli/ # Commander.js CLI -│ ├── index.ts # Main entry (#!/usr/bin/env node) -│ └── commands/ # Each file exports registerCommands(program) -├── mcp/ # MCP server -│ ├── server.ts # MCP entry point -│ └── tools/ # Each file exports registerTools(server, db, provider) -├── lite/ # libscope/lite — embeddable semantic search (no CLI/MCP/connectors) -│ ├── index.ts # Public entrypoint — exports LibScopeLite + types -│ ├── core.ts # LibScopeLite class (index, search, getContext, ask, rate) -│ ├── types.ts # LiteOptions, LiteDoc, LiteSearchResult, etc. -│ ├── normalize.ts # Raw input → markdown (dispatches to core/parsers/) -│ └── chunker-treesitter.ts # Optional tree-sitter code chunker (TS/JS/Python) -├── core/ # Business logic (documents, search, indexing, packs, topics, etc.) -│ └── parsers/ # File format parsers (markdown, pdf, docx, html, epub, pptx, csv, yaml, json) -├── api/ # REST API server (routes, middleware, openapi spec) -├── web/ # Web UI / dashboard -├── db/ # SQLite via better-sqlite3 (schema, migrations, connection) -├── providers/ # Embedding providers (local/xenova, ollama, openai) -├── registry/ # Git-backed pack registries (config, git, sync, publish, search, checksum) -├── connectors/ # Third-party syncs (notion, slack, confluence, onenote, obsidian) -├── config.ts # Config loading: env vars > project .libscope.json > user ~/.libscope/config.json > defaults -├── errors.ts # Error hierarchy -├── logger.ts # Pino logger -└── LibScope.ts # Main public API class +├── core/ +│ ├── operations/ # The operation layer: one file per group (documents, search, links, graph, tags, +│ │ # topics, searches, packs, registries, connectors, admin, analytics, webhooks, tasks) +│ │ ├── types.ts # defineOperation, runOperation, startOperationTask, createOperationContext, ListResult +│ │ ├── schemas.ts # Shared zod field schemas (documentId, topic, sourceType, limit, tags, ...) +│ │ └── index.ts # OPERATIONS (every operation, in surface order), getOperation(name) +│ ├── bootstrap.ts # One startup for every surface: config -> db path -> migrate -> provider -> vector table +│ ├── ingest.ts # add: content, file, directory, URL (+ spider) or repository +│ ├── tasks.ts # Background task registry (AbortSignal, progress, 1 h retention) +│ ├── overview.ts # Counts, topics, packs, index model, health +│ ├── document-view.ts # get-document view (paged content, tags, links, ratings) +│ ├── rag.ts # answer() for ask (LLM or passthrough) +│ ├── index.ts # Package root exports: LibScope class, option/result types, errors +│ ├── parsers/ # File format parsers (markdown, pdf, docx, html, epub, pptx, csv, yaml, json, text) +│ └── ... # Business logic used by operations (documents, search, indexing, packs, topics, ...) +├── cli/ +│ ├── index.ts # Program, global options (--json, --verbose, --log-level, --workspace), error handler; exports `program` +│ └── commands/ # One file per command group; each exports register(program) +├── mcp/ +│ ├── server.ts # createMcpServer(): tools generated from operations (11 core + admin toolset); runStdioServer() +│ ├── format.ts # Compact text output of each tool +│ ├── errors.ts # withErrorHandling(), errorResponse() +│ └── main.ts # stdio entry point (npm run serve) +├── api/ +│ ├── routes.ts # API_ROUTES: one route per operation with `http`, plus /openapi.json and /api/v1/health +│ ├── adapter.ts # HTTP input -> operation input; 202 + task for long-running operations; error -> status +│ ├── openapi.ts # buildOpenApiSpec(API_ROUTES) from the operation schemas +│ ├── middleware.ts +│ └── server.ts # startApiServer({ db, provider, config }, options) +├── web/ # Dashboard server (server.ts) and page (dashboard.ts: do not edit) +├── lite/ # libscope/lite: createLite() preset of LibScope, createCodeChunker(), TreeSitterChunker, normalizeRawInput() +├── db/ # SQLite (schema.ts migrations, connection.ts, index-meta.ts embedding model of the index) +├── providers/ # Embedding providers (local/xenova, ollama, openai) +├── registry/ # Git-backed pack registries (config, git, sync, publish, search, resolve, checksum) +├── connectors/ # notion, slack, confluence, onenote, obsidian, docs; registry.ts (types, saved connections, sync) +├── config-schema.ts # The zod config schema: keys, defaults, env var names, docs table (getConfigKeyTable) +├── config.ts # loadConfig(), config get/set/unset, secrets.json +├── errors.ts # Error hierarchy +├── logger.ts # Pino logger (stderr) +└── LibScope.ts # SDK class: add/search/ask/askStream/overview + namespaces bound to operations +scripts/gen-docs.mjs # Fills the generated blocks in docs/reference/*.md from dist/ tests/ -├── unit/ # Fast, mocked — test modules in isolation -├── integration/ # Real SQLite DB, full workflows -└── fixtures/ # test-db.ts, mock-provider.ts, helpers.ts +├── unit/ # Fast, mocked — test modules in isolation (operations: tests/unit/operations/) +├── integration/ # Real SQLite DB, full workflows +└── fixtures/ # test-db.ts, mock-provider.ts, helpers.ts ``` +### Adding or changing a feature + +1. Add or change the operation in `src/core/operations/<group>.ts` with `defineOperation({ name, group, summary, input, annotations, http, handler })`. Every input field has `.describe()`; defaults live in the schema only. Add it to the group's exported list (it then appears in `OPERATIONS`). +2. REST: setting `http: { method, path }` creates the route and its OpenAPI entry. `annotations.longRunning` makes the route answer `202` with a task. +3. MCP: tools are registered in `src/mcp/server.ts` (`registerCoreTools` / `registerAdminTools`) with a formatter from `format.ts`. Not every operation is a tool: saved searches, webhooks, analytics and registries are CLI/REST/SDK only. +4. CLI: add or change the command in `src/cli/commands/<group>.ts`; it calls the operation and prints the result. +5. SDK: add the operation to its namespace in `NAMESPACES` in `src/LibScope.ts`. +6. Run `npm run build && npm run docs:gen` and commit the updated references. List removed or renamed names in `docs/migration-v2.md`. + ## Error Handling All custom errors extend `LibScopeError` with a `code` property and optional `cause`: @@ -107,22 +139,23 @@ All custom errors extend `LibScopeError` with a `code` property and optional `ca LibScopeError ├── DatabaseError ├── EmbeddingError - ├── ValidationError - ├── FetchError + ├── ValidationError (REST 400) + ├── FetchError (REST 502) ├── ConfigError - ├── DocumentNotFoundError - ├── ChunkNotFoundError - └── TopicNotFoundError + └── NotFoundError (REST 404; code names the resource, e.g. LINK_NOT_FOUND) + ├── DocumentNotFoundError + ├── ChunkNotFoundError + └── TopicNotFoundError ``` -Always use the appropriate error subclass. MCP tool handlers must be wrapped with `withErrorHandling()` from `src/mcp/errors.ts`. +Always use the appropriate error subclass. MCP tool handlers are wrapped with `withErrorHandling()` from `src/mcp/errors.ts` (`registerOperationTool` does this). The CLI prints every error once in `src/cli/errors.ts` (`✗ message` and a hint). ## Database - **Engine:** better-sqlite3 + sqlite-vec for vector search -- **Schema version:** 18 (migrations in `src/db/schema.ts`) +- **Schema version:** 19 (migrations in `src/db/schema.ts`) - **Adding migrations:** Increment `SCHEMA_VERSION`, add entry to `MIGRATIONS` object with the new version number as key -- **Key tables:** documents, chunks, chunks_fts (FTS5), chunk_embeddings (vector), topics, ratings, schema_version +- **Key tables:** documents, chunks, chunks_fts (FTS5), chunk_embeddings (vector), index_meta (embedding model of the index), topics, tags, ratings, document_links, saved_searches, webhooks, packs, schema_version. `connector_configs` is unused. ## Testing Patterns @@ -159,7 +192,7 @@ packs/<pack-name>/ ``` - Registry names: `/^[a-zA-Z0-9_-]+$/`, 2-64 chars -- Git URLs: https://, ssh://, or git@host:path (no embedded credentials) +- Registry URLs: https://, ssh://, SCP-style SSH (user@host:path) or file:/// (no embedded credentials) - Path segment validation: reject `..`, `/`, `\`, null bytes, non-alphanumeric (except `._-`) ## Logging @@ -168,6 +201,6 @@ Use `getLogger()` from `src/logger.ts` (pino). Create child loggers with `create ## Configuration -`loadConfig()` merges with precedence: env vars > project `.libscope.json` > user `~/.libscope/config.json` > defaults. 30-second cache TTL. +`src/config-schema.ts` is the one zod schema for every config key: defaults, validation, `config get/set/unset`, env var names and the docs table. `loadConfig()` merges with precedence: env vars > project `.libscope.json` > user `~/.libscope/config.json` > defaults. API keys (`openai.apiKey`, `anthropic.apiKey`) are read only from env vars and `~/.libscope/secrets.json`. 30-second cache TTL. -Key env vars: `LIBSCOPE_EMBEDDING_PROVIDER` (local|ollama|openai), `LIBSCOPE_LLM_PROVIDER` (openai|ollama|anthropic|passthrough), `LIBSCOPE_OPENAI_API_KEY`, `LIBSCOPE_ANTHROPIC_API_KEY`, `LIBSCOPE_OLLAMA_URL`, `LIBSCOPE_OLLAMA_MODEL`. +Every key `section.field` has the env var `LIBSCOPE_<SECTION>_<FIELD>`. Key env vars: `LIBSCOPE_EMBEDDING_PROVIDER` (local|ollama|openai), `LIBSCOPE_EMBEDDING_MODEL`, `LIBSCOPE_EMBEDDING_URL`, `LIBSCOPE_LLM_PROVIDER` (auto|openai|anthropic|ollama|passthrough; default auto), `LIBSCOPE_LLM_MODEL`, `LIBSCOPE_LLM_URL`, `LIBSCOPE_OPENAI_API_KEY` (or `OPENAI_API_KEY`), `LIBSCOPE_ANTHROPIC_API_KEY` (or `ANTHROPIC_API_KEY`), `LIBSCOPE_MCP_TOOLSETS` (admin|all). Not config keys: `LIBSCOPE_WORKSPACE`, `LIBSCOPE_API_KEY` (REST auth), `LIBSCOPE_SECRET_KEY` (webhook secrets), `LIBSCOPE_VERBOSE`, `LIBSCOPE_GIT_TIMEOUT_MS`. Full list: `docs/reference/configuration.md`. From f391c105cfb374886f61ba343b7432e23e721798 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 19:08:46 +0000 Subject: [PATCH 64/83] docs: bring the README, guides and agent notes in line with 2.0 README: shorter, with the 2.0 quick start (CLI, MCP config, SDK) and links to the generated references and the migration guide. Guides: connections (connect/sync/connections/disconnect), admin commands, doctor, MCP core/admin tools, generated REST routes, createLite and createCodeChunker, scope.links.graph and scope.registries. Architecture describes the operation layer and the surface adapters (schema version 19). agents.md, CONTRIBUTING.md and .env.example use the 2.0 names. --- .env.example | 29 +- CONTRIBUTING.md | 65 ++-- README.md | 599 +++---------------------------- agents.md | 233 ++++++------ docs/contributing.md | 71 ++-- docs/guide/architecture.md | 516 +++++++++++++------------- docs/guide/code-indexing.md | 104 +++--- docs/guide/configuration.md | 50 +-- docs/guide/connectors.md | 189 +++++----- docs/guide/dashboard.md | 2 +- docs/guide/getting-started.md | 6 +- docs/guide/how-search-works.md | 51 ++- docs/guide/mcp-setup.md | 2 +- docs/guide/programmatic-usage.md | 53 +-- docs/guide/troubleshooting.md | 81 ++--- docs/guide/webhooks.md | 14 +- docs/index.md | 6 +- docs/reference/lite-api.md | 2 +- docs/reference/registry.md | 2 +- 19 files changed, 813 insertions(+), 1262 deletions(-) diff --git a/.env.example b/.env.example index 17615e42..15393eb4 100644 --- a/.env.example +++ b/.env.example @@ -1,21 +1,36 @@ # LibScope environment variables # LibScope does not load .env files. Export these in your shell, or put them in # the "env" block of your MCP client config. This file is a reference only. +# Every config key has a variable LIBSCOPE_<SECTION>_<FIELD>; see +# docs/reference/configuration.md for the full list. # Embedding provider: "local" | "ollama" | "openai" (default: "local") LIBSCOPE_EMBEDDING_PROVIDER=local -# Ollama settings (used when the provider is "ollama") -LIBSCOPE_OLLAMA_URL=http://localhost:11434 -LIBSCOPE_OLLAMA_MODEL=nomic-embed-text +# Embedding model and server URL (the local provider ignores both) +# LIBSCOPE_EMBEDDING_MODEL=nomic-embed-text +# LIBSCOPE_EMBEDDING_URL=http://localhost:11434 # OpenAI key (used for embeddings and for the LLM). -# Precedence: LIBSCOPE_OPENAI_API_KEY > OPENAI_API_KEY > config file. +# Precedence: LIBSCOPE_OPENAI_API_KEY > OPENAI_API_KEY > ~/.libscope/secrets.json. LIBSCOPE_OPENAI_API_KEY= -# LLM for `ask` / RAG: "openai" | "ollama" | "anthropic" | "passthrough" -# LIBSCOPE_LLM_PROVIDER= +# LLM for `ask`: "auto" (default) | "openai" | "anthropic" | "ollama" | "passthrough" +# LIBSCOPE_LLM_PROVIDER=auto # LIBSCOPE_LLM_MODEL= +# LIBSCOPE_LLM_URL= -# Anthropic key. Precedence: LIBSCOPE_ANTHROPIC_API_KEY > ANTHROPIC_API_KEY > config file. +# Anthropic key. Precedence: LIBSCOPE_ANTHROPIC_API_KEY > ANTHROPIC_API_KEY > ~/.libscope/secrets.json. # LIBSCOPE_ANTHROPIC_API_KEY= + +# Fetching from private networks and self-signed TLS certificates (default: false) +# LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS=false +# LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS=false + +# MCP server: optional toolsets ("admin" adds sync, install-pack, list-packs, reindex-documents) +# LIBSCOPE_MCP_TOOLSETS=admin + +# Not config keys: active workspace, REST API key, key that encrypts stored webhook secrets +# LIBSCOPE_WORKSPACE=default +# LIBSCOPE_API_KEY= +# LIBSCOPE_SECRET_KEY= diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dc5d039b..003b1974 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,8 +12,8 @@ cd libscope # Install dependencies npm install -# Initialize the database -npm run build && node dist/cli/index.js init +# Build, then check the setup (creates the database) +npm run build && node dist/cli/index.js doctor --fix # Run in development mode (watch) npm run dev @@ -21,37 +21,55 @@ npm run dev ## Scripts -| Command | Description | -| ----------------------- | -------------------------------- | -| `npm run build` | Compile TypeScript to `dist/` | -| `npm run dev` | Watch mode compilation | -| `npm run lint` | Run ESLint | -| `npm run lint:fix` | Run ESLint with auto-fix | -| `npm run format` | Format code with Prettier | -| `npm run format:check` | Check formatting without changes | -| `npm run typecheck` | Type-check without emitting | -| `npm test` | Run tests | -| `npm run test:watch` | Run tests in watch mode | -| `npm run test:coverage` | Run tests with coverage report | +| Command | Description | +| ----------------------- | ----------------------------------------------------------------------- | +| `npm run build` | Compile TypeScript to `dist/` | +| `npm run dev` | Watch mode compilation | +| `npm run lint` | Run ESLint | +| `npm run lint:fix` | Run ESLint with auto-fix | +| `npm run format` | Format code with Prettier | +| `npm run format:check` | Check formatting without changes | +| `npm run typecheck` | Type-check without emitting | +| `npm test` | Run tests | +| `npm run test:watch` | Run tests in watch mode | +| `npm run test:coverage` | Run tests with coverage report | +| `npm run docs:gen` | Update the generated blocks in `docs/reference` (after `npm run build`) | +| `npm run docs:check` | Fail when a generated block is out of date (runs in CI) | +| `npm run docs:dev` | Docs site dev server (VitePress) | +| `npm run docs:build` | Build the docs site | ## Project Structure ``` src/ -├── core/ # Business logic (indexing, search, ratings, topics, documents) -├── db/ # SQLite schema, migrations, connection management +├── core/ +│ ├── operations/ # The operation layer: every command, tool, route and SDK method calls one +│ ├── parsers/ # File format parsers +│ └── ... # Business logic (ingest, indexing, search, rag, documents, tasks, ...) +├── cli/ # CLI: index.ts and one file per command group in commands/ +├── mcp/ # MCP server: tools generated from operations +├── api/ # REST API: routes generated from operations, OpenAPI +├── web/ # Web dashboard +├── lite/ # libscope/lite: createLite(), code chunker, normalizeRawInput() +├── db/ # SQLite schema, migrations, connection ├── providers/ # Embedding providers (local, Ollama, OpenAI) -├── mcp/ # MCP server and tool definitions -├── cli/ # CLI entry point and commands -├── config.ts # Configuration management +├── connectors/ # Notion, Slack, Confluence, Obsidian, OneNote, doc sites +├── registry/ # Git pack registries +├── config-schema.ts # Config schema (keys, defaults, env vars) +├── config.ts # Config loading +├── LibScope.ts # Node.js SDK class ├── logger.ts # Structured logging (pino) -└── errors.ts # Custom error hierarchy +└── errors.ts # Error hierarchy +scripts/ +└── gen-docs.mjs # Fills the generated blocks of docs/reference tests/ ├── unit/ # Fast isolated tests with mocked dependencies -├── integration/ # Tests with real SQLite DB +├── integration/ # Tests with a real SQLite database └── fixtures/ # Test helpers, mock providers, sample data ``` +See the [Architecture Guide](docs/guide/architecture.md) for the operation layer and how to add an operation. The CLI, MCP tool, configuration and REST references in `docs/reference` are partly generated from the code: after you change a command, tool, config key or route, run `npm run build && npm run docs:gen` and commit the result. + ## Making Changes 1. **Create a branch** from `main` @@ -59,7 +77,8 @@ tests/ 3. **Add tests** for new functionality 4. **Run the full check suite:** ```bash - npm run lint && npm run typecheck && npm test + npm run format:check && npm run lint && npm run typecheck && npm test + npm run build && npm run docs:check ``` 5. **Commit** with a descriptive message following [Conventional Commits](https://www.conventionalcommits.org/): - `feat: add bulk import from directory` @@ -93,4 +112,4 @@ Open an issue on GitHub with: ## License -By contributing, you agree that your contributions will be licensed under the MIT License. +By contributing, you agree that your contributions will be licensed under the project's license (Business Source License 1.1, see `LICENSE`). diff --git a/README.md b/README.md index c5342a4e..442fa48a 100644 --- a/README.md +++ b/README.md @@ -9,51 +9,37 @@ [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=RobertLD_libscope&metric=bugs)](https://sonarcloud.io/summary/new_code?id=RobertLD_libscope) [![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=RobertLD_libscope&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=RobertLD_libscope) -LibScope is a local knowledge base that makes your documentation searchable by AI assistants. Point it at markdown files, URLs, or connect it to Obsidian/Notion/Confluence/Slack, and it chunks, embeds, and indexes everything into a local SQLite database. Your AI tools query it through [MCP](https://modelcontextprotocol.io/) (Model Context Protocol) or a REST API. +LibScope is a local knowledge base that makes your documentation searchable by AI assistants. Add markdown files, PDFs, web pages, repositories, or connect Obsidian, Notion, Confluence, Slack and OneNote. LibScope chunks, embeds and indexes everything into a local SQLite database. Your AI tools query it through [MCP](https://modelcontextprotocol.io/) (Model Context Protocol), the CLI, a REST API or the Node.js SDK. -Everything runs on your machine. No cloud services required for basic usage — just `npm install` and go. +Everything runs on your machine. The default embedding model runs locally, so no cloud service is needed for indexing and search. -## Getting Started +**Upgrading from 1.x?** LibScope 2.0 renames many commands, MCP tools, REST routes, config keys and SDK methods. See [Migrating to LibScope 2.0](docs/migration-v2.md). + +## Quick start + +### CLI ```bash npm install -g libscope -# Set up the database -libscope init +# Add files, directories, web pages or GitHub/GitLab repositories +libscope add ./docs --library my-lib +libscope add https://docs.example.com/guide --spider --max-pages 50 +libscope add https://github.com/org/repo -# Index some docs -libscope add ./docs/getting-started.md --library my-lib -libscope add https://docs.example.com/guide -libscope import ./docs/ --library my-lib --extensions .md,.mdx - -# Search +# Search, and ask questions (ask needs an LLM: set an OpenAI or Anthropic key, or use Ollama) libscope search "how to authenticate" +libscope ask "How do I configure OAuth2?" -# Start the MCP server so your AI assistant can query it -libscope serve +# Check the setup: config, database, embedding model, index and LLM +libscope doctor ``` -On first run with the default embedding provider, LibScope downloads the [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) model (~80 MB). Subsequent runs use the cached model. - -### Document Format Support - -LibScope supports **Markdown** (`.md`, `.mdx`) and **plain text** natively. Additional formats are available via optional dependencies: - -| Format | Extension | Optional Dependency | Node.js Requirement | -| ------------ | --------- | ------------------- | ------------------- | -| PDF | `.pdf` | `pdf-parse` (v2) | ≥ 20.16 or ≥ 22.3 | -| Word | `.docx` | `mammoth` | Any | -| EPUB | `.epub` | `epub2` | Any | -| PowerPoint | `.pptx` | `pizzip` | Any | -| CSV | `.csv` | Built-in | Any | - -The `pdf-parse`, `mammoth`, `epub2`, and `pizzip` packages are listed as `optionalDependencies` and install automatically when the Node.js version is compatible. Note: binary `.ppt` files are not supported — only `.pptx`. +The database is created on first use. The first command that embeds text downloads the local model [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) (about 80 MB, first run only). -## Using with AI Assistants +### MCP (Claude, Cursor, VS Code, ...) -LibScope exposes an MCP server over stdio. Point your MCP-compatible client at it: - -**Cursor** — add to `~/.cursor/mcp.json`: +Add LibScope to your MCP client config: ```json { @@ -66,543 +52,74 @@ LibScope exposes an MCP server over stdio. Point your MCP-compatible client at i } ``` -**Claude Code:** +For Claude Code: ```bash claude mcp add --scope user libscope -- npx -y libscope serve ``` -Once connected, your assistant can search docs, submit new documents, rate content quality, and ask RAG-powered questions against your knowledge base. - -<details> -<summary>Full list of MCP tools</summary> - -| Tool | What it does | -| ---------------------- | --------------------------------------------------------- | -| `search-docs` | Semantic search with topic/library/version/rating filters | -| `ask-question` | RAG question-answering with source citations | -| `get-document` | Retrieve a document by ID | -| `list-documents` | List docs with filters | -| `list-topics` | Browse the topic hierarchy | -| `submit-document` | Index new content (raw text or a URL to fetch) | -| `update-document` | Update a document's title, content, or metadata | -| `delete-document` | Remove a document | -| `rate-document` | Rate a doc 1–5 with optional feedback and corrections | -| `suggest-tags` | Auto-suggest tags based on content analysis | -| `save-search` | Save a named search query with filters | -| `list-saved-searches` | List all saved searches | -| `run-saved-search` | Execute a saved search by name or ID | -| `delete-saved-search` | Delete a saved search | -| `link-documents` | Create a cross-reference between two documents | -| `get-document-links` | List all incoming and outgoing links for a document | -| `delete-link` | Remove a cross-reference link | -| `reindex-documents` | Re-embed chunks (useful after switching providers) | -| `health-check` | DB status, doc/chunk counts | -| `sync-obsidian-vault` | Sync an Obsidian vault | -| `sync-onenote` | Sync OneNote notebooks via Microsoft Graph | -| `sync-notion` | Sync Notion pages and databases | -| `sync-confluence` | Sync Confluence spaces | -| `sync-slack` | Sync Slack channels and threads | -| `install-pack` | Install a knowledge pack | -| `list-packs` | List installed or registry packs | - -</details> - -## Connectors - -LibScope can pull documentation from several platforms. Each connector handles incremental syncing so re-runs only process what changed. - -Each `connect` command saves its settings, including credentials, to `~/.libscope/connectors/<name>.json` (mode 0600; the name defaults to the connector type, set it with `--name`). `--sync` re-runs a saved config, and `libscope schedule set <name> "<cron>"` syncs it on a schedule while `libscope serve --api` runs. - -```bash -# Obsidian — parses wikilinks, frontmatter, embeds, tags -libscope connect obsidian /path/to/vault -libscope connect obsidian --sync # re-sync the saved vault config - -# Notion -libscope connect notion --token secret_abc123 - -# Confluence -libscope connect confluence \ - --url https://acme.atlassian.net \ - --email user@acme.com \ - --token $CONFLUENCE_TOKEN - -# Slack — index channel messages and threads -libscope connect slack --token xoxb-... --channels general,engineering - -# OneNote — device code auth via Microsoft Graph -export ONENOTE_CLIENT_ID=your-client-id -libscope connect onenote - -# GitHub / GitLab repos -libscope add-repo https://github.com/org/repo --branch main --path docs/ - -# Sync a saved connector every 6 hours (runs inside `libscope serve --api`) -libscope schedule set notion "0 */6 * * *" - -# Remove a connector's data and its saved config -libscope disconnect obsidian /path/to/vault -``` - -<details> -<summary>Connector options reference</summary> - -**All connect commands:** `--sync` (re-run the saved config), `--name <name>` (saved config name; default: connector type) - -**Obsidian:** `--topic-mapping frontmatter`, `--exclude "templates/*" "daily/*"` - -**Notion:** `--exclude page-id-1 db-id-2` - -**Confluence:** `--spaces ENG,DEVOPS`, `--exclude-spaces ARCHIVE`, `--type server` - -**Slack:** `--thread-mode aggregate|separate`, `--exclude random` - -**OneNote:** `--notebook "Work Notes"`, `--token <access-token>` - -**GitHub/GitLab:** `--token`, `--branch`, `--path`, `--extensions .md,.mdx,.rst` - -</details> - -## Search and RAG - -```bash -# Semantic search -libscope search "authentication best practices" -libscope search "API rate limiting" --library my-lib --topic security --limit 10 - -# Ask questions (needs an LLM provider configured — see Configuration below) -libscope ask "How do I configure OAuth2?" --library my-lib - -# Interactive REPL for iterative searching -libscope repl -``` - -Search uses sqlite-vec for vector similarity when available, with FTS5 full-text search as a fallback. Results automatically get a 1.5× title boost when the document title matches query words. You can also pass `--diversity 0.5` (0–1) for MMR-based diversity reranking. - -### Programmatic SDK +The server has 11 core tools (`search`, `ask`, `get-document`, `list-documents`, `overview`, `submit-document`, `update-document`, `delete-document`, `rate-document`, `link-documents`, `task`). Set `LIBSCOPE_MCP_TOOLSETS=admin` to add `sync`, `install-pack`, `list-packs` and `reindex-documents`. Under MCP, `ask` returns the retrieved context by default, and your assistant writes the answer. -LibScope exports two embeddable APIs: - -**`LibScope`** — full SDK with all features (connectors, topics, packs, etc.): +### SDK (Node.js) ```ts import { LibScope } from "libscope"; -const scope = LibScope.create(); -await scope.index({ title: "My Doc", content: "..." }); -const results = await scope.search("query"); -scope.close(); -``` - -**`LibScopeLite`** — lightweight embeddable class for external applications. No CLI, no MCP server, no connectors. Designed for embedding semantic search directly into other tools (MCP servers, VS Code extensions, CI scripts): - -```ts -import { LibScopeLite } from "libscope/lite"; - -const lite = new LibScopeLite({ dbPath: ":memory:" }); +const scope = LibScope.create(); // the same database as the CLI and MCP server -// Index documents (or code files via tree-sitter chunking) -await lite.indexBatch(docs, { concurrency: 4 }); +await scope.add({ + title: "Auth Guide", + content: "# Authentication\n\nUse OAuth2...", + library: "my-lib", +}); +await scope.add("./docs"); -// Hybrid vector + FTS5 search -const results = await lite.search("how to authenticate"); +const { items } = await scope.search("how to authenticate"); +for (const hit of items) console.log(hit.documentId, hit.title, hit.score); -// Get RAG context for injection into an external LLM prompt -const context = await lite.getContext("How does auth work?"); +const result = await scope.ask("How do I configure OAuth2?"); +if (result.mode === "answer") console.log(result.answer); -lite.close(); -``` - -Tree-sitter powered code indexing splits TypeScript, JavaScript, and Python files at function/class boundaries: - -```ts -import { TreeSitterChunker } from "libscope/lite"; - -const chunker = new TreeSitterChunker(); -const chunks = await chunker.chunk(sourceCode, "typescript"); -// Each chunk is a complete function or class with 1-based line numbers -``` - -See the [LibScope Lite guide](https://libscope.com/guide/lite) and [API reference](https://libscope.com/reference/lite-api) for the full documentation. - -## Organizing Content - -**Topics** give your docs a hierarchy. **Tags** give them flexible labels. **Workspaces** give you isolated databases. - -```bash -# Topics -libscope topics create "backend" -libscope topics create "auth" --parent backend --description "Auth & identity" - -# Tags -libscope tag add <doc-id> typescript,api,v2 - -# Workspaces — separate knowledge bases entirely -libscope workspace create my-project -libscope workspace use my-project -libscope --workspace my-project search "deploy steps" -``` - -Documents also keep version history, so you can roll back if a re-index goes wrong: - -```bash -libscope docs history <doc-id> -libscope docs rollback <doc-id> 3 -``` - -## REST API - -For programmatic access outside of MCP: - -```bash -libscope serve --api --port 3378 -``` - -OpenAPI 3.0 spec at `GET /openapi.json`. Key endpoints: - -| Method | Endpoint | Description | -| --------------- | --------------------------------- | ---------------------------------- | -| `GET` | `/api/v1/search?q=...` | Semantic search | -| `POST` | `/api/v1/batch-search` | Batch search (up to 20 queries) | -| `POST` | `/api/v1/ask` | RAG question-answering | -| `GET/POST` | `/api/v1/documents` | List or create documents | -| `GET/PATCH/DELETE` | `/api/v1/documents/:id` | Get, update, or delete a document | -| `POST` | `/api/v1/documents/url` | Index from a URL | -| `POST` | `/api/v1/documents/:id/tags` | Add tags | -| `GET` | `/api/v1/documents/:id/suggest-tags` | Auto-suggest tags | -| `GET/POST` | `/api/v1/documents/:id/links` | List or create cross-references | -| `DELETE` | `/api/v1/links/:id` | Delete a cross-reference | -| `GET/POST` | `/api/v1/topics` | List or create topics | -| `GET` | `/api/v1/tags` | List tags | -| `GET/POST` | `/api/v1/searches` | List or create saved searches | -| `POST` | `/api/v1/searches/:id/run` | Run a saved search | -| `DELETE` | `/api/v1/searches/:id` | Delete a saved search | -| `POST` | `/api/v1/bulk/delete` | Bulk delete documents | -| `POST` | `/api/v1/bulk/retag` | Bulk add/remove tags | -| `POST` | `/api/v1/bulk/move` | Bulk move to a topic | -| `GET/POST` | `/api/v1/webhooks` | List or create webhooks | -| `DELETE` | `/api/v1/webhooks/:id` | Delete a webhook | -| `POST` | `/api/v1/webhooks/:id/test` | Send a test ping to a webhook | -| `GET` | `/api/v1/analytics/searches` | Search analytics and knowledge gaps| -| `GET` | `/api/v1/connectors/status` | Connector sync status and history | -| `GET` | `/api/v1/stats` | Usage statistics | -| `GET` | `/api/v1/health` | Health check | - -## Configuration - -LibScope reads config from (highest priority first): **environment variables** → **`.libscope.json`** in your project → **`~/.libscope/config.json`** → built-in defaults. - -### Embedding Providers - -| Provider | Default? | Requirements | Model | -| -------- | -------- | ----------------------------------------- | ---------------------- | -| `local` | Yes | None (~80 MB model download on first run) | all-MiniLM-L6-v2 | -| `ollama` | | Ollama running locally | nomic-embed-text | -| `openai` | | `LIBSCOPE_OPENAI_API_KEY` | text-embedding-3-small | - -```bash -libscope config set embedding.provider ollama -# or -export LIBSCOPE_EMBEDDING_PROVIDER=openai -export LIBSCOPE_OPENAI_API_KEY=sk-... -``` - -The vector index is tied to the model that built it. After you change the provider or model, run `libscope reindex --rebuild`. For a model whose vector size LibScope does not know, set `embedding.dimensions` in the config file. - -### LLM for RAG - -The `ask` command and `ask-question` MCP tool need an LLM. Configure one with: - -```bash -export LIBSCOPE_LLM_PROVIDER=openai # or ollama, anthropic -export LIBSCOPE_LLM_MODEL=gpt-4o-mini # optional -``` - -<details> -<summary>All environment variables</summary> - -| Variable | Description | Default | -| ---------------------------------- | ---------------------------------------- | ------------------------ | -| `LIBSCOPE_EMBEDDING_PROVIDER` | `local`, `ollama`, or `openai` | `local` | -| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key (embeddings and LLM) | — | -| `OPENAI_API_KEY` | Used when `LIBSCOPE_OPENAI_API_KEY` is not set | — | -| `LIBSCOPE_OLLAMA_URL` | Ollama server URL | `http://localhost:11434` | -| `LIBSCOPE_OLLAMA_MODEL` | Ollama embedding model | `nomic-embed-text` | -| `LIBSCOPE_LLM_PROVIDER` | LLM for RAG (`openai` / `ollama` / `anthropic`) | — | -| `LIBSCOPE_LLM_MODEL` | LLM model override | — | -| `LIBSCOPE_ANTHROPIC_API_KEY` | Anthropic API key (for Claude models) | — | -| `ANTHROPIC_API_KEY` | Used when `LIBSCOPE_ANTHROPIC_API_KEY` is not set | — | -| `LIBSCOPE_ALLOW_PRIVATE_URLS` | Allow fetching from private/internal IPs | `false` | -| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | Accept self-signed TLS certificates | `false` | -| `LIBSCOPE_WORKSPACE` | Active workspace for this shell | `default` | -| `LIBSCOPE_API_KEY` | REST API key (`Authorization: Bearer <key>`) | — | -| `LIBSCOPE_SECRET_KEY` | Encrypts stored webhook secrets | — | -| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (stderr) in the CLI | — | -| `ONENOTE_CLIENT_ID` | Microsoft app registration client ID | — | -| `ONENOTE_TENANT_ID` | Microsoft tenant ID | `common` | -| `NOTION_TOKEN` | Notion integration token | — | -| `CONFLUENCE_URL` | Confluence base URL | — | -| `CONFLUENCE_EMAIL` | Confluence user email | — | -| `CONFLUENCE_TOKEN` | Confluence API token | — | - -API keys use one rule for embeddings and the LLM: `LIBSCOPE_<PROVIDER>_API_KEY`, then `<PROVIDER>_API_KEY`, then the key in a config file. LibScope does not load `.env` files. - -</details> - -<details> -<summary>Example config file (~/.libscope/config.json)</summary> - -```json -{ - "embedding": { - "provider": "local", - "ollamaUrl": "http://localhost:11434", - "ollamaModel": "nomic-embed-text", - "openaiModel": "text-embedding-3-small" - }, - "llm": { - "provider": "openai", - "model": "gpt-4o-mini" - }, - "logging": { - "level": "info" - }, - "indexing": { - "allowPrivateUrls": false, - "allowSelfSignedCerts": false - } -} -``` - -</details> - -### Corporate / Internal Networks - -If you're indexing docs from internal servers (Confluence, wikis, etc.), you may need: - -```bash -# Allow fetching from private/internal IP addresses -libscope config set indexing.allowPrivateUrls true - -# Accept self-signed or corporate TLS certificates -libscope config set indexing.allowSelfSignedCerts true -``` - -Or via environment variables: - -```bash -export LIBSCOPE_ALLOW_PRIVATE_URLS=true -export LIBSCOPE_ALLOW_SELF_SIGNED_CERTS=true -``` - -## Webhooks - -LibScope can push events to any HTTP endpoint. Useful for triggering CI pipelines, Slack notifications, or custom workflows whenever documents are created or updated. - -Events fire from every surface (CLI, MCP server, REST API, SDK) when that process changes the knowledge base. Delivery runs in the background and never fails or delays the operation that caused it. - -```bash -# Create a webhook (CLI) -libscope webhooks create https://hooks.example.com/libscope --events document.created,document.updated - -# Or via the REST API -curl -X POST http://localhost:3378/api/v1/webhooks \ - -H "Content-Type: application/json" \ - -d '{"url": "https://hooks.example.com/libscope", "events": ["document.created", "document.updated"], "secret": "my-hmac-secret"}' - -# Send a test ping -libscope webhooks test <id> -curl -X POST http://localhost:3378/api/v1/webhooks/<id>/test +scope.close(); ``` -Webhook payloads are signed with HMAC-SHA256 when a secret is set (storing a secret requires `LIBSCOPE_SECRET_KEY`). The hex signature is in the `X-LibScope-Signature` header. +To embed LibScope in another application without reading config files, use `createLite()` from `libscope/lite`. -Supported events: `document.created`, `document.updated`, `document.deleted`, `document.rated`, `search.executed`. +## Documentation -## Other Tools +The full documentation is at [libscope.com](https://libscope.com). -LibScope ships with a few more utilities beyond the core index-and-search loop: +Guides: -```bash -# Watch a directory and auto-reindex on changes -libscope watch ./docs/ +- [Getting started](https://libscope.com/guide/getting-started) +- [Configuration](https://libscope.com/guide/configuration): embedding providers, LLM for `ask`, API keys +- [MCP setup](https://libscope.com/guide/mcp-setup) +- [Connectors](https://libscope.com/guide/connectors): Obsidian, Notion, Confluence, Slack, OneNote, documentation sites +- [Knowledge packs](https://libscope.com/guide/knowledge-packs) and [pack registries](https://libscope.com/guide/pack-registries) +- [Programmatic usage](https://libscope.com/guide/programmatic-usage) and [LibScope Lite](https://libscope.com/guide/lite) +- [Web dashboard](https://libscope.com/guide/dashboard), [webhooks](https://libscope.com/guide/webhooks), [troubleshooting](https://libscope.com/guide/troubleshooting) -# Rebuild the vector index after switching embedding provider or model -libscope reindex --rebuild +Reference (generated from the code): -# Find duplicate documents -libscope dedupe +- [CLI commands](docs/reference/cli.md) +- [MCP tools](docs/reference/mcp-tools.md) +- [REST API](docs/reference/rest-api.md) (`libscope serve api`) +- [Configuration keys and environment variables](docs/reference/configuration.md) -# Export / import the whole knowledge base -libscope export ./backup.json -libscope import-backup ./backup.json +Client SDKs for the REST API: [Python](sdk/python/README.md) and [Go](sdk/go/README.md). -# Usage analytics -libscope stats # overview -libscope stats popular # most-searched docs -libscope stats stale --days 90 # docs nobody searches for +## Supported formats -# Knowledge packs — portable document bundles -libscope pack create --name "react-docs" --topic react -libscope pack install ./react-docs.json -``` - -### Pack Registries +Markdown (`.md`, `.mdx`, `.markdown`), plain text, HTML, JSON, YAML, CSV, EPUB and PowerPoint (`.pptx`) are built in. PDF (`pdf-parse`) and Word `.docx` (`mammoth`) need optional dependencies, which npm installs when your Node.js version supports them. -Share and discover knowledge packs through git-based registries. A registry is a git repo with a defined folder structure managed by libscope. +## How it works -```bash -# Add a registry -libscope registry add https://github.com/org/libscope-registry.git --name official - -# Search for packs across all registries -libscope registry search "react" - -# Install a pack by name (resolves from registries) -libscope pack install react-docs -libscope pack install react-docs@1.2.0 # specific version - -# Create your own registry -libscope registry create ./my-registry - -# Publish a pack file to your registry -libscope registry publish ./my-pack.json --registry my-registry --pack-version 1.0.0 - -# Submit a pack to someone else's registry (creates a feature branch) -libscope registry publish ./my-pack.json --registry community --submit -``` +LibScope stores everything in one SQLite database per workspace (`~/.libscope/workspaces/default/libscope.db` for the default workspace). The CLI, MCP server, REST API and SDK all open the same file and call the same operations, so they take the same parameters and return the same results. -Authentication is delegated to git — SSH keys and HTTPS credential helpers work automatically. Search and install read the local copies of the registries; `libscope registry sync` updates them. See the [Pack Registries guide](/guide/pack-registries) for full details. - -There's also a web dashboard at `http://localhost:3377` when you run `libscope serve`, with search, document browsing, topic navigation, and a knowledge graph visualization at `/graph`. - -<details> -<summary>Full CLI reference</summary> - -**Core** - -| Command | Description | -| ----------------------------------- | ----------------------------------- | -| `libscope init` | Initialize the database | -| `libscope add <fileOrUrl>` | Index a file or URL | -| `libscope import <directory>` | Bulk import from a directory | -| `libscope import-batch <directory>` | Parallel batch import | -| `libscope search <query>` | Search | -| `libscope ask <question>` | RAG question-answering | -| `libscope repl` | Interactive search REPL | -| `libscope serve` | Start MCP server (`--api` for REST) | - -**Documents** - -| Command | Description | -| --------------------------------------- | ---------------------------- | -| `libscope docs list` | List documents | -| `libscope docs show <id>` | Show a document | -| `libscope docs update <id>` | Update title/content/metadata| -| `libscope docs delete <id>` | Delete a document | -| `libscope docs history <id>` | Version history | -| `libscope docs rollback <id> <ver>` | Roll back to a prior version | - -**Organization** - -| Command | Description | -| ------------------------------------ | -------------------------------- | -| `libscope topics list` | List topics | -| `libscope topics create <name>` | Create a topic | -| `libscope tag add <id> <tags...>` | Add tags | -| `libscope tag remove <id> <tag>` | Remove a tag | -| `libscope tag list` | List tags | -| `libscope workspace create <name>` | Create workspace | -| `libscope workspace list` | List workspaces | -| `libscope workspace use <name>` | Switch workspace | -| `libscope workspace delete <name>` | Delete workspace | - -**Saved Searches** - -| Command | Description | -| ------------------------------------ | ----------------------------- | -| `libscope searches list` | List all saved searches | -| `libscope searches run <name>` | Re-run a saved search | -| `libscope searches delete <name>` | Delete a saved search | -| `libscope search <q> --save <name>` | Save a search while running it| - -**Document Links** - -| Command | Description | -| ------------------------------------- | -------------------------------- | -| `libscope link <srcId> <tgtId>` | Create a cross-reference | -| `libscope links <docId>` | Show all links for a document | -| `libscope unlink <linkId>` | Remove a link | -| `libscope prereqs <docId>` | Show prerequisite reading chain | - -**Bulk Operations** - -| Command | Description | -| ---------------------------- | ----------------------------------- | -| `libscope bulk delete` | Delete all matching documents | -| `libscope bulk retag` | Add/remove tags on matching docs | -| `libscope bulk move` | Move matching docs to a topic | - -**Connectors** - -| Command | Description | -| ---------------------------------- | -------------------------- | -| `libscope connect obsidian [path]` | Sync Obsidian vault | -| `libscope connect onenote` | Sync OneNote | -| `libscope connect notion` | Sync Notion | -| `libscope connect confluence` | Sync Confluence | -| `libscope connect slack` | Sync Slack | -| `libscope add-repo <url>` | Index a GitHub/GitLab repo | -| `libscope disconnect <name>` | Remove connector data | -| `libscope schedule set <name> <cron>` | Schedule a saved connector | - -**Registries** - -| Command | Description | -| ----------------------------------------------------- | ---------------------------------------- | -| `libscope registry add <url> [-n <alias>]` | Register a git repo as a pack registry | -| `libscope registry remove <name>` | Unregister a registry | -| `libscope registry list` | List configured registries | -| `libscope registry sync [<name>]` | Sync one or all registries | -| `libscope registry search <query> [--registry <name>]` | Search registry pack indexes | -| `libscope registry create <path>` | Initialize a new registry repo | -| `libscope registry publish <file> --registry <name>` | Publish a pack file to a registry | -| `libscope registry unpublish <name@version> --registry <name>` | Remove a pack version from a registry | - -**Utilities** - -| Command | Description | -| --------------------------------- | ---------------------------- | -| `libscope watch <dir>` | Auto-reindex on file changes | -| `libscope reindex` | Re-embed all chunks | -| `libscope dedupe` | Find duplicates | -| `libscope export <path>` | Export to JSON | -| `libscope import-backup <path>` | Import from backup | -| `libscope stats` | Usage overview | -| `libscope pack install <name>` | Install a knowledge pack | -| `libscope pack create` | Create a knowledge pack | -| `libscope config set <key> <val>` | Set any config key (not API keys) | -| `libscope config get <key>` | Print one effective value | -| `libscope config unset <key>` | Remove a key from the user config file | -| `libscope config path` | Print the user config file path | -| `libscope config show` | Show config (API keys masked) | - -**Global flags:** `--verbose`, `--log-level <level>`, `--workspace <name>` - -</details> - -## How It Works - -LibScope stores everything in a local SQLite database, one per workspace (`~/.libscope/workspaces/<workspace>/libscope.db`; the default workspace is `default`). Set `database.path` to use a specific file instead. The CLI, MCP server, REST API, and `LibScope.create()` all open the same file: - -- Documents are split into chunks by heading boundaries -- Each chunk is embedded into a vector using the configured provider -- Vector search is done via [sqlite-vec](https://github.com/asg017/sqlite-vec); FTS5 full-text search is used as a fallback -- The MCP server reads from this same database over stdio -- Connectors fetch content from external platforms and feed it through the same indexing pipeline - -The stack: [better-sqlite3](https://github.com/WiseLibs/better-sqlite3) + [sqlite-vec](https://github.com/asg017/sqlite-vec) + [@modelcontextprotocol/sdk](https://github.com/modelcontextprotocol/typescript-sdk) + [@xenova/transformers](https://github.com/xenova/transformers.js) for local embeddings. +- Documents are split into chunks at heading boundaries. +- Each chunk is embedded with the configured provider (local, Ollama or OpenAI). +- Search combines vector similarity ([sqlite-vec](https://github.com/asg017/sqlite-vec)) with FTS5 full-text search. +- Connectors fetch content from other platforms and feed it through the same indexing pipeline. ## Contributing diff --git a/agents.md b/agents.md index 1b21baa8..757c08e6 100644 --- a/agents.md +++ b/agents.md @@ -13,67 +13,66 @@ LibScope is an **AI-powered knowledge base with MCP (Model Context Protocol) int ## Quick Reference — Commands -| Task | Command | -| ----------------------- | ----------------------- | -| Build | `npm run build` | -| Typecheck (no emit) | `npm run typecheck` | -| Run all tests | `npm test` | -| Run tests in watch mode | `npm run test:watch` | -| Run tests with coverage | `npm run test:coverage` | -| Lint | `npm run lint` | -| Lint and auto-fix | `npm run lint:fix` | -| Format check | `npm run format:check` | -| Format and write | `npm run format` | -| Start MCP server | `npm run serve` | -| TypeScript watch | `npm run dev` | +| Task | Command | +| ------------------------- | ----------------------------------- | +| Build | `npm run build` | +| Typecheck (no emit) | `npm run typecheck` | +| Run all tests | `npm test` | +| Run tests in watch mode | `npm run test:watch` | +| Run tests with coverage | `npm run test:coverage` | +| Lint | `npm run lint` | +| Lint and auto-fix | `npm run lint:fix` | +| Format check | `npm run format:check` | +| Format and write | `npm run format` | +| Start MCP server | `npm run serve` | +| TypeScript watch | `npm run dev` | +| Regenerate reference docs | `npm run build && npm run docs:gen` | +| Check reference docs | `npm run docs:check` | **Always run `npm run typecheck` and `npm test` before committing.** ## Architecture +One operation layer, four thin surfaces. Every operation (add, search, ask, get-document, ...) is defined once in `src/core/operations/`: a name, a zod input schema (parameter names, defaults, validation, descriptions), annotations, an optional REST mapping (`http`) and a handler. The CLI, the MCP server, the REST API and the SDK are adapters: they parse input, call `runOperation()` (or `startOperationTask()` for long-running operations) and format the result. + ``` src/ -├── cli/index.ts # CLI entry point (commander). All commands in one file. -├── mcp/server.ts # MCP server (stdio transport, @modelcontextprotocol/sdk) +├── core/ +│ ├── operations/ # The operation layer: types.ts (defineOperation, runOperation, +│ │ # startOperationTask, createOperationContext), schemas.ts (shared +│ │ # input fields), one file per group, index.ts (OPERATIONS) +│ ├── bootstrap.ts # One startup for every surface: config, database, provider, vector table +│ ├── ingest.ts # Add inline content, files, directories, URLs, crawls, repositories +│ ├── search.ts, rag.ts # Hybrid search (vector + FTS5, RRF) and answer() for ask +│ ├── tasks.ts # In-memory background task registry (AbortSignal, progress) +│ ├── overview.ts, document-view.ts, documents.ts, topics.ts, tags.ts, links.ts, packs.ts, ... +│ └── parsers/ # File format parsers (markdown, pdf, docx, html, epub, pptx, csv, yaml, json) +├── cli/ +│ ├── index.ts # Commander program, global options, one error handler +│ └── commands/ # One file per command group; each exports register(program) +├── mcp/ +│ ├── server.ts # createMcpServer(): tools generated from operations (core + admin toolsets) +│ ├── format.ts # Compact text output for tools +│ └── main.ts # Executable entry point (stdio) ├── api/ -│ ├── server.ts # HTTP server bootstrap (createServer, listen) -│ ├── routes.ts # All REST route handlers in one function (~700 lines) -│ ├── middleware.ts # Auth (checkApiKey), rate limiting, body parsing, sendError/sendJson -│ └── openapi.ts # OpenAPI spec generation -├── core/ # Business logic — framework-agnostic, no side effects -│ ├── indexing.ts # Document parsing, chunking by heading, embedding + storage -│ ├── search.ts # Semantic (vector) + FTS5 + LIKE fallback search -│ ├── ratings.ts # Rating storage, aggregation, correction suggestions -│ ├── documents.ts # Document CRUD -│ ├── topics.ts # Topic hierarchy management -│ ├── bulk.ts # Bulk delete/move/retag operations with selector resolution -│ ├── tags.ts # Tag CRUD and document–tag associations -│ ├── dedup.ts # Content deduplication helpers -│ ├── rag.ts # Retrieval-augmented generation (ask a question over the index) -│ ├── url-fetcher.ts # Fetch URL → convert HTML to markdown-like text (SSRF-protected) -│ └── index.ts # Public re-exports (barrel file) -├── connectors/ # External-service sync connectors -│ ├── obsidian.ts # Obsidian vault sync (reads .md files + YAML frontmatter) -│ ├── confluence.ts # Confluence Cloud sync (REST API) -│ ├── notion.ts # Notion sync (official API) -│ ├── onenote.ts # OneNote sync (Microsoft Graph API) -│ ├── slack.ts # Slack channel sync (Web API) -│ ├── sync-tracker.ts # Tracks last-synced state per connector in SQLite -│ ├── http-utils.ts # Authenticated fetch helper shared by connectors (respects allowSelfSignedCerts) -│ └── index.ts # Re-exports -├── db/ -│ ├── connection.ts # SQLite connection + sqlite-vec extension loading -│ ├── schema.ts # Migrations (versioned) + vector table creation -│ └── index.ts # Re-exports -├── providers/ -│ ├── embedding.ts # EmbeddingProvider interface -│ ├── local.ts # all-MiniLM-L6-v2 via @xenova/transformers (384 dims) -│ ├── ollama.ts # Ollama API provider (768 dims default) -│ ├── openai.ts # OpenAI text-embedding-3-small (1536 dims) -│ └── index.ts # Factory function + re-exports -├── config.ts # 3-tier config: env > project .libscope.json > user ~/.libscope/config.json > defaults -├── logger.ts # pino structured logging wrapper -└── errors.ts # Custom error hierarchy +│ ├── routes.ts # API_ROUTES: one route per operation with `http`, plus /openapi.json and /health +│ ├── openapi.ts # buildOpenApiSpec() from the same routes +│ ├── adapter.ts # Request to operation input; response and error mapping +│ ├── middleware.ts # Auth, rate limiting, CORS, body parsing +│ └── server.ts # HTTP server +├── web/ # Dashboard server; dashboard.ts is one large template literal (do not edit) +├── LibScope.ts # SDK: LibScope.create(), add/search/ask/askStream/overview and namespaces +│ # (docs, topics, tags, links, searches, packs, registries, connectors, +│ # tasks, admin, analytics, webhooks) bound to operations +├── lite/ # libscope/lite: createLite() preset, tree-sitter code chunker, normalizeRawInput +├── connectors/ # Notion, Slack, Confluence, Obsidian, OneNote, docs sites; registry.ts lists them +├── registry/ # Git-backed pack registries +├── db/ # better-sqlite3 + sqlite-vec; schema.ts has the migrations (schema version 19) +├── providers/ # Embedding providers: local (384 dims), ollama, openai +├── config-schema.ts # One zod schema: keys, defaults, env var names, docs table +├── config.ts # loadConfig(), secrets.json, config set/unset +├── logger.ts # pino +└── errors.ts # Error hierarchy ``` ## Critical Conventions @@ -109,15 +108,18 @@ All errors extend `LibScopeError` from `src/errors.ts`: LibScopeError (base) ├── DatabaseError ├── EmbeddingError -├── ValidationError +├── ValidationError (REST 400) +├── FetchError (REST 502) ├── ConfigError -├── DocumentNotFoundError -└── ChunkNotFoundError +└── NotFoundError (REST 404; the code names the resource, e.g. LINK_NOT_FOUND) + ├── DocumentNotFoundError + ├── ChunkNotFoundError + └── TopicNotFoundError ``` - Public functions should throw typed errors from this hierarchy, never raw `Error`. -- MCP tool handlers catch errors and return structured error responses. -- CLI shows user-friendly messages; `--verbose` enables full stack traces. +- MCP tools return the error message with `isError: true`. REST answers `{ "error": { "code", "message" } }`. +- The CLI prints `✗ <message>` and a next step; `--verbose` adds the stack trace. ### Logging @@ -125,7 +127,7 @@ Uses **pino** for structured JSON logging via `src/logger.ts`. - Call `initLogger(level)` once at startup. - Use `getLogger()` everywhere else to obtain the singleton. -- Default level: `info` for CLI, `warn` for MCP server (to avoid polluting stdio). +- The CLI logs nothing by default (`--verbose` or `--log-level` turns logs on). The MCP server logs at `logging.level` to stderr, so stdout carries only JSON-RPC. - Never use `console.log` in `src/core/`, `src/db/`, or `src/providers/`. Use the logger. (`console.log` is acceptable in `src/cli/` for user-facing output.) ### Database @@ -141,7 +143,7 @@ Uses **pino** for structured JSON logging via `src/logger.ts`. 1. Increment `SCHEMA_VERSION` at the top of `src/db/schema.ts`. 2. Add a new key to the `MIGRATIONS` record with the SQL. 3. The migration must insert its version into `schema_version`. -4. Update the schema version assertion in `tests/unit/schema.test.ts`. +4. Update the tests that assert the schema version. ### Embedding Providers @@ -158,11 +160,13 @@ interface EmbeddingProvider { The factory in `src/providers/index.ts` selects the provider based on config. Default is `local` (runs in-process, downloads model on first use). -### Config Precedence +### Configuration + +One zod schema (`src/config-schema.ts`) defines every key, its default, its validation and its environment variable (`LIBSCOPE_<SECTION>_<FIELD>`, for example `embedding.model` becomes `LIBSCOPE_EMBEDDING_MODEL`). The key table in `docs/reference/configuration.md` is generated from it. -Environment variables > project `.libscope.json` > user `~/.libscope/config.json` > hardcoded defaults. +Precedence: environment variables > `~/.libscope/secrets.json` (API keys only) > project `.libscope.json` > user `~/.libscope/config.json` > defaults. -Env vars: `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_OPENAI_API_KEY`, `LIBSCOPE_OLLAMA_URL`, `LIBSCOPE_ALLOW_PRIVATE_URLS`, `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS`. +Main env vars: `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_EMBEDDING_MODEL`, `LIBSCOPE_EMBEDDING_URL`, `LIBSCOPE_LLM_PROVIDER`, `LIBSCOPE_LLM_MODEL`, `LIBSCOPE_LLM_URL`, `LIBSCOPE_OPENAI_API_KEY` (or `OPENAI_API_KEY`), `LIBSCOPE_ANTHROPIC_API_KEY` (or `ANTHROPIC_API_KEY`), `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS`, `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS`, `LIBSCOPE_MCP_TOOLSETS`. Not config keys: `LIBSCOPE_WORKSPACE`, `LIBSCOPE_API_KEY` (REST auth), `LIBSCOPE_SECRET_KEY`. ## Security Patterns @@ -205,23 +209,6 @@ const response = await fetch(url, { process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0"; ``` -### SSE streaming — check backpressure - -`res.write()` returns `false` when the socket buffer is full or the client has disconnected. Ignoring the return value wastes compute and holds open connections indefinitely. - -```typescript -// ✅ Correct -for await (const event of stream) { - const ok = res.write(`data: ${JSON.stringify(event)}\n\n`); - if (!ok) break; -} - -// ❌ Wrong — no backpressure handling -for await (const event of stream) { - res.write(`data: ${JSON.stringify(event)}\n\n`); -} -``` - ## Testing ### Framework @@ -233,28 +220,20 @@ for await (const event of stream) { ``` tests/ ├── fixtures/ -│ ├── mock-provider.ts # Deterministic 4D embedding provider (use in all unit tests) -│ ├── test-db.ts # In-memory SQLite with migrations (no sqlite-vec) -│ ├── sample-api-docs.md # Sample library documentation -│ └── sample-topic-docs.md # Sample topic documentation -├── unit/ # Fast, isolated, no I/O or network -│ ├── chunking.test.ts -│ ├── config.test.ts -│ ├── documents.test.ts -│ ├── errors.test.ts -│ ├── ratings.test.ts -│ ├── schema.test.ts -│ └── topics.test.ts -└── integration/ - └── workflow.test.ts # Full index → search → rate → query flow +│ ├── mock-provider.ts # Deterministic 4D embedding provider (use in all unit tests) +│ ├── test-db.ts # createTestDb(): in-memory SQLite with migrations; createTestDbWithVec() +│ ├── helpers.ts # insertDoc, insertChunk, seedTestDocument +│ └── sample-*.md # Sample documents +├── unit/ # Fast, isolated, no network (unit/operations/ tests the operations) +└── integration/ # Real SQLite, full workflows (index, search, rate) ``` ### Writing Tests - **Use `MockEmbeddingProvider`** from `tests/fixtures/mock-provider.ts` for all tests that need embeddings. It returns deterministic 4D vectors — no model download, no network. - **Use `createTestDb()`** from `tests/fixtures/test-db.ts` for an in-memory SQLite instance with all migrations applied. -- **sqlite-vec is NOT available in tests.** The test DB is plain SQLite. Vector search tests exercise the FTS5/LIKE fallback path. This is by design. -- **Coverage thresholds** (enforced in `vitest.config.ts`): statements ≥ 75%, branches ≥ 74%, functions ≥ 75%, lines ≥ 75%. CLI code (`src/cli/`) is excluded from coverage. +- **`createTestDb()` does not load sqlite-vec.** Vector search tests use `createTestDbWithVec()` or exercise the FTS5/keyword path. +- **Coverage thresholds** (enforced in `vitest.config.ts`): statements ≥ 75%, branches ≥ 74%, functions ≥ 75%, lines ≥ 75%. `src/cli/`, `src/mcp/main.ts` and the network embedding providers are excluded from coverage. - **Always run `npm run test:coverage`** (not just `npm test`) before pushing. CI runs `test:coverage`, which fails if any threshold is missed. `npm test` alone does NOT check coverage. - When adding new source files, ensure adequate test coverage so global thresholds are not violated. New files with many uncovered branches will drag the overall percentage down. - Tests should be fast (< 1 second total), deterministic, and not depend on ordering. @@ -276,25 +255,23 @@ SQLite `datetime('now')` has **second-level precision**. If you insert multiple ## CI/CD -Three GitHub Actions workflows in `.github/workflows/`: +Workflows in `.github/workflows/`: -| Workflow | Trigger | What it does | -| ------------- | ------------- | --------------------------------------------------- | -| `ci.yml` | Push & PR | Lint, typecheck, test (Node 18/20/22 matrix), build | -| `release.yml` | Tags `v*.*.*` | Full CI then `npm publish --provenance` | -| `codeql.yml` | Weekly + PR | CodeQL security scanning | +| Workflow | What it does | +| ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | +| `ci.yml` | Lint, format check, typecheck, `test:coverage` (Node 20 and 22), build, `docs:check` (the generated reference docs must be current) | +| `codeql.yml` | CodeQL security scanning | +| `release-please.yml` | Release PRs and the changelog | +| `docker.yml` | Docker image | +| `sdk-python.yml`, `sdk-go.yml` | Python and Go client SDKs | +| `release-python.yml` | Python SDK release | -## MCP Server +## Surfaces -The MCP server (`src/mcp/server.ts`) uses **stdio transport** and exposes 5 tools: - -1. `search-docs` — Semantic search with optional topic/library/version/rating filters -2. `get-document` — Retrieve a document by ID -3. `rate-document` — Rate a document or suggest corrections -4. `submit-document` — Submit a new document for indexing -5. `list-topics` — List available topics - -To test the MCP server locally: `npm run build && npm run serve` +- **MCP** (`src/mcp/server.ts`): 11 core tools (`search`, `ask`, `get-document`, `list-documents`, `overview`, `submit-document`, `update-document`, `delete-document`, `rate-document`, `link-documents`, `task`). `ask` is registered only with passthrough or a configured LLM. The admin toolset (`mcp.toolsets` or `LIBSCOPE_MCP_TOOLSETS=admin`) adds `sync`, `install-pack`, `list-packs` and `reindex-documents`. See `docs/reference/mcp-tools.md`. Test locally: `npm run build && npm run serve`. +- **CLI** (`src/cli/commands/*`): see `docs/reference/cli.md` or `libscope --help`. +- **REST** (`src/api/routes.ts`): every route under `/api/v1` comes from an operation's `http` mapping. Long-running operations answer `202` with a task ID. See `docs/reference/rest-api.md`. +- **SDK** (`src/LibScope.ts`): methods call the same operations. The package root exports only the class, its types, the provider interfaces and the errors. ## Parallel Agent Work — Git Worktrees @@ -367,9 +344,9 @@ Every PR must follow this complete lifecycle. **Do not consider a PR done until ## Adding a New Feature — Checklist -1. Add business logic in `src/core/` (no framework dependencies). +1. Add business logic in `src/core/` (no surface dependencies). 2. If it needs new DB tables/columns, add a migration in `src/db/schema.ts`. -3. Expose via MCP tool in `src/mcp/server.ts` and/or CLI command in `src/cli/index.ts`. +3. Add or extend an operation in `src/core/operations/` (an input schema with `.describe()` on every field, defaults only in the schema, an `http` mapping for REST). The REST route and the OpenAPI entry follow from it. Add it to an SDK namespace in `src/LibScope.ts`, a CLI command in `src/cli/commands/`, and, only if assistants need it, an MCP tool in `src/mcp/server.ts`. 4. Write unit tests in `tests/unit/` using `MockEmbeddingProvider` and `createTestDb()`. 5. Add integration coverage in `tests/integration/workflow.test.ts` if it's a core flow. 6. Run `npm run typecheck && npm run test:coverage && npm run lint` — all must pass. **Use `test:coverage`, not `test`** — CI enforces coverage thresholds and will fail if new code drops coverage below the configured minimums (see `vitest.config.ts` thresholds). @@ -380,21 +357,17 @@ Every PR must follow this complete lifecycle. **Do not consider a PR done until ## Documentation -Every user-facing change **must** update all relevant documentation. Documentation lives in multiple places — check each one: - -| Location | What it covers | -| --------------------------------- | ------------------------------------------------------------------ | -| `README.md` | Top-level overview, quickstart, config tables, CLI summary | -| `docs/guide/getting-started.md` | First-run walkthrough | -| `docs/guide/configuration.md` | Config guide with env var table and examples | -| `docs/reference/cli.md` | Full CLI command reference | -| `docs/reference/configuration.md` | Complete config key reference, env vars, example config | -| `agents.md` | Agent/Copilot guide — architecture, conventions, config precedence | +Every user-facing change **must** update the documentation. The CLI, MCP tool, config key and REST route tables in `docs/reference/` are generated from the code: run `npm run build && npm run docs:gen` and commit the result. CI runs `npm run docs:check`, which fails when a generated block is out of date. -**What to update for common change types:** +| Location | What it covers | +| --------------------------------- | ---------------------------------------------------------------------------- | +| `README.md` | Pitch and quick start (CLI, MCP, SDK); links to the docs site | +| `docs/guide/*.md` | Guides (getting started, configuration, connectors, programmatic usage, ...) | +| `docs/reference/cli.md` | CLI reference (generated option tables, hand-written examples) | +| `docs/reference/mcp-tools.md` | MCP tools (generated) | +| `docs/reference/rest-api.md` | REST routes (generated) and conventions | +| `docs/reference/configuration.md` | Config keys (generated), config files, other env vars | +| `docs/migration-v2.md` | Every name removed or renamed in 2.0 | +| `agents.md`, `CLAUDE.md` | Agent guides: architecture, conventions | -- **New config key:** `src/config.ts` (interface + defaults + env override) → `README.md` (env var table, example config) → `docs/guide/configuration.md` (env var table) → `docs/reference/configuration.md` (config keys table, env vars table, example config) → `agents.md` (config precedence env var list) -- **New CLI command:** `src/cli/index.ts` → `README.md` (CLI table) → `docs/reference/cli.md` (command reference) -- **New MCP tool:** `src/mcp/server.ts` → `README.md` (MCP tools list) → `agents.md` (MCP Server section) -- **New connector:** `src/connectors/` → `README.md` (connectors section) → `docs/guide/` (new guide page) → `docs/reference/cli.md` (sync/disconnect commands) -- **New env var:** All env var tables: `README.md`, `docs/guide/configuration.md`, `docs/reference/configuration.md` +Do not edit `CHANGELOG.md` or `docs/changelog.md`; release-please writes them. diff --git a/docs/contributing.md b/docs/contributing.md index f1b9e5aa..c075e0b4 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -8,44 +8,56 @@ Thanks for your interest in contributing! Here's how to get started. git clone https://github.com/RobertLD/libscope.git cd libscope npm install -npm run build && node dist/cli/index.js init +npm run build && node dist/cli/index.js doctor --fix npm run dev # TypeScript watch mode ``` ## Scripts -| Command | Description | -| ----------------------- | ----------------------------- | -| `npm run build` | Compile TypeScript to `dist/` | -| `npm run dev` | Watch mode compilation | -| `npm run lint` | Run ESLint | -| `npm run lint:fix` | ESLint with auto-fix | -| `npm run format` | Format with Prettier | -| `npm run format:check` | Check formatting | -| `npm run typecheck` | Type-check without emitting | -| `npm test` | Run tests | -| `npm run test:watch` | Tests in watch mode | -| `npm run test:coverage` | Tests with coverage report | +| Command | Description | +| ----------------------- | ----------------------------------------------------------------------- | +| `npm run build` | Compile TypeScript to `dist/` | +| `npm run dev` | Watch mode compilation | +| `npm run lint` | Run ESLint | +| `npm run lint:fix` | ESLint with auto-fix | +| `npm run format` | Format with Prettier | +| `npm run format:check` | Check formatting | +| `npm run typecheck` | Type-check without emitting | +| `npm test` | Run tests | +| `npm run test:watch` | Tests in watch mode | +| `npm run test:coverage` | Tests with coverage report | +| `npm run docs:gen` | Update the generated blocks in `docs/reference` (after `npm run build`) | +| `npm run docs:check` | Fail when a generated block is out of date (runs in CI) | +| `npm run docs:dev` | Docs site dev server (VitePress) | +| `npm run docs:build` | Build the docs site | ## Project Structure ``` src/ -├── core/ # Business logic (indexing, search, ratings, topics, documents) -├── db/ # SQLite schema, migrations, connection management +├── core/ +│ ├── operations/ # The operation layer: every command, tool, route and SDK method calls one +│ ├── parsers/ # File format parsers +│ └── ... # Business logic (ingest, indexing, search, rag, documents, tasks, ...) +├── cli/ # CLI: index.ts and one file per command group in commands/ +├── mcp/ # MCP server: tools generated from operations +├── api/ # REST API: routes generated from operations, OpenAPI +├── web/ # Web dashboard +├── lite/ # libscope/lite: createLite(), code chunker, normalizeRawInput() +├── db/ # SQLite schema, migrations, connection ├── providers/ # Embedding providers (local, Ollama, OpenAI) -├── mcp/ # MCP server and tool definitions -├── cli/ # CLI entry point and commands -├── api/ # REST API server and routes -├── web/ # Web dashboard and knowledge graph -├── connectors/ # Third-party syncs (Obsidian, Notion, Confluence, Slack, OneNote) -├── registry/ # Git-backed pack registry system -├── config.ts # Configuration management +├── connectors/ # Notion, Slack, Confluence, Obsidian, OneNote, doc sites +├── registry/ # Git pack registries +├── config-schema.ts # Config schema (keys, defaults, env vars) +├── config.ts # Config loading +├── LibScope.ts # Node.js SDK class ├── logger.ts # Structured logging (pino) -└── errors.ts # Custom error hierarchy +└── errors.ts # Error hierarchy +scripts/ +└── gen-docs.mjs # Fills the generated blocks of docs/reference tests/ ├── unit/ # Fast isolated tests with mocked dependencies -├── integration/ # Tests with real SQLite DB +├── integration/ # Tests with a real SQLite database └── fixtures/ # Test helpers, mock providers, sample data ``` @@ -53,9 +65,9 @@ For a full module-by-module breakdown and data flow diagrams, see the [Architect ## Design Principles -**Entry points are thin.** The CLI, MCP server, and REST API are adapters. They parse input and format output. All business logic lives in `src/core/`. +**Operations first.** New functionality is an operation in `src/core/operations/` (a zod input schema and a handler). The CLI, MCP server, REST API and SDK are thin adapters over the operations: they parse input and format output. Business logic lives in `src/core/`. See [How to Add an Operation](/guide/architecture#how-to-add-an-operation). -**Errors are typed.** Use the appropriate `LibScopeError` subclass (`DatabaseError`, `ValidationError`, `FetchError`, etc.) rather than throwing plain `Error`. MCP tool handlers must be wrapped with `withErrorHandling()` from `src/mcp/errors.ts`. +**Errors are typed.** Use the appropriate `LibScopeError` subclass (`DatabaseError`, `ValidationError`, `FetchError`, etc.) rather than throwing plain `Error`. Each surface maps errors in one place (`src/cli/errors.ts`, `withErrorHandling()` in `src/mcp/errors.ts`, `describeError()` in `src/api/adapter.ts`). **Core modules are testable.** They accept `db` and `provider` as parameters — never import them directly inside a function. This makes it easy to swap in `createTestDb()` and `MockEmbeddingProvider` in tests. @@ -63,6 +75,8 @@ For a full module-by-module breakdown and data flow diagrams, see the [Architect **No `any`.** Use `unknown` and narrow with type guards. The ESLint rule `no-explicit-any: "error"` is enforced. +**Generated references.** The CLI, MCP tool, configuration and REST references are partly generated from the code. After you change a command, tool, config key or route, run `npm run build && npm run docs:gen` and commit the result. + ## Making Changes 1. **Create a branch** from `main` @@ -70,7 +84,8 @@ For a full module-by-module breakdown and data flow diagrams, see the [Architect 3. **Add tests** for new functionality 4. **Run the full check suite:** ```bash - npm run lint && npm run typecheck && npm test + npm run format:check && npm run lint && npm run typecheck && npm test + npm run build && npm run docs:check ``` 5. **Commit** with [Conventional Commits](https://www.conventionalcommits.org/): - `feat: add bulk import from directory` @@ -104,4 +119,4 @@ Open an issue on GitHub with: ## License -By contributing, you agree that your contributions will be licensed under the MIT License. +By contributing, you agree that your contributions will be licensed under the project's license (Business Source License 1.1, see `LICENSE`). diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index e938045e..1d65857f 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -1,367 +1,375 @@ # Architecture -This guide explains how LibScope is structured internally. It is intended for contributors and developers who want to understand or extend the codebase. +This guide explains how LibScope is structured internally. It is for contributors and developers who want to understand or extend the code. ## System Layers -LibScope is organized into four distinct layers: - ``` -┌─────────────────────────────────────────────────────────┐ -│ Entry Points │ -│ CLI (Commander.js) MCP Server REST API LibScopeLite │ -└──────────────────────────┬──────────────────────────────┘ - │ -┌──────────────────────────▼──────────────────────────────┐ -│ Core Business Logic │ -│ indexing · search · rag · documents · parsers · … │ -└──────────────────────────┬──────────────────────────────┘ - │ -┌──────────────────────────▼──────────────────────────────┐ -│ Infrastructure │ -│ db/ (SQLite + sqlite-vec) providers/ (embeddings)│ -└─────────────────────────────────────────────────────────┘ +┌──────────────────────────────────────────────────────────────────┐ +│ Surfaces (adapters) │ +│ CLI MCP server REST API Web dashboard SDK (LibScope) │ +└────────────────────────────────┬─────────────────────────────────┘ + │ runOperation / startOperationTask +┌────────────────────────────────▼─────────────────────────────────┐ +│ Operation layer (src/core/operations) │ +│ name + zod input schema + handler, one file per group │ +└────────────────────────────────┬─────────────────────────────────┘ + │ +┌────────────────────────────────▼─────────────────────────────────┐ +│ Core logic (src/core) │ +│ ingest · indexing · search · rag · documents · tasks · parsers │ +└────────────────────────────────┬─────────────────────────────────┘ + │ +┌────────────────────────────────▼─────────────────────────────────┐ +│ Infrastructure │ +│ db/ (SQLite + sqlite-vec) providers/ (embeddings) connectors│ +└──────────────────────────────────────────────────────────────────┘ ``` -**Entry points** (`src/cli/`, `src/mcp/`, `src/api/`, `src/lite/`) are thin adapters. They parse input, call core functions, and format output. They contain no business logic. +**Operations** (`src/core/operations/`) are the single entry into the core. An operation has a kebab-case `name`, a `group`, a one-line `summary`, a zod `input` schema (every field has a description and its default), optional `annotations` (`readOnly`, `destructive`, `idempotent`, `longRunning`), an optional `http` mapping (method and path), and a `handler(ctx, input)`. The handler receives an `OperationContext` with the database, the embedding provider, the config, the calling surface, an optional `AbortSignal` and progress callback, and the LLM. -**Core** (`src/core/`) contains all business logic. Core modules are plain TypeScript functions — they don't know whether they were called from the CLI, an MCP tool, the REST API, or `LibScopeLite`. +**Surfaces** are thin adapters. They parse their input, call `runOperation(op, ctx, input)` (or `startOperationTask` for a background task), and format the result. They contain no business logic. Because every surface uses the same schema, parameter names, defaults, validation and results are the same in the CLI, the MCP server, the REST API and the SDK. -**Infrastructure** (`src/db/`, `src/providers/`) handles persistence and external services. The database layer uses better-sqlite3 (synchronous). The provider layer abstracts embedding models behind a common interface. +**Core** (`src/core/`) contains the business logic as plain functions over a database and a provider. -### LibScope Lite Layer +**Infrastructure** (`src/db/`, `src/providers/`, `src/connectors/`, `src/registry/`) handles storage, embedding models, third-party services and git pack registries. -`src/lite/` is a separate entry point that exposes a minimal embeddable API built on top of the same core and infrastructure modules: +### The operation layer -``` -libscope/lite → src/lite/index.ts → LibScopeLite class - ├── core/indexing.ts - ├── core/search.ts - ├── core/rag.ts - ├── core/ratings.ts - ├── db/connection.ts - └── providers/ -``` +| File | Content | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `types.ts` | `defineOperation`, `runOperation`, `startOperationTask`, `parseOperationInput`, `createOperationContext`, `OperationContext`, `ListResult` | +| `schemas.ts` | Shared input fields (`documentId`, `topic`, `sourceType`, `tags`, `limit`, `offset`, ...) so every operation uses the same names | +| `index.ts` | `OPERATIONS` (every operation, in display order) and `getOperation(name)` | +| `documents.ts`, `search.ts`, `links.ts` | One file per group: documents, search and ask, links | +| `graph.ts`, `tags.ts`, `topics.ts` | Knowledge graph, tags, topics | +| `searches.ts`, `packs.ts` | Saved searches, knowledge packs | +| `registries.ts`, `connectors.ts` | Git pack registries, saved connector connections | +| `admin.ts`, `analytics.ts` | Overview, reindex, dedupe, backup/restore, prune, bulk changes; search analytics | +| `webhooks.ts`, `tasks.ts` | Webhooks, background tasks | + +`runOperation` validates the input (a zod error becomes a `ValidationError` that names the operation and the field), runs the operation's `validate` hook, and calls the handler. `startOperationTask` does the same validation at once, then runs the handler as a background task in the shared task registry (`src/core/tasks.ts`) and returns the task. + +### Surfaces + +| Surface | Files | How it uses the operations | +| --------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CLI | `src/cli/index.ts`, `src/cli/commands/*.ts` | Each command file exports `register(program)`. A command maps its arguments and flags to the operation input and calls `run(op, input, format)` from `src/cli/run.ts`, which prints JSON with `--json` or the human format. `src/cli/errors.ts` is the one error handler. `program` is exported for tests and docs. | +| MCP server | `src/mcp/server.ts`, `src/mcp/main.ts`, `src/mcp/format.ts` | `createMcpServer()` registers each tool from an operation: the input schema, the description (the summary) and the annotations come from the operation. It registers the 11 core tools, and the admin toolset when `mcp.toolsets` enables it. `main.ts` runs it on stdio. Importing `server.ts` starts nothing. | +| REST API | `src/api/routes.ts`, `src/api/adapter.ts`, `src/api/openapi.ts` | `API_ROUTES` has one route per operation that declares `http`, plus `/openapi.json` and `/api/v1/health`. A `longRunning` operation answers `202` with a task ID. `buildOpenApiSpec(API_ROUTES)` generates the OpenAPI 3.1 document from the input schemas. | +| Web dashboard | `src/web/server.ts`, `src/web/dashboard.ts` | The dashboard's JSON routes run operations and reshape the results for the dashboard page. | +| SDK | `src/LibScope.ts`, `src/core/index.ts` | `LibScope` has top-level methods (`add`, `search`, `ask`, `askStream`, `overview`) and namespaces (`docs`, `topics`, `tags`, `links`, `searches`, `packs`, `registries`, `connectors`, `tasks`, `admin`, `analytics`, `webhooks`). The `NAMESPACES` table binds each method to an operation, and the types come from its schema. | +| `libscope/lite` | `src/lite/` | `createLite()` is a preset of `LibScope` that does not read config files. It also exports a tree-sitter code chunker and `normalizeRawInput()`. It does not import the CLI, the MCP server or the connectors. | -`LibScopeLite` deliberately omits connectors, topics, packs, webhooks, and registry — keeping the API surface small and the import footprint minimal for embedding in external applications. +### Startup + +Every surface starts with `bootstrap()` (`src/core/bootstrap.ts`): load the config, resolve the database path (explicit path, `database.path`, or the workspace database), open and migrate the database, create the embedding provider, and create or check the vector table. `createOperationContext()` then builds the context for the surface. + +### Shared core pieces + +| File | Purpose | +| ---------------------------- | --------------------------------------------------------------------------------------------------------- | +| `src/core/ingest.ts` | One `add` path for inline content, files, directories, URLs (optionally crawled) and repositories | +| `src/core/tasks.ts` | Background task registry with `AbortSignal` and progress; tasks are kept in memory for one hour | +| `src/core/overview.ts` | Counts, topics, installed packs, index identity and health (`overview` operation, `admin stats`) | +| `src/core/document-view.ts` | A document with its tags, links and ratings, with content paging (`get-document`) | +| `src/core/rag.ts` | `answer()`: LLM answer or passthrough context; LLM providers | +| `src/connectors/registry.ts` | One table of connector types: config shape, secret fields, sync and disconnect; schedules for `serve api` | +| `src/config-schema.ts` | The zod config schema: every key, its default, its env vars, and the generated config reference | ## Module Map ``` src/ ├── cli/ -│ ├── index.ts # main CLI entry (#!/usr/bin/env node) -│ └── commands/ # each file exports registerCommands(program) +│ ├── index.ts # program, global options, error handler; register() per command file +│ ├── commands/ # add, search (+ ask), docs, topics (+ tags), searches, bulk, connectors, +│ │ # pack, registry, serve, config, workspace, webhooks, admin, doctor +│ ├── run.ts # run an operation and print JSON or the human format +│ ├── context.ts # the command's OperationContext (opened on first use) +│ ├── options.ts # shared flags (document filters) and parsing helpers +│ ├── errors.ts # "✗ message" + next-step hint; stack only with --verbose +│ ├── reporter.ts # progress line on stderr +│ ├── repl.ts # interactive search (search with no query) +│ └── confirm.ts # confirmation prompts (-y skips them) ├── mcp/ -│ ├── server.ts # MCP entry point — registers all tools -│ ├── tools/ # each file exports registerTools(server, db, provider) -│ └── errors.ts # withErrorHandling() wrapper for MCP tool handlers +│ ├── server.ts # createMcpServer(), runStdioServer(); tools generated from operations +│ ├── main.ts # executable: MCP server on stdio +│ ├── format.ts # compact text output of each tool +│ └── errors.ts # withErrorHandling(): errors become isError results ├── api/ -│ ├── server.ts # Express app factory -│ ├── routes/ # route handlers -│ └── openapi.ts # OpenAPI 3.0 spec +│ ├── server.ts # startApiServer(): node:http server, rate limit, auth, schedules +│ ├── routes.ts # API_ROUTES generated from the operations; handleRequest() +│ ├── adapter.ts # request -> operation input; operation result/error -> HTTP response +│ ├── openapi.ts # buildOpenApiSpec(): OpenAPI 3.1 from the routes +│ └── middleware.ts # CORS, API key, rate limit, security headers, JSON responses ├── web/ -│ ├── server.ts # HTTP server (port 3377) -│ ├── dashboard.ts # self-contained dashboard HTML -│ └── graph-api.ts # knowledge graph data API +│ ├── server.ts # dashboard server (port 3377) and its JSON routes +│ └── dashboard.ts # dashboard page and scripts ├── core/ -│ ├── indexing.ts # document chunking and embedding +│ ├── operations/ # the operation layer (see above) +│ ├── bootstrap.ts # one startup sequence for every surface +│ ├── ingest.ts # add content, files, directories, URLs, repositories +│ ├── indexing.ts # chunking and embedding │ ├── search.ts # hybrid vector + FTS5 search -│ ├── rag.ts # LLM-based question answering +│ ├── rag.ts # ask: LLM answer or passthrough context │ ├── documents.ts # document CRUD +│ ├── document-view.ts # document with tags, links, ratings; content paging +│ ├── overview.ts # counts, topics, packs, index, health +│ ├── tasks.ts # background task registry │ ├── versioning.ts # document version history │ ├── topics.ts # topic hierarchy -│ ├── tags.ts # tag management and auto-suggest -│ ├── links.ts # cross-document references -│ ├── ratings.ts # document/chunk ratings +│ ├── tags.ts # tags and tag suggestions +│ ├── links.ts # typed links between documents +│ ├── ratings.ts # document and chunk ratings │ ├── dedup.ts # duplicate detection -│ ├── bulk.ts # bulk operations +│ ├── bulk.ts # bulk delete, retag, move │ ├── analytics.ts # search analytics, knowledge gaps -│ ├── graph.ts # knowledge graph with cluster detection -│ ├── packs.ts # knowledge pack create/install -│ ├── batch.ts # parallel batch import -│ ├── batch-search.ts # concurrent multi-query search -│ ├── url-fetcher.ts # HTTP fetch with retry, proxy, cert handling -│ ├── spider.ts # recursive web crawler -│ ├── link-extractor.ts # extract links from HTML/Markdown/Wikilinks -│ ├── repo.ts # GitHub/GitLab repo cloning and indexing -│ ├── watcher.ts # file system watching for auto-reindex -│ ├── reindex.ts # re-embedding after provider switch -│ ├── scheduler.ts # background task scheduling -│ ├── webhooks.ts # event webhooks with HMAC signing -│ ├── saved-searches.ts # named query persistence -│ ├── workspace.ts # workspace isolation -│ ├── export.ts # full knowledge base backup/restore -│ ├── ttl.ts # document expiry management -│ └── parsers/ # file format parsers -│ ├── markdown.ts # Markdown + MDX -│ ├── text.ts # plain text -│ ├── pdf.ts # PDF (optional: pdf-parse) -│ ├── word.ts # DOCX (optional: mammoth) -│ ├── epub.ts # EPUB (optional: epub2) -│ ├── pptx.ts # PowerPoint (optional: pizzip) -│ ├── html.ts # HTML -│ ├── csv.ts # CSV -│ ├── json-parser.ts # JSON -│ └── yaml.ts # YAML +│ ├── graph.ts # knowledge graph with clusters +│ ├── packs.ts # knowledge pack create/install/remove +│ ├── url-fetcher.ts # HTTP fetch with retry, proxy and certificate handling +│ ├── spider.ts # web crawler +│ ├── link-extractor.ts # links from HTML, Markdown and wikilinks +│ ├── repo.ts # GitHub/GitLab repository indexing +│ ├── watcher.ts # file watching for `add --watch` +│ ├── reindex.ts # re-embed chunks, rebuild the vector index +│ ├── scheduler.ts # cron schedules of saved connections +│ ├── webhooks.ts # webhooks with HMAC signing +│ ├── events.ts # document and search events (webhooks subscribe to them) +│ ├── saved-searches.ts # saved searches +│ ├── workspace.ts # workspaces +│ ├── export.ts # backup and restore +│ ├── ttl.ts # expiry of documents (prune-expired) +│ ├── index.ts # package root exports +│ └── parsers/ # markdown, text, pdf, word, epub, pptx, html, csv, json, yaml ├── db/ -│ ├── connection.ts # SQLite connection factory (WAL mode, sqlite-vec) -│ └── schema.ts # SCHEMA_VERSION, MIGRATIONS, createSchema() +│ ├── connection.ts # database path resolution, SQLite connection (WAL, sqlite-vec) +│ ├── schema.ts # SCHEMA_VERSION, MIGRATIONS, runMigrations(), createVectorTable() +│ ├── index-meta.ts # embedding model and vector size recorded for the vector index +│ └── validate.ts # row validation helpers ├── providers/ -│ ├── index.ts # EmbeddingProvider interface + factory +│ ├── embedding.ts # EmbeddingProvider interface +│ ├── index.ts # createEmbeddingProvider() from config +│ ├── dimensions.ts # vector sizes of known models │ ├── local.ts # @xenova/transformers (all-MiniLM-L6-v2) │ ├── ollama.ts # Ollama HTTP API │ └── openai.ts # OpenAI embeddings API -├── registry/ -│ ├── types.ts # RegistryEntry, PackSummary, PackManifest -│ ├── config.ts # registry list in ~/.libscope/config.json -│ ├── git.ts # git clone/pull/commit/push -│ ├── sync.ts # registry syncing (on request only) -│ ├── search.ts # pack search across registries -│ ├── resolve.ts # pack resolution (version conflicts) -│ ├── publish.ts # publishing packs to registries -│ └── checksum.ts # SHA-256 verification +├── registry/ # git pack registries: config, git, sync, search, resolve, publish, checksum ├── connectors/ -│ ├── obsidian.ts # Obsidian vault sync -│ ├── notion.ts # Notion API sync -│ ├── confluence.ts # Confluence API sync -│ ├── slack.ts # Slack API sync -│ ├── onenote.ts # Microsoft Graph API sync -│ ├── http-utils.ts # shared retry logic with exponential backoff -│ └── sync-tracker.ts # sync history and status in database +│ ├── registry.ts # connector registry (types, secrets, sync, disconnect, schedules) +│ ├── saved-config.ts # saved connections in ~/.libscope/connectors/<name>.json +│ ├── notion.ts, slack.ts, confluence.ts, obsidian.ts, onenote.ts, docs.ts +│ ├── http-utils.ts # retry with exponential backoff +│ └── sync-tracker.ts # sync history and status in the database ├── lite/ -│ ├── index.ts # public entrypoint — exports LibScopeLite + types -│ ├── core.ts # LibScopeLite class implementation -│ ├── types.ts # LiteOptions, LiteDoc, LiteSearchResult, etc. -│ ├── normalize.ts # raw input → markdown (dispatches to core/parsers/) -│ └── chunker-treesitter.ts # optional tree-sitter code chunker (TS/JS/Python) -├── config.ts # loadConfig() — merges env, project, user, defaults +│ ├── index.ts # libscope/lite: package root + createLite, chunker, normalizeRawInput +│ ├── core.ts # createLite(), createCodeChunker() +│ ├── normalize.ts # raw input -> { title, content } (uses core/parsers) +│ └── chunker-treesitter.ts # optional tree-sitter code chunker +├── utils/ # glob, retry, row validation +├── config-schema.ts # zod config schema (keys, defaults, env vars, docs table) +├── config.ts # loadConfig(), config files, secrets.json, config get/set ├── errors.ts # LibScopeError hierarchy -├── logger.ts # pino logger with child logger support -└── LibScope.ts # main public API class +├── logger.ts # pino logger +└── LibScope.ts # the SDK class ``` ## Key Design Patterns ### Error Hierarchy -All errors extend `LibScopeError`, which carries a `code` string and an optional `cause`: - -```typescript -// src/errors.ts -class LibScopeError extends Error { - constructor(message: string, public readonly code: string, options?: ErrorOptions) -} +All errors extend `LibScopeError`, which has a `code` string and an optional `cause`: -// Subclasses: -DatabaseError // SQLite failures -EmbeddingError // provider failures -ValidationError // bad input -FetchError // HTTP fetch failures -ConfigError // misconfiguration -DocumentNotFoundError -ChunkNotFoundError -TopicNotFoundError +``` +LibScopeError + ├── DatabaseError + ├── EmbeddingError + ├── ValidationError # bad input; REST 400 + ├── FetchError # a fetched URL failed; REST 502 + ├── ConfigError # misconfiguration, with the setting to change + └── NotFoundError # REST 404; code names the resource (LINK_NOT_FOUND, ...) + ├── DocumentNotFoundError + ├── ChunkNotFoundError + └── TopicNotFoundError ``` -Always throw the most specific subclass. MCP tool handlers must be wrapped with `withErrorHandling()` from `src/mcp/errors.ts` — this converts `LibScopeError` to well-structured MCP error responses. +Throw the most specific class. The surfaces map errors in one place each: `src/cli/errors.ts`, `withErrorHandling()` in `src/mcp/errors.ts`, and `describeError()` in `src/api/adapter.ts`. ### Embedding Provider Interface -All embedding providers implement the same interface: - ```typescript +// src/providers/embedding.ts interface EmbeddingProvider { - readonly dimensions: number; - embed(texts: string[]): Promise<number[][]>; + readonly name: string; + readonly model?: string | undefined; + readonly dimensions: number; // 0 until the first embedding when unknown + embed(text: string): Promise<number[]>; + embedBatch(texts: string[]): Promise<number[][]>; } ``` -The factory function in `src/providers/index.ts` returns the correct implementation based on config. Core modules accept an `EmbeddingProvider` and never import a specific provider directly — this makes them testable with the `MockEmbeddingProvider` fixture. +`createEmbeddingProvider(config)` in `src/providers/index.ts` returns the configured provider. Core modules receive a provider and never import a specific one, so tests use the `MockEmbeddingProvider` fixture. The vector index records the provider, model and vector size (`src/db/index-meta.ts`); a mismatch is a `ConfigError` that suggests `libscope admin reindex --rebuild`. ### Database Migrations -Schema migrations in `src/db/schema.ts` follow this pattern: +Migrations are in `src/db/schema.ts`. The current schema version is 19. ```typescript -export const SCHEMA_VERSION = 17; +const SCHEMA_VERSION = 19; -export const MIGRATIONS: Record<number, string> = { - 1: "CREATE TABLE ...", +const MIGRATIONS: Record<number, string> = { // ... - 17: "ALTER TABLE chunks ADD COLUMN ...", + 19: "...", }; ``` -To add a migration: increment `SCHEMA_VERSION` and add a new entry in `MIGRATIONS` with the new version number as the key. `createSchema()` runs all missing migrations on startup. +To add a migration, increment `SCHEMA_VERSION` and add an entry to `MIGRATIONS` with the new version number as its key. `runMigrations()` applies the missing migrations at startup. -### Configuration Merging +### Configuration -`loadConfig()` in `src/config.ts` merges four tiers, highest priority first: +`src/config-schema.ts` defines every config key once (type, default, environment variables, secret or not, description). `loadConfig()` in `src/config.ts` merges, highest priority first: -1. Environment variables (`LIBSCOPE_*`) -2. Project `.libscope.json` (current working directory) +1. Environment variables (`LIBSCOPE_<SECTION>_<FIELD>`) +2. Project `.libscope.json` 3. User `~/.libscope/config.json` -4. Defaults - -The result is cached for 30 seconds to avoid repeated file reads. Configuration is passed down to core modules — they never read environment variables directly. +4. Schema defaults -### Connectors Pattern +API keys come only from environment variables and `~/.libscope/secrets.json`. The result is cached for 30 seconds. The same schema drives `libscope config get/set/unset` and the [configuration reference](/reference/configuration). -Each connector in `src/connectors/` follows the same structure: +### Connectors -1. **Auth config** stored in `~/.libscope/connectors/<name>.json` (permissions 0o600) -2. **Incremental sync** tracked via `sync-tracker.ts` (last sync timestamp in DB) -3. **Retry logic** from `http-utils.ts` (exponential backoff, configurable retries) -4. **Disconnect** removes all documents with matching `sourceType` + source identifier +Each connector type is one entry in `src/connectors/registry.ts`. `libscope connect <type>` saves a connection in `~/.libscope/connectors/<name>.json` (mode `0600`) and syncs it. The `sync` and `disconnect` operations, the MCP admin `sync` tool, the REST routes and the scheduler all read the saved connection, so credentials are never operation parameters. Sync state is tracked by `sync-tracker.ts`, and HTTP calls retry with `http-utils.ts`. -## Data Flow: Indexing +## Data Flow: Adding content -When you index a document (`libscope add` or `submit-document` MCP tool): +When you run `libscope add`, the MCP tool `submit-document`, `POST /api/v1/documents` or `scope.add()`, the same `add` operation runs: ``` -Input (file/URL/text) +Input (inline content / file / directory / URL / repository URL) + → ingest.ts: detect the kind, fetch or read, crawl (spider) or list files → Parser (markdown.ts / html.ts / pdf.ts / ...) - → Raw text - → Chunker (indexing.ts) — paragraph-aware, heading breadcrumbs, overlap - → Chunks[] - → EmbeddingProvider.embed(chunks) - → Vectors[] - → DB: documents table + chunks table + chunk_embeddings (vector) + chunks_fts (FTS5) + → Chunker (indexing.ts, or a custom Chunker): heading-aware chunks with breadcrumbs + → EmbeddingProvider.embedBatch(chunks) + → DB: documents + chunks + chunk_embeddings (vector) + chunks_fts (FTS5) ``` -The chunker in `src/core/indexing.ts` splits on paragraph boundaries while respecting heading structure. Each chunk carries a breadcrumb of its parent headings so context is preserved across chunk boundaries. - ## Data Flow: Search ``` Query string - → EmbeddingProvider.embed([query]) → query vector + → EmbeddingProvider.embed(query) → query vector ↓ - ┌── Vector search (sqlite-vec ANN cosine similarity) → ranked chunks - │ + ┌── Vector search (sqlite-vec, cosine distance) → ranked chunks └── FTS5 search (BM25, AND then OR fallback) → ranked chunks ↓ Reciprocal Rank Fusion (RRF, k=60) → merged ranked list ↓ -Title boost (1.5× if title contains query words) - ↓ -MMR diversity reranking (optional, diversity param 0–1) +Title boost ↓ -Filter by: library, topic, tags, minRating, maxChunksPerDocument +MMR diversity reranking (optional, diversity 0–1) ↓ -Paginate (limit, offset) +Filters: topic, library, version, sourceType, tags, minRating, maxChunksPerDocument ↓ -Results with scoreExplanation +Paging (limit, offset) → { items, total, limit, offset } ``` See [How Search Works](/guide/how-search-works) for more detail. -## Data Flow: RAG (Ask) +## Data Flow: Ask ``` Question - → search() — retrieve top-K relevant chunks - → Build context string from chunks - → LLM prompt: "Answer based only on this context: ..." - → LLM response (streaming or buffered) - → Return { text, sources: cited chunks } + → search() — top-K chunks + → context prompt from the chunks + → LLM (OpenAI, Anthropic or Ollama) → { mode: "answer", answer, sources, model } + or passthrough (no LLM call) → { mode: "context", contextPrompt, sources } ``` -The LLM integration in `src/core/rag.ts` supports OpenAI, Ollama, Anthropic, and a `passthrough` mode where the application handles the LLM call externally. +`answer()` in `src/core/rag.ts` handles both modes. With `llm.provider` `auto` (the default), the MCP server uses passthrough: the calling assistant writes the answer. ## Database Schema Key tables (schema version 19): -| Table | Purpose | -| ------------------- | -------------------------------------------------- | -| `documents` | Document metadata: title, content, library, topic | -| `chunks` | Document chunks: content, chunk_index, document_id | -| `chunk_embeddings` | Vector table (sqlite-vec): embedding per chunk | -| `chunks_fts` | FTS5 virtual table: full-text search index | -| `topics` | Topic hierarchy (id, name, parent_id) | -| `tags` | Tag definitions | -| `document_tags` | Many-to-many document ↔ tag | -| `ratings` | Document and chunk ratings (1–5) | -| `document_versions` | Version history for rollback | -| `document_links` | Typed cross-references between documents | -| `search_log` | Query analytics | -| `document_hits` | Per-document result hit analytics | -| `saved_searches` | Named query persistence | -| `connector_configs` | Connector state (tokens, last sync) | -| `webhooks` | Event webhook configuration | -| `schema_version` | Current migration version | - -## How to Add a New CLI Command - -1. Add a new file in `src/cli/commands/` (or add to an existing file) -2. Export a `registerCommands(program: Command): void` function -3. Import and call it in `src/cli/index.ts` -4. Call the appropriate `src/core/` functions — don't implement logic in the CLI layer - -Example skeleton: +| Table | Purpose | +| ------------------- | --------------------------------------------------- | +| `documents` | Document metadata and content | +| `chunks` | Document chunks | +| `chunk_embeddings` | Vector table (sqlite-vec): one embedding per chunk | +| `chunks_fts` | FTS5 full-text index | +| `index_meta` | Provider, model and vector size of the vector index | +| `topics` | Topic hierarchy | +| `tags` | Tags | +| `document_tags` | Document ↔ tag | +| `ratings` | Document and chunk ratings (1–5) | +| `document_versions` | Version history for rollback | +| `document_links` | Typed links between documents | +| `packs` | Installed knowledge packs | +| `search_log` | One row per search (method, result count, latency) | +| `search_queries` | Query log with result count and top score | +| `document_hits` | How often each document is returned | +| `saved_searches` | Saved searches | +| `connector_syncs` | Connector sync history | +| `connector_configs` | Not used (connections are saved as files) | +| `webhooks` | Webhooks | +| `schema_version` | Applied migrations | + +## How to Add an Operation + +New functionality starts as an operation. The surfaces then expose it with little or no code. + +1. Write the logic in `src/core/` as a function over the database and provider. +2. Define the operation in the file of its group in `src/core/operations/` with `defineOperation({ name, group, summary, input, annotations, http, handler })`. Use the shared fields from `schemas.ts`, give every field a `.describe()`, and put defaults in the schema. Mark it `longRunning` if it can take long (surfaces then offer a background task). +3. Add it to the group's exported list (for example `topicOperations`), so it is in `OPERATIONS`. +4. Expose it: + - **REST**: automatic when the operation has `http`. The OpenAPI document follows. + - **SDK**: add it to `NAMESPACES` in `src/LibScope.ts` (a test checks that the SDK reaches every operation). + - **CLI**: add a command in the right `src/cli/commands/*.ts` file that calls `run(op, input, format)`. A test checks that every command calls an operation. + - **MCP**: only for tools that an assistant needs. Add it to `registerCoreTools` or `registerAdminTools` in `src/mcp/server.ts` with a formatter in `src/mcp/format.ts`. +5. Run `npm run build && npm run docs:gen` to update the generated references, and commit them. + +Example (a hypothetical `rename-topic` operation; `renameTopic` is not in the code): ```typescript -// src/cli/commands/my-feature.ts -import { Command } from "commander"; -import { myFeatureCore } from "../../core/my-feature.js"; -import { getDb } from "../../db/connection.js"; - -export function registerCommands(program: Command): void { - program - .command("my-feature <arg>") - .description("Does something useful") - .option("--flag <value>", "An option") - .action(async (arg, opts) => { - const db = getDb(); - const result = await myFeatureCore(db, arg, opts); - console.log(result); - }); -} +// src/core/operations/topics.ts +export const renameTopicOperation = defineOperation({ + name: "rename-topic", + group: "topics", + summary: "Rename a topic", + input: z.object({ topic: s.topic, name: z.string().min(1).describe("New name") }), + annotations: { idempotent: true }, + http: { method: "PATCH", path: "/topics/:topic" }, + handler: (ctx, input) => renameTopic(ctx.db, resolveTopicId(ctx.db, input.topic), input.name), +}); ``` -## How to Add a New MCP Tool +```typescript +// src/cli/commands/topics.ts, inside register(program) +topics + .command("rename <topic> <name>") + .description("Rename a topic") + .action(async (topic: string, name: string) => { + await run(renameTopicOperation, { topic, name }, (t) => console.log(`✓ Renamed to ${t.name}`)); + }); +``` -1. Add the tool in an existing file under `src/mcp/tools/` (or create a new file) -2. Register the tool via `server.tool(name, description, schema, handler)` -3. Wrap the handler with `withErrorHandling()` from `src/mcp/errors.ts` -4. Export a `registerTools(server, db, provider)` function and call it in `src/mcp/server.ts` +## How to Add a Connector -Example skeleton: +1. Write the connector in `src/connectors/my-connector.ts`: its config type, a sync function and a disconnect function. Use `http-utils.ts` for HTTP calls with retry. +2. Add an entry to `CONNECTORS` in `src/connectors/registry.ts` with the config schema and the secret fields. +3. Add its flags to `libscope connect` in `src/cli/commands/connectors.ts`. -```typescript -import { withErrorHandling } from "../errors.js"; -import { z } from "zod"; - -export function registerTools(server, db, provider) { - server.tool( - "my-tool", - "Does something useful", - { param: z.string().describe("A parameter") }, - withErrorHandling(async ({ param }) => { - const result = await myCore(db, param); - return { content: [{ type: "text", text: JSON.stringify(result) }] }; - }), - ); -} -``` +`sync`, `disconnect`, `connections`, the MCP admin `sync` tool, the REST routes and the scheduler then work for it without more code. -## How to Add a New Connector +## How to Add an Embedding Provider -1. Create `src/connectors/my-connector.ts` -2. Use `http-utils.ts` for HTTP calls with retry -3. Track sync state with `sync-tracker.ts` -4. Store auth config in `~/.libscope/connectors/my-connector.json` (chmod 0o600) -5. Add a CLI command in `src/cli/commands/` under the `connect` subcommand -6. Add an MCP tool in `src/mcp/tools/` +1. Create `src/providers/my-provider.ts` that implements `EmbeddingProvider`. +2. Add it to `createEmbeddingProvider()` in `src/providers/index.ts`. +3. Add the provider name to `EMBEDDING_PROVIDERS` in `src/config-schema.ts`. -## How to Add a New Embedding Provider +## Generated Documentation -1. Create `src/providers/my-provider.ts` implementing `EmbeddingProvider` -2. Export a class with `dimensions: number` and `embed(texts: string[]): Promise<number[][]>` -3. Add it to the provider factory in `src/providers/index.ts` -4. Add the provider name to the config type in `src/config.ts` +The CLI, MCP tool, configuration and REST references contain blocks between `<!-- generated:start NAME -->` and `<!-- generated:end NAME -->` markers. `scripts/gen-docs.mjs` fills them from the built code: the commander `program`, the tools of `createMcpServer()`, `getConfigKeyTable()` and `API_ROUTES`. Run `npm run build && npm run docs:gen` after changing a command, a tool, a config key or a route. CI runs `npm run docs:check`, which fails when a block is out of date. ## Testing Approach @@ -370,15 +378,15 @@ import { createTestDb } from "../fixtures/test-db.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; import { insertDoc, insertChunk, seedTestDocument } from "../fixtures/helpers.js"; -// Fresh in-memory DB with all migrations applied +// Fresh in-memory database with all migrations applied const db = createTestDb(); -// Deterministic 4-dimensional vectors — no real embedding model required +// Deterministic 4-dimensional vectors: no embedding model needed const provider = new MockEmbeddingProvider(); ``` -- **Unit tests** (`tests/unit/`) — mock all dependencies, test one module at a time, run fast -- **Integration tests** (`tests/integration/`) — real SQLite DB, full indexing → search → rate workflow -- Use `createTestDbWithVec()` when you need the vector table (requires sqlite-vec) +- **Unit tests** (`tests/unit/`): mocked dependencies, one module at a time. Operations are tested in `tests/unit/operations/`. +- **Integration tests** (`tests/integration/`): a real SQLite database and full workflows. +- Use `createTestDbWithVec()` when you need the vector table. -Coverage thresholds enforced in CI: 75% statements, 74% branches, 75% functions, 75% lines. +Coverage thresholds in CI: 75% statements, 74% branches, 75% functions, 75% lines. diff --git a/docs/guide/code-indexing.md b/docs/guide/code-indexing.md index 176adf3d..9942ff74 100644 --- a/docs/guide/code-indexing.md +++ b/docs/guide/code-indexing.md @@ -1,6 +1,6 @@ # Code Indexing -LibScope Lite includes a tree-sitter powered code chunker that splits source files at function and class boundaries. This produces semantically meaningful chunks that are far better for embedding than naive line-count splits. +`libscope/lite` includes a tree-sitter code chunker that splits source files at function and class boundaries. These chunks embed much better than splits at a fixed line count. Use `createCodeChunker()` to index code with `LibScope`, or `TreeSitterChunker` to get the chunks with line numbers. ## Why Code-Aware Chunking Matters @@ -30,19 +30,19 @@ npm install tree-sitter-c # C npm install tree-sitter-go # Go ``` -If tree-sitter is not installed, `TreeSitterChunker.chunk()` throws a `ValidationError` with a clear install message. All other LibScope Lite features work normally without tree-sitter. +If tree-sitter is not installed, `TreeSitterChunker.chunk()` throws a `ValidationError` with the install command, and `createCodeChunker()` falls back to the built-in chunker. Everything else works without tree-sitter. ## Supported Languages -| Language | Aliases | Grammar Package | -|---|---|---| -| TypeScript | `typescript`, `ts`, `tsx` | `tree-sitter-typescript` | +| Language | Aliases | Grammar Package | +| ---------- | --------------------------------------- | ------------------------ | +| TypeScript | `typescript`, `ts`, `tsx` | `tree-sitter-typescript` | | JavaScript | `javascript`, `js`, `jsx`, `mjs`, `cjs` | `tree-sitter-javascript` | -| Python | `python`, `py` | `tree-sitter-python` | -| C# | `csharp`, `cs` | `tree-sitter-c-sharp` | -| C++ | `cpp`, `cc`, `cxx`, `hpp`, `h` | `tree-sitter-cpp` | -| C | `c` | `tree-sitter-c` | -| Go | `go` | `tree-sitter-go` | +| Python | `python`, `py` | `tree-sitter-python` | +| C# | `csharp`, `cs` | `tree-sitter-c-sharp` | +| C++ | `cpp`, `cc`, `cxx`, `hpp`, `h` | `tree-sitter-cpp` | +| C | `c` | `tree-sitter-c` | +| Go | `go` | `tree-sitter-go` | Aliases are case-insensitive: `"TS"`, `"ts"`, `"TypeScript"` all resolve to TypeScript. @@ -90,21 +90,18 @@ Each chunk in the result: ```ts interface CodeChunk { - content: string; // source text of the chunk - startLine: number; // 1-based start line in the original file - endLine: number; // 1-based end line in the original file - nodeType: string; // tree-sitter node type (see below) + content: string; // source text of the chunk + startLine: number; // 1-based start line in the original file + endLine: number; // 1-based end line in the original file + nodeType: string; // tree-sitter node type (see below) } ``` -For the example above, you'd get chunks like: +For the example above, the class fits in one chunk (under 1500 characters). The import before it is prepended to it as a preamble: ``` -chunk[0]: "import { EventEmitter } from 'events';" - startLine: 2, endLine: 2, nodeType: "preamble" - -chunk[1]: "export class AuthService extends EventEmitter { ... }" - startLine: 4, endLine: 25, nodeType: "class_declaration" +chunk[0]: "import { EventEmitter } from \"events\";\n\nexport class AuthService ... }" + startLine: 2, endLine: 23, nodeType: "export_statement" ``` ## Node Types @@ -112,6 +109,7 @@ chunk[1]: "export class AuthService extends EventEmitter { ... }" The chunker extracts these node types per language: **TypeScript / TSX:** + - `function_declaration` — `function foo() {}` - `class_declaration` — `class Foo {}` - `method_definition` — methods inside a class @@ -122,14 +120,17 @@ The chunker extracts these node types per language: - `enum_declaration` — TypeScript enums **JavaScript / JSX:** + - `function_declaration`, `class_declaration`, `method_definition`, `export_statement`, `lexical_declaration` **Python:** + - `function_definition` — `def foo():` - `class_definition` — `class Foo:` - `decorated_definition` — `@decorator\ndef foo():` **C#:** + - `method_declaration` — `public void Foo() {}` - `class_declaration` — `class Foo {}` - `interface_declaration` — `interface IFoo {}` @@ -138,16 +139,19 @@ The chunker extracts these node types per language: - `constructor_declaration` — `public Foo() {}` **C++:** + - `function_definition` — `void foo() {}` - `class_specifier` — `class Foo {}` - `struct_specifier` — `struct Point {}` - `namespace_definition` — `namespace MyNS {}` **C:** + - `function_definition` — `void foo() {}` - `struct_specifier` — `struct Point {}` **Go:** + - `function_declaration` — `func Foo() {}` - `method_declaration` — `func (r *Receiver) Foo() {}` - `type_declaration` — `type Foo struct {}` / `type Bar interface {}` @@ -184,58 +188,35 @@ When a class is split, each method becomes its own chunk. If a single method is If the source has no declaration nodes (e.g., a config file, a `.d.ts` with only type exports), the entire source is returned as a single chunk with `nodeType: "module"`. -## Integrating with LibScope Lite +## Indexing code with LibScope -The typical pattern for indexing a codebase: +`createCodeChunker()` returns a `Chunker` for `createLite()` or `LibScope.create()`. `add()` then splits each document with tree-sitter. The language comes from the extension of the file path, URL or title (or the `language` option). Documents in other languages use the built-in chunker. ```ts -import { LibScopeLite, TreeSitterChunker } from "libscope/lite"; +import { createLite, createCodeChunker } from "libscope/lite"; import { readdir, readFile } from "node:fs/promises"; -import { join, extname } from "node:path"; +import { join } from "node:path"; -const chunker = new TreeSitterChunker(); -const lite = new LibScopeLite({ dbPath: "./my-project.db" }); - -async function indexDirectory(dir: string): Promise<void> { - const entries = await readdir(dir, { recursive: true, withFileTypes: true }); +const scope = createLite({ dbPath: "./my-project.db", chunker: createCodeChunker() }); - const tasks = entries +const entries = await readdir("./src", { recursive: true, withFileTypes: true }); +await Promise.all( + entries .filter((e) => e.isFile()) - .map(async (entry) => { - const filePath = join(entry.parentPath, entry.name); - const ext = extname(entry.name).slice(1); // "ts", "py", etc. - const source = await readFile(filePath, "utf8"); - - if (chunker.supports(ext)) { - // Code-aware chunking - const chunks = await chunker.chunk(source, ext); - return chunks.map((c) => ({ - title: `${filePath}:${c.startLine}-${c.endLine}`, - content: c.content, - url: filePath, - library: "src", - })); - } - - // Plain text fallback for unsupported files - return [{ title: filePath, content: source, url: filePath, library: "src" }]; - }); - - const docGroups = await Promise.all(tasks); - await lite.indexBatch(docGroups.flat(), { concurrency: 4 }); -} + .map(async (e) => { + const path = join(e.parentPath, e.name); + await scope.add({ title: path, content: await readFile(path, "utf8"), library: "src" }); + }), +); -await indexDirectory("./src"); -console.log("Indexed. Searching..."); +const { items } = await scope.search("authentication token generation"); +for (const r of items) console.log(`${r.title} (score: ${r.score.toFixed(3)})`); -const results = await lite.search("authentication token generation"); -for (const r of results) { - console.log(`${r.title} (score: ${r.score.toFixed(3)})`); -} - -lite.close(); +scope.close(); ``` +`scope.add(path)` does not accept source-code extensions such as `.ts` or `.py` (only document formats). Read the file and pass its `content` with the path as `title`, as above. + ## Caching `TreeSitterChunker` lazily initializes the tree-sitter parser and grammar modules on first use and caches them for the lifetime of the instance. Create one `TreeSitterChunker` instance and reuse it rather than creating a new one per file: @@ -271,6 +252,7 @@ try { ``` Two error conditions: + 1. **Unsupported language** — throws immediately with the list of supported aliases 2. **tree-sitter not installed** — throws with the exact `npm install` command diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 028ff141..54a2fae3 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -57,7 +57,7 @@ By default the database is the active workspace's file: `~/.libscope/workspaces/ 1. `database.path`, if set in `.libscope.json` or `~/.libscope/config.json` (or `dbPath` passed to `LibScope.create()`). A leading `~` is expanded. 2. Otherwise the workspace: `--workspace` / `LibScope.create({ workspace })`, then `LIBSCOPE_WORKSPACE`, then `"workspace"` in `.libscope.json`, then `libscope workspace use`, then `default`. -`libscope init` and `libscope stats` show the file in use. +`libscope doctor` shows the workspace and the database file in use. Earlier versions of the SDK stored data in `~/.libscope/libscope.db`. If that file exists and the workspace database does not, LibScope prints a warning once. To keep using the old file, run `libscope config set database.path ~/.libscope/libscope.db`. LibScope does not move or delete it. @@ -89,14 +89,14 @@ libscope config set openai.apiKey sk-... # or export OPENAI_API LibScope records the provider, model, and vector size that built the vector index. If you change the provider, the model, or `embedding.dimensions` after you index documents, LibScope stops with an error that names the old and the new model. To rebuild the vector index with the new model, run: ```bash -libscope reindex --rebuild +libscope admin reindex --rebuild ``` -LibScope knows the vector size of common models (for example `nomic-embed-text`, `mxbai-embed-large`, `all-minilm`, `text-embedding-3-small`, and `text-embedding-3-large`). For a different Ollama or OpenAI model, set `embedding.dimensions` to the vector size of the model. If you do not set it, `libscope reindex --rebuild` embeds one probe text to find the size. For `text-embedding-3-*` models, `embedding.dimensions` also asks OpenAI for shorter vectors. +LibScope knows the vector size of common models (for example `nomic-embed-text`, `mxbai-embed-large`, `all-minilm`, `text-embedding-3-small`, and `text-embedding-3-large`). For a different Ollama or OpenAI model, set `embedding.dimensions` to the vector size of the model. If you do not set it, `libscope admin reindex --rebuild` embeds one probe text to find the size. For `text-embedding-3-*` models, `embedding.dimensions` also asks OpenAI for shorter vectors. ## LLM Configuration -The `ask` command and the `ask-question` MCP tool use an LLM to write answers from search results (RAG). +The `ask` command, the `ask` MCP tool and `POST /api/v1/ask` use an LLM to write answers from search results (RAG). `llm.provider` is `auto` by default. LibScope selects the LLM when `ask` runs, without network calls: @@ -128,7 +128,7 @@ export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-... # or: libscope config set anthrop With `llm.provider` set to `auto`, this key alone selects Anthropic (when no OpenAI key is set). You can set `llm.model` to choose a specific Claude model. -The `passthrough` provider does not call an LLM. The MCP `ask-question` tool then returns the retrieved context, and the calling assistant writes the answer. +The `passthrough` provider does not call an LLM. `ask` then returns the retrieved context, and the caller (for example the assistant that called the MCP `ask` tool) writes the answer. `libscope doctor` shows which LLM `ask` will use. ## API keys @@ -142,34 +142,18 @@ The OpenAI key is used for OpenAI embeddings and the OpenAI LLM. LibScope ignore ## Environment Variables -| Variable | Config key | Default | -| ------------------------------------------------- | --------------------------------------------------- | ------------------------ | -| `LIBSCOPE_EMBEDDING_PROVIDER` | `embedding.provider` | `local` | -| `LIBSCOPE_EMBEDDING_MODEL` | `embedding.model` | provider default | -| `LIBSCOPE_EMBEDDING_URL` | `embedding.url` | `http://localhost:11434` | -| `LIBSCOPE_EMBEDDING_DIMENSIONS` | `embedding.dimensions` | size of the model | -| `LIBSCOPE_LLM_PROVIDER` | `llm.provider` | `auto` | -| `LIBSCOPE_LLM_MODEL` | `llm.model` | provider default | -| `LIBSCOPE_LLM_URL` | `llm.url` | `embedding.url` | -| `LIBSCOPE_OPENAI_API_KEY`, `OPENAI_API_KEY` | `openai.apiKey` | — | -| `LIBSCOPE_ANTHROPIC_API_KEY`, `ANTHROPIC_API_KEY` | `anthropic.apiKey` | — | -| `LIBSCOPE_DATABASE_PATH` | `database.path` | workspace database | -| `LIBSCOPE_LOGGING_LEVEL` | `logging.level` | `info` | -| `LIBSCOPE_INDEXING_MAX_DOCUMENT_SIZE` | `indexing.maxDocumentSize` | `104857600` | -| `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS` | `indexing.allowPrivateUrls` | `false` | -| `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS` | `indexing.allowSelfSignedCerts` | `false` | -| `LIBSCOPE_WORKSPACE` | Active workspace for this shell | `default` | -| `LIBSCOPE_API_KEY` | REST API key (`Authorization: Bearer <key>`) | — | -| `LIBSCOPE_SECRET_KEY` | Encrypts stored webhook secrets | — | -| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (stderr) in the CLI | — | -| `ONENOTE_CLIENT_ID` | Microsoft app registration client ID | — | -| `ONENOTE_TENANT_ID` | Microsoft tenant ID | `common` | -| `NOTION_TOKEN` | Notion integration token | — | -| `CONFLUENCE_URL` | Confluence base URL | — | -| `CONFLUENCE_EMAIL` | Confluence user email | — | -| `CONFLUENCE_TOKEN` | Confluence API token | — | - -Environment variables take precedence over config files. LibScope does not load `.env` files; export the variables in your shell or MCP client config. +Every config key has an environment variable `LIBSCOPE_<SECTION>_<FIELD>`, for example `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_LLM_MODEL` or `LIBSCOPE_MCP_TOOLSETS`. The API keys also accept `OPENAI_API_KEY` and `ANTHROPIC_API_KEY`. The [configuration reference](../reference/configuration.md#all-config-keys) lists every key with its variable and default. + +These variables are not config keys: + +| Variable | Description | +| --------------------- | ---------------------------------------------------------- | +| `LIBSCOPE_WORKSPACE` | Active workspace for this shell (default `default`) | +| `LIBSCOPE_API_KEY` | REST API key; requests must send `Authorization: Bearer …` | +| `LIBSCOPE_SECRET_KEY` | Encrypts stored webhook secrets | +| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (stderr) in the CLI | + +Environment variables take precedence over config files. LibScope does not load `.env` files; export the variables in your shell or MCP client config. Connector credentials are not read from environment variables: give them to `libscope connect` (see [Connectors](./connectors.md)). ## Workspaces diff --git a/docs/guide/connectors.md b/docs/guide/connectors.md index 2188339f..fed3f5d5 100644 --- a/docs/guide/connectors.md +++ b/docs/guide/connectors.md @@ -1,181 +1,190 @@ # Connectors -Connectors pull documents from external tools into your LibScope knowledge base. Each connector handles authentication, pagination, and incremental sync so you don't have to think about it. +Connectors pull documents from external tools into your LibScope knowledge base: Notion, Slack, Confluence, Obsidian, OneNote and documentation sites. Each connector handles authentication, pagination and incremental sync. -## Saved Configs, Re-sync and Schedules +## Connections -Every `libscope connect <type>` command saves its settings as a named connector config in `~/.libscope/connectors/<name>.json` (file mode 0600). The default name is the connector type. Use `--name` to keep more than one config of the same type: +`libscope connect <type>` saves a **connection** and syncs it. A connection is the connector's settings, including credentials, in `~/.libscope/connectors/<name>.json` (file mode `0600`). The default name is the connector type. Use `--name` to keep more than one connection of the same type: ```bash libscope connect slack --token xoxb-work-token --name work-slack ``` -A saved config holds what a later run needs, including credentials: +| Command | What it does | +| -------------------------------------------- | --------------------------------------------------------------- | +| `libscope connect <type> [source] [options]` | Create or update a connection, then sync it (`--no-sync` skips) | +| `libscope sync <name>` | Sync one connection with its saved settings | +| `libscope sync --all` | Sync every connection | +| `libscope connections` | List connections with their schedule, last sync and last run | +| `libscope disconnect <name>` | Delete the connection's documents and its saved settings | -| Connector | Saved fields | -| ---------- | ---------------------------------------------------------------------------------------------------- | -| Notion | `token`, `excludePages`, `lastSync` | -| Slack | `token`, `channels`, `excludeChannels`, `threadMode`, `lastSync` | -| Confluence | `baseUrl`, `type` (cloud/server), `email`, `token`, `spaces`, `excludeSpaces`, `lastSync` | -| Obsidian | `vaultPath`, `topicMapping`, `excludePatterns`, `lastSync` | -| OneNote | `clientId`, `tenantId`, `accessToken`, `refreshToken`, `tokenExpiry`, `notebooks`, `excludeSections`, `lastSync` | +Running `connect` again for a saved connection changes only the settings you give and keeps the rest. `disconnect --keep-documents` deletes only the saved settings. The [CLI reference](/reference/cli#connectors) lists every option. -Each file also stores `connectorType` and, after `libscope schedule set`, a `schedule`. +A connection holds these settings: -`--sync` on any connect command re-runs the saved config (`--name` selects which one). Other options on the command line are ignored with `--sync`. After each successful sync, `lastSync` is set to the time the sync started. +| Type | Saved settings | +| ------------ | ---------------------------------------------------------------------------------------------------- | +| `notion` | `token`, `excludePages` | +| `slack` | `token`, `channels`, `excludeChannels`, `threadMode` | +| `confluence` | `baseUrl`, `type` (cloud or server), `email`, `token`, `spaces`, `excludeSpaces` | +| `obsidian` | `vaultPath`, `topicMapping`, `excludePatterns` | +| `onenote` | `clientId`, `tenantId`, `accessToken`, `refreshToken`, `tokenExpiry`, `notebooks`, `excludeSections` | +| `docs` | `url`, `type` (site generator), `library`, `version`, `maxPages`, `maxDepth`, `pathPrefix` | -To sync a saved config on a schedule, set a cron expression. Schedules run while the API server is running (`libscope serve api`): +Each file also stores `connectorType`, the `schedule` (if set) and `lastSync`. After each successful sync, `lastSync` is set to the time the sync started. Notion, Slack and OneNote use it to fetch only what changed since then. Obsidian re-indexes only files whose modification time changed. + +### Schedules + +`--schedule <cron>` syncs a connection on a cron schedule. Schedules run while the REST API server runs (`libscope serve api`): ```bash -libscope schedule set notion "0 */6 * * *" -libscope schedule list -libscope schedule remove notion +libscope connect notion --schedule "0 */6 * * *" --no-sync # every 6 hours +libscope connections # shows the schedule +libscope connect notion --schedule off --no-sync # remove the schedule ``` -Each run, scheduled or manual, writes one entry to the connector sync history under the config name. `libscope disconnect <type>` also deletes the saved config (`--name` selects which one). +Each run, scheduled or manual, records its status. `libscope connections` shows the last run. ## Obsidian -Sync an entire Obsidian vault. Parses frontmatter, wikilinks, embeds, and tags. Folder structure maps to topics. +Sync an Obsidian vault. LibScope parses frontmatter, wikilinks, embeds and tags. The folder structure maps to topics. ```bash -# Initial sync +# First sync libscope connect obsidian /path/to/vault -# Map topics from frontmatter instead of folder structure +# Map topics from frontmatter instead of folders libscope connect obsidian /path/to/vault --topic-mapping frontmatter -# Re-sync the saved vault config (only changed files are re-indexed) -libscope connect obsidian --sync +# Skip folders (comma-separated globs) +libscope connect obsidian /path/to/vault --exclude "templates/*,daily/*" -# Exclude folders -libscope connect obsidian /path/to/vault --exclude "templates/*" "daily/*" +# Sync again (only changed files are re-indexed) +libscope sync obsidian -# Remove vault data from LibScope -libscope disconnect obsidian /path/to/vault +# Remove the vault's documents and the connection +libscope disconnect obsidian ``` ## Notion -Sync pages and databases from your Notion workspace. Requires a [Notion integration token](https://www.notion.so/my-integrations). +Sync pages and databases from your Notion workspace. You need a [Notion integration token](https://www.notion.so/my-integrations). ```bash -# Sync with integration token +# Sync with an integration token libscope connect notion --token secret_abc123 -# Exclude specific pages or databases -libscope connect notion --token $NOTION_TOKEN --exclude page-id-1 db-id-2 +# Skip pages or databases (comma-separated IDs) +libscope connect notion --exclude page-id-1,db-id-2 -# Re-sync with the saved config (pages edited since the last sync) -libscope connect notion --sync +# Sync again (pages edited since the last sync) +libscope sync notion -# Disconnect libscope disconnect notion ``` ## Confluence -Sync Confluence Cloud spaces and pages. Requires an [API token](https://id.atlassian.com/manage-profile/security/api-tokens). +Sync Confluence spaces and pages. For Confluence Cloud you need an [API token](https://id.atlassian.com/manage-profile/security/api-tokens). The base URL is the `[source]` argument. ```bash -# Sync all spaces -libscope connect confluence \ - --url https://acme.atlassian.net \ +# All spaces (Cloud) +libscope connect confluence https://acme.atlassian.net \ --email user@acme.com \ - --token $CONFLUENCE_TOKEN + --token "$CONFLUENCE_API_TOKEN" -# Sync specific spaces -libscope connect confluence \ - --url https://acme.atlassian.net \ +# Some spaces, skip one +libscope connect confluence https://acme.atlassian.net \ --email user@acme.com \ - --token $CONFLUENCE_TOKEN \ + --token "$CONFLUENCE_API_TOKEN" \ --spaces ENG,DEVOPS \ - --exclude-spaces ARCHIVE + --exclude ARCHIVE -# Re-sync with the saved config -libscope connect confluence --sync +# Server or Data Center with a personal access token +libscope connect confluence https://wiki.internal.example.com --server --token "$CONFLUENCE_PAT" -# Disconnect +libscope sync confluence libscope disconnect confluence ``` +The examples read the token from a shell variable to keep it out of your shell history. LibScope does not read connector credentials from environment variables. + ## Slack -Index Slack channel messages and threads. Requires a [Slack bot token](https://api.slack.com/authentication/token-types#bot) with appropriate scopes. +Index Slack channel messages and threads. You need a [Slack bot token](https://api.slack.com/authentication/token-types#bot) with the read scopes for the channels. ```bash -# Sync all channels +# All channels libscope connect slack --token xoxb-your-bot-token -# Sync specific channels -libscope connect slack \ - --token xoxb-... \ - --channels general,engineering \ - --thread-mode aggregate +# Some channels +libscope connect slack --token xoxb-... --channels general,engineering --thread-mode aggregate # Thread modes: -# aggregate — combines thread replies into one document (default) -# separate — one document per reply +# aggregate: one document per thread (default) +# separate: one document per reply -# Re-sync with the saved config (messages since the last sync) -libscope connect slack --sync +# Sync again (messages since the last sync) +libscope sync slack -# Disconnect libscope disconnect slack ``` ## OneNote -Sync OneNote notebooks via the Microsoft Graph API. Uses device code authentication — you'll be prompted to open a browser and log in. +Sync OneNote notebooks through the Microsoft Graph API. LibScope signs in with a device code: it prints a URL and a code, and you sign in in a browser. ```bash -# Set your app registration client ID -export ONENOTE_CLIENT_ID=your-client-id +# Sign in and sync (needs an Azure app registration client ID) +libscope connect onenote --client-id your-client-id -# Authenticate and sync -libscope connect onenote - -# Sync a specific notebook +# One notebook only libscope connect onenote --notebook "Work Notes" -# Re-sync with the saved config (refreshes the access token when it has expired) -libscope connect onenote --sync +# Sync again (the access token is refreshed when it has expired) +libscope sync onenote -# Use an existing access token instead of signing in (it cannot be refreshed) -libscope connect onenote --token <access-token> +# Use an access token instead of signing in (it cannot be refreshed) +libscope connect onenote --token "$GRAPH_ACCESS_TOKEN" -# Disconnect libscope disconnect onenote ``` -You'll need an Azure AD app registration with `Notes.Read` permission. See [Microsoft's guide](https://learn.microsoft.com/en-us/graph/auth-register-app-v2) for setup. +You need an Azure AD app registration with the `Notes.Read` permission. See [Microsoft's guide](https://learn.microsoft.com/en-us/graph/auth-register-app-v2). `--tenant-id` defaults to `common`. -## GitHub / GitLab +## Documentation sites -Index documentation from any GitHub or GitLab repository. +Crawl a documentation site (Sphinx, VitePress, Doxygen or a generic site). The site type is detected unless you set `--site-type`. ```bash -# Public repo -libscope add-repo https://github.com/org/repo +libscope connect docs https://docs.example.com --library example --lib-version 2.0 +libscope connect docs https://docs.example.com --name example-guide --path-prefix /guide --max-pages 200 +libscope sync example-guide +``` + +The defaults are 500 pages and a link depth of 10. -# Private repo with token, specific branch and path -libscope add-repo https://github.com/org/private-repo \ - --token $GITHUB_TOKEN \ +## GitHub and GitLab repositories + +A repository is not a connector. Add it with `libscope add`: + +```bash +# Public repository +libscope add https://github.com/org/repo + +# Private repository, one branch, some folders and file types +libscope add https://github.com/org/private-repo \ + --token "$GITHUB_TOKEN" \ --branch develop \ - --path docs/ \ + --path docs \ --extensions .md,.mdx,.rst ``` -## MCP Usage - -All connectors are also available as MCP tools, so your AI assistant can trigger syncs directly: +See [Code Indexing](/guide/code-indexing) for source code. -- `sync-obsidian-vault` -- `sync-notion` -- `sync-confluence` -- `sync-slack` -- `sync-onenote` +## MCP and REST -Tokens and other settings are optional when a saved config exists: the tools read the config named by their `name` parameter (default: the connector type), so the assistant does not need secrets in its context. Run a sync with `async: true` to get a task ID; `cancel-task` stops the sync between pages or items. +Credentials are never tool or request parameters. Save the connection with `libscope connect` first. Then: -See the [MCP Tools Reference](/reference/mcp-tools) for parameter details. +- **MCP**: the `sync` tool in the admin toolset (`LIBSCOPE_MCP_TOOLSETS=admin`) syncs one connection (`{"name": "notion"}`) or all of them (`{"all": true}`). With `async: true` it returns a task ID; `task {"action": "cancel", "taskId": "..."}` stops the sync. See the [MCP Tools Reference](/reference/mcp-tools#sync). +- **REST**: `POST /api/v1/sync` with `{ "name": "notion" }` or `{ "all": true }` starts a task. `GET /api/v1/connections` lists the connections (secrets masked), and `DELETE /api/v1/connections/:name` disconnects one. See the [REST API reference](/reference/rest-api). diff --git a/docs/guide/dashboard.md b/docs/guide/dashboard.md index b19cac28..179de667 100644 --- a/docs/guide/dashboard.md +++ b/docs/guide/dashboard.md @@ -51,7 +51,7 @@ The **Topics** panel shows your topic hierarchy. Clicking a topic filters the do Navigate to `/graph` (e.g. `http://localhost:3377/graph`) to view an interactive visualization of your knowledge base: - **Nodes** represent documents -- **Edges** represent cross-reference links (`link-documents`, `libscope link`) +- **Edges** represent cross-reference links (`libscope docs link`, the `link-documents` MCP tool, `POST /api/v1/documents/:documentId/links`) - **Clusters** are automatically detected and color-coded by topic - Hover a node to see the document title; click to open it in the document browser diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index fca72ab5..9b02d4b4 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -38,7 +38,7 @@ libscope add https://github.com/org/repo --path docs Each added document is printed with its ID. Add `--dry-run` to see what would be added. -LibScope supports **Markdown** (`.md`, `.mdx`) and **plain text** natively. Additional formats — **PDF** (`.pdf`), **Word** (`.docx`), **EPUB** (`.epub`), and **PowerPoint** (`.pptx`) — are available via optional dependencies that install automatically. See the [README](/) for the full format table. +LibScope reads **Markdown** (`.md`, `.markdown`, `.mdx`), **plain text** (`.txt`), **HTML** (`.html`, `.htm`), **EPUB** (`.epub`), **PowerPoint** (`.pptx`), **CSV**, **JSON** and **YAML**. **PDF** (`.pdf`) and **Word** (`.docx`) need the optional dependencies `pdf-parse` and `mammoth`, which npm installs by default. Each document gets chunked by heading, embedded into vectors, and stored in the database. @@ -58,7 +58,7 @@ libscope search libscope search "API rate limiting" --json ``` -Results are ranked by vector similarity, with optional FTS5 boosting for keyword matches. Each result shows its document ID and chunk ID; `libscope docs show <documentId>` prints the whole document. +Results combine vector similarity and keyword (FTS5) matches; see [How Search Works](/guide/how-search-works). Each result shows its document ID and chunk ID; `libscope docs show <documentId>` prints the whole document. ## Ask Questions @@ -122,7 +122,7 @@ For programmatic access, start the REST API instead of the MCP server: libscope serve api --port 3378 ``` -The OpenAPI 3.0 spec is served at `GET /openapi.json`. See [REST API Reference](/reference/rest-api) for full documentation. +The OpenAPI 3.1 document is served at `GET /openapi.json`. See [REST API Reference](/reference/rest-api) for full documentation. ## What's Next diff --git a/docs/guide/how-search-works.md b/docs/guide/how-search-works.md index 4f2e9061..afa7ba05 100644 --- a/docs/guide/how-search-works.md +++ b/docs/guide/how-search-works.md @@ -5,24 +5,29 @@ LibScope uses a hybrid search strategy combining vector (semantic) search with f ## Search Pipeline ### 1. Query Embedding -Your search query is converted to a vector embedding using the configured embedding provider (local model or OpenAI). This captures the semantic meaning of your query. + +Your search query is converted to a vector embedding using the configured embedding provider (local model, Ollama or OpenAI). This captures the semantic meaning of your query. ### 2. Vector Search (ANN) + The query vector is matched against all indexed chunk embeddings using approximate nearest-neighbour (ANN) search via `sqlite-vec`. Results are ranked by cosine similarity — chunks semantically related to your query rank highest, even if they use different words. ### 3. Full-Text Search (FTS5) + Simultaneously, SQLite's FTS5 (BM25 ranking) searches for chunks containing your query terms. LibScope first tries AND logic (all terms must match) for precision, then falls back to OR logic if no results are found. ### 4. Hybrid Fusion (RRF) + Vector and FTS5 results are merged using **Reciprocal Rank Fusion (RRF)** — a technique that combines ranked lists without needing calibrated scores: ``` RRF_score(chunk) = Σ 1 / (k + rank_in_list) ``` -where `k = 60` (standard constant). Chunks that rank well in *both* vector and FTS5 lists get the highest fused scores. +where `k = 60` (standard constant). Chunks that rank well in _both_ vector and FTS5 lists get the highest fused scores. ### 5. Title Boost + Chunks whose document title contains any query word receive a 1.5× score multiplier, lifting exact-title matches to the top. ### 6. MMR Diversity Reranking @@ -35,21 +40,24 @@ When you set the `diversity` option (0–1), results are reranked using **Maxima MMR is applied after title boost and score sorting. It's useful when you want to cover different aspects of a topic rather than getting multiple chunks from the same document. +`diversity` is a parameter of the `search` MCP tool, `GET /api/v1/search` and `scope.search()`. The CLI has no flag for it. + ```bash -libscope search "authentication" --diversity 0.5 +curl "http://localhost:3378/api/v1/search?query=authentication&diversity=0.5" ``` ### 7. Pagination & Deduplication -Results are optionally deduplicated by document (`maxChunksPerDocument`) and paginated. Use `offset` and `limit` in your search options for pagination. + +Results are optionally deduplicated by document (`maxChunksPerDocument`) and paginated. Use `offset` and `limit` for pagination (CLI: `--offset` and `-n, --limit`). ## Search Methods -| Method | When Used | Best For | -|--------|-----------|---------| -| `hybrid` | sqlite-vec available + FTS5 match | Most queries — best precision & recall | -| `vector` | sqlite-vec available, FTS5 returns nothing | Conceptual/semantic queries | -| `fts5` | Part of hybrid pipeline | Keyword-heavy queries | -| `keyword` | sqlite-vec unavailable | Fallback — exact word matching only | +| Method | When Used | Best For | +| --------- | ------------------------------------------ | -------------------------------------- | +| `hybrid` | sqlite-vec available + FTS5 match | Most queries — best precision & recall | +| `vector` | sqlite-vec available, FTS5 returns nothing | Conceptual/semantic queries | +| `fts5` | Part of hybrid pipeline | Keyword-heavy queries | +| `keyword` | sqlite-vec unavailable | Fallback — exact word matching only | The active method is returned in each result's `scoreExplanation.method` field. @@ -68,12 +76,17 @@ Every search result includes a `scoreExplanation` object: ## Tuning Search -| Option | Default | Effect | -|--------|---------|--------| -| `limit` | 10 | Results per page | -| `offset` | 0 | Pagination offset | -| `maxChunksPerDocument` | unlimited | Max chunks returned per document | -| `contextChunks` | 0 | Adjacent chunks to include for context (max 2) | -| `diversity` | 0 | MMR diversity factor (0 = relevance only, 1 = max diversity) | -| `minRating` | none | Filter by minimum avg document rating | -| `tags` | none | Filter by document tags (AND logic) | +These are the parameters of the `search` operation: the MCP `search` tool, `GET /api/v1/search` and `scope.search()` use these names. The CLI flag is in the last column. + +| Option | Default | Effect | CLI flag | +| ---------------------- | --------- | ------------------------------------------------------------------------- | --------------- | +| `limit` | 10 | Results per page (max 100) | `-n, --limit` | +| `offset` | 0 | Pagination offset | `--offset` | +| `maxChunksPerDocument` | unlimited | Max chunks returned per document | `--max-per-doc` | +| `contextChunks` | 0 | Adjacent chunks to include for context (max 2) | `--context` | +| `diversity` | 0 | MMR diversity factor (0 = relevance only, 1 = max diversity) | — | +| `minRating` | none | Filter by minimum avg document rating | `--min-rating` | +| `tags` | none | Filter by document tags (AND logic) | `--tags` | +| `relatedTo` | none | Find content similar to a document or chunk ID instead of running a query | `--related` | + +See the [MCP Tools Reference](/reference/mcp-tools#search) and the [CLI reference](/reference/cli#libscope-search) for all parameters. diff --git a/docs/guide/mcp-setup.md b/docs/guide/mcp-setup.md index 8daff622..2c05d185 100644 --- a/docs/guide/mcp-setup.md +++ b/docs/guide/mcp-setup.md @@ -128,7 +128,7 @@ Your AI assistant calls these tools when it needs information from your docs. Th ## Logs -The MCP server writes JSON-RPC messages to stdout and its logs to stderr. Set the log level with `logging.level` in the config file or `LIBSCOPE_LOGGING_LEVEL`. +The MCP server writes JSON-RPC messages to stdout and its logs to stderr. Set the log level with `logging.level` in the config file, `LIBSCOPE_LOGGING_LEVEL`, or `libscope serve --log-level debug` (`--verbose` also sets `debug`). ## Embedding the Server diff --git a/docs/guide/programmatic-usage.md b/docs/guide/programmatic-usage.md index ca0e9df8..44415a28 100644 --- a/docs/guide/programmatic-usage.md +++ b/docs/guide/programmatic-usage.md @@ -111,34 +111,47 @@ for await (const event of scope.askStream("How does OAuth2 work?")) { The other operations are grouped in namespaces. Each method takes one input object and returns a promise. The input type is the operation's schema input, so your editor shows each field and its description. -| Method | Operation | -| ----------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | -| `scope.overview()` | Counts, topics, packs, index, health | -| `docs.get({ documentId, offset?, maxLength? })` | Document with tags, links, ratings | -| `docs.list({ topic?, library?, version?, sourceType?, tags?, limit?, offset? })` | List documents | -| `docs.update({ documentId, title?, content?, library?, version?, url?, topic?, tags? })` | Update a document | -| `docs.delete({ documentId })` | Delete a document | -| `docs.rate({ documentId, rating, chunkId?, feedback?, suggestedCorrection? })` | Rate a document | -| `docs.history({ documentId })`, `docs.rollback({ documentId, version })` | Saved versions | -| `topics.list()`, `topics.create({ name })`, `topics.delete({ topic })` | Topics | -| `tags.add`, `tags.remove` (`{ documentId, tags }`), `tags.list()`, `tags.suggest({ documentId })` | Tags | -| `links.create({ documentId, targetDocumentId, linkType })`, `links.delete({ linkId })`, `links.list()`, `links.prerequisites({ documentId })` | Document links | -| `searches.save`, `searches.list`, `searches.run`, `searches.delete` | Saved searches | -| `packs.install`, `packs.remove`, `packs.list`, `packs.create` | Knowledge packs | -| `connectors.list()`, `connectors.sync({ name } \| { all: true })`, `connectors.disconnect({ name })` | Saved connections | -| `tasks.get`, `tasks.cancel`, `tasks.list` | Background tasks | -| `admin.reindex`, `admin.dedupe`, `admin.backup`, `admin.restore`, `admin.pruneExpired`, `admin.bulkDelete`, `admin.bulkRetag`, `admin.bulkMove` | Maintenance | -| `analytics.popular`, `analytics.stale`, `analytics.topQueries`, `analytics.searches` | Search analytics | -| `webhooks.create`, `webhooks.list`, `webhooks.delete`, `webhooks.test` | Webhooks | +| Method | Operation | +| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | +| `scope.overview()` | Counts, topics, packs, index, health | +| `docs.get({ documentId, offset?, maxLength? })` | Document with tags, links, ratings | +| `docs.list({ topic?, library?, version?, sourceType?, tags?, limit?, offset? })` | List documents | +| `docs.update({ documentId, title?, content?, library?, version?, url?, topic?, tags? })` | Update a document | +| `docs.delete({ documentId })` | Delete a document | +| `docs.rate({ documentId, rating, chunkId?, feedback?, suggestedCorrection? })` | Rate a document | +| `docs.history({ documentId })`, `docs.rollback({ documentId, version })` | Saved versions | +| `topics.list({ parent? })`, `topics.create({ name, description?, parent? })`, `topics.delete({ topic, deleteDocuments? })` | Topics | +| `tags.add`, `tags.remove` (`{ documentId, tags }`), `tags.list()`, `tags.suggest({ documentId })` | Tags | +| `links.create({ documentId, targetDocumentId, linkType, label? })`, `links.delete({ linkId })`, `links.list({ documentId?, linkType? })`, `links.prerequisites({ documentId })` | Document links | +| `links.graph({ topic?, tag?, threshold?, maxNodes? })` | Knowledge graph (nodes and edges) | +| `searches.save`, `searches.list`, `searches.run`, `searches.delete` | Saved searches | +| `packs.install({ pack, registry? })`, `packs.remove({ pack })`, `packs.list({ available?, registry? })`, `packs.create({ name, ... })` | Knowledge packs | +| `registries.list()`, `registries.add({ url, name? })`, `registries.remove({ name })`, `registries.sync({ name? })`, `registries.search({ query, registry? })`, `registries.create({ path })`, `registries.publish({ file, registry, version? })`, `registries.unpublish({ pack, registry })` | Git pack registries | +| `connectors.list()`, `connectors.sync({ name } \| { all: true })`, `connectors.disconnect({ name })` | Saved connections | +| `tasks.get`, `tasks.cancel`, `tasks.list` | Background tasks | +| `admin.reindex`, `admin.dedupe`, `admin.backup`, `admin.restore`, `admin.pruneExpired`, `admin.bulkDelete`, `admin.bulkRetag`, `admin.bulkMove` | Maintenance | +| `analytics.popular`, `analytics.stale`, `analytics.topQueries`, `analytics.searches` | Search analytics | +| `webhooks.create`, `webhooks.list`, `webhooks.delete`, `webhooks.test` | Webhooks | ```ts const doc = await scope.docs.get({ documentId: "doc-id" }); await scope.tags.add({ documentId: "doc-id", tags: ["reviewed"] }); await scope.docs.update({ documentId: "doc-id", topic: "security" }); const { stats } = await scope.overview(); + +// Knowledge graph of one topic +const graph = await scope.links.graph({ topic: "security", maxNodes: 100 }); +console.log(graph.nodes.length, graph.edges.length); + +// Git pack registries +await scope.registries.add({ url: "https://github.com/org/packs.git", name: "team" }); +const found = await scope.registries.search({ query: "react" }); +await scope.packs.install({ pack: "react-docs", registry: "team" }); ``` -Long-running methods (`add`, `admin.reindex`, `packs.install`, `connectors.sync`) take a second argument with an `AbortSignal` and a progress callback: +See [Pack Registries](/guide/pack-registries) for the registry layout and publishing. + +Every method except `askStream` takes an optional second argument with an `AbortSignal` and a progress callback. The long-running methods (`add`, `admin.reindex`, `admin.dedupe`, `packs.install`, `connectors.sync`) report progress: ```ts const controller = new AbortController(); diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index d62b43ed..db016577 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -9,6 +9,7 @@ **Cause:** `sqlite-vec` requires a native Node.js addon that must be built for your platform. **Fix:** + ```bash npm rebuild sqlite-vec # or @@ -21,11 +22,12 @@ If you're on an unsupported platform, LibScope will fall back to keyword-only se ### Model download fails or hangs -**Symptom:** First run hangs at "Downloading model..." or fails with a network error. +**Symptom:** First run hangs at "Downloading embedding model ... (first run only)..." or fails with a network error. **Cause:** The local embedding model (~80MB) downloads from Hugging Face on first use. **Fix:** + - Check your internet connection - If behind a proxy, set `HTTPS_PROXY` environment variable - To use OpenAI embeddings instead (no download required): `libscope config set embedding.provider openai` @@ -39,9 +41,11 @@ If you're on an unsupported platform, LibScope will fall back to keyword-only se **Cause:** LibScope records the provider, model, and vector size that built the vector index. Different models produce different vectors, so the existing vectors cannot be used with the new model. **Fix:** Rebuild the vector index with the configured model: + ```bash -libscope reindex --rebuild +libscope admin reindex --rebuild ``` + This command drops and recreates the vector table with the size of the new model. Then it re-embeds every chunk. Your documents, topics, and tags are kept. ### `Expected embedding dimension N, got M` @@ -50,7 +54,9 @@ This command drops and recreates the vector table with the size of the new model **Cause:** The model returns vectors of a different size than LibScope expects. LibScope knows the size of common OpenAI and Ollama models. For other models, it uses `embedding.dimensions` from the config, or the size of the first vector the model returns. -**Fix:** Set `embedding.dimensions` to M in `~/.libscope/config.json` or `.libscope.json`, then run `libscope reindex --rebuild`. +**Fix:** Set `embedding.dimensions` to M in `~/.libscope/config.json` or `.libscope.json`, then run `libscope admin reindex --rebuild`. + +`libscope doctor` shows the configured embedding model and the model the vector index was built with. --- @@ -58,10 +64,11 @@ This command drops and recreates the vector table with the size of the new model ### Search returns no results -1. Check that documents are indexed: `libscope list` +1. Check that documents are indexed: `libscope docs list` (or `libscope admin stats` for counts) 2. Try a simpler query — FTS5 AND logic requires all terms to match 3. Check your filters — `--library`, `--topic`, `--tags` may be too restrictive -4. Run `libscope search "test" --limit 5` to verify basic search works +4. Run `libscope search "test" -n 5` to verify basic search works +5. Run `libscope doctor` to check the database, the embedding model and the vector index ### Results seem irrelevant @@ -71,17 +78,14 @@ This command drops and recreates the vector table with the size of the new model --- -## API / MCP Issues - -### `401 Unauthorized` from API - -When the server was started with `LIBSCOPE_API_KEY` set, every request must send `Authorization: Bearer <key>` with that value. `libscope config show` does not show this key. +## MCP Issues ### MCP tools not appearing in Claude / Cursor -1. Verify the MCP server is running: `libscope mcp start` -2. Check the MCP config path in your client settings points to the libscope server -3. Restart your AI client after adding the MCP server +1. Check that the server starts: run `libscope serve` in a terminal. It waits for JSON-RPC on stdin; any startup error is printed on stderr. Press Ctrl+C to stop it. +2. Check that your client config runs `npx -y libscope serve` (or `libscope serve`). See [MCP Setup](/guide/mcp-setup). +3. Restart your AI client after you change its MCP config. +4. `ask` is registered only when an LLM or passthrough is available, and `sync`, `install-pack`, `list-packs` and `reindex-documents` only with the admin toolset (`LIBSCOPE_MCP_TOOLSETS=admin`). See the [MCP Tools Reference](/reference/mcp-tools). --- @@ -90,6 +94,7 @@ When the server was started with `LIBSCOPE_API_KEY` set, every request must send ### Database locked errors LibScope uses SQLite WAL mode which supports concurrent reads but only one writer. If you see lock errors: + - Ensure only one libscope process is running - Check for stuck processes: `ps aux | grep libscope` @@ -99,10 +104,10 @@ There is no reset command. To remove all indexed content and keep your config, s ```bash rm ~/.libscope/workspaces/default/libscope.db* # replace "default" with your workspace name -libscope init +libscope doctor --fix # optional: the next command also creates it ``` -To replace only the vectors (for example, after you change the embedding model), use `libscope reindex --rebuild`. +`libscope config path` and `libscope doctor` show which files are in use. To replace only the vectors (for example, after you change the embedding model), use `libscope admin reindex --rebuild`. --- @@ -112,13 +117,14 @@ To replace only the vectors (for example, after you change the embedding model), - Verify your integration token has access to the pages you want to sync — in Notion, you must explicitly share pages with your integration - Check that the token starts with `secret_` (internal integrations) or is a valid OAuth token -- Try syncing a specific page ID first: `libscope connect notion --token $NOTION_TOKEN` +- Run `libscope connections` to see the last run and its error ### Confluence sync fails with 401 - Ensure you are using an API token (not your password) — generate one at [id.atlassian.com](https://id.atlassian.com/manage-profile/security/api-tokens) - The `--email` flag must match the Atlassian account that owns the token -- Verify the `--url` uses your full Atlassian domain: `https://your-org.atlassian.net` +- Verify the base URL (the `[source]` argument of `connect confluence`) is your full Atlassian domain: `https://your-org.atlassian.net` +- For Confluence Server or Data Center, add `--server` and use a personal access token ### Slack sync fails with `missing_scope` error @@ -133,16 +139,16 @@ Reinstall the app to your workspace after adding scopes. ### OneNote sync prompts for authentication every time -OneNote uses device code auth which caches a refresh token. If the cache is missing or expired: +OneNote uses device code sign-in and saves a refresh token in the connection. If the token is missing or no longer valid: -1. Ensure `ONENOTE_CLIENT_ID` is set to your Azure AD app registration client ID -2. The token is cached in `~/.libscope/connectors/onenote.json` — delete this file to force re-authentication +1. Run `libscope connect onenote --client-id <id>` with your Azure AD app registration client ID (add `--name` if the connection has another name) +2. The tokens are saved in `~/.libscope/connectors/onenote.json` (or `<name>.json`). `libscope disconnect onenote --keep-documents` deletes it and forces a new sign-in 3. Ensure your app registration has `Notes.Read` (or `Notes.ReadWrite`) permission granted in Azure ### Obsidian sync misses some notes -- Check your `--exclude` patterns — glob patterns use `/` as separator even on Windows -- Obsidian notes in the `.obsidian/` config folder are automatically excluded +- Check your `--exclude` patterns (comma-separated globs) — glob patterns use `/` as separator even on Windows +- The `.obsidian/`, `.trash/` and `templates/` folders are always excluded - Notes without the `.md` extension are skipped; check your vault for unusual extensions --- @@ -151,7 +157,7 @@ OneNote uses device code auth which caches a refresh token. If the cache is miss ### `401 Unauthorized` from API -Authentication is on only when the server process has `LIBSCOPE_API_KEY` set. Then every request must send that value as a bearer token: +Authentication is on only when the server process has `LIBSCOPE_API_KEY` set. Then every request must send that value as a bearer token. `libscope config show` does not show this key. ```bash curl -H "Authorization: Bearer $LIBSCOPE_API_KEY" http://localhost:3378/api/v1/documents @@ -159,17 +165,17 @@ curl -H "Authorization: Bearer $LIBSCOPE_API_KEY" http://localhost:3378/api/v1/d ### `429 Too Many Requests` -The API applies rate limiting per IP. If you're hitting limits during bulk operations, use the batch endpoints: +The API allows 120 requests per minute from one address. If you hit the limit during bulk operations: -- `POST /api/v1/batch-search` — up to 20 queries at once -- `POST /api/v1/bulk/delete`, `/bulk/retag`, `/bulk/move` — process many documents in one request +- Use `POST /api/v1/bulk/delete`, `/bulk/retag` and `/bulk/move` to change up to 1000 documents in one request +- Send one `POST /api/v1/documents` with `url` and `spider: true` to crawl a site, instead of one request per page ### CORS errors when calling from a browser -The REST API allows CORS by default for localhost origins. If you're calling from a different origin, you can either: +By default, the REST API allows browser requests from `http://localhost` and `http://localhost:3000`. Write requests from other origins get `403` (`FORBIDDEN_ORIGIN`). If you call it from a different origin, you can: -- Run your frontend on the same host as the API -- Use a reverse proxy that adds the appropriate `Access-Control-Allow-Origin` header +- Run your frontend on the same origin as the API +- Start the server from code with `startApiServer` and list your origin in `corsOrigins` (see the [REST API reference](/reference/rest-api)) --- @@ -182,10 +188,9 @@ Optional parser dependencies may not be installed. Install them: ```bash npm install -g pdf-parse # for .pdf files npm install -g mammoth # for .docx files -npm install -g epub2 # for .epub files ``` -Or install all optional parsers at once: +Or reinstall LibScope with its optional dependencies: ```bash npm install -g libscope --include=optional @@ -211,13 +216,9 @@ export LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS=true ### Import is very slow for large directories -Use `import-batch` with parallelism instead of `import`: - -```bash -libscope import-batch ./docs/ --concurrency 8 --library my-lib -``` - -The default `import` command is sequential. `import-batch` processes files in parallel. +- Limit the files with `--include` and `--exclude` (comma-separated globs): `libscope add ./docs --include "**/*.md" --exclude "node_modules/**"` +- Run `libscope add ./docs --dry-run` first to see which files would be added +- The local embedding model runs on the CPU. An Ollama or OpenAI embedding provider can be faster for large imports --- @@ -225,7 +226,7 @@ The default `import` command is sequential. `import-batch` processes files in pa ### `ask` returns "No LLM provider configured" -The `ask` command requires an LLM provider. Configure one: +The `ask` command needs an LLM. With `llm.provider` set to `auto` (the default), LibScope uses OpenAI if an OpenAI key is set, else Anthropic if an Anthropic key is set, else Ollama if `llm.url` is set or the embedding provider is `ollama`. `libscope doctor` shows which LLM `ask` will use. To choose one: ```bash # OpenAI (LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY) @@ -243,6 +244,6 @@ export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-... ### Answers are low quality or hallucinated - Index more relevant documents — the quality of RAG answers depends on what's in the knowledge base -- Increase `--top-k` to retrieve more context chunks: `libscope ask "..." --top-k 10` +- Use more context chunks: `libscope ask "..." -n 10` (default 5) - Switch to a more capable embedding provider (OpenAI `text-embedding-3-small` outperforms the local model) - Switch to a more capable LLM model: `libscope config set llm.model gpt-4o` diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index aa38f584..4d23446b 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -26,6 +26,8 @@ curl -X POST http://localhost:3378/api/v1/webhooks \ }' ``` +In the SDK, use `scope.webhooks.create({ url, events, secret })`. Webhooks are not available as MCP tools. + Storing a secret requires the `LIBSCOPE_SECRET_KEY` environment variable. LibScope encrypts the secret at rest with AES-256-GCM. Webhook URLs must use http or https and must not resolve to a private or internal IP address. ## Supported Events @@ -57,12 +59,12 @@ Every webhook delivery sends a `POST` request with `Content-Type: application/js The `data` object depends on the event: -| Event | `data` fields | -| --------------------------------------- | -------------------------------------------------- | -| `document.created`, `document.updated` | `documentId`, `title`, `library`?, `version`? | -| `document.deleted` | `documentId` | -| `document.rated` | `documentId`, `rating`, `feedback`? | -| `search.executed` | `query`, `resultCount`, `topicId`? | +| Event | `data` fields | +| -------------------------------------- | --------------------------------------------- | +| `document.created`, `document.updated` | `documentId`, `title`, `library`?, `version`? | +| `document.deleted` | `documentId` | +| `document.rated` | `documentId`, `rating`, `feedback`? | +| `search.executed` | `query`, `resultCount`, `topicId`? | Fields marked `?` are omitted when they have no value. diff --git a/docs/index.md b/docs/index.md index fea84d38..dce51613 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,13 +16,13 @@ hero: features: - icon: 🔍 title: Semantic Search - details: Vector similarity search powered by sqlite-vec, with FTS5 full-text fallback. Find what you need, even when you don't know the exact words. + details: Hybrid search that combines vector similarity (sqlite-vec) with FTS5 full-text search. Find what you need, even when you don't know the exact words. - icon: 🤖 title: MCP Integration - details: 31 tools for AI assistants out of the box. Works with Claude, Cursor, VS Code, and any MCP-compatible client. + details: 11 core tools for AI assistants (search, ask, read, add, rate, link), plus an optional admin toolset. Works with Claude, Cursor, VS Code, and any MCP-compatible client. - icon: 🔗 title: Connectors - details: Pull in docs from Obsidian, Notion, Confluence, Slack, OneNote, and GitHub/GitLab. Keep everything in one place. + details: Pull in docs from Obsidian, Notion, Confluence, Slack, OneNote, documentation sites, and GitHub/GitLab repositories. Keep everything in one place. - icon: 📦 title: Knowledge Packs details: Install pre-built documentation bundles or create your own. Share curated collections with your team. diff --git a/docs/reference/lite-api.md b/docs/reference/lite-api.md index e5478224..f1e80e80 100644 --- a/docs/reference/lite-api.md +++ b/docs/reference/lite-api.md @@ -60,7 +60,7 @@ Returns `LibScope.create({ ...options, useConfigFile: false })`: config files, ` `SearchResult` has `documentId`, `chunkId`, `title`, `content`, `score`, `sourceType`, `library`, `version`, `topicId`, `url`, `avgRating` and `scoreExplanation`. -`docs`, `topics`, `tags`, `links`, `searches`, `packs`, `connectors`, `tasks`, `admin`, `analytics` and `webhooks` have the same methods as on `LibScope.create()`. +`docs`, `topics`, `tags`, `links`, `searches`, `packs`, `registries`, `connectors`, `tasks`, `admin`, `analytics` and `webhooks` have the same methods as on `LibScope.create()`. --- diff --git a/docs/reference/registry.md b/docs/reference/registry.md index 97584c31..1cd4cb8d 100644 --- a/docs/reference/registry.md +++ b/docs/reference/registry.md @@ -339,7 +339,7 @@ Each registry is cloned to `~/.libscope/registries/<name>/`. Searching, listing Every pack version includes a `checksum.sha256` file containing the SHA-256 hex hash of the pack file. - **On publish**: libscope generates the checksum automatically and writes it alongside the pack file. -- **On install**: libscope verifies the checksum before extracting. A mismatch fails with: "Checksum verification failed — the pack file may have been tampered with or corrupted." +- **On install**: libscope verifies the checksum before extracting. A mismatch fails with a `ValidationError`: "Checksum verification failed for ...: expected ..., got .... The pack file may have been tampered with or corrupted." --- From a2c7e61e20c650da2be9852e9c5eb9174158626a Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 19:11:16 +0000 Subject: [PATCH 65/83] fix(core): a custom chunker can index files that have no parser libscope/lite in 1.x indexed source files by path with the tree-sitter chunker; 2.0 rejected them as an unsupported format. indexFile now reads such files as UTF-8 when a chunker is set and indexes them if the chunker returns chunks. --- src/core/indexing.ts | 16 ++++++++-------- tests/unit/libscope.test.ts | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/core/indexing.ts b/src/core/indexing.ts index 8984e727..a140a972 100644 --- a/src/core/indexing.ts +++ b/src/core/indexing.ts @@ -630,20 +630,20 @@ export async function indexFile( : undefined; const effectiveName = normalizedFormat ? `file${normalizedFormat}` : filePath; const parser = getParserForFile(effectiveName); - - if (!parser) { - const supported = getSupportedExtensions().join(", "); - throw new ValidationError( - `Unsupported file format: "${filePath}". Supported extensions: ${supported}`, + const unsupported = (): ValidationError => + new ValidationError( + `Unsupported file format: "${filePath}". Supported extensions: ${getSupportedExtensions().join(", ")}`, ); - } + // Without a parser, a custom chunker may still take the file as UTF-8 text (e.g. source code). + if (!parser && !options.chunker) throw unsupported(); - log.info({ filePath, parser: parser.extensions[0] }, "Parsing file for indexing"); + log.info({ filePath, parser: parser?.extensions[0] ?? "chunker" }, "Parsing file for indexing"); const buffer = readFileSync(filePath); - const content = await parser.parse(buffer); + const content = parser ? await parser.parse(buffer) : buffer.toString("utf-8"); const title = options.title ?? basename(filePath).replace(/\.[^.]+$/, ""); const preChunked = await options.chunker?.({ content, title, source: filePath }); + if (!parser && !preChunked) throw unsupported(); return indexDocument(db, provider, { title, diff --git a/tests/unit/libscope.test.ts b/tests/unit/libscope.test.ts index b28cdb6a..f1df0ef3 100644 --- a/tests/unit/libscope.test.ts +++ b/tests/unit/libscope.test.ts @@ -308,6 +308,25 @@ describe("LibScope options", () => { } }); + it("indexes a file with no parser (source code) when the chunker returns chunks", async () => { + const chunker = vi.fn(({ source }: { source: string }) => + source.endsWith(".ts") ? ["function a() {}", "function b() {}"] : undefined, + ); + const scope = create({ chunker }); + try { + const code = join(tempHome, "util.ts"); + writeFileSync(code, "function a() {}\nfunction b() {}\n"); + const result = await scope.add(code); + expect(result.documents[0]!.chunkCount).toBe(2); + + const binary = join(tempHome, "image.png"); + writeFileSync(binary, "not really a png"); + await expect(scope.add(binary)).rejects.toThrow(/Unsupported file format/); + } finally { + scope.close(); + } + }); + it("leaves an injected database open on close", async () => { const db = createTestDb(); const scope = LibScope.create({ From 56366d30c8ba9f37e1d1b4015b57d11471e2edce Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 19:13:40 +0000 Subject: [PATCH 66/83] refactor(core): remove unused batch import and batch search modules Directory ingest replaces batchImport; searchBatch had no callers after the root exports shrank. --- src/core/batch-search.ts | 52 ------ src/core/batch.ts | 147 --------------- tests/integration/parsed-file-import.test.ts | 21 ++- tests/unit/batch-search.test.ts | 81 --------- tests/unit/batch.test.ts | 181 ------------------- 5 files changed, 11 insertions(+), 471 deletions(-) delete mode 100644 src/core/batch-search.ts delete mode 100644 src/core/batch.ts delete mode 100644 tests/unit/batch-search.test.ts delete mode 100644 tests/unit/batch.test.ts diff --git a/src/core/batch-search.ts b/src/core/batch-search.ts deleted file mode 100644 index 7785c18f..00000000 --- a/src/core/batch-search.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import { searchDocuments, type SearchOptions, type SearchResponse } from "./search.js"; -import { ValidationError } from "../errors.js"; - -export const BATCH_SEARCH_MAX_REQUESTS = 20; - -export interface BatchSearchRequest { - /** The query string for this search. */ - query: string; - /** Per-request overrides — all SearchOptions except `query` which comes from above. */ - options?: Omit<SearchOptions, "query">; -} - -export interface BatchSearchResponse { - /** Results keyed by the original query string. */ - results: Record<string, SearchResponse>; -} - -/** - * Execute multiple search queries concurrently. - * Results are keyed by the query string. - */ -export async function searchBatch( - db: Database.Database, - provider: EmbeddingProvider, - requests: BatchSearchRequest[], -): Promise<BatchSearchResponse> { - if (!Array.isArray(requests) || requests.length === 0) { - throw new ValidationError("At least one search request is required"); - } - if (requests.length > BATCH_SEARCH_MAX_REQUESTS) { - throw new ValidationError( - `Batch size ${requests.length} exceeds maximum of ${BATCH_SEARCH_MAX_REQUESTS}`, - ); - } - - const entries = await Promise.all( - requests.map(async (req) => { - const searchOpts: SearchOptions = { - ...req.options, - query: req.query, - }; - const response = await searchDocuments(db, provider, searchOpts); - return [req.query, response] as const; - }), - ); - - return { - results: Object.fromEntries(entries), - }; -} diff --git a/src/core/batch.ts b/src/core/batch.ts deleted file mode 100644 index fdb87054..00000000 --- a/src/core/batch.ts +++ /dev/null @@ -1,147 +0,0 @@ -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import { indexFile } from "./indexing.js"; -import { getParserForFile } from "./parsers/index.js"; -import { basename, extname } from "node:path"; -import { getLogger } from "../logger.js"; - -export interface BatchImportOptions { - sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; - library?: string | undefined; - version?: string | undefined; - topicId?: string | undefined; - concurrency?: number | undefined; - onProgress?: ((progress: BatchProgress) => void) | undefined; -} - -export interface BatchProgress { - total: number; - completed: number; - failed: number; - /** Files skipped because no parser handles their extension. */ - skipped: number; - currentFile?: string; -} - -export interface BatchFileResult { - file: string; - success: boolean; - /** True when the file was not imported because its extension is unsupported. */ - skipped?: boolean; - chunkCount?: number; - documentId?: string; - error?: string; -} - -export interface BatchImportResult { - total: number; - completed: number; - failed: number; - skipped: number; - results: BatchFileResult[]; -} - -/** - * Import multiple files in parallel with configurable concurrency. - * Uses a semaphore pattern to limit concurrent operations. - */ -export async function batchImport( - db: Database.Database, - provider: EmbeddingProvider, - files: string[], - options: BatchImportOptions = {}, -): Promise<BatchImportResult> { - const logger = getLogger(); - const concurrency = options.concurrency ?? 5; - const progress: BatchProgress = { total: files.length, completed: 0, failed: 0, skipped: 0 }; - const results: BatchFileResult[] = []; - - logger.debug(`Starting batch import: ${files.length} files, concurrency ${concurrency}`); - - // Semaphore-based parallel processing - let activeCount = 0; - let fileIndex = 0; - - await new Promise<void>((resolve) => { - if (files.length === 0) { - resolve(); - return; - } - - function runNext(): void { - while (activeCount < concurrency && fileIndex < files.length) { - const currentIndex = fileIndex++; - const file = files[currentIndex]!; - activeCount++; - progress.currentFile = file; - - processFile(db, provider, file, options) - .then((result) => { - results[currentIndex] = result; - if (result.success) { - progress.completed++; - } else if (result.skipped) { - progress.skipped++; - } else { - progress.failed++; - } - options.onProgress?.({ ...progress }); - }) - .catch((err) => { - results[currentIndex] = { - file, - success: false, - error: err instanceof Error ? err.message : String(err), - }; - progress.failed++; - options.onProgress?.({ ...progress }); - }) - .finally(() => { - activeCount--; - if (progress.completed + progress.failed + progress.skipped === files.length) { - resolve(); - } else { - runNext(); - } - }); - } - } - - runNext(); - }); - - return { - total: files.length, - completed: progress.completed, - failed: progress.failed, - skipped: progress.skipped, - results, - }; -} - -async function processFile( - db: Database.Database, - provider: EmbeddingProvider, - file: string, - options: BatchImportOptions, -): Promise<BatchFileResult> { - if (!getParserForFile(file)) { - const error = `Unsupported file type "${extname(file) || basename(file)}"`; - getLogger().warn({ file }, `Skipping ${file}: ${error}`); - return { file, success: false, skipped: true, error }; - } - - const result = await indexFile(db, provider, file, { - sourceType: options.sourceType, - library: options.library, - version: options.version, - topic: options.topicId, - }); - - return { - file, - success: true, - chunkCount: result.chunkCount, - documentId: result.id, - }; -} diff --git a/tests/integration/parsed-file-import.test.ts b/tests/integration/parsed-file-import.test.ts index dee798a6..cbc20caf 100644 --- a/tests/integration/parsed-file-import.test.ts +++ b/tests/integration/parsed-file-import.test.ts @@ -6,7 +6,7 @@ import PizZip from "pizzip"; import type Database from "better-sqlite3"; import { createTestDbWithVec } from "../fixtures/test-db.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; -import { batchImport } from "../../src/core/batch.js"; +import { ingest } from "../../src/core/ingest.js"; import { FileWatcher, readWatchedFile } from "../../src/core/watcher.js"; import { initLogger } from "../../src/logger.js"; @@ -47,7 +47,7 @@ function documentContent(db: Database.Database, id: string): { content: string; .get(id) as { content: string; source: string }; } -describe("integration: import-batch and watch use the parser registry", () => { +describe("integration: directory ingest and watch use the parser registry", () => { let db: Database.Database; let provider: MockEmbeddingProvider; let dir: string; @@ -67,19 +67,20 @@ describe("integration: import-batch and watch use the parser registry", () => { rmSync(dir, { recursive: true, force: true }); }); - it("batchImport indexes parsed DOCX text and skips unsupported files", async () => { + it("directory ingest indexes parsed DOCX text and skips unsupported files", async () => { const pngPath = join(dir, "logo.png"); writeFileSync(pngPath, Buffer.from([0x89, 0x50, 0x4e, 0x47])); - const result = await batchImport(db, provider, [docxPath, pngPath], { - sourceType: "model-generated", - }); + const result = await ingest( + { db, provider }, + { source: dir, kind: "directory", sourceType: "model-generated" }, + ); - expect(result.completed).toBe(1); - expect(result.failed).toBe(0); - expect(result.skipped).toBe(1); + expect(result.documents).toHaveLength(1); + expect(result.errors).toEqual([]); + expect(result.skipped).toEqual([{ source: pngPath, reason: "unsupported file format" }]); - const doc = documentContent(db, result.results[0]!.documentId!); + const doc = documentContent(db, result.documents[0]!.documentId); expect(doc.content).toContain(DOCX_TEXT); expect(doc.content).not.toContain("word/document.xml"); expect(doc.source).toBe("model-generated"); diff --git a/tests/unit/batch-search.test.ts b/tests/unit/batch-search.test.ts deleted file mode 100644 index b887d809..00000000 --- a/tests/unit/batch-search.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import type Database from "better-sqlite3"; -import { createTestDbWithVec } from "../fixtures/test-db.js"; -import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; -import { seedTestDocument, insertChunk } from "../fixtures/helpers.js"; -import { - searchBatch, - BATCH_SEARCH_MAX_REQUESTS, - type BatchSearchRequest, -} from "../../src/core/batch-search.js"; - -describe("searchBatch", () => { - let db: Database.Database; - let provider: MockEmbeddingProvider; - - beforeEach(() => { - db = createTestDbWithVec(); - provider = new MockEmbeddingProvider(); - - // Seed a document with a chunk so searches have something to find - seedTestDocument(db, "doc-1", { title: "TypeScript Guide", content: "TypeScript basics" }); - insertChunk(db, "chunk-1", "doc-1", "TypeScript is a typed superset of JavaScript", 0); - }); - - afterEach(() => { - db.close(); - }); - - it("should throw ValidationError for empty request array", async () => { - await expect(searchBatch(db, provider, [])).rejects.toThrow( - "At least one search request is required", - ); - }); - - it("should throw ValidationError for non-array input", async () => { - await expect( - searchBatch(db, provider, null as unknown as BatchSearchRequest[]), - ).rejects.toThrow("At least one search request is required"); - }); - - it("should throw ValidationError when exceeding max requests", async () => { - const requests: BatchSearchRequest[] = Array.from( - { length: BATCH_SEARCH_MAX_REQUESTS + 1 }, - (_, i) => ({ query: `query-${i}` }), - ); - await expect(searchBatch(db, provider, requests)).rejects.toThrow( - `Batch size ${BATCH_SEARCH_MAX_REQUESTS + 1} exceeds maximum of ${BATCH_SEARCH_MAX_REQUESTS}`, - ); - }); - - it("should accept exactly the maximum number of requests", async () => { - const requests: BatchSearchRequest[] = Array.from( - { length: BATCH_SEARCH_MAX_REQUESTS }, - (_, i) => ({ query: `query-${i}` }), - ); - const result = await searchBatch(db, provider, requests); - expect(Object.keys(result.results)).toHaveLength(BATCH_SEARCH_MAX_REQUESTS); - }); - - it("should return results keyed by query string", async () => { - const requests: BatchSearchRequest[] = [{ query: "TypeScript" }, { query: "JavaScript" }]; - const result = await searchBatch(db, provider, requests); - - expect(result.results).toHaveProperty("TypeScript"); - expect(result.results).toHaveProperty("JavaScript"); - }); - - it("should execute a single request and return results", async () => { - const result = await searchBatch(db, provider, [{ query: "TypeScript" }]); - expect(Object.keys(result.results)).toHaveLength(1); - expect(result.results["TypeScript"]).toBeDefined(); - expect(result.results["TypeScript"]!.results).toBeDefined(); - }); - - it("should pass per-request options through to search", async () => { - const result = await searchBatch(db, provider, [ - { query: "TypeScript", options: { limit: 1 } }, - ]); - expect(result.results["TypeScript"]).toBeDefined(); - }); -}); diff --git a/tests/unit/batch.test.ts b/tests/unit/batch.test.ts deleted file mode 100644 index b45fd997..00000000 --- a/tests/unit/batch.test.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; -import { batchImport } from "../../src/core/batch.js"; -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../../src/providers/embedding.js"; - -// Mock indexing module -vi.mock("../../src/core/indexing.js", () => ({ - indexFile: vi.fn().mockResolvedValue({ id: "doc-1", chunkCount: 3 }), -})); - -// Mock logger -const mockLogger = vi.hoisted(() => ({ - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), -})); -vi.mock("../../src/logger.js", () => ({ - getLogger: (): typeof mockLogger => mockLogger, -})); - -function createMockDb(): Database.Database { - return {} as Database.Database; -} - -function createMockProvider(): EmbeddingProvider { - return { - name: "mock", - dimensions: 384, - embed: vi.fn().mockResolvedValue(new Array(384).fill(0)), - embedBatch: vi.fn().mockResolvedValue([new Array(384).fill(0)]), - }; -} - -describe("batchImport", () => { - beforeEach(() => { - mockLogger.warn.mockClear(); - }); - - it("should return empty results for no files", async () => { - const result = await batchImport(createMockDb(), createMockProvider(), []); - - expect(result.total).toBe(0); - expect(result.completed).toBe(0); - expect(result.failed).toBe(0); - expect(result.skipped).toBe(0); - expect(result.results).toEqual([]); - }); - - it("should process files and return results", async () => { - const files = ["file1.md", "file2.md", "file3.md"]; - const result = await batchImport(createMockDb(), createMockProvider(), files); - - expect(result.total).toBe(3); - expect(result.completed).toBe(3); - expect(result.failed).toBe(0); - expect(result.results).toHaveLength(3); - for (const r of result.results) { - expect(r.success).toBe(true); - expect(r.chunkCount).toBe(3); - expect(r.documentId).toBe("doc-1"); - } - }); - - it("should respect concurrency option", async () => { - const files = Array.from({ length: 10 }, (_, i) => `file${i}.md`); - const result = await batchImport(createMockDb(), createMockProvider(), files, { - concurrency: 2, - }); - - expect(result.total).toBe(10); - expect(result.completed).toBe(10); - }); - - it("should call onProgress callback", async () => { - const files = ["a.md", "b.md"]; - const progressCalls: Array<{ completed: number; failed: number }> = []; - - await batchImport(createMockDb(), createMockProvider(), files, { - onProgress: (progress) => { - progressCalls.push({ completed: progress.completed, failed: progress.failed }); - }, - }); - - expect(progressCalls.length).toBe(2); - const lastCall = progressCalls[progressCalls.length - 1]; - expect(lastCall.completed + lastCall.failed).toBe(2); - }); - - it("should handle failed files gracefully", async () => { - const { indexFile } = await import("../../src/core/indexing.js"); - const mockedIndex = vi.mocked(indexFile); - - mockedIndex.mockRejectedValueOnce(new Error("bad file")); - mockedIndex.mockResolvedValueOnce({ id: "doc-2", chunkCount: 1 }); - - const files = ["bad.md", "good.md"]; - const result = await batchImport(createMockDb(), createMockProvider(), files, { - concurrency: 1, - }); - - expect(result.total).toBe(2); - expect(result.completed).toBe(1); - expect(result.failed).toBe(1); - expect(result.results[0].success).toBe(false); - expect(result.results[0].error).toBe("bad file"); - expect(result.results[1].success).toBe(true); - }); - - it("should pass library and topic options through", async () => { - const { indexFile } = await import("../../src/core/indexing.js"); - const mockedIndex = vi.mocked(indexFile); - mockedIndex.mockResolvedValue({ id: "doc-3", chunkCount: 2 }); - - await batchImport(createMockDb(), createMockProvider(), ["test.md"], { - library: "react", - version: "18.0", - topicId: "frontend", - }); - - expect(mockedIndex).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - "test.md", - expect.objectContaining({ - library: "react", - version: "18.0", - topic: "frontend", - }), - ); - }); - - it("should pass the sourceType option through", async () => { - const { indexFile } = await import("../../src/core/indexing.js"); - const mockedIndex = vi.mocked(indexFile); - mockedIndex.mockClear(); - - await batchImport(createMockDb(), createMockProvider(), ["notes.md"], { - sourceType: "model-generated", - }); - - expect(mockedIndex).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - "notes.md", - expect.objectContaining({ sourceType: "model-generated" }), - ); - }); - - it("should skip unsupported file types with a warning", async () => { - const { indexFile } = await import("../../src/core/indexing.js"); - const mockedIndex = vi.mocked(indexFile); - mockedIndex.mockClear(); - - const progressSkipped: number[] = []; - const result = await batchImport(createMockDb(), createMockProvider(), ["logo.png", "a.md"], { - concurrency: 1, - onProgress: (progress) => progressSkipped.push(progress.skipped), - }); - - expect(result.total).toBe(2); - expect(result.completed).toBe(1); - expect(result.failed).toBe(0); - expect(result.skipped).toBe(1); - expect(result.results[0]).toMatchObject({ file: "logo.png", success: false, skipped: true }); - expect(progressSkipped.at(-1)).toBe(1); - expect(mockedIndex).toHaveBeenCalledTimes(1); - expect(mockLogger.warn).toHaveBeenCalledWith( - { file: "logo.png" }, - expect.stringContaining("Unsupported file type"), - ); - }); - - it("should default concurrency to 5", async () => { - const files = Array.from({ length: 8 }, (_, i) => `file${i}.md`); - const result = await batchImport(createMockDb(), createMockProvider(), files); - - expect(result.total).toBe(8); - expect(result.completed).toBe(8); - }); -}); From 3f0ec43b7f531a2e063e5abcc07855786315fa21 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Wed, 7 Oct 2026 19:13:44 +0000 Subject: [PATCH 67/83] fix: correct stale command names in messages; remove the unreachable provider plugin registry registerProvider was not exported in 2.0 (custom providers are passed as an instance), so the open-enum config path for plugin provider names is removed. --- docs/reference/configuration.md | 2 +- docs/reference/mcp-tools.md | 2 +- docs/reference/rest-api.md | 2 +- src/api/openapi.ts | 2 +- src/config-schema.ts | 15 +----- src/config.ts | 1 - src/connectors/confluence.ts | 2 +- src/core/operations/packs.ts | 4 +- src/providers/index.ts | 15 ------ tests/unit/config-layers.test.ts | 4 +- tests/unit/plugin-provider.test.ts | 77 ------------------------------ 11 files changed, 10 insertions(+), 116 deletions(-) delete mode 100644 tests/unit/plugin-provider.test.ts diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 426c0ff5..70b22a6b 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -24,7 +24,7 @@ Every key `section.field` has the environment variable `LIBSCOPE_<SECTION>_<FIEL | Key | Type | Default | Environment variables | Secret | Description | | ------------------------------- | -------------------------------------------------------------- | --------------------------------------------------------------------------- | ------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `embedding.provider` | `local` \| `ollama` \| `openai` | `local` | `LIBSCOPE_EMBEDDING_PROVIDER` | | Embedding provider. A config file may also name a provider registered in code. | +| `embedding.provider` | `local` \| `ollama` \| `openai` | `local` | `LIBSCOPE_EMBEDDING_PROVIDER` | | Embedding provider. | | `embedding.model` | string | ollama: nomic-embed-text; openai: text-embedding-3-small | `LIBSCOPE_EMBEDDING_MODEL` | | Embedding model. Ignored by the local provider (always Xenova/all-MiniLM-L6-v2). | | `embedding.url` | string | `http://localhost:11434` | `LIBSCOPE_EMBEDDING_URL` | | Server URL of the embedding provider. Used by the ollama provider. | | `embedding.dimensions` | integer | size of the model | `LIBSCOPE_EMBEDDING_DIMENSIONS` | | Vector size, for models whose size is not built in. | diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 8cd4eb10..ba823d7d 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -255,7 +255,7 @@ Annotations: not destructive. ### install-pack -Install a knowledge pack from a registry (name or name@version) or a local file +Install a knowledge pack from a registry (name or name@version) Annotations: not destructive, idempotent. diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index 6edc7e85..c34c2353 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -108,7 +108,7 @@ This list is generated from the routes (`npm run docs:gen`). The parameters of e | Method | Path | Operation | Status | Description | | -------- | --------------------------- | ------------------- | ------------ | ----------------------------------------------------------------------------------- | -| `POST` | `/api/v1/packs` | `install-pack` | `202` (task) | Install a knowledge pack from a registry (name or name@version) or a local file | +| `POST` | `/api/v1/packs` | `install-pack` | `202` (task) | Install a knowledge pack from a registry (name or name@version) | | `DELETE` | `/api/v1/packs/:pack` | `remove-pack` | `200` | Remove an installed pack and its documents | | `GET` | `/api/v1/packs` | `list-packs` | `200` | List installed packs, or the packs available in the configured registries | | `GET` | `/api/v1/registries` | `list-registries` | `200` | List the configured pack registries | diff --git a/src/api/openapi.ts b/src/api/openapi.ts index 2aba9dac..19d90409 100644 --- a/src/api/openapi.ts +++ b/src/api/openapi.ts @@ -203,7 +203,7 @@ export function buildOpenApiSpec(routes: readonly ApiRoute[]): Record<string, un description: "Every /api/v1 route runs one LibScope operation (the same operations as the CLI, MCP server and SDK). Responses are `{ data, meta }`; errors are `{ error: { code, message } }`. Long-running operations answer 202 with a task ID.", }, - servers: [{ url: "http://localhost:3378", description: "Default `libscope serve --api`" }], + servers: [{ url: "http://localhost:3378", description: "Default `libscope serve api`" }], security: [{}, { bearerAuth: [] }], paths, components: COMPONENTS, diff --git a/src/config-schema.ts b/src/config-schema.ts index 0e7b4d30..e5a5badb 100644 --- a/src/config-schema.ts +++ b/src/config-schema.ts @@ -14,11 +14,6 @@ export interface ConfigKeyMeta { env?: readonly string[]; /** Secret values are masked on display and stored in secrets.json, never in config.json. */ secret?: boolean; - /** - * Config files may hold values outside the enum (names of providers registered at runtime - * with registerProvider). `config set` and env vars accept only the listed values. - */ - openEnum?: boolean; /** Default text for the docs table when the default depends on other keys. */ defaultText?: string; } @@ -47,13 +42,7 @@ const nonEmpty = (): z.ZodString => z.string().trim().min(1); export const ConfigSchema = z.object({ embedding: z.object({ - provider: leaf( - z - .enum(EMBEDDING_PROVIDERS) - .default("local") - .describe("Embedding provider. A config file may also name a provider registered in code."), - { openEnum: true }, - ), + provider: leaf(z.enum(EMBEDDING_PROVIDERS).default("local").describe("Embedding provider.")), model: leaf( nonEmpty() .optional() @@ -167,7 +156,6 @@ export interface ConfigKeySpec { /** Env var names in priority order (canonical name first). */ env: readonly string[]; secret: boolean; - openEnum: boolean; description: string; /** Default value, or undefined when the key has no fixed default. */ defaultValue: unknown; @@ -226,7 +214,6 @@ function buildSpecs(): ConfigKeySpec[] { schema, env: [canonicalEnvName(section, field), ...(meta.env ?? [])], secret: meta.secret ?? false, - openEnum: meta.openEnum ?? false, description: schema.description ?? "", defaultValue: parsedDefault.success ? parsedDefault.data : undefined, defaultText: meta.defaultText, diff --git a/src/config.ts b/src/config.ts index 84e8b096..8455766c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -246,7 +246,6 @@ function warnIgnored(what: string, err: unknown): void { * lists (comma-separated). */ function readFileValue(spec: ConfigKeySpec, value: unknown, source: string): unknown { - if (spec.openEnum && typeof value === "string" && value.trim() !== "") return value; try { if ( typeof value === "string" && diff --git a/src/connectors/confluence.ts b/src/connectors/confluence.ts index 1f3a41c9..8ecca84b 100644 --- a/src/connectors/confluence.ts +++ b/src/connectors/confluence.ts @@ -513,7 +513,7 @@ function validateConfluenceConfig(config: ConfluenceConfig): void { const confluenceType = config.type ?? "cloud"; if (confluenceType === "cloud" && !config.email?.trim()) { throw new ValidationError( - "Confluence email is required for Cloud. For Server/Data Center, use --type server", + "Confluence email is required for Cloud. For Server/Data Center, use --server", ); } } diff --git a/src/core/operations/packs.ts b/src/core/operations/packs.ts index 2c377a09..5813ead4 100644 --- a/src/core/operations/packs.ts +++ b/src/core/operations/packs.ts @@ -32,9 +32,9 @@ export function assertLocalFilesAllowed(ctx: OperationContext, what: string): vo export const installPackOperation = defineOperation({ name: "install-pack", group: "packs", - summary: "Install a knowledge pack from a registry (name or name@version) or a local file", + summary: "Install a knowledge pack from a registry (name or name@version)", description: - "Packs are looked up in the local copies of the configured registries (run sync-registries to update them). Local .json/.json.gz files are accepted from the CLI and the Node.js API only.", + "Packs are looked up in the local copies of the configured registries (sync the registries to update them). The CLI and the Node.js API also accept a local .json/.json.gz file.", input: z.object({ pack: z .string() diff --git a/src/providers/index.ts b/src/providers/index.ts index 4d6a05b8..b041fc74 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -11,23 +11,8 @@ import { LocalEmbeddingProvider } from "./local.js"; import { OllamaEmbeddingProvider } from "./ollama.js"; import { OpenAIEmbeddingProvider } from "./openai.js"; -/** Factory function that creates an EmbeddingProvider from config. */ -export type ProviderFactory = (config: LibScopeConfig) => EmbeddingProvider; - -const providerRegistry = new Map<string, ProviderFactory>(); - -/** Register a custom embedding provider factory. */ -export function registerProvider(name: string, factory: ProviderFactory): void { - providerRegistry.set(name, factory); -} - /** Create an embedding provider based on config. */ export function createEmbeddingProvider(config: LibScopeConfig): EmbeddingProvider { - const registered = providerRegistry.get(config.embedding.provider); - if (registered) { - return registered(config); - } - const { embedding } = config; switch (embedding.provider) { case "local": diff --git a/tests/unit/config-layers.test.ts b/tests/unit/config-layers.test.ts index a2891bae..fdac032f 100644 --- a/tests/unit/config-layers.test.ts +++ b/tests/unit/config-layers.test.ts @@ -304,9 +304,9 @@ describe("config layering (temp HOME and cwd)", () => { expect(messages(warn).some((m) => m.includes("LIBSCOPE_MCP_TOOLSETS"))).toBe(true); }); - it("accepts a custom embedding provider name in a file", () => { + it("ignores an unknown embedding provider name in a file", () => { writeUserConfig({ embedding: { provider: "my-plugin" } }); - expect(load().embedding.provider).toBe("my-plugin"); + expect(load().embedding.provider).toBe("local"); }); it("drops unknown and old keys from the loaded config", () => { diff --git a/tests/unit/plugin-provider.test.ts b/tests/unit/plugin-provider.test.ts deleted file mode 100644 index 580a286f..00000000 --- a/tests/unit/plugin-provider.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { describe, it, expect, beforeEach } from "vitest"; -import type { EmbeddingProvider } from "../../src/providers/embedding.js"; -import type { ProviderFactory } from "../../src/providers/index.js"; -import { registerProvider, createEmbeddingProvider } from "../../src/providers/index.js"; -import type { LibScopeConfig } from "../../src/config.js"; - -function makeConfig(provider: string): LibScopeConfig { - return { - embedding: { - provider, - }, - database: { path: ":memory:" }, - indexing: { maxDocumentSize: 1024 }, - logging: { level: "silent" }, - }; -} - -class StubProvider implements EmbeddingProvider { - readonly name = "stub"; - readonly dimensions = 3; - - embed(_text: string): Promise<number[]> { - return Promise.resolve([0.1, 0.2, 0.3]); - } - - embedBatch(texts: string[]): Promise<number[][]> { - return Promise.resolve(texts.map(() => [0.1, 0.2, 0.3])); - } -} - -describe("plugin provider registration", () => { - beforeEach(() => { - // Register a fresh custom provider before each test - registerProvider("stub", () => new StubProvider()); - }); - - it("should use a registered custom provider", () => { - const provider = createEmbeddingProvider(makeConfig("stub")); - expect(provider.name).toBe("stub"); - expect(provider.dimensions).toBe(3); - }); - - it("should return embeddings from a custom provider", async () => { - const provider = createEmbeddingProvider(makeConfig("stub")); - const result = await provider.embed("hello"); - expect(result).toEqual([0.1, 0.2, 0.3]); - }); - - it("should return batch embeddings from a custom provider", async () => { - const provider = createEmbeddingProvider(makeConfig("stub")); - const results = await provider.embedBatch(["a", "b"]); - expect(results).toHaveLength(2); - }); - - it("should pass config to the factory function", () => { - let receivedConfig: LibScopeConfig | undefined; - const factory: ProviderFactory = (config) => { - receivedConfig = config; - return new StubProvider(); - }; - registerProvider("config-test", factory); - const config = makeConfig("config-test"); - createEmbeddingProvider(config); - expect(receivedConfig).toBe(config); - }); - - it("should still support built-in providers", () => { - const provider = createEmbeddingProvider(makeConfig("local")); - expect(provider.name).toBe("local"); - }); - - it("should throw for unknown unregistered providers", () => { - expect(() => createEmbeddingProvider(makeConfig("nonexistent"))).toThrow( - "Unknown embedding provider", - ); - }); -}); From 5293a2cc786dbbe82f8de2cf5bc000ba32140f66 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo <rderienzo@voloridge.com> Date: Thu, 8 Oct 2026 16:48:20 +0000 Subject: [PATCH 68/83] fix(core): decode HTML character references in page titles Titles taken from <title> or <h1> kept references such as —, so a URL add, a spider crawl and the docs connector stored titles like "textwrap — Python documentation". --- src/connectors/docs.ts | 5 ++-- src/core/html-entities.ts | 42 +++++++++++++++++++++++++++++++ src/core/spider.ts | 5 ++-- src/core/url-fetcher.ts | 4 ++- tests/unit/docs-connector.test.ts | 12 +++++++++ tests/unit/html-entities.test.ts | 21 ++++++++++++++++ tests/unit/spider.test.ts | 12 +++++++++ tests/unit/url-fetcher.test.ts | 14 +++++++++++ 8 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 src/core/html-entities.ts create mode 100644 tests/unit/html-entities.test.ts diff --git a/src/connectors/docs.ts b/src/connectors/docs.ts index 76a7f383..ee60c467 100644 --- a/src/connectors/docs.ts +++ b/src/connectors/docs.ts @@ -10,6 +10,7 @@ import { NodeHtmlMarkdown } from "node-html-markdown"; import { ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; import { fetchRaw } from "../core/url-fetcher.js"; +import { decodeHtmlEntities } from "../core/html-entities.js"; import type { FetchOptions } from "../core/url-fetcher.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; @@ -354,7 +355,7 @@ export function extractDocTitle(html: string, url: string): string { .slice(innerStart, h1CloseIdx) .replaceAll(/<[^>]{1,2000}>/g, "") .trim(); - if (title) return title; + if (title) return decodeHtmlEntities(title); } } @@ -362,7 +363,7 @@ export function extractDocTitle(html: string, url: string): string { const titleTagMatch = /<title[^>]{0,2000}>([^<]+)<\/title>/i.exec(html); if (titleTagMatch?.[1]) { const title = titleTagMatch[1].trim(); - if (title) return title; + if (title) return decodeHtmlEntities(title); } // Last resort: derive from URL path diff --git a/src/core/html-entities.ts b/src/core/html-entities.ts new file mode 100644 index 00000000..88ed5f1d --- /dev/null +++ b/src/core/html-entities.ts @@ -0,0 +1,42 @@ +/** Decoding of HTML character references in text taken from raw HTML. */ + +const NAMED_ENTITIES: Readonly<Record<string, string>> = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: " ", + ndash: "–", + mdash: "—", + hellip: "…", + lsquo: "‘", + rsquo: "’", + ldquo: "“", + rdquo: "”", + laquo: "«", + raquo: "»", + middot: "·", + bull: "•", + copy: "©", + reg: "®", + trade: "™", +}; + +/** + * Decode HTML character references in text taken from raw HTML (for example a page title): + * numeric references (`—`, `—`) and common named ones (`&`, `—`). + * Unknown or invalid references are kept as they are. + */ +export function decodeHtmlEntities(text: string): string { + return text.replaceAll( + /&(#\d{1,7}|#x[\da-f]{1,6}|[a-z]{2,8});/gi, + (ref: string, body: string) => { + if (!body.startsWith("#")) return NAMED_ENTITIES[body.toLowerCase()] ?? ref; + const hex = body[1] === "x" || body[1] === "X"; + const codePoint = Number.parseInt(body.slice(hex ? 2 : 1), hex ? 16 : 10); + if (codePoint === 0 || codePoint > 0x10ffff) return ref; + return String.fromCodePoint(codePoint); + }, + ); +} diff --git a/src/core/spider.ts b/src/core/spider.ts index a7081773..1987a903 100644 --- a/src/core/spider.ts +++ b/src/core/spider.ts @@ -13,6 +13,7 @@ import { setTimeout as sleep } from "node:timers/promises"; import { getLogger } from "../logger.js"; import { FetchError } from "../errors.js"; import { fetchRaw, type FetchOptions } from "./url-fetcher.js"; +import { decodeHtmlEntities } from "./html-entities.js"; import { extractLinks } from "./link-extractor.js"; import { NodeHtmlMarkdown } from "node-html-markdown"; @@ -269,11 +270,11 @@ function stripTags(input: string): string { function extractTitle(html: string, url: string): string { // Try <title> tag const match = /<title[^>]*>([^<]+)<\/title>/i.exec(html); - if (match?.[1]) return match[1].trim(); + if (match?.[1]) return decodeHtmlEntities(match[1].trim()); // Try first <h1> const h1 = /<h1[^>]*>([\s\S]*?)<\/h1>/i.exec(html); if (h1?.[1]) { - return stripTags(h1[1]).trim(); + return decodeHtmlEntities(stripTags(h1[1]).trim()); } // Fall back to URL path try { diff --git a/src/core/url-fetcher.ts b/src/core/url-fetcher.ts index 050c6e8c..d3765e3e 100644 --- a/src/core/url-fetcher.ts +++ b/src/core/url-fetcher.ts @@ -4,6 +4,7 @@ import { Agent } from "undici"; import { NodeHtmlMarkdown } from "node-html-markdown"; import { FetchError } from "../errors.js"; import { getLogger } from "../logger.js"; +import { decodeHtmlEntities } from "./html-entities.js"; /** Lazy singleton undici Agent that skips TLS certificate verification. */ let _insecureAgent: Agent | undefined; @@ -325,7 +326,8 @@ export async function fetchAndConvert( /** Extract title from HTML <title> tag. */ function extractTitleFromHtml(html: string): string | null { const match = /<title[^>]*>([^<]+)<\/title>/i.exec(html); - return match?.[1]?.trim() ?? null; + const title = match?.[1]?.trim(); + return title ? decodeHtmlEntities(title) : null; } /** Extract title from first markdown heading. */ diff --git a/tests/unit/docs-connector.test.ts b/tests/unit/docs-connector.test.ts index 4f164781..bf8e626c 100644 --- a/tests/unit/docs-connector.test.ts +++ b/tests/unit/docs-connector.test.ts @@ -276,6 +276,18 @@ describe("extractDocTitle", () => { "https://example.com/", "example.com", ], + [ + "H1 with character references", + "<h1>Tom & Jerry — API</h1>", + "https://example.com/docs/api", + "Tom & Jerry — API", + ], + [ + "<title> with character references", + "<html><head><title>textwrap — Python", + "https://example.com/docs", + "textwrap — Python", + ], [ "H1 precedence over title", "Page Title

Real Title

", diff --git a/tests/unit/html-entities.test.ts b/tests/unit/html-entities.test.ts new file mode 100644 index 00000000..106065f5 --- /dev/null +++ b/tests/unit/html-entities.test.ts @@ -0,0 +1,21 @@ +import { describe, it, expect } from "vitest"; +import { decodeHtmlEntities } from "../../src/core/html-entities.js"; + +describe("decodeHtmlEntities", () => { + it.each([ + ["decimal reference", "textwrap — Python docs", "textwrap — Python docs"], + ["hex reference", "A — B A", "A — B A"], + [ + "named references", + "Tom & Jerry <3 "hi" — ok", + 'Tom & Jerry <3 "hi" — ok', + ], + ["named reference in upper case", "A & B", "A & B"], + ["decodes only once", "&lt;", "<"], + ["unknown named reference kept", "&unknownent; x", "&unknownent; x"], + ["out-of-range code point kept", "� �", "� �"], + ["text without references", "Plain title", "Plain title"], + ])("%s", (_label, input, expected) => { + expect(decodeHtmlEntities(input)).toBe(expected); + }); +}); diff --git a/tests/unit/spider.test.ts b/tests/unit/spider.test.ts index 8cf59070..2d93fb69 100644 --- a/tests/unit/spider.test.ts +++ b/tests/unit/spider.test.ts @@ -81,6 +81,18 @@ describe("spiderUrl", () => { expect(page.depth).toBe(0); }); + it("decodes HTML character references in the page title", async () => { + mockFetchRaw.mockImplementation((url: string) => { + if (url.endsWith("/robots.txt")) return Promise.reject(new Error("404")); + return Promise.resolve(pageResponse(htmlPage("Docs — A & B"), url)); + }); + + const gen = spiderUrl("https://example.com/", { maxPages: 1, requestDelay: 0 }); + const result = await gen.next(); + const page = result.value as { title: string }; + expect(page.title).toBe("Docs — A & B"); + }); + it("follows links up to maxDepth", async () => { mockFetchRaw.mockImplementation((url: string) => { if (url.endsWith("/robots.txt")) return Promise.reject(new Error("404")); diff --git a/tests/unit/url-fetcher.test.ts b/tests/unit/url-fetcher.test.ts index 59f340cb..5c956e64 100644 --- a/tests/unit/url-fetcher.test.ts +++ b/tests/unit/url-fetcher.test.ts @@ -94,6 +94,20 @@ describe("fetchAndConvert", () => { expect(result.content).toContain("World"); }); + it("decodes HTML character references in the ", async () => { + const html = + "<html><head><title>textwrap — Text & wrapping

x

"; + mockFetch.mockResolvedValue({ + ok: true, + headers: new Headers({ "content-type": "text/html" }), + body: bodyStream(html), + text: () => Promise.resolve(html), + }); + + const result = await fetchAndConvert("https://example.com/page"); + expect(result.title).toBe("textwrap — Text & wrapping"); + }); + it("should return markdown/plain text as-is", async () => { const md = "# Markdown Title\n\nSome content here."; mockFetch.mockResolvedValue({ From cdd574bc0c545a0d93d1ef22f9617de4e5b3cb5a Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 16:49:43 +0000 Subject: [PATCH 69/83] fix(connectors): docs disconnect deletes only the pages the connection indexed disconnect of a docs connection deleted every document whose URL started with the site URL, including pages added with `libscope add `. The URL was also matched with LIKE, so "_" and "%" in it acted as wildcards. Match the prefix exactly and only for pages the docs connector stored (source type library, added by the crawler). --- src/connectors/docs.ts | 14 ++++++---- tests/unit/docs-connector.test.ts | 43 +++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/connectors/docs.ts b/src/connectors/docs.ts index ee60c467..4483d841 100644 --- a/src/connectors/docs.ts +++ b/src/connectors/docs.ts @@ -781,8 +781,9 @@ async function runDocSiteSync( /** * Remove all documents that were indexed from a given documentation site. * - * Identifies documents by URL prefix (`siteUrl + "%"`) so only pages that - * originated from the specified site are removed. + * Removes only the pages this connector indexed (source type "library", added by the crawler) + * whose URL starts with the site URL, so documents added another way under the same site + * (for example with `libscope add `) are kept. * * @param db The database connection. * @param siteUrl Root URL of the documentation site (used as URL prefix filter). @@ -800,10 +801,13 @@ export function disconnectDocSite(db: Database.Database, siteUrl: string): numbe throw new ValidationError(`Invalid site URL for disconnect: ${siteUrl}`); } - // Parameterised LIKE — the prefix is derived from a validated URL, not user input. + // An exact prefix comparison: LIKE would treat "_" and "%" in the URL as wildcards. const rows = db - .prepare("SELECT id FROM documents WHERE url LIKE ?") - .all(`${basePrefix}%`) as Array<{ id: string }>; + .prepare( + `SELECT id FROM documents + WHERE source_type = ? AND submitted_by = 'crawler' AND substr(url, 1, ?) = ?`, + ) + .all(SOURCE_TYPE, basePrefix.length, basePrefix) as Array<{ id: string }>; let removed = 0; for (const row of rows) { diff --git a/tests/unit/docs-connector.test.ts b/tests/unit/docs-connector.test.ts index bf8e626c..6d129c4b 100644 --- a/tests/unit/docs-connector.test.ts +++ b/tests/unit/docs-connector.test.ts @@ -918,13 +918,13 @@ describe("disconnectDocSite", () => { it("removes all documents from the given site URL prefix", () => { // Seed some docs manually db.prepare( - "INSERT INTO documents (id, source_type, title, content, url) VALUES (?, 'library', ?, ?, ?)", + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', ?, ?, ?, 'crawler')", ).run("doc-1", "Page 1", "Content 1", "https://docs.example.com/docs/page1"); db.prepare( - "INSERT INTO documents (id, source_type, title, content, url) VALUES (?, 'library', ?, ?, ?)", + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', ?, ?, ?, 'crawler')", ).run("doc-2", "Page 2", "Content 2", "https://docs.example.com/docs/page2"); db.prepare( - "INSERT INTO documents (id, source_type, title, content, url) VALUES (?, 'library', ?, ?, ?)", + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', ?, ?, ?, 'crawler')", ).run("doc-3", "Other", "Content 3", "https://other.example.com/docs/page"); const removed = disconnectDocSite(db, "https://docs.example.com/docs/"); @@ -935,6 +935,39 @@ describe("disconnectDocSite", () => { expect(remaining.n).toBe(1); // doc-3 should remain }); + it("keeps documents under the site URL that the connector did not index", () => { + const insert = db.prepare( + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, ?, ?, ?, ?, ?)", + ); + insert.run("synced", "library", "Synced", "Body", "https://docs.example.com/docs/a", "crawler"); + insert.run("added", "manual", "Added", "Body", "https://docs.example.com/docs/a/", "manual"); + insert.run("by-model", "library", "Model", "Body", "https://docs.example.com/docs/b", "model"); + + const removed = disconnectDocSite(db, "https://docs.example.com/docs/"); + + expect(removed).toBe(1); + const ids = ( + db.prepare("SELECT id FROM documents ORDER BY id").all() as Array<{ id: string }> + ).map((r) => r.id); + expect(ids).toEqual(["added", "by-model"]); + }); + + it("matches the site URL exactly, not as a LIKE pattern", () => { + const insert = db.prepare( + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', 'T', 'B', ?, 'crawler')", + ); + insert.run("inside", "https://docs.example.com/my_docs/page"); + insert.run("lookalike", "https://docs.example.com/myXdocs/page"); + + const removed = disconnectDocSite(db, "https://docs.example.com/my_docs/"); + + expect(removed).toBe(1); + const ids = (db.prepare("SELECT id FROM documents").all() as Array<{ id: string }>).map( + (r) => r.id, + ); + expect(ids).toEqual(["lookalike"]); + }); + it("returns 0 when no matching documents exist", () => { const removed = disconnectDocSite(db, "https://docs.example.com/docs/"); expect(removed).toBe(0); @@ -946,7 +979,7 @@ describe("disconnectDocSite", () => { it("does not remove documents from other sites", () => { db.prepare( - "INSERT INTO documents (id, source_type, title, content, url) VALUES (?, 'library', ?, ?, ?)", + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', ?, ?, ?, 'crawler')", ).run("doc-1", "Page 1", "Content 1", "https://other.example.com/docs/page"); const removed = disconnectDocSite(db, "https://docs.example.com/docs/"); @@ -958,7 +991,7 @@ describe("disconnectDocSite", () => { it("removes associated chunks", () => { db.prepare( - "INSERT INTO documents (id, source_type, title, content, url) VALUES (?, 'library', 'Title', 'Body', ?)", + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', 'Title', 'Body', ?, 'crawler')", ).run("doc-1", "https://docs.example.com/docs/page"); db.prepare( "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", From a39bd9370837fa4f7a489b1f9b4001d1a553e58a Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 16:52:33 +0000 Subject: [PATCH 70/83] fix(lite): load the TypeScript grammar from tree-sitter-typescript's default export Under ESM, import() of the CommonJS grammar packages returns module.exports as the default export, so TreeSitterChunker passed { typescript, tsx } to setLanguage and every TypeScript chunk failed with "Invalid language object". The docs and the install hints now pin tree-sitter@0.21 and the matching grammar versions: the latest grammars need a newer tree-sitter and npm refuses them next to libscope's peer range. The code-indexing example no longer nests a template literal inside its template-literal source string. --- docs/guide/code-indexing.md | 26 +++++++++++----------- docs/guide/lite.md | 8 +++---- src/lite/chunker-treesitter.ts | 19 +++++----------- tests/unit/code-chunker-grammar.test.ts | 29 +++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 30 deletions(-) create mode 100644 tests/unit/code-chunker-grammar.test.ts diff --git a/docs/guide/code-indexing.md b/docs/guide/code-indexing.md index 9942ff74..b92ef6c7 100644 --- a/docs/guide/code-indexing.md +++ b/docs/guide/code-indexing.md @@ -14,20 +14,20 @@ The tree-sitter chunker uses the Abstract Syntax Tree (AST) to split at **semant ## Installation -Tree-sitter is an **optional peer dependency**. Install the packages for the languages you need: +Tree-sitter is an **optional peer dependency**. Install the packages for the languages you need, with the versions that libscope supports (the latest grammar releases need a newer tree-sitter, and npm refuses to install them next to libscope): ```bash # Core tree-sitter parser -npm install tree-sitter +npm install tree-sitter@0.21 # Language grammars (install only what you need) -npm install tree-sitter-typescript # TypeScript + TSX -npm install tree-sitter-javascript # JavaScript, JSX, MJS, CJS -npm install tree-sitter-python # Python -npm install tree-sitter-c-sharp # C# -npm install tree-sitter-cpp # C++ (also used for .h/.hpp headers) -npm install tree-sitter-c # C -npm install tree-sitter-go # Go +npm install tree-sitter-typescript@0.21 # TypeScript + TSX +npm install tree-sitter-javascript@0.21 # JavaScript, JSX, MJS, CJS +npm install tree-sitter-python@0.21 # Python +npm install tree-sitter-c-sharp@0.21 # C# +npm install tree-sitter-cpp@0.22 # C++ (also used for .h/.hpp headers) +npm install tree-sitter-c@0.21 # C +npm install tree-sitter-go@0.21 # Go ``` If tree-sitter is not installed, `TreeSitterChunker.chunk()` throws a `ValidationError` with the install command, and `createCodeChunker()` falls back to the built-in chunker. Everything else works without tree-sitter. @@ -78,7 +78,7 @@ export class AuthService extends EventEmitter { private async generateToken(userId: string): Promise { // ... token generation logic - return `tok_${userId}_${Date.now()}`; + return "tok_" + userId + "_" + Date.now(); } } `; @@ -244,8 +244,8 @@ try { const chunks = await chunker.chunk(source, "rust"); } catch (err) { if (err instanceof ValidationError) { - // "Unsupported language for code chunking: 'rust'" - // "Code chunking requires the 'tree-sitter' package. Install it with: ..." + // Unsupported language for code chunking: "rust" + // Code chunking requires the "tree-sitter" package. Install it with: ... console.warn(err.message); } } @@ -253,7 +253,7 @@ try { Two error conditions: -1. **Unsupported language** — throws immediately with the list of supported aliases +1. **Unsupported language** — throws immediately (the aliases are in [Supported Languages](#supported-languages)) 2. **tree-sitter not installed** — throws with the exact `npm install` command Both are `ValidationError` from LibScope's error hierarchy. diff --git a/docs/guide/lite.md b/docs/guide/lite.md index 6fcd92a8..cfd61a8b 100644 --- a/docs/guide/lite.md +++ b/docs/guide/lite.md @@ -20,12 +20,12 @@ npm install libscope ``` -For code chunking, also install tree-sitter and the grammars that you need (optional peer dependencies): +For code chunking, also install tree-sitter and the grammars that you need (optional peer dependencies). Use these versions: npm refuses the latest grammar releases next to libscope. ```bash -npm install tree-sitter -npm install tree-sitter-typescript tree-sitter-javascript tree-sitter-python -npm install tree-sitter-c-sharp tree-sitter-cpp tree-sitter-c tree-sitter-go +npm install tree-sitter@0.21 +npm install tree-sitter-typescript@0.21 tree-sitter-javascript@0.21 tree-sitter-python@0.21 +npm install tree-sitter-c-sharp@0.21 tree-sitter-cpp@0.22 tree-sitter-c@0.21 tree-sitter-go@0.21 ``` If they are not installed, code is chunked with the built-in text chunker. diff --git a/src/lite/chunker-treesitter.ts b/src/lite/chunker-treesitter.ts index 4c5af272..eeb56bdf 100644 --- a/src/lite/chunker-treesitter.ts +++ b/src/lite/chunker-treesitter.ts @@ -322,7 +322,7 @@ export class TreeSitterChunker { } catch (err: unknown) { throw new ValidationError( 'Code chunking requires the "tree-sitter" package. ' + - "Install it with: npm install tree-sitter tree-sitter-typescript tree-sitter-javascript tree-sitter-python", + "Install it with: npm install tree-sitter@0.21 tree-sitter-typescript@0.21 tree-sitter-javascript@0.21 tree-sitter-python@0.21", err, ); } @@ -341,24 +341,17 @@ export class TreeSitterChunker { try { const mod = (await import(packageName)) as Record; - // Grammar packages typically export the language as the default export. - // tree-sitter-typescript exports { typescript, tsx } as named exports. - let grammar: unknown; - if (language === "typescript" && "typescript" in mod) { - grammar = mod["typescript"]; - } else if ("default" in mod) { - grammar = mod["default"]; - } else { - // Fallback: use the module itself (some packages export the grammar directly) - grammar = mod; - } + // The grammar packages are CommonJS: import() returns their module.exports as `default`. + const exported = ("default" in mod ? mod["default"] : mod) as Record; + // tree-sitter-typescript exports { typescript, tsx }; the others export the language. + const grammar = language === "typescript" ? exported["typescript"] : exported; this.grammarCache.set(language, grammar); return grammar; } catch (err: unknown) { throw new ValidationError( `Code chunking for ${language} requires the "${packageName}" package. ` + - `Install it with: npm install ${packageName}`, + `Install it with: npm install ${packageName}@${language === "cpp" ? "0.22" : "0.21"}`, err, ); } diff --git a/tests/unit/code-chunker-grammar.test.ts b/tests/unit/code-chunker-grammar.test.ts new file mode 100644 index 00000000..f2e84e64 --- /dev/null +++ b/tests/unit/code-chunker-grammar.test.ts @@ -0,0 +1,29 @@ +import { describe, it, expect, vi } from "vitest"; +import { TreeSitterChunker } from "../../src/lite/chunker-treesitter.js"; + +// The grammar packages are CommonJS. import() of them yields a namespace whose `default` is +// module.exports, with no named exports (tree-sitter-typescript's are not statically detectable). +const typescriptLanguage = { name: "typescript" }; +const tsxLanguage = { name: "tsx" }; +const pythonLanguage = { name: "python", language: {} }; + +vi.mock("tree-sitter-typescript", () => ({ + default: { typescript: typescriptLanguage, tsx: tsxLanguage }, +})); +vi.mock("tree-sitter-python", () => ({ default: pythonLanguage })); + +interface GrammarLoader { + loadGrammar(language: string): Promise; +} + +describe("TreeSitterChunker grammar loading", () => { + it("picks the typescript language out of tree-sitter-typescript's default export", async () => { + const loader = new TreeSitterChunker() as unknown as GrammarLoader; + await expect(loader.loadGrammar("typescript")).resolves.toBe(typescriptLanguage); + }); + + it("uses the default export of other grammar packages", async () => { + const loader = new TreeSitterChunker() as unknown as GrammarLoader; + await expect(loader.loadGrammar("python")).resolves.toBe(pythonLanguage); + }); +}); From e77ea84e17423634f161c66b5ac746be94d04c42 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 16:52:39 +0000 Subject: [PATCH 71/83] fix(search): leave function words out of the keyword OR fallback When no chunk contains every query word, the keyword search retried with any word. "how to authenticate" then matched every chunk containing "to", and the fused ranking put those chunks above the semantic match. The OR retry now skips stopwords (shared with tag suggestions in core/stopwords.ts) unless the query has nothing else. --- src/core/search.ts | 9 ++- src/core/stopwords.ts | 120 ++++++++++++++++++++++++++++++++++++++ src/core/tags.ts | 102 +------------------------------- tests/unit/search.test.ts | 11 ++++ 4 files changed, 139 insertions(+), 103 deletions(-) create mode 100644 src/core/stopwords.ts diff --git a/src/core/search.ts b/src/core/search.ts index 6ad8fbca..9ee5ac97 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -7,6 +7,7 @@ import { validateRow, validateRows } from "../db/validate.js"; import { REBUILD_VECTOR_INDEX_HINT, isVectorDimensionError } from "../db/index-meta.js"; import { logSearch, recordSearchQuery } from "./analytics.js"; import { emitEvent } from "./events.js"; +import { STOPWORDS } from "./stopwords.js"; import { performance } from "node:perf_hooks"; import { ChunkNotFoundError, DocumentNotFoundError } from "../errors.js"; @@ -1082,9 +1083,13 @@ function fts5Search( const quoted = words.map((w) => quoteFtsWord(w)); let fts = runFtsQuery(db, options, quoted.join(" AND "), limit, offset, "fts5Search.rows"); - // If AND returned nothing, retry with OR for recall (the lazy count then uses the OR query) + // If AND returned nothing, retry with OR for recall (the lazy count then uses the OR query). + // Leave out function words: "how to authenticate" must not match every chunk containing "to". if (fts.rows.length === 0 && words.length > 1) { - fts = runFtsQuery(db, options, quoted.join(" OR "), limit, offset, "fts5Search.orRows"); + const meaningful = words.filter((w) => !STOPWORDS.has(w.toLowerCase())); + const orWords = meaningful.length > 0 ? meaningful : words; + const orQuery = orWords.map((w) => quoteFtsWord(w)).join(" OR "); + fts = runFtsQuery(db, options, orQuery, limit, offset, "fts5Search.orRows"); } const totalCount = lazyCount( diff --git a/src/core/stopwords.ts b/src/core/stopwords.ts new file mode 100644 index 00000000..2ff88ea4 --- /dev/null +++ b/src/core/stopwords.ts @@ -0,0 +1,120 @@ +/** + * English function words that carry no meaning on their own: left out of tag suggestions, and + * out of the any-word keyword search that runs when no chunk contains every query word. + */ +export const STOPWORDS: ReadonlySet = new Set([ + "the", + "a", + "an", + "is", + "are", + "was", + "were", + "be", + "been", + "being", + "have", + "has", + "had", + "do", + "does", + "did", + "will", + "would", + "shall", + "should", + "may", + "might", + "must", + "can", + "could", + "that", + "this", + "with", + "from", + "for", + "not", + "but", + "and", + "or", + "nor", + "so", + "yet", + "both", + "either", + "neither", + "each", + "every", + "all", + "any", + "few", + "more", + "most", + "other", + "some", + "such", + "than", + "too", + "very", + "just", + "about", + "above", + "after", + "again", + "against", + "below", + "between", + "during", + "into", + "through", + "under", + "until", + "also", + "how", + "what", + "when", + "where", + "which", + "while", + "who", + "whom", + "why", + "its", + "our", + "their", + "your", + "his", + "her", + "out", + "then", + "there", + "these", + "those", + "them", + "they", + "you", + "only", + "own", + "same", + "here", + "over", + "once", + "use", + "used", + "to", + "of", + "in", + "on", + "at", + "by", + "as", + "it", + "if", + "we", + "i", + "me", + "my", + "am", + "no", + "up", +]); diff --git a/src/core/tags.ts b/src/core/tags.ts index 9e4548e8..007a669e 100644 --- a/src/core/tags.ts +++ b/src/core/tags.ts @@ -5,6 +5,7 @@ import { DocumentNotFoundError, ValidationError } from "../errors.js"; import { validateRow, validateRows } from "../db/validate.js"; import { createChildLogger } from "../logger.js"; import { rowToDocument, type Document } from "./documents.js"; +import { STOPWORDS } from "./stopwords.js"; const TagRowSchema = z.object({ id: z.string(), @@ -48,107 +49,6 @@ const TitleContentRowSchema = z.object({ content: z.string(), }); -const STOPWORDS = new Set([ - "the", - "a", - "an", - "is", - "are", - "was", - "were", - "be", - "been", - "being", - "have", - "has", - "had", - "do", - "does", - "did", - "will", - "would", - "shall", - "should", - "may", - "might", - "must", - "can", - "could", - "that", - "this", - "with", - "from", - "for", - "not", - "but", - "and", - "or", - "nor", - "so", - "yet", - "both", - "either", - "neither", - "each", - "every", - "all", - "any", - "few", - "more", - "most", - "other", - "some", - "such", - "than", - "too", - "very", - "just", - "about", - "above", - "after", - "again", - "against", - "below", - "between", - "during", - "into", - "through", - "under", - "until", - "also", - "how", - "what", - "when", - "where", - "which", - "while", - "who", - "whom", - "why", - "its", - "our", - "their", - "your", - "his", - "her", - "out", - "then", - "there", - "these", - "those", - "them", - "they", - "you", - "only", - "own", - "same", - "here", - "over", - "once", - "use", - "used", -]); - export interface Tag { id: string; name: string; diff --git a/tests/unit/search.test.ts b/tests/unit/search.test.ts index fa092412..f7350e7b 100644 --- a/tests/unit/search.test.ts +++ b/tests/unit/search.test.ts @@ -854,6 +854,17 @@ describe("FTS5 AND-by-default logic (issue #362)", () => { expect(results.length).toBeGreaterThan(0); expect(results[0]!.content).toContain("TypeScript"); }); + + it("should leave function words out of the OR fallback", async () => { + insertDoc(db, "doc1", "Deploy Guide"); + insertChunk(db, "c1", "doc1", "Deploy to Kubernetes using Helm charts"); + insertDoc(db, "doc2", "Auth Guide"); + insertChunk(db, "c2", "doc2", "Authenticate every request with an OAuth2 token"); + + const { results } = await searchDocuments(db, provider, { query: "how to authenticate" }); + + expect(results.map((r) => r.documentId)).toEqual(["doc2"]); + }); }); describe("title boosting (issue #362)", () => { From 62528806d33369a26f8d5a5d45eccdb28a8a3595 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 16:52:53 +0000 Subject: [PATCH 72/83] fix(registry): publish --submit pushes the feature branch and restores the clone publish --submit created feature/add- in the local registry clone and then ran a plain `git push`, which fails for a new branch without an upstream (git's default settings). The cleanup then ran `git checkout main`, which fails for a registry whose branch is master, so the clone stayed on the feature branch with a dirty tree; after a successful --submit it also stayed there, so the next normal publish went to the feature branch. Push the commit explicitly to refs/heads/feature/add-, always check out the clone's previous branch again and delete the local feature branch, and use the published version (not the pack file's) in the default commit message. --- src/registry/git.ts | 9 +- src/registry/publish.ts | 95 +++++++++------ .../registry/registry-publish.test.ts | 112 +++++++++++++++++- 3 files changed, 179 insertions(+), 37 deletions(-) diff --git a/src/registry/git.ts b/src/registry/git.ts index b6cff0dd..3fbfa62d 100644 --- a/src/registry/git.ts +++ b/src/registry/git.ts @@ -228,12 +228,17 @@ export async function createRegistryRepo(path: string): Promise { log.info({ path }, "Registry repo initialized"); } -/** Add, commit, and push changes in a registry repo. */ -export async function commitAndPush(repoPath: string, message: string): Promise { +/** Add and commit all changes in a registry repo. */ +export async function commitAll(repoPath: string, message: string): Promise { await git(["add", "."], { cwd: repoPath }); await git( ["-c", "user.name=libscope", "-c", "user.email=libscope@localhost", "commit", "-m", message], { cwd: repoPath }, ); +} + +/** Add, commit, and push changes in a registry repo. */ +export async function commitAndPush(repoPath: string, message: string): Promise { + await commitAll(repoPath, message); await git(["push"], { cwd: repoPath }); } diff --git a/src/registry/publish.ts b/src/registry/publish.ts index 8c36fbc1..80cf85dc 100644 --- a/src/registry/publish.ts +++ b/src/registry/publish.ts @@ -22,7 +22,7 @@ import { getRegistryCacheDir, } from "./types.js"; import { requireRegistry } from "./config.js"; -import { commitAndPush, fetchRegistry, git, clearIndexCache } from "./git.js"; +import { commitAll, commitAndPush, fetchRegistry, git, clearIndexCache } from "./git.js"; import { computeChecksum, writeChecksumFile } from "./checksum.js"; import { readPackFile, type KnowledgePack } from "../core/packs.js"; @@ -244,18 +244,35 @@ function rollbackVersionDir(versionDir: string, created: boolean): void { } } +/** Where a publish goes: its default commit message and how the commit is pushed. */ +interface PublishTarget { + defaultMessage: (packName: string, version: string) => string; + push: (cacheDir: string) => Promise; +} + /** * Publish a pack to a registry. * Creates the canonical folder structure, generates checksum, updates index and manifest, commits and pushes. */ export async function publishPack(options: PublishOptions): Promise { - const log = getLogger(); - const { registryName, packFilePath, commitMessage } = options; - - const cacheDir = validateRegistryCache(registryName); + const cacheDir = validateRegistryCache(options.registryName); await tryFetchLatest(cacheDir, "publish"); + const pack = readAndValidatePack(options.packFilePath); + return publishInCache(options, cacheDir, pack, { + defaultMessage: (name, version) => `publish: ${name}@${version}`, + push: (dir) => git(["push"], { cwd: dir }), + }); +} - const pack = readAndValidatePack(packFilePath); +/** Write the pack's version into the registry clone at `cacheDir`, commit it and push it. */ +async function publishInCache( + options: PublishOptions, + cacheDir: string, + pack: KnowledgePack, + target: PublishTarget, +): Promise { + const log = getLogger(); + const { registryName, packFilePath, commitMessage } = options; const packDir = join(cacheDir, PACKS_DIR, pack.name); const manifestPath = join(packDir, PACK_MANIFEST_FILE); @@ -309,8 +326,8 @@ export async function publishPack(options: PublishOptions): Promise` branch of the registry instead of its main branch (for a + * pull request). The branch starts at the remote's latest main branch and is pushed to the + * remote. The local clone is left on its own branch, so later publishes and syncs are not + * affected. */ export async function publishPackToBranch( options: PublishOptions, ): Promise { const log = getLogger(); - const { registryName, packFilePath } = options; + const { registryName } = options; const cacheDir = validateRegistryCache(registryName); - - const pack = readPackJson(packFilePath); + await tryFetchLatest(cacheDir, "publish"); + const pack = readAndValidatePack(options.packFilePath); const branchName = `feature/add-${pack.name}`; - // Create and checkout branch - await git(["checkout", "-b", branchName], { cwd: cacheDir }); + const startRef = await currentRef(cacheDir); + await git(["checkout", "-B", branchName], { cwd: cacheDir }); try { - // Reuse the normal publish flow (which commits) - const result = await publishPack({ - ...options, - commitMessage: options.commitMessage ?? `feat: add ${pack.name}@${pack.version}`, + const result = await publishInCache(options, cacheDir, pack, { + defaultMessage: (name, version) => `feat: add ${name}@${version}`, + push: (dir) => git(["push", "origin", `HEAD:refs/heads/${branchName}`], { cwd: dir }), }); - - // Push the branch - await git(["push", "-u", "origin", branchName], { cwd: cacheDir }); - log.info({ branch: branchName, registry: registryName }, "Pack published to feature branch"); - return { ...result, branch: branchName }; + } finally { + await leaveFeatureBranch(cacheDir, startRef, branchName); + } +} + +/** The checked-out branch name, or the commit when HEAD is detached. */ +async function currentRef(cacheDir: string): Promise { + const branch = await git(["rev-parse", "--abbrev-ref", "HEAD"], { cwd: cacheDir }); + return branch === "HEAD" ? git(["rev-parse", "HEAD"], { cwd: cacheDir }) : branch; +} + +/** Check out `startRef` again, dropping the publish's files, and delete the local branch. */ +async function leaveFeatureBranch( + cacheDir: string, + startRef: string, + featureBranch: string, +): Promise { + try { + await git(["checkout", "-f", startRef], { cwd: cacheDir }); + await git(["branch", "-D", featureBranch], { cwd: cacheDir }); } catch (err) { - // Try to go back to main branch on failure - try { - await git(["checkout", "main"], { cwd: cacheDir }); - await git(["branch", "-D", branchName], { cwd: cacheDir }); - } catch (cleanupErr) { - log.warn( - { err: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr) }, - "Failed to clean up feature branch after publish failure", - ); - } - throw err; + getLogger().warn( + { err: err instanceof Error ? err.message : String(err) }, + "Failed to return the registry clone to its branch after publishing to a feature branch", + ); } } diff --git a/tests/integration/registry/registry-publish.test.ts b/tests/integration/registry/registry-publish.test.ts index 64b98b5c..1daf3812 100644 --- a/tests/integration/registry/registry-publish.test.ts +++ b/tests/integration/registry/registry-publish.test.ts @@ -29,7 +29,8 @@ vi.mock("node:os", async (importOriginal) => { const { loadRegistries, saveRegistries, getRegistry } = await import("../../../src/registry/config.js"); const { syncRegistry } = await import("../../../src/registry/sync.js"); -const { publishPack, unpublishPack } = await import("../../../src/registry/publish.js"); +const { publishPack, publishPackToBranch, unpublishPack } = + await import("../../../src/registry/publish.js"); const { verifyChecksum } = await import("../../../src/registry/checksum.js"); const { getRegistryCacheDir } = await import("../../../src/registry/types.js"); @@ -331,3 +332,112 @@ describe("integration: registry publish", () => { ).rejects.toThrow(/not found/); }); }); + +describe("integration: registry publish to a feature branch (--submit)", () => { + let tempDir: string; + let savedGlobalConfig: string | undefined; + + /** Run git in `cwd` and return its trimmed output. */ + function gitOut(args: string, cwd: string): string { + return execSync(`git ${args}`, { cwd, stdio: "pipe" }).toString().trim(); + } + + beforeEach(() => { + initLogger("silent"); + tempDir = mkdtempSync(join(tmpdir(), "libscope-submit-")); + tempHome = join(tempDir, "home"); + mkdirSync(tempHome, { recursive: true }); + // Default git settings: no push.autoSetupRemote or other options from the user's config. + savedGlobalConfig = process.env["GIT_CONFIG_GLOBAL"]; + const emptyConfig = join(tempDir, "gitconfig"); + writeFileSync(emptyConfig, "", "utf-8"); + process.env["GIT_CONFIG_GLOBAL"] = emptyConfig; + }); + + afterEach(() => { + if (savedGlobalConfig === undefined) delete process.env["GIT_CONFIG_GLOBAL"]; + else process.env["GIT_CONFIG_GLOBAL"] = savedGlobalConfig; + rmSync(tempDir, { recursive: true, force: true }); + }); + + async function setUpRegistry(name: string): Promise<{ bareRepo: string; cacheDir: string }> { + const bareRepo = createBareRepo(tempDir); + addTestRegistry(makeEntry(name, bareRepo)); + await syncRegistry(getRegistry(name)!); + return { bareRepo, cacheDir: getRegistryCacheDir(name) }; + } + + it("pushes the pack to feature/add- and leaves the main branch unchanged", async () => { + const { bareRepo, cacheDir } = await setUpRegistry("submit-reg"); + const mainBranch = gitOut("rev-parse --abbrev-ref HEAD", cacheDir); + const mainBefore = gitOut(`rev-parse ${mainBranch}`, bareRepo); + + const result = await publishPackToBranch({ + registryName: "submit-reg", + packFilePath: createPackFile(tempDir, "sub-pack", "1.2.0"), + }); + + expect(result).toMatchObject({ + packName: "sub-pack", + version: "1.2.0", + branch: "feature/add-sub-pack", + }); + // The remote has the branch with the pack files and the index entry. + const branch = "feature/add-sub-pack"; + expect(gitOut(`log -1 --format=%s ${branch}`, bareRepo)).toBe("feat: add sub-pack@1.2.0"); + expect(gitOut(`ls-tree -r --name-only ${branch}`, bareRepo).split("\n")).toEqual( + expect.arrayContaining([ + "packs/sub-pack/pack.json", + "packs/sub-pack/1.2.0/sub-pack.json", + "packs/sub-pack/1.2.0/checksum.sha256", + ]), + ); + const index = JSON.parse(gitOut(`show ${branch}:index.json`, bareRepo)) as PackSummary[]; + expect(index.map((p) => p.name)).toEqual(["sub-pack"]); + // The main branch did not move, and the local clone is back on it, clean. + expect(gitOut(`rev-parse ${mainBranch}`, bareRepo)).toBe(mainBefore); + expect(gitOut("rev-parse --abbrev-ref HEAD", cacheDir)).toBe(mainBranch); + expect(gitOut("status --porcelain", cacheDir)).toBe(""); + expect(gitOut("branch --list feature/*", cacheDir)).toBe(""); + expect(existsSync(join(cacheDir, "packs", "sub-pack"))).toBe(false); + }); + + it("a later normal publish goes to the main branch", async () => { + const { bareRepo, cacheDir } = await setUpRegistry("submit-then-main"); + const mainBranch = gitOut("rev-parse --abbrev-ref HEAD", cacheDir); + await publishPackToBranch({ + registryName: "submit-then-main", + packFilePath: createPackFile(tempDir, "first-pack"), + }); + + await publishPack({ + registryName: "submit-then-main", + packFilePath: createPackFile(tempDir, "second-pack"), + }); + + expect(gitOut(`log -1 --format=%s ${mainBranch}`, bareRepo)).toBe("publish: second-pack@1.0.0"); + expect(gitOut("log -1 --format=%s feature/add-first-pack", bareRepo)).toBe( + "feat: add first-pack@1.0.0", + ); + }); + + it("returns the clone to its branch when the publish fails", async () => { + const { bareRepo, cacheDir } = await setUpRegistry("submit-fail"); + const mainBranch = gitOut("rev-parse --abbrev-ref HEAD", cacheDir); + const packFile = createPackFile(tempDir, "dup-pack"); + await publishPack({ registryName: "submit-fail", packFilePath: packFile, version: "1.0.0" }); + + await expect( + publishPackToBranch({ + registryName: "submit-fail", + packFilePath: packFile, + version: "1.0.0", + }), + ).rejects.toThrow(/already exists/); + + expect(gitOut("rev-parse --abbrev-ref HEAD", cacheDir)).toBe(mainBranch); + expect(gitOut("status --porcelain", cacheDir)).toBe(""); + expect(gitOut("branch --list feature/*", cacheDir)).toBe(""); + expect(gitOut("branch --list feature/*", bareRepo)).toBe(""); + }); +}); From b0c06ed3b265cda94df0f267366af4533fb4c7cd Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 16:56:58 +0000 Subject: [PATCH 73/83] fix(registry): create registries on main and push new branches with --submit `registry create` committed on git's default branch (often master), so the documented `git push -u origin main` failed and the remote stayed empty. `registry publish --submit` ran a plain `git push` on the new feature branch, which has no upstream, and failed before its own `push -u`. commitAndPush now pushes HEAD with -u, and the clone returns to its base branch after a submit. --- src/registry/git.ts | 7 +-- src/registry/publish.ts | 13 +++--- .../registry/registry-publish.test.ts | 24 +++++++++- tests/unit/registry/git.test.ts | 45 +++++++++++++++++++ 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/src/registry/git.ts b/src/registry/git.ts index b6cff0dd..1b7cc72e 100644 --- a/src/registry/git.ts +++ b/src/registry/git.ts @@ -196,8 +196,9 @@ export async function createRegistryRepo(path: string): Promise { mkdirSync(path, { recursive: true }); - // Initialize git repo + // Initialize git repo on "main" whatever init.defaultBranch is (the docs push "main") await git(["init"], { cwd: path }); + await git(["symbolic-ref", "HEAD", "refs/heads/main"], { cwd: path }); // Create canonical structure const indexPath = join(path, INDEX_FILE); @@ -228,12 +229,12 @@ export async function createRegistryRepo(path: string): Promise { log.info({ path }, "Registry repo initialized"); } -/** Add, commit, and push changes in a registry repo. */ +/** Add, commit, and push the current branch of a registry repo (also when the branch is new). */ export async function commitAndPush(repoPath: string, message: string): Promise { await git(["add", "."], { cwd: repoPath }); await git( ["-c", "user.name=libscope", "-c", "user.email=libscope@localhost", "commit", "-m", message], { cwd: repoPath }, ); - await git(["push"], { cwd: repoPath }); + await git(["push", "-u", "origin", "HEAD"], { cwd: repoPath }); } diff --git a/src/registry/publish.ts b/src/registry/publish.ts index 8c36fbc1..20359223 100644 --- a/src/registry/publish.ts +++ b/src/registry/publish.ts @@ -334,26 +334,25 @@ export async function publishPackToBranch( const pack = readPackJson(packFilePath); const branchName = `feature/add-${pack.name}`; - // Create and checkout branch + // Create and checkout branch; the clone goes back to its branch afterwards + const baseBranch = await git(["rev-parse", "--abbrev-ref", "HEAD"], { cwd: cacheDir }); await git(["checkout", "-b", branchName], { cwd: cacheDir }); try { - // Reuse the normal publish flow (which commits) + // Reuse the normal publish flow, which commits and pushes the feature branch const result = await publishPack({ ...options, commitMessage: options.commitMessage ?? `feat: add ${pack.name}@${pack.version}`, }); - - // Push the branch - await git(["push", "-u", "origin", branchName], { cwd: cacheDir }); + await git(["checkout", baseBranch], { cwd: cacheDir }); log.info({ branch: branchName, registry: registryName }, "Pack published to feature branch"); return { ...result, branch: branchName }; } catch (err) { - // Try to go back to main branch on failure + // Try to go back to the base branch on failure try { - await git(["checkout", "main"], { cwd: cacheDir }); + await git(["checkout", "--force", baseBranch], { cwd: cacheDir }); await git(["branch", "-D", branchName], { cwd: cacheDir }); } catch (cleanupErr) { log.warn( diff --git a/tests/integration/registry/registry-publish.test.ts b/tests/integration/registry/registry-publish.test.ts index 64b98b5c..f7c0aab0 100644 --- a/tests/integration/registry/registry-publish.test.ts +++ b/tests/integration/registry/registry-publish.test.ts @@ -29,7 +29,8 @@ vi.mock("node:os", async (importOriginal) => { const { loadRegistries, saveRegistries, getRegistry } = await import("../../../src/registry/config.js"); const { syncRegistry } = await import("../../../src/registry/sync.js"); -const { publishPack, unpublishPack } = await import("../../../src/registry/publish.js"); +const { publishPack, publishPackToBranch, unpublishPack } = + await import("../../../src/registry/publish.js"); const { verifyChecksum } = await import("../../../src/registry/checksum.js"); const { getRegistryCacheDir } = await import("../../../src/registry/types.js"); @@ -166,6 +167,27 @@ describe("integration: registry publish", () => { expect(await verifyChecksum(publishedFile, result.checksum)).toBe(true); }); + it("should submit a pack on a pushed feature branch and return to the base branch", async () => { + const bareRepo = createBareRepo(tempDir); + addTestRegistry(makeEntry("submit-reg", bareRepo)); + await syncRegistry(getRegistry("submit-reg")!); + const cacheDir = getRegistryCacheDir("submit-reg"); + const baseBranch = execSync("git rev-parse --abbrev-ref HEAD", { cwd: cacheDir }).toString(); + + const packFile = createPackFile(tempDir, "submitted-pack"); + const result = await publishPackToBranch({ + registryName: "submit-reg", + packFilePath: packFile, + version: "1.0.0", + }); + + expect(result.branch).toBe("feature/add-submitted-pack"); + const remoteBranches = execSync("git branch --list", { cwd: bareRepo }).toString(); + expect(remoteBranches).toContain("feature/add-submitted-pack"); + const current = execSync("git rev-parse --abbrev-ref HEAD", { cwd: cacheDir }).toString(); + expect(current).toBe(baseBranch); + }); + it("should unpublish a pack version", async () => { const bareRepo = createBareRepo(tempDir); addTestRegistry(makeEntry("unpub-reg", bareRepo)); diff --git a/tests/unit/registry/git.test.ts b/tests/unit/registry/git.test.ts index e62fe69f..fdf218b4 100644 --- a/tests/unit/registry/git.test.ts +++ b/tests/unit/registry/git.test.ts @@ -227,6 +227,27 @@ describe("registry git helpers", () => { const log = execSync("git log --oneline", { cwd: repoPath, encoding: "utf-8" }); expect(log).toContain("Initial registry structure"); }); + + it("should commit on main even when git's default branch is master", async () => { + const repoPath = join(tempDir, "main-registry"); + const saved = { ...process.env }; + Object.assign(process.env, { + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "init.defaultBranch", + GIT_CONFIG_VALUE_0: "master", + }); + try { + await createRegistryRepo(repoPath); + } finally { + process.env = saved; + } + + const branch = execSync("git rev-parse --abbrev-ref HEAD", { + cwd: repoPath, + encoding: "utf-8", + }); + expect(branch.trim()).toBe("main"); + }); }); describe("cloneRegistry + fetchRegistry", () => { @@ -347,6 +368,30 @@ describe("registry git helpers", () => { const content = readFileSync(join(verifyDir, "file.txt"), "utf-8"); expect(content).toBe("updated"); }); + + it("should push a new branch that has no upstream yet", async () => { + const bareDir = join(tempDir, "branch-bare.git"); + execSync(`git init --bare "${bareDir}"`, { stdio: "pipe" }); + const workDir = join(tempDir, "branch-work"); + execSync(`git clone "${bareDir}" "${workDir}"`, { stdio: "pipe" }); + writeFileSync(join(workDir, "file.txt"), "initial", "utf-8"); + execSync("git add . && git commit -m 'init'", { cwd: workDir, stdio: "pipe", env: gitEnv }); + execSync("git push origin HEAD", { cwd: workDir, stdio: "pipe" }); + + execSync("git checkout -b feature/add-pack", { cwd: workDir, stdio: "pipe" }); + writeFileSync(join(workDir, "file.txt"), "on branch", "utf-8"); + const saved = { ...process.env }; + // Ignore the user's git config (push.autoSetupRemote would hide the missing upstream) + Object.assign(process.env, { GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }); + try { + await commitAndPush(workDir, "branch commit"); + } finally { + process.env = saved; + } + + const branches = execSync("git branch --list", { cwd: bareDir, encoding: "utf-8" }); + expect(branches).toContain("feature/add-pack"); + }); }); // Robustness: LIBSCOPE_GIT_TIMEOUT_MS controls git timeout From b650ed144a9e17d1e2265143bcded8ee2f642884 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 17:04:33 +0000 Subject: [PATCH 74/83] fix(ingest): decode character references in page titles Titles taken from and <h1> kept their HTML character references, so https://docs.python.org/3/tutorial/ was indexed as "The Python Tutorial — Python 3.14 documentation". URL ingestion, the spider and the docs connector now decode numeric and common named references. --- src/connectors/docs.ts | 5 +++-- src/core/html-entities.ts | 33 +++++++++++++++++++++++++++++++ src/core/spider.ts | 5 +++-- src/core/url-fetcher.ts | 4 +++- tests/unit/docs-connector.test.ts | 6 ++++++ tests/unit/html-entities.test.ts | 15 ++++++++++++++ tests/unit/spider.test.ts | 12 +++++++++++ tests/unit/url-fetcher.test.ts | 14 +++++++++++++ 8 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 src/core/html-entities.ts create mode 100644 tests/unit/html-entities.test.ts diff --git a/src/connectors/docs.ts b/src/connectors/docs.ts index 76a7f383..ee60c467 100644 --- a/src/connectors/docs.ts +++ b/src/connectors/docs.ts @@ -10,6 +10,7 @@ import { NodeHtmlMarkdown } from "node-html-markdown"; import { ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; import { fetchRaw } from "../core/url-fetcher.js"; +import { decodeHtmlEntities } from "../core/html-entities.js"; import type { FetchOptions } from "../core/url-fetcher.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; @@ -354,7 +355,7 @@ export function extractDocTitle(html: string, url: string): string { .slice(innerStart, h1CloseIdx) .replaceAll(/<[^>]{1,2000}>/g, "") .trim(); - if (title) return title; + if (title) return decodeHtmlEntities(title); } } @@ -362,7 +363,7 @@ export function extractDocTitle(html: string, url: string): string { const titleTagMatch = /<title[^>]{0,2000}>([^<]+)<\/title>/i.exec(html); if (titleTagMatch?.[1]) { const title = titleTagMatch[1].trim(); - if (title) return title; + if (title) return decodeHtmlEntities(title); } // Last resort: derive from URL path diff --git a/src/core/html-entities.ts b/src/core/html-entities.ts new file mode 100644 index 00000000..498ce724 --- /dev/null +++ b/src/core/html-entities.ts @@ -0,0 +1,33 @@ +/** Character references in text taken from HTML. */ + +const NAMED_ENTITIES: Readonly<Record<string, string>> = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: " ", + ndash: "–", + mdash: "—", + hellip: "…", + lsquo: "‘", + rsquo: "’", + ldquo: "“", + rdquo: "”", + laquo: "«", + raquo: "»", + middot: "·", + copy: "©", + reg: "®", + trade: "™", +}; + +/** Decode character references (`&`, `—`, `—`) in text taken from HTML, e.g. a title. */ +export function decodeHtmlEntities(text: string): string { + return text.replaceAll(/&(#x[\da-f]{1,6}|#\d{1,7}|[a-z]{2,8});/gi, (entity, ref: string) => { + if (!ref.startsWith("#")) return NAMED_ENTITIES[ref.toLowerCase()] ?? entity; + const hex = ref[1] === "x" || ref[1] === "X"; + const code = Number.parseInt(ref.slice(hex ? 2 : 1), hex ? 16 : 10); + return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : entity; + }); +} diff --git a/src/core/spider.ts b/src/core/spider.ts index a7081773..1987a903 100644 --- a/src/core/spider.ts +++ b/src/core/spider.ts @@ -13,6 +13,7 @@ import { setTimeout as sleep } from "node:timers/promises"; import { getLogger } from "../logger.js"; import { FetchError } from "../errors.js"; import { fetchRaw, type FetchOptions } from "./url-fetcher.js"; +import { decodeHtmlEntities } from "./html-entities.js"; import { extractLinks } from "./link-extractor.js"; import { NodeHtmlMarkdown } from "node-html-markdown"; @@ -269,11 +270,11 @@ function stripTags(input: string): string { function extractTitle(html: string, url: string): string { // Try <title> tag const match = /<title[^>]*>([^<]+)<\/title>/i.exec(html); - if (match?.[1]) return match[1].trim(); + if (match?.[1]) return decodeHtmlEntities(match[1].trim()); // Try first <h1> const h1 = /<h1[^>]*>([\s\S]*?)<\/h1>/i.exec(html); if (h1?.[1]) { - return stripTags(h1[1]).trim(); + return decodeHtmlEntities(stripTags(h1[1]).trim()); } // Fall back to URL path try { diff --git a/src/core/url-fetcher.ts b/src/core/url-fetcher.ts index 050c6e8c..d3765e3e 100644 --- a/src/core/url-fetcher.ts +++ b/src/core/url-fetcher.ts @@ -4,6 +4,7 @@ import { Agent } from "undici"; import { NodeHtmlMarkdown } from "node-html-markdown"; import { FetchError } from "../errors.js"; import { getLogger } from "../logger.js"; +import { decodeHtmlEntities } from "./html-entities.js"; /** Lazy singleton undici Agent that skips TLS certificate verification. */ let _insecureAgent: Agent | undefined; @@ -325,7 +326,8 @@ export async function fetchAndConvert( /** Extract title from HTML <title> tag. */ function extractTitleFromHtml(html: string): string | null { const match = /<title[^>]*>([^<]+)<\/title>/i.exec(html); - return match?.[1]?.trim() ?? null; + const title = match?.[1]?.trim(); + return title ? decodeHtmlEntities(title) : null; } /** Extract title from first markdown heading. */ diff --git a/tests/unit/docs-connector.test.ts b/tests/unit/docs-connector.test.ts index 4f164781..5c480281 100644 --- a/tests/unit/docs-connector.test.ts +++ b/tests/unit/docs-connector.test.ts @@ -252,6 +252,12 @@ describe("extractDocTitle", () => { "https://example.com/docs", "My Library — Docs", ], + [ + "character references decoded", + "<html><head><title>Guide — Q&A", + "https://example.com/docs", + "Guide — Q&A", + ], [ "URL-derived fallback", "

content

", diff --git a/tests/unit/html-entities.test.ts b/tests/unit/html-entities.test.ts new file mode 100644 index 00000000..68a13308 --- /dev/null +++ b/tests/unit/html-entities.test.ts @@ -0,0 +1,15 @@ +import { describe, it, expect } from "vitest"; +import { decodeHtmlEntities } from "../../src/core/html-entities.js"; + +describe("decodeHtmlEntities", () => { + it.each([ + ["The Python Tutorial — Python 3", "The Python Tutorial — Python 3"], + ["A — B", "A — B"], + ["Q&A <tags> "quoted" it's", `Q&A "quoted" it's`], + ["Docs — Guide…", "Docs — Guide…"], + ["Unknown &foo; stays, so does a bare & sign", "Unknown &foo; stays, so does a bare & sign"], + ["Out of range � stays", "Out of range � stays"], + ])("decodes %j", (input, expected) => { + expect(decodeHtmlEntities(input)).toBe(expected); + }); +}); diff --git a/tests/unit/spider.test.ts b/tests/unit/spider.test.ts index 8cf59070..f4d28993 100644 --- a/tests/unit/spider.test.ts +++ b/tests/unit/spider.test.ts @@ -81,6 +81,18 @@ describe("spiderUrl", () => { expect(page.depth).toBe(0); }); + it("decodes character references in the page title", async () => { + mockFetchRaw.mockImplementation((url: string) => { + if (url.endsWith("/robots.txt")) return Promise.reject(new Error("404")); + return Promise.resolve(pageResponse(htmlPage("Guide — Q&A"), url)); + }); + + const gen = spiderUrl("https://example.com/", { maxPages: 1, requestDelay: 0 }); + const result = await gen.next(); + const page = result.value as { title: string }; + expect(page.title).toBe("Guide — Q&A"); + }); + it("follows links up to maxDepth", async () => { mockFetchRaw.mockImplementation((url: string) => { if (url.endsWith("/robots.txt")) return Promise.reject(new Error("404")); diff --git a/tests/unit/url-fetcher.test.ts b/tests/unit/url-fetcher.test.ts index 59f340cb..f551c985 100644 --- a/tests/unit/url-fetcher.test.ts +++ b/tests/unit/url-fetcher.test.ts @@ -94,6 +94,20 @@ describe("fetchAndConvert", () => { expect(result.content).toContain("World"); }); + it("should decode character references in the ", async () => { + const html = + "<html><head><title>The Python Tutorial — Python & more

x

"; + mockFetch.mockResolvedValue({ + ok: true, + headers: new Headers({ "content-type": "text/html" }), + body: bodyStream(html), + text: () => Promise.resolve(html), + }); + + const result = await fetchAndConvert("https://example.com/page"); + expect(result.title).toBe("The Python Tutorial — Python & more"); + }); + it("should return markdown/plain text as-is", async () => { const md = "# Markdown Title\n\nSome content here."; mockFetch.mockResolvedValue({ From 7055a1201362e666dc758fcaf7df47893cf305ed Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 17:10:13 +0000 Subject: [PATCH 75/83] docs: fix examples that fail when run as written - Python SDK quick start: create the "python" topic before add_text uses it (the server answers 404 for an unknown topic); note that ask needs an LLM. - README SDK example: note that ask needs an LLM. - Programmatic usage: askStream throws a ConfigError without an LLM, including with passthrough. - Connectors: a --schedule-only connect works on a saved connection; a new one also needs its settings. - REST reference: a task result string starts with "kind". - Architecture: renameTopic exists in core/topics.ts; only the operation is hypothetical. --- README.md | 2 +- docs/guide/architecture.md | 2 +- docs/guide/connectors.md | 2 +- docs/guide/programmatic-usage.md | 2 +- docs/reference/rest-api.md | 2 +- sdk/python/README.md | 2 ++ 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 442fa48a..6089f3cf 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ await scope.add("./docs"); const { items } = await scope.search("how to authenticate"); for (const hit of items) console.log(hit.documentId, hit.title, hit.score); -const result = await scope.ask("How do I configure OAuth2?"); +const result = await scope.ask("How do I configure OAuth2?"); // needs an LLM, like the CLI if (result.mode === "answer") console.log(result.answer); scope.close(); diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 1d65857f..ff26c820 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -328,7 +328,7 @@ New functionality starts as an operation. The surfaces then expose it with littl - **MCP**: only for tools that an assistant needs. Add it to `registerCoreTools` or `registerAdminTools` in `src/mcp/server.ts` with a formatter in `src/mcp/format.ts`. 5. Run `npm run build && npm run docs:gen` to update the generated references, and commit them. -Example (a hypothetical `rename-topic` operation; `renameTopic` is not in the code): +Example (a hypothetical `rename-topic` operation over the existing `renameTopic` in `src/core/topics.ts`): ```typescript // src/core/operations/topics.ts diff --git a/docs/guide/connectors.md b/docs/guide/connectors.md index fed3f5d5..a61985b3 100644 --- a/docs/guide/connectors.md +++ b/docs/guide/connectors.md @@ -35,7 +35,7 @@ Each file also stores `connectorType`, the `schedule` (if set) and `lastSync`. A ### Schedules -`--schedule ` syncs a connection on a cron schedule. Schedules run while the REST API server runs (`libscope serve api`): +`--schedule ` syncs a connection on a cron schedule. Schedules run while the REST API server runs (`libscope serve api`). For a saved connection, give only the schedule (a new connection also needs its other settings, such as `--token`): ```bash libscope connect notion --schedule "0 */6 * * *" --no-sync # every 6 hours diff --git a/docs/guide/programmatic-usage.md b/docs/guide/programmatic-usage.md index 44415a28..6865c895 100644 --- a/docs/guide/programmatic-usage.md +++ b/docs/guide/programmatic-usage.md @@ -98,7 +98,7 @@ if (result.mode === "answer") { `ask()` uses `llmProvider`, or the LLM from `llm.provider` (default `auto`: OpenAI if an OpenAI key is set, else Anthropic if an Anthropic key is set, else Ollama if `llm.url` is set or the embedding provider is `ollama`). If there is no LLM, `ask()` throws a `ConfigError` that tells you what to set. -Streaming: +Streaming needs an LLM: with `passthrough` or no LLM, `askStream()` throws a `ConfigError`. ```ts for await (const event of scope.askStream("How does OAuth2 work?")) { diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index c34c2353..33c56f0f 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -179,7 +179,7 @@ curl -X POST http://localhost:3378/api/v1/documents \ # -> 202 { "data": { "taskId": "6f1c...", "operation": "add", "status": "running" } } curl http://localhost:3378/api/v1/tasks/6f1c... -# -> { "data": { "status": "completed", "result": "{\"documents\":[{\"documentId\":\"...\"}],...}", ... } } +# -> { "data": { "status": "completed", "result": "{\"kind\":\"content\",\"documents\":[{\"documentId\":\"...\",...}],...}", ... } } ``` ### Add a web page, a crawled site or a repository diff --git a/sdk/python/README.md b/sdk/python/README.md index f027ac96..e7e18f16 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -28,6 +28,7 @@ with LibscopeClient() as client: done = client.wait_for_task(task.id) doc_id = done.result["documents"][0]["documentId"] + client.create_topic("python") # a topic must exist before documents use it client.add_text("My Notes", "Some useful content...", topic="python") page = client.search("how to use decorators", limit=5) @@ -36,6 +37,7 @@ with LibscopeClient() as client: client.add_tags(doc_id, ["python", "tutorial"]) + # ask needs an LLM on the server (otherwise it raises ServerError with the setting to change) answer = client.ask("What is the best practice for error handling?") print(answer.answer) ``` From 2291602b509538d26e22356c14f8d04a07a94ac6 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 17:30:22 +0000 Subject: [PATCH 76/83] test: use specific matchers, parameterize repeated cases, assert in schema tests --- .../registry/registry-lifecycle.test.ts | 2 +- tests/unit/confirm.test.ts | 38 ++---- tests/unit/docs-connector.test.ts | 4 +- .../operations/searches-analytics.test.ts | 2 +- tests/unit/packs.test.ts | 16 +-- tests/unit/registry/config.test.ts | 31 ++--- tests/unit/registry/publish.test.ts | 38 ++---- tests/unit/schema.test.ts | 108 ++---------------- tests/unit/search.test.ts | 102 ++++++++--------- tests/unit/spider.test.ts | 8 +- tests/unit/topics.test.ts | 20 ++-- tests/unit/web-server.test.ts | 4 +- 12 files changed, 123 insertions(+), 250 deletions(-) diff --git a/tests/integration/registry/registry-lifecycle.test.ts b/tests/integration/registry/registry-lifecycle.test.ts index 24ea4485..adc837ff 100644 --- a/tests/integration/registry/registry-lifecycle.test.ts +++ b/tests/integration/registry/registry-lifecycle.test.ts @@ -282,7 +282,7 @@ describe("integration: registry lifecycle", () => { // Search across both const { results } = searchRegistries("pack"); - expect(results.length).toBe(2); + expect(results).toHaveLength(2); expect(results.map((r) => r.name).sort((a, b) => a.localeCompare(b))).toEqual([ "pack-from-reg1", "pack-from-reg2", diff --git a/tests/unit/confirm.test.ts b/tests/unit/confirm.test.ts index 61a71ad8..43121c7a 100644 --- a/tests/unit/confirm.test.ts +++ b/tests/unit/confirm.test.ts @@ -18,34 +18,16 @@ describe("confirmAction", () => { expect(result).toBe(true); }); - it("returns true when user answers 'y'", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("y")); - expect(result).toBe(true); - }); - - it("returns true when user answers 'yes'", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("yes")); - expect(result).toBe(true); - }); - - it("returns true when user answers 'YES' (case-insensitive)", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("YES")); - expect(result).toBe(true); - }); - - it("returns false when user answers 'n'", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("n")); - expect(result).toBe(false); - }); - - it("returns false when user answers empty string", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("")); - expect(result).toBe(false); - }); - - it("returns false when user answers arbitrary text", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("maybe")); - expect(result).toBe(false); + it.each([ + { answer: "y", expected: true }, + { answer: "yes", expected: true }, + { answer: "YES", expected: true }, + { answer: "n", expected: false }, + { answer: "", expected: false }, + { answer: "maybe", expected: false }, + ])("returns $expected when user answers '$answer'", async ({ answer, expected }) => { + const result = await confirmAction("Delete?", false, createMockInterface(answer)); + expect(result).toBe(expected); }); it("closes the readline interface after prompting", async () => { diff --git a/tests/unit/docs-connector.test.ts b/tests/unit/docs-connector.test.ts index 2c783fc3..510da342 100644 --- a/tests/unit/docs-connector.test.ts +++ b/tests/unit/docs-connector.test.ts @@ -359,7 +359,7 @@ describe("extractDocLinks", () => { 'Three', ].join("\n"); const links = extractDocLinks(html, BASE, "/docs/"); - expect(links.filter((l) => l.includes("/docs/page")).length).toBe(1); + expect(links.filter((l) => l.includes("/docs/page"))).toHaveLength(1); }); it("returns empty array when no anchors found", () => { @@ -420,7 +420,7 @@ describe("extractSitemapUrls", () => { https://docs.example.com/docs/page `; const urls = extractSitemapUrls(xml, BASE, "/docs/"); - expect(urls.length).toBe(1); + expect(urls).toHaveLength(1); }); it("returns empty array for empty sitemap", () => { diff --git a/tests/unit/operations/searches-analytics.test.ts b/tests/unit/operations/searches-analytics.test.ts index a41123dd..943349fa 100644 --- a/tests/unit/operations/searches-analytics.test.ts +++ b/tests/unit/operations/searches-analytics.test.ts @@ -42,7 +42,7 @@ describe("saved search and analytics operations", () => { expect(list).toMatchObject({ total: 1, limit: 50, offset: 0 }); const ran = await run(runSavedSearchOperation, t.ctx, { search: "react-hooks" }); - expect(ran.items.length).toBe(1); + expect(ran.items).toHaveLength(1); expect(ran.search.resultCount).toBe(1); expect(ran.search.lastRunAt).not.toBeNull(); diff --git a/tests/unit/packs.test.ts b/tests/unit/packs.test.ts index 6a501ffc..7be080ae 100644 --- a/tests/unit/packs.test.ts +++ b/tests/unit/packs.test.ts @@ -75,7 +75,7 @@ describe("knowledge packs", () => { const docs = db .prepare("SELECT id, pack_name FROM documents WHERE pack_name = ?") .all("test-pack") as Array<{ id: string; pack_name: string }>; - expect(docs.length).toBe(2); + expect(docs).toHaveLength(2); // Verify pack is recorded const packs = db.prepare("SELECT * FROM packs WHERE name = ?").all("test-pack") as Array<{ @@ -83,7 +83,7 @@ describe("knowledge packs", () => { version: string; doc_count: number; }>; - expect(packs.length).toBe(1); + expect(packs).toHaveLength(1); expect(packs[0]!.version).toBe("1.0.0"); expect(packs[0]!.doc_count).toBe(2); }); @@ -224,17 +224,17 @@ describe("knowledge packs", () => { const docsBefore = db .prepare("SELECT id FROM documents WHERE pack_name = ?") .all("test-pack"); - expect(docsBefore.length).toBe(2); + expect(docsBefore).toHaveLength(2); removePack(db, "test-pack"); // Verify docs removed const docsAfter = db.prepare("SELECT id FROM documents WHERE pack_name = ?").all("test-pack"); - expect(docsAfter.length).toBe(0); + expect(docsAfter).toHaveLength(0); // Verify pack record removed const packRecord = db.prepare("SELECT * FROM packs WHERE name = ?").all("test-pack"); - expect(packRecord.length).toBe(0); + expect(packRecord).toHaveLength(0); }); it("should throw when removing a non-existent pack", () => { @@ -260,7 +260,7 @@ describe("knowledge packs", () => { await installPack(db, provider, path2); const packs = listInstalledPacks(db); - expect(packs.length).toBe(2); + expect(packs).toHaveLength(2); expect(packs.map((p) => p.name)).toEqual(["pack-a", "pack-b"]); expect(packs[0]!.version).toBe("1.0.0"); expect(packs[0]!.docCount).toBe(2); @@ -288,7 +288,7 @@ describe("knowledge packs", () => { }); expect(pack.name).toBe("exported"); - expect(pack.documents.length).toBe(2); + expect(pack.documents).toHaveLength(2); expect(pack.version).toBe("1.0.0"); expect(pack.metadata.author).toBe("libscope"); @@ -1006,7 +1006,7 @@ describe("knowledge packs", () => { const docs = db .prepare("SELECT id FROM documents WHERE pack_name = ?") .all("concurrent-4") as Array<{ id: string }>; - expect(docs.length).toBe(4); + expect(docs).toHaveLength(4); }); it("should make multiple embedBatch calls with small batchSize and high concurrency", async () => { diff --git a/tests/unit/registry/config.test.ts b/tests/unit/registry/config.test.ts index 5e0d8743..54cad00c 100644 --- a/tests/unit/registry/config.test.ts +++ b/tests/unit/registry/config.test.ts @@ -117,19 +117,24 @@ describe("registry config", () => { }); // Security/robustness fixes: whitespace trimming and trailing slash normalisation - it("should trim leading and trailing whitespace", () => { - const result = validateGitUrl(" https://github.com/org/repo.git "); - expect(result).toBe("https://github.com/org/repo.git"); - }); - - it("should strip trailing slashes", () => { - const result = validateGitUrl("https://github.com/org/repo/"); - expect(result).toBe("https://github.com/org/repo"); - }); - - it("should strip both whitespace and trailing slashes together", () => { - const result = validateGitUrl(" https://github.com/org/repo.git "); - expect(result).toBe("https://github.com/org/repo.git"); + it.each([ + { + name: "trims leading and trailing whitespace", + input: " https://github.com/org/repo.git ", + expected: "https://github.com/org/repo.git", + }, + { + name: "strips trailing slashes", + input: "https://github.com/org/repo/", + expected: "https://github.com/org/repo", + }, + { + name: "strips both whitespace and trailing slashes together", + input: " https://github.com/org/repo/ ", + expected: "https://github.com/org/repo", + }, + ])("should normalise the URL: $name", ({ input, expected }) => { + expect(validateGitUrl(input)).toBe(expected); }); // Security: reject embedded credentials diff --git a/tests/unit/registry/publish.test.ts b/tests/unit/registry/publish.test.ts index 91b316b7..8cfefa99 100644 --- a/tests/unit/registry/publish.test.ts +++ b/tests/unit/registry/publish.test.ts @@ -223,42 +223,20 @@ describe("registry publish", () => { // ---------------------------------------------------------------- describe("semver version validation", () => { - it("should reject a version string that is not semver ('abc')", async () => { + it.each([ + { label: "a version string that is not semver", version: "abc", file: "bad-ver.json" }, + { label: "a version with only two parts", version: "1.2", file: "two-part.json" }, + { label: "a version with four parts", version: "1.2.3.4", file: "four-part.json" }, + ])("should reject $label ('$version')", async ({ version, file }) => { const regName = `reg-${randomUUID().slice(0, 8)}`; const { bareUrl } = createRegistryCache(tempDir, regName); addTestRegistry(makeEntry(regName, bareUrl)); - const packFile = join(tempDir, "bad-ver.json"); - writeFileSync(packFile, JSON.stringify(makePackJson("good-pack", "abc")), "utf-8"); + const packFile = join(tempDir, file); + writeFileSync(packFile, JSON.stringify(makePackJson("good-pack", version)), "utf-8"); await expect( - publishPack({ registryName: regName, packFilePath: packFile, version: "abc" }), - ).rejects.toThrow(/semver/i); - }); - - it("should reject a version with only two parts ('1.2')", async () => { - const regName = `reg-${randomUUID().slice(0, 8)}`; - const { bareUrl } = createRegistryCache(tempDir, regName); - addTestRegistry(makeEntry(regName, bareUrl)); - - const packFile = join(tempDir, "two-part.json"); - writeFileSync(packFile, JSON.stringify(makePackJson("good-pack", "1.2")), "utf-8"); - - await expect( - publishPack({ registryName: regName, packFilePath: packFile, version: "1.2" }), - ).rejects.toThrow(/semver/i); - }); - - it("should reject a version with four parts ('1.2.3.4')", async () => { - const regName = `reg-${randomUUID().slice(0, 8)}`; - const { bareUrl } = createRegistryCache(tempDir, regName); - addTestRegistry(makeEntry(regName, bareUrl)); - - const packFile = join(tempDir, "four-part.json"); - writeFileSync(packFile, JSON.stringify(makePackJson("good-pack", "1.2.3.4")), "utf-8"); - - await expect( - publishPack({ registryName: regName, packFilePath: packFile, version: "1.2.3.4" }), + publishPack({ registryName: regName, packFilePath: packFile, version }), ).rejects.toThrow(/semver/i); }); diff --git a/tests/unit/schema.test.ts b/tests/unit/schema.test.ts index 97865943..3b1cd2e8 100644 --- a/tests/unit/schema.test.ts +++ b/tests/unit/schema.test.ts @@ -131,22 +131,14 @@ describe("database schema", () => { runMigrations(db); }); - it("should throw DatabaseError for zero dimensions", () => { - expect(() => createVectorTable(db, 0)).toThrow(DatabaseError); - expect(() => createVectorTable(db, 0)).toThrow("Invalid vector dimensions"); - }); - - it("should throw DatabaseError for negative dimensions", () => { - expect(() => createVectorTable(db, -1)).toThrow(DatabaseError); - }); - - it("should throw DatabaseError for dimensions > 10000", () => { - expect(() => createVectorTable(db, 10001)).toThrow(DatabaseError); - expect(() => createVectorTable(db, 10001)).toThrow("Invalid vector dimensions"); - }); - - it("should throw DatabaseError for non-integer dimensions", () => { - expect(() => createVectorTable(db, 3.5)).toThrow(DatabaseError); + it.each([ + { label: "zero", dimensions: 0 }, + { label: "negative", dimensions: -1 }, + { label: "> 10000", dimensions: 10001 }, + { label: "non-integer", dimensions: 3.5 }, + ])("should throw DatabaseError for $label dimensions", ({ dimensions }) => { + expect(() => createVectorTable(db, dimensions)).toThrow(DatabaseError); + expect(() => createVectorTable(db, dimensions)).toThrow("Invalid vector dimensions"); }); it("should not throw for valid dimensions (vec0 may not be available)", () => { @@ -154,88 +146,4 @@ describe("database schema", () => { expect(() => createVectorTable(db, 384)).not.toThrow(); }); }); - - describe("createVectorTable", () => { - it("should throw on invalid dimensions (zero)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 0)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (negative)", () => { - runMigrations(db); - expect(() => createVectorTable(db, -1)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (too large)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 10001)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on non-integer dimensions", () => { - runMigrations(db); - expect(() => createVectorTable(db, 3.5)).toThrow("Invalid vector dimensions"); - }); - - it("should not throw on valid dimensions (vec0 will fail without sqlite-vec but catches)", () => { - runMigrations(db); - // This won't create a real vector table (no sqlite-vec) but should not throw - createVectorTable(db, 384); - }); - }); - - describe("createVectorTable", () => { - it("should throw on invalid dimensions (zero)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 0)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (negative)", () => { - runMigrations(db); - expect(() => createVectorTable(db, -1)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (too large)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 10001)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on non-integer dimensions", () => { - runMigrations(db); - expect(() => createVectorTable(db, 3.5)).toThrow("Invalid vector dimensions"); - }); - - it("should not throw on valid dimensions (vec0 will fail without sqlite-vec but catches)", () => { - runMigrations(db); - // This won't create a real vector table (no sqlite-vec) but should not throw - createVectorTable(db, 384); - }); - }); - - describe("createVectorTable", () => { - it("should throw on invalid dimensions (zero)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 0)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (negative)", () => { - runMigrations(db); - expect(() => createVectorTable(db, -1)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (too large)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 10001)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on non-integer dimensions", () => { - runMigrations(db); - expect(() => createVectorTable(db, 3.5)).toThrow("Invalid vector dimensions"); - }); - - it("should not throw on valid dimensions (vec0 will fail without sqlite-vec but catches)", () => { - runMigrations(db); - // This won't create a real vector table (no sqlite-vec) but should not throw - createVectorTable(db, 384); - }); - }); }); diff --git a/tests/unit/search.test.ts b/tests/unit/search.test.ts index f7350e7b..eee9c619 100644 --- a/tests/unit/search.test.ts +++ b/tests/unit/search.test.ts @@ -28,7 +28,7 @@ describe("searchDocuments (FTS5 fallback)", () => { const { results, totalCount } = await searchDocuments(db, provider, { query: "TypeScript" }); - expect(results.length).toBe(2); + expect(results).toHaveLength(2); expect(totalCount).toBe(2); expect(results[0].content).toContain("TypeScript"); expect(results[0].score).toBeGreaterThan(0); @@ -46,7 +46,7 @@ describe("searchDocuments (FTS5 fallback)", () => { library: "react", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].library).toBe("react"); }); @@ -65,7 +65,7 @@ describe("searchDocuments (FTS5 fallback)", () => { topic: "testing", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].topicId).toBe("testing"); }); @@ -97,7 +97,7 @@ describe("searchDocuments (FTS5 fallback)", () => { }); expect(page1.totalCount).toBe(3); - expect(page1.results.length).toBe(2); + expect(page1.results).toHaveLength(2); const page2 = await searchDocuments(db, provider, { query: "TypeScript", @@ -106,7 +106,7 @@ describe("searchDocuments (FTS5 fallback)", () => { }); expect(page2.totalCount).toBe(3); - expect(page2.results.length).toBe(1); + expect(page2.results).toHaveLength(1); }); it("should filter by dateFrom", async () => { @@ -121,7 +121,7 @@ describe("searchDocuments (FTS5 fallback)", () => { dateFrom: "2024-01-01T00:00:00.000Z", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].title).toBe("New Doc"); }); @@ -137,7 +137,7 @@ describe("searchDocuments (FTS5 fallback)", () => { dateTo: "2023-12-31T23:59:59.000Z", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].title).toBe("Old Doc"); }); @@ -157,7 +157,7 @@ describe("searchDocuments (FTS5 fallback)", () => { dateTo: "2024-12-31T23:59:59.000Z", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].title).toBe("Mid Doc"); }); @@ -173,7 +173,7 @@ describe("searchDocuments (FTS5 fallback)", () => { source: "library", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].sourceType).toBe("library"); }); @@ -195,7 +195,7 @@ describe("searchDocuments (FTS5 fallback)", () => { version: "18.0.0", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].version).toBe("18.0.0"); }); @@ -221,7 +221,7 @@ describe("searchDocuments (FTS5 fallback)", () => { minRating: 4, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].title).toBe("Good Doc"); }); @@ -239,7 +239,7 @@ describe("searchDocuments (FTS5 fallback)", () => { tags: ["tutorial"], }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].title).toBe("Tagged Doc"); }); @@ -252,7 +252,7 @@ describe("searchDocuments (FTS5 fallback)", () => { analyticsEnabled: false, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); // Verify no search log entry was created const logCount = db.prepare("SELECT COUNT(*) as cnt FROM search_log").get() as { cnt: number }; expect(logCount.cnt).toBe(0); @@ -337,7 +337,7 @@ describe("LIKE wildcard escaping in keyword search (issue #79)", () => { insertChunk(db, "c2", "doc2", "something completely different here"); const { results } = await searchDocuments(db, provider, { query: "100%" }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.content).toContain("100%"); }); @@ -349,7 +349,7 @@ describe("LIKE wildcard escaping in keyword search (issue #79)", () => { insertChunk(db, "c2", "doc2", "the username field is optional"); const { results } = await searchDocuments(db, provider, { query: "user_name" }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.content).toContain("user_name"); }); }); @@ -406,7 +406,7 @@ describe("search result scoring explanation (issue #89)", () => { const { results } = await searchDocuments(db, provider, { query: "consistency" }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); const result = results[0]!; // FTS5: score = -rawScore (BM25 rank is negative) expect(result.score).toBe(-result.scoreExplanation.rawScore); @@ -454,7 +454,7 @@ describe("LIKE fallback with filters", () => { version: "18.0.0", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.version).toBe("18.0.0"); }); @@ -482,7 +482,7 @@ describe("LIKE fallback with filters", () => { minRating: 4, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.title).toBe("Rated Doc"); }); @@ -500,7 +500,7 @@ describe("LIKE fallback with filters", () => { dateFrom: "2024-01-01T00:00:00.000Z", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.title).toBe("New Doc"); }); @@ -518,7 +518,7 @@ describe("LIKE fallback with filters", () => { dateTo: "2023-12-31T23:59:59.000Z", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.title).toBe("Old Doc"); }); @@ -536,7 +536,7 @@ describe("LIKE fallback with filters", () => { source: "library", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.sourceType).toBe("library"); }); @@ -556,7 +556,7 @@ describe("LIKE fallback with filters", () => { tags: ["howto"], }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.title).toBe("Tagged Doc"); }); @@ -583,7 +583,7 @@ describe("LIKE fallback with filters", () => { maxChunksPerDocument: 1, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); }); it("should attach context chunks in LIKE fallback", async () => { @@ -599,11 +599,11 @@ describe("LIKE fallback with filters", () => { contextChunks: 1, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.contextBefore).toBeDefined(); - expect(results[0]!.contextBefore!.length).toBe(1); + expect(results[0]!.contextBefore!).toHaveLength(1); expect(results[0]!.contextAfter).toBeDefined(); - expect(results[0]!.contextAfter!.length).toBe(1); + expect(results[0]!.contextAfter!).toHaveLength(1); }); }); @@ -628,7 +628,7 @@ describe("deduplicate search results by document (issue #245)", () => { const { results } = await searchDocuments(db, provider, { query: "TypeScript" }); - expect(results.length).toBe(3); + expect(results).toHaveLength(3); expect(results.every((r) => r.documentId === "doc1")).toBe(true); }); @@ -650,8 +650,8 @@ describe("deduplicate search results by document (issue #245)", () => { const doc1Results = results.filter((r) => r.documentId === "doc1"); const doc2Results = results.filter((r) => r.documentId === "doc2"); - expect(doc1Results.length).toBe(1); - expect(doc2Results.length).toBe(1); + expect(doc1Results).toHaveLength(1); + expect(doc2Results).toHaveLength(1); }); it("should limit to at most 2 chunks per document with maxChunksPerDocument=2", async () => { @@ -665,7 +665,7 @@ describe("deduplicate search results by document (issue #245)", () => { maxChunksPerDocument: 2, }); - expect(results.length).toBe(2); + expect(results).toHaveLength(2); expect(results.every((r) => r.documentId === "doc1")).toBe(true); }); }); @@ -695,14 +695,14 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 1, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); const result = results[0]!; expect(result.content).toContain("Core concepts"); expect(result.contextBefore).toBeDefined(); - expect(result.contextBefore!.length).toBe(1); + expect(result.contextBefore!).toHaveLength(1); expect(result.contextBefore![0]!.content).toContain("Introduction"); expect(result.contextAfter).toBeDefined(); - expect(result.contextAfter!.length).toBe(1); + expect(result.contextAfter!).toHaveLength(1); expect(result.contextAfter![0]!.content).toContain("Advanced patterns"); }); @@ -719,12 +719,12 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 2, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); const result = results[0]!; - expect(result.contextBefore!.length).toBe(2); + expect(result.contextBefore!).toHaveLength(2); expect(result.contextBefore![0]!.content).toContain("Section zero"); expect(result.contextBefore![1]!.content).toContain("Section one"); - expect(result.contextAfter!.length).toBe(2); + expect(result.contextAfter!).toHaveLength(2); expect(result.contextAfter![0]!.content).toContain("Section three"); expect(result.contextAfter![1]!.content).toContain("Section four"); }); @@ -739,9 +739,9 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 1, }); - expect(results.length).toBe(1); - expect(results[0]!.contextBefore!.length).toBe(0); - expect(results[0]!.contextAfter!.length).toBe(1); + expect(results).toHaveLength(1); + expect(results[0]!.contextBefore!).toHaveLength(0); + expect(results[0]!.contextAfter!).toHaveLength(1); }); it("should handle last chunk with no context after", async () => { @@ -754,9 +754,9 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 1, }); - expect(results.length).toBe(1); - expect(results[0]!.contextBefore!.length).toBe(1); - expect(results[0]!.contextAfter!.length).toBe(0); + expect(results).toHaveLength(1); + expect(results[0]!.contextBefore!).toHaveLength(1); + expect(results[0]!.contextAfter!).toHaveLength(0); }); it("should not include context when contextChunks is 0 or unset", async () => { @@ -769,7 +769,7 @@ describe("context chunk expansion (issue #247)", () => { query: "keyword", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.contextBefore).toBeUndefined(); expect(results[0]!.contextAfter).toBeUndefined(); }); @@ -789,10 +789,10 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 5, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); // Capped at 2 - expect(results[0]!.contextBefore!.length).toBe(2); - expect(results[0]!.contextAfter!.length).toBe(2); + expect(results[0]!.contextBefore!).toHaveLength(2); + expect(results[0]!.contextAfter!).toHaveLength(2); }); it("should include chunkIndex in context chunks", async () => { @@ -806,7 +806,7 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 1, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.contextBefore![0]!.chunkIndex).toBe(0); expect(results[0]!.contextAfter![0]!.chunkIndex).toBe(2); }); @@ -892,7 +892,7 @@ describe("title boosting (issue #362)", () => { }); // Both match the keyword, but doc1's title matches the query - expect(results.length).toBe(2); + expect(results).toHaveLength(2); const doc1Result = results.find((r) => r.documentId === "doc1"); const doc2Result = results.find((r) => r.documentId === "doc2"); expect(doc1Result).toBeDefined(); @@ -912,8 +912,8 @@ describe("title boosting (issue #362)", () => { query: "TypeScript", }); - expect(results.length).toBe(1); - expect(results[0]!.scoreExplanation.boostFactors.length).toBe(0); + expect(results).toHaveLength(1); + expect(results[0]!.scoreExplanation.boostFactors).toHaveLength(0); }); }); @@ -959,7 +959,7 @@ describe("lazy count optimization (issue #362)", () => { }); expect(page1.totalCount).toBeGreaterThanOrEqual(5); - expect(page1.results.length).toBe(2); + expect(page1.results).toHaveLength(2); }); }); diff --git a/tests/unit/spider.test.ts b/tests/unit/spider.test.ts index 2d93fb69..f64c5aaa 100644 --- a/tests/unit/spider.test.ts +++ b/tests/unit/spider.test.ts @@ -177,8 +177,8 @@ describe("spiderUrl", () => { // Should only visit a and b once each const urls = pages.map((p) => p.url); - expect(urls.filter((u) => u === "https://example.com/a").length).toBe(1); - expect(urls.filter((u) => u === "https://example.com/b").length).toBe(1); + expect(urls.filter((u) => u === "https://example.com/a")).toHaveLength(1); + expect(urls.filter((u) => u === "https://example.com/b")).toHaveLength(1); }); it("filters cross-domain links when sameDomain=true (default)", async () => { @@ -437,7 +437,7 @@ describe("spiderUrl", () => { const gen = spiderUrl("https://example.com/", { maxDepth: 0, maxPages: 10, requestDelay: 0 }); const { pages } = await collectPages(gen); - expect(pages.length).toBe(1); + expect(pages).toHaveLength(1); expect(pages[0]!.url).toBe("https://example.com/"); }); @@ -488,7 +488,7 @@ describe("spiderUrl", () => { const gen = spiderUrl("https://example.com/notes.txt", { maxDepth: 0, requestDelay: 0 }); const { pages } = await collectPages(gen); - expect(pages.length).toBe(1); + expect(pages).toHaveLength(1); expect(pages[0]!.title).toBe("Plain Text"); }); diff --git a/tests/unit/topics.test.ts b/tests/unit/topics.test.ts index f5590a4f..22717028 100644 --- a/tests/unit/topics.test.ts +++ b/tests/unit/topics.test.ts @@ -70,7 +70,7 @@ describe("topics", () => { createTopic(db, { name: "API" }); const topics = listTopics(db); - expect(topics.length).toBe(3); + expect(topics).toHaveLength(3); // Should be alphabetical expect(topics[0]!.name).toBe("API"); expect(topics[1]!.name).toBe("Auth"); @@ -84,12 +84,12 @@ describe("topics", () => { createTopic(db, { name: "Unrelated" }); const children = listTopics(db, parent.id); - expect(children.length).toBe(2); + expect(children).toHaveLength(2); }); it("should return empty array when no topics exist", () => { const topics = listTopics(db); - expect(topics.length).toBe(0); + expect(topics).toHaveLength(0); }); }); @@ -217,7 +217,7 @@ describe("topics", () => { ).run("d2", "topic", "Doc B", "content B", "docs", "manual"); const docs = getDocumentsByTopic(db, "docs"); - expect(docs.length).toBe(2); + expect(docs).toHaveLength(2); expect(docs[0]!.title).toBeDefined(); }); @@ -230,19 +230,19 @@ describe("topics", () => { } const page1 = getDocumentsByTopic(db, "paged", { limit: 2, offset: 0 }); - expect(page1.length).toBe(2); + expect(page1).toHaveLength(2); const page2 = getDocumentsByTopic(db, "paged", { limit: 2, offset: 2 }); - expect(page2.length).toBe(2); + expect(page2).toHaveLength(2); const page3 = getDocumentsByTopic(db, "paged", { limit: 2, offset: 4 }); - expect(page3.length).toBe(1); + expect(page3).toHaveLength(1); }); it("should return empty array for topic with no documents", () => { createTopic(db, { name: "Empty" }); const docs = getDocumentsByTopic(db, "empty"); - expect(docs.length).toBe(0); + expect(docs).toHaveLength(0); }); it("should throw TopicNotFoundError for nonexistent topic", () => { @@ -262,7 +262,7 @@ describe("topics", () => { ).run("s2", "topic", "Doc 2", "content", "alpha", "manual"); const stats = getTopicStats(db); - expect(stats.length).toBe(2); + expect(stats).toHaveLength(2); const alpha = stats.find((s) => s.id === "alpha"); const beta = stats.find((s) => s.id === "beta"); expect(alpha!.documentCount).toBe(2); @@ -271,7 +271,7 @@ describe("topics", () => { it("should return empty array when no topics exist", () => { const stats = getTopicStats(db); - expect(stats.length).toBe(0); + expect(stats).toHaveLength(0); }); }); }); diff --git a/tests/unit/web-server.test.ts b/tests/unit/web-server.test.ts index b9d48344..703a1d50 100644 --- a/tests/unit/web-server.test.ts +++ b/tests/unit/web-server.test.ts @@ -71,7 +71,7 @@ describe("web server", () => { expect(status).toBe(200); const docs = body as unknown as Array<{ title: string }>; expect(Array.isArray(docs)).toBe(true); - expect(docs.length).toBe(2); + expect(docs).toHaveLength(2); expect(docs[0]!.title).toBeDefined(); }); @@ -93,7 +93,7 @@ describe("web server", () => { expect(status).toBe(200); const topics = body as unknown as Array<{ id: string; documentCount: number }>; expect(Array.isArray(topics)).toBe(true); - expect(topics.length).toBe(1); + expect(topics).toHaveLength(1); expect(topics[0]!.id).toBe("ts"); expect(topics[0]!.documentCount).toBe(1); }); From 4767beea03e1efdc7993027389f61425ad3883b6 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 17:36:26 +0000 Subject: [PATCH 77/83] ci: install npm packages with --ignore-scripts and rebuild only native addons Lint/typecheck, the npm publish job and the Docker builder stage need no native addon. The test job rebuilds better-sqlite3 and sharp (loaded by @xenova/transformers in the neural retrieval suite); the build job rebuilds better-sqlite3 for docs:check. Sort the apt package names in the Dockerfile. --- .github/workflows/ci.yml | 12 +++++++++--- .github/workflows/release-please.yml | 3 ++- Dockerfile | 7 ++++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fee8f0c..fa5a339b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,8 @@ jobs: with: node-version: 22 cache: npm - - run: npm ci + # Lint, format and typecheck load no native addon. + - run: npm ci --ignore-scripts - run: npm run lint - run: npm run format:check - run: npm run typecheck @@ -36,7 +37,10 @@ jobs: with: node-version: ${{ matrix.node-version }} cache: npm - - run: npm ci + - run: npm ci --ignore-scripts + # Build only the native addons the tests load: better-sqlite3, and sharp + # (@xenova/transformers loads it in the neural retrieval-quality suite). + - run: npm rebuild better-sqlite3 sharp - run: npm run test:coverage - name: Upload coverage if: matrix.node-version == 22 @@ -65,7 +69,9 @@ jobs: with: node-version: 22 cache: npm - - run: npm ci + - run: npm ci --ignore-scripts + # docs:check opens an in-memory better-sqlite3 database. + - run: npm rebuild better-sqlite3 - run: npm run build - name: Verify dist output run: | diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index af8c40f2..94969281 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -36,7 +36,8 @@ jobs: node-version: 22 cache: npm registry-url: "https://registry.npmjs.org" - - run: npm ci + # tsc and npm publish load no native addon. + - run: npm ci --ignore-scripts - run: npm run build - run: npm publish --provenance --access public env: diff --git a/Dockerfile b/Dockerfile index e4ed9f77..ebb0dd66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ FROM node:22-bookworm-slim AS builder WORKDIR /app COPY package*.json ./ -RUN npm ci +# tsc needs no native addon; the production stage builds them. +RUN npm ci --ignore-scripts COPY . . RUN npm run build @@ -16,11 +17,11 @@ RUN groupadd -g 1001 libscope && \ mkdir /data && chown libscope:libscope /data COPY package*.json ./ RUN apt-get update && \ - apt-get install -y --no-install-recommends python3 make g++ && \ + apt-get install -y --no-install-recommends g++ make python3 && \ npm ci --omit=dev --ignore-scripts && \ npm rebuild better-sqlite3 sharp && \ npm cache clean --force && \ - apt-get purge -y python3 make g++ && \ + apt-get purge -y g++ make python3 && \ apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* COPY --from=builder /app/dist ./dist From e76a94306579075a92aa6ee4fee4c5aa60dbd002 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 17:36:30 +0000 Subject: [PATCH 78/83] refactor(scripts): split gen-docs main into per-file helpers Format the blocks of a file and the four doc files with Promise.all instead of awaiting in a loop, and use String.raw for the escaped pipe. Output, problems and exit code are unchanged. --- scripts/gen-docs.mjs | 119 ++++++++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 47 deletions(-) diff --git a/scripts/gen-docs.mjs b/scripts/gen-docs.mjs index 90fb1d02..c8715f4d 100644 --- a/scripts/gen-docs.mjs +++ b/scripts/gen-docs.mjs @@ -39,12 +39,16 @@ const DOCS = { // --------------------------------------------------------------------------------------------- // Markdown helpers +/** A `|` escaped for a table cell, and the same between two alternatives. */ +const PIPE = String.raw`\|`; +const OR = ` ${PIPE} `; + /** Text for a table cell or a paragraph: no HTML or Vue interpolation, no broken tables. */ function text(value) { return String(value ?? "") .replaceAll(homedir(), "~") .replaceAll("\n", " ") - .replaceAll("|", "\\|") + .replaceAll("|", PIPE) .replaceAll("<", "<") .replaceAll(">", ">") .replaceAll("{{", "{ {"); @@ -52,7 +56,7 @@ function text(value) { /** Inline code; the content keeps `<` and `|` (escaped for tables). */ function code(value) { - return `\`${String(value).replaceAll(homedir(), "~").replaceAll("|", "\\|")}\``; + return `\`${String(value).replaceAll(homedir(), "~").replaceAll("|", PIPE)}\``; } /** Text that ends with a period when `withPeriod` (when more text follows in the cell). */ @@ -150,8 +154,8 @@ async function cliBlocks() { function schemaType(s) { if (!s || typeof s !== "object") return "any"; - if (s.enum) return s.enum.map((v) => code(v)).join(" \\| "); - if (s.anyOf) return s.anyOf.map(schemaType).join(" \\| "); + if (s.enum) return s.enum.map((v) => code(v)).join(OR); + if (s.anyOf) return s.anyOf.map(schemaType).join(OR); if (s.type === "array") return `${schemaType(s.items)}[]`; if (s.type === "string" && s.format) return `string (${s.format})`; return s.type ?? "any"; @@ -245,7 +249,7 @@ function configType(type) { const list = type.startsWith("list of "); const values = list ? type.slice("list of ".length) : type; if (!list && !values.includes(" | ")) return values; - const choices = values.split(" | ").map(code).join(" \\| "); + const choices = values.split(" | ").map(code).join(OR); return list ? `list of ${choices}` : choices; } @@ -339,6 +343,60 @@ async function formatBlock(content, file) { return format(content, { ...options, parser: "markdown", filepath: file }); } +const REQUIRED_BLOCKS = { + cli: ["cli:global"], + mcp: ["mcp:core", "mcp:admin"], + config: ["config:keys"], + rest: ["rest:routes"], +}; + +/** The generated text of a block, with blank lines around it as prettier writes them. */ +async function blockBody(generator, name, file) { + return `\n${await formatBlock(generator.render(name), file)}\n`; +} + +/** Each top-level command needs a `cli:` block. */ +function uncoveredCommands(relPath, seen, commands) { + const covered = new Set( + [...seen].filter((n) => n !== "cli:global").flatMap((n) => n.slice(4).split(",")), + ); + return commands + .filter((command) => !covered.has(command)) + .map((command) => `${relPath}: no block documents "libscope ${command}"`); +} + +/** Regenerate the blocks of one doc file: its original and updated text, and its problems. */ +async function updateFile(kind, relPath, generator, check) { + const file = join(ROOT, relPath); + const original = readFileSync(file, "utf8"); + const matches = [...original.matchAll(BLOCK)]; + const bodies = await Promise.all( + matches.map(([, name]) => + name.split(":")[0] === kind ? blockBody(generator, name, file) : null, + ), + ); + const problems = []; + const seen = new Set(); + let updated = original; + matches.forEach(([whole, name, current], i) => { + const body = bodies[i]; + if (body === null) { + problems.push(`${relPath}: block "${name}" does not belong in this file`); + return; + } + seen.add(name); + if (current === body) return; + if (check) problems.push(`${relPath}: block "${name}" is out of date`); + const block = `\n${body}`; + updated = updated.replace(whole, () => block); + }); + for (const name of REQUIRED_BLOCKS[kind]) { + if (!seen.has(name)) problems.push(`${relPath}: missing block "${name}"`); + } + if (kind === "cli") problems.push(...uncoveredCommands(relPath, seen, generator.commands)); + return { file, original, updated, problems }; +} + async function main() { const check = process.argv.includes("--check"); const generators = { @@ -347,53 +405,20 @@ async function main() { config: await configBlocks(), rest: await restBlocks(), }; - const problems = []; - const required = { - cli: ["cli:global"], - mcp: ["mcp:core", "mcp:admin"], - config: ["config:keys"], - rest: ["rest:routes"], - }; - - for (const [kind, relPath] of Object.entries(DOCS)) { - const file = join(ROOT, relPath); - const original = readFileSync(file, "utf8"); - const seen = new Set(); - let updated = original; - for (const match of original.matchAll(BLOCK)) { - const [whole, name, current] = match; - const prefix = name.split(":")[0]; - if (prefix !== kind) { - problems.push(`${relPath}: block "${name}" does not belong in this file`); - continue; - } - seen.add(name); - // Blank lines around the markers, as prettier writes them. - const body = `\n${await formatBlock(generators[kind].render(name), file)}\n`; - if (current !== body) { - if (check) problems.push(`${relPath}: block "${name}" is out of date`); - const block = `\n${body}`; - updated = updated.replace(whole, () => block); - } - } - for (const name of required[kind]) { - if (!seen.has(name)) problems.push(`${relPath}: missing block "${name}"`); - } - if (kind === "cli") { - const covered = new Set( - [...seen].filter((n) => n !== "cli:global").flatMap((n) => n.slice(4).split(",")), - ); - for (const command of generators.cli.commands) { - if (!covered.has(command)) - problems.push(`${relPath}: no block documents "libscope ${command}"`); - } - } - if (!check && updated !== original) { + const results = await Promise.all( + Object.entries(DOCS).map(([kind, relPath]) => + updateFile(kind, relPath, generators[kind], check), + ), + ); + if (!check) { + for (const { file, original, updated } of results) { + if (updated === original) continue; writeFileSync(file, updated); console.log(`updated ${relative(ROOT, file)}`); } } + const problems = results.flatMap((r) => r.problems); if (problems.length > 0) { for (const p of problems) console.error(`✗ ${p}`); if (check) console.error("Run `npm run build && npm run docs:gen` and commit the result."); From 8ca1700ec6a51e4b28f237035566be234f1cf3bb Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 17:36:32 +0000 Subject: [PATCH 79/83] refactor(sdk/go): use a constant for the documents path --- sdk/go/documents.go | 13 ++++++++----- sdk/go/tags.go | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/sdk/go/documents.go b/sdk/go/documents.go index 72883783..2b55eb64 100644 --- a/sdk/go/documents.go +++ b/sdk/go/documents.go @@ -5,32 +5,35 @@ import ( "net/http" ) +// documentsPath is the REST collection of documents. +const documentsPath = "/documents" + // AddText starts adding a document from text and returns the background task. Params: // WithTopic, WithLibrary, WithVersion, WithSourceType, WithTags, WithURL. // Use WaitForTask and Task.DecodeResult (*IngestResult) to get the new document. func (c *Client) AddText(ctx context.Context, title, content string, params ...Param) (*Task, error) { body := fields(map[string]any{"title": title, "content": content}, params) - return c.startTask(ctx, "/documents", body) + return c.startTask(ctx, documentsPath, body) } // AddURL starts adding a web page, a crawled site (WithSpider, WithMaxPages) or a public // GitHub/GitLab repository, and returns the background task. func (c *Client) AddURL(ctx context.Context, url string, params ...Param) (*Task, error) { - return c.startTask(ctx, "/documents", fields(map[string]any{"url": url}, params)) + return c.startTask(ctx, documentsPath, fields(map[string]any{"url": url}, params)) } // GetDocument returns a document with its content, tags, links and rating summary. func (c *Client) GetDocument(ctx context.Context, documentID string) (*DocumentView, error) { - return get[DocumentView](ctx, c, "/documents"+segment(documentID), nil) + return get[DocumentView](ctx, c, documentsPath+segment(documentID), nil) } // ListDocuments lists documents, newest first. Params: WithLimit, WithOffset and the filters // WithTopic, WithLibrary, WithVersion, WithSourceType, WithTags. func (c *Client) ListDocuments(ctx context.Context, params ...Param) (*Page[Document], error) { - return get[Page[Document]](ctx, c, "/documents", query(fields(map[string]any{}, params))) + return get[Page[Document]](ctx, c, documentsPath, query(fields(map[string]any{}, params))) } // DeleteDocument deletes a document. func (c *Client) DeleteDocument(ctx context.Context, documentID string) error { - return c.call(ctx, http.MethodDelete, "/documents"+segment(documentID), nil, nil, nil) + return c.call(ctx, http.MethodDelete, documentsPath+segment(documentID), nil, nil, nil) } diff --git a/sdk/go/tags.go b/sdk/go/tags.go index 81fb1058..5d338be4 100644 --- a/sdk/go/tags.go +++ b/sdk/go/tags.go @@ -21,7 +21,7 @@ func (c *Client) ListTags(ctx context.Context) ([]Tag, error) { // AddTags adds tags to a document and returns the document's tags. func (c *Client) AddTags(ctx context.Context, documentID string, tags []string) ([]string, error) { - path := "/documents" + segment(documentID) + "/tags" + path := documentsPath + segment(documentID) + "/tags" out, err := send[documentTags](ctx, c, http.MethodPost, path, documentTags{Tags: tags}) if err != nil { return nil, err @@ -32,7 +32,7 @@ func (c *Client) AddTags(ctx context.Context, documentID string, tags []string) // RemoveTags removes tags from a document and returns the document's remaining tags. func (c *Client) RemoveTags(ctx context.Context, documentID string, tags []string) ([]string, error) { var out documentTags - path := "/documents" + segment(documentID) + "/tags" + path := documentsPath + segment(documentID) + "/tags" if err := c.call(ctx, http.MethodDelete, path, url.Values{"tags": tags}, nil, &out); err != nil { return nil, err } From 890dbc44db82eece2851685e299e0837755cb407 Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 17:36:34 +0000 Subject: [PATCH 80/83] refactor(sdk/python): let callers bound the async wait_for_task AsyncLibscopeClient.wait_for_task no longer takes timeout; wrap it in asyncio.wait_for or asyncio.timeout. The synchronous client keeps timeout. Use a constant for the /tags path. --- docs/migration-v2.md | 2 ++ sdk/python/README.md | 11 +++++++++-- sdk/python/src/pylibscope/client.py | 17 ++++++++--------- sdk/python/tests/test_client.py | 9 +++++++++ 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/docs/migration-v2.md b/docs/migration-v2.md index a1a9646c..1eb2edac 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -475,6 +475,8 @@ The Python (`pylibscope`) and Go (`sdk/go`) clients now call the 2.0 `/api/v1` r | `pylibscope.connectors.build_connector_config` | Removed | | — | New: `get_task`, `cancel_task`, `wait_for_task`, `TaskFailedError`, `Tag`, `Overview`, `DocumentView`, `Page` | +`AsyncLibscopeClient.wait_for_task(task_id, *, interval)` has no `timeout` parameter. Set the limit around the call: `await asyncio.wait_for(client.wait_for_task(task.id), timeout=300)`, or `async with asyncio.timeout(300):` on Python 3.11 and later. The synchronous `LibscopeClient.wait_for_task` keeps `timeout`. + ### Go (`github.com/RobertLD/libscope/sdk/go`) | 1.x | 2.0 | diff --git a/sdk/python/README.md b/sdk/python/README.md index e7e18f16..86640d5b 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -57,7 +57,14 @@ async def main(): asyncio.run(main()) ``` -`AsyncLibscopeClient` has the same methods as `LibscopeClient`. +`AsyncLibscopeClient` has the same methods as `LibscopeClient`, except that its `wait_for_task` has no `timeout` parameter. Set the time limit around the call: + +```python +task = await client.add_url("https://example.com/docs") +done = await asyncio.wait_for(client.wait_for_task(task.id), timeout=300) +``` + +On Python 3.11 and later, `async with asyncio.timeout(300):` also works. At the limit, `asyncio.wait_for` raises `asyncio.TimeoutError` (the same class as `TimeoutError` on Python 3.11 and later). ## Configuration @@ -71,7 +78,7 @@ client = LibscopeClient( ## Background tasks -`add_text`, `add_url`, `sync` and `sync_all` return a `Task` at once (the server answers `202`). Call `wait_for_task(task.id, timeout=300, interval=1.0)` to poll `GET /api/v1/tasks/:taskId` until the task finishes. It returns the task, with `result` holding the operation result. It raises `TaskFailedError` when the task failed or was cancelled, and `TimeoutError` after `timeout` seconds. `get_task` and `cancel_task` are also available. +`add_text`, `add_url`, `sync` and `sync_all` return a `Task` at once (the server answers `202`). Call `wait_for_task(task.id, timeout=300, interval=1.0)` to poll `GET /api/v1/tasks/:taskId` until the task finishes. It returns the task, with `result` holding the operation result. It raises `TaskFailedError` when the task failed or was cancelled, and `TimeoutError` after `timeout` seconds. The async `wait_for_task(task.id, interval=1.0)` has no `timeout` (see [Async Usage](#async-usage)). `get_task` and `cancel_task` are also available. ## API Reference diff --git a/sdk/python/src/pylibscope/client.py b/sdk/python/src/pylibscope/client.py index 1c1ca155..c28968cf 100644 --- a/sdk/python/src/pylibscope/client.py +++ b/sdk/python/src/pylibscope/client.py @@ -36,6 +36,7 @@ ) _API = "/api/v1" +_TAGS = "/tags" DEFAULT_BASE_URL = "http://localhost:3378" @@ -154,17 +155,17 @@ def create_topic(name: str, parent: Optional[str], description: Optional[str]) - @staticmethod def add_tags(document_id: str, tags: List[str]) -> _Call: - path = _doc(document_id, "/tags") + path = _doc(document_id, _TAGS) return _Call("POST", path, lambda d: list(d["tags"]), json={"tags": tags}) @staticmethod def remove_tags(document_id: str, tags: List[str]) -> _Call: - path = _doc(document_id, "/tags") + path = _doc(document_id, _TAGS) return _Call("DELETE", path, lambda d: list(d["tags"]), params={"tags": tags}) @staticmethod def list_tags() -> _Call: - return _Call("GET", "/tags", _items(Tag)) + return _Call("GET", _TAGS, _items(Tag)) @staticmethod def graph( @@ -573,14 +574,12 @@ async def get_task(self, task_id: str) -> Task: async def cancel_task(self, task_id: str) -> bool: return await self._run(_Calls.cancel_task(task_id)) - async def wait_for_task( - self, task_id: str, *, timeout: float = 300.0, interval: float = 1.0 - ) -> Task: - deadline = time.monotonic() + timeout + async def wait_for_task(self, task_id: str, *, interval: float = 1.0) -> Task: + """Poll a task until it finishes. Raises TaskFailedError when it failed or was + cancelled. It has no time limit: set one with ``asyncio.wait_for(..., timeout)``, or + ``async with asyncio.timeout(...)`` on Python 3.11+.""" while True: task = await self.get_task(task_id) if task.done: return _finished(task) - if time.monotonic() >= deadline: - raise TimeoutError(f"Task {task_id} did not finish in {timeout} s") await asyncio.sleep(interval) diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index aa45fd99..ace308c7 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -1,5 +1,6 @@ """Tests for the libscope sync and async clients (HTTP mocked with respx).""" +import asyncio import json import httpx @@ -304,6 +305,14 @@ async def test_add_wait_search(self): assert done.result["kind"] == "content" assert page.items[0].title == "Doc 1" + @pytest.mark.asyncio + @respx.mock + async def test_wait_for_task_is_bounded_by_the_caller(self): + respx.get(f"{API}/tasks/t1").mock(return_value=ok(task("running"))) + async with AsyncLibscopeClient() as client: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(client.wait_for_task("t1", interval=0), timeout=0.05) + @pytest.mark.asyncio @respx.mock async def test_errors(self): From fcf62a6ee0238fd581658eb9b57dfa1a824dbcee Mon Sep 17 00:00:00 2001 From: Robert DeRienzo Date: Thu, 8 Oct 2026 17:59:32 +0000 Subject: [PATCH 81/83] refactor: fix SonarCloud issues in src - Replace await-in-loop (S9382) with shared helpers in src/utils/async.ts (forEachSequential, mapSequential, runConcurrent) or recursion for pagination, retries, polling and crawl loops; order is kept everywhere. - Rewrite super-linear regexes (S8786) as linear scanners with the same matches: Obsidian wikilinks, first markdown heading, trailing slashes and embedded credentials in registry URLs. - Split connectSettings, syncConfluenceSpace and globToRegExp (S3776). - Remove nested ternaries and nested template literals (S3358, S4624). - Use top-level await in the CLI and MCP entry points (S7785); the CLI still runs only when it is the process entry point. - Combine Array#push calls, use Number directly, a Set for watch extensions, replaceAll, a readonly scheduler map and an async wrapper instead of a trivial promise. --- src/cli/commands/add.ts | 15 +-- src/cli/commands/admin.ts | 11 +- src/cli/commands/connectors.ts | 134 +++++++++++-------- src/cli/commands/docs.ts | 15 +-- src/cli/commands/pack.ts | 8 +- src/cli/commands/registry.ts | 4 +- src/cli/commands/search.ts | 22 ++-- src/cli/commands/searches.ts | 14 +- src/cli/commands/serve.ts | 6 +- src/cli/index.ts | 2 +- src/cli/options.ts | 7 +- src/cli/repl.ts | 46 ++++--- src/cli/run.ts | 5 + src/connectors/confluence.ts | 53 ++++---- src/connectors/docs.ts | 111 ++++++++++------ src/connectors/http-utils.ts | 12 +- src/connectors/notion.ts | 22 ++-- src/connectors/obsidian-wikilinks.ts | 107 ++++++++++++++++ src/connectors/obsidian.ts | 28 ++-- src/connectors/onenote.ts | 99 ++++++++------ src/connectors/slack.ts | 185 +++++++++++++++------------ src/core/ingest.ts | 21 +-- src/core/operations/connectors.ts | 14 +- src/core/packs.ts | 86 ++++++++----- src/core/reindex.ts | 110 ++++++++++------ src/core/repo.ts | 84 +++++++----- src/core/scheduler.ts | 2 +- src/core/spider.ts | 114 ++++++++++++----- src/core/tasks.ts | 6 +- src/core/url-fetcher.ts | 28 ++-- src/core/watcher.ts | 4 +- src/mcp/format.ts | 36 ++++-- src/mcp/main.ts | 6 +- src/providers/local.ts | 7 +- src/registry/config.ts | 29 ++++- src/registry/sync.ts | 21 +-- src/utils/async.ts | 55 ++++++++ src/utils/glob.ts | 48 +++---- src/utils/markdown.ts | 49 +++++++ src/utils/strings.ts | 8 ++ 40 files changed, 1041 insertions(+), 593 deletions(-) create mode 100644 src/connectors/obsidian-wikilinks.ts create mode 100644 src/utils/async.ts create mode 100644 src/utils/markdown.ts create mode 100644 src/utils/strings.ts diff --git a/src/cli/commands/add.ts b/src/cli/commands/add.ts index f562f6f0..1bea84b8 100644 --- a/src/cli/commands/add.ts +++ b/src/cli/commands/add.ts @@ -7,13 +7,13 @@ import { resolveTopicId } from "../../core/topics.js"; import { FileWatcher } from "../../core/watcher.js"; import type { SourceType } from "../../core/indexing.js"; import { ValidationError } from "../../errors.js"; +import { mapSequential } from "../../utils/async.js"; import { getContext, untilInterrupted } from "../context.js"; import { addDocumentOptions, defined, documentInput, splitList, - toNumber, type DocumentFlags, } from "../options.js"; import { call, plural, print } from "../run.js"; @@ -138,8 +138,8 @@ export function register(program: Command): void { .option("--exclude ", "Directory: skip files matching these globs (comma-separated)") .option("--watch", "Directory: keep running and re-index files when they change") .option("--spider", "URL: also crawl linked pages") - .option("--max-pages ", "Crawl: page limit (default 25, max 200)", toNumber) - .option("--max-depth ", "Crawl: link depth (default 2, max 5)", toNumber) + .option("--max-pages ", "Crawl: page limit (default 25, max 200)", Number) + .option("--max-depth ", "Crawl: link depth (default 2, max 5)", Number) .option("--no-same-domain", "Crawl: also follow links to other domains") .option("--path-prefix ", "Crawl: only follow links under this path") .option("--exclude-urls ", "Crawl: skip URLs matching these globs (comma-separated)") @@ -153,12 +153,9 @@ export function register(program: Command): void { if (flags.watch && directories.length !== sources.length) { throw new ValidationError("--watch works with directories only"); } - const results: IngestResult[] = []; - for (const source of sources) { - results.push( - await call(addOperation, addInput(source, flags), { embeddings: !flags.dryRun }), - ); - } + const results: IngestResult[] = await mapSequential(sources, (source) => + call(addOperation, addInput(source, flags), { embeddings: !flags.dryRun }), + ); print(results, (all) => all.forEach(printResult)); if (results.every((r) => r.documents.length === 0 && r.errors.length > 0)) { process.exitCode = 1; diff --git a/src/cli/commands/admin.ts b/src/cli/commands/admin.ts index b1b8ba74..37d423b3 100644 --- a/src/cli/commands/admin.ts +++ b/src/cli/commands/admin.ts @@ -14,7 +14,7 @@ import { import type { Overview } from "../../core/overview.js"; import { describeEmbeddingIdentity } from "../../db/index-meta.js"; import { confirmOrCancel } from "../confirm.js"; -import { defined, toNumber } from "../options.js"; +import { defined } from "../options.js"; import { call, plural, print, printList, run } from "../run.js"; /** Counts, topics, packs and index identity (shared with `doctor`). */ @@ -28,7 +28,8 @@ export function printOverview(o: Overview): void { const stored = describeEmbeddingIdentity(o.index.stored); console.log(`Index: ${o.index.vectorTableDimensions === undefined ? "none" : stored}`); if (o.packs.length > 0) { - console.log(`Packs: ${o.packs.map((p) => `${p.name} v${p.version}`).join(", ")}`); + const packs = o.packs.map((p) => `${p.name} v${p.version}`).join(", "); + console.log(`Packs: ${packs}`); } } @@ -68,7 +69,7 @@ function registerMaintenance(admin: Command): void { .option("--doc ", "Only these documents") .option("--since ", "Only documents created on or after (ISO 8601)") .option("--before ", "Only documents created on or before (ISO 8601)") - .option("--batch-size ", "Chunks per embedding call (default 50)", toNumber) + .option("--batch-size ", "Chunks per embedding call (default 50)", Number) .action( async (flags: { rebuild?: boolean; @@ -101,7 +102,7 @@ function registerMaintenance(admin: Command): void { admin .command("dedupe") .description("Find duplicate and near-duplicate documents") - .option("--threshold ", "Similarity threshold 0-1 (default 0.95)", toNumber) + .option("--threshold ", "Similarity threshold 0-1 (default 0.95)", Number) .option("--strategy ", "exact, semantic or both (default)") .action(async (flags: { threshold?: number; strategy?: string }) => { await run( @@ -159,7 +160,7 @@ export function register(program: Command): void { admin .command("stats") .description("Counts, index, most returned and stale documents, and search analytics") - .option("--days ", "Look-back days for stale documents and search analytics", toNumber) + .option("--days ", "Look-back days for stale documents and search analytics", Number) .action(async (flags: { days?: number }) => { const days = defined({ days: flags.days }); const stats: Stats = { diff --git a/src/cli/commands/connectors.ts b/src/cli/commands/connectors.ts index 4d24d4e4..1f796062 100644 --- a/src/cli/commands/connectors.ts +++ b/src/cli/commands/connectors.ts @@ -16,7 +16,7 @@ import { import { authenticateDeviceCode } from "../../connectors/onenote.js"; import { ValidationError } from "../../errors.js"; import { confirmOrCancel } from "../confirm.js"; -import { defined, splitList, toNumber } from "../options.js"; +import { defined, splitList } from "../options.js"; import { plural, printList, run } from "../run.js"; export interface ConnectFlags { @@ -50,6 +50,82 @@ function requireSource(type: string, source: string | undefined, what: string): return source; } +/** Settings given on the command line for each connector type; undefined where not given. */ +const FLAG_SETTINGS: Record< + ConnectorType, + (source: string | undefined, flags: ConnectFlags) => Record +> = { + notion: (_source, flags) => ({ token: flags.token, excludePages: list(flags.exclude) }), + slack: (_source, flags) => ({ + token: flags.token, + channels: list(flags.channels), + excludeChannels: list(flags.exclude), + threadMode: flags.threadMode, + }), + confluence: (source, flags) => ({ + baseUrl: source, + type: flags.server ? "server" : undefined, + email: flags.email, + token: flags.token, + spaces: list(flags.spaces), + excludeSpaces: list(flags.exclude), + }), + obsidian: (source, flags) => ({ + vaultPath: source && resolve(source), + topicMapping: flags.topicMapping, + excludePatterns: list(flags.exclude), + }), + onenote: (_source, flags) => ({ + clientId: flags.clientId, + tenantId: flags.tenantId, + notebooks: flags.notebook === undefined ? undefined : [flags.notebook], + excludeSections: undefined, + // A token given by hand cannot be refreshed. + ...(flags.token === undefined + ? {} + : { accessToken: flags.token, refreshToken: undefined, tokenExpiry: undefined }), + }), + docs: (source, flags) => ({ + url: source, + type: flags.siteType, + library: flags.library, + version: flags.libVersion, + maxPages: flags.maxPages, + maxDepth: flags.maxDepth, + pathPrefix: flags.pathPrefix, + }), +}; + +/** What the source argument is, for connector types that need it for a new connection. */ +const REQUIRED_SOURCE: Partial> = { + confluence: "base URL", + obsidian: "vault path", + docs: "site URL", +}; + +/** Values a new connection gets for the settings that no flag sets (fresh objects each call). */ +const NEW_CONNECTION_DEFAULTS: Partial Record>> = { + slack: () => ({ channels: ["all"], threadMode: "aggregate" }), + confluence: () => ({ type: "cloud", spaces: ["all"] }), + obsidian: () => ({ topicMapping: "folder", excludePatterns: [] }), + onenote: () => ({ clientId: "", tenantId: "common", notebooks: ["all"], excludeSections: [] }), +}; + +/** `settings` of a new connection: checks the source and fills in the defaults. */ +function newConnectionSettings( + type: ConnectorType, + source: string | undefined, + settings: Record, +): Record { + const what = REQUIRED_SOURCE[type]; + if (what) requireSource(type, source, what); + const merged = { ...settings }; + for (const [key, value] of Object.entries(NEW_CONNECTION_DEFAULTS[type]?.() ?? {})) { + merged[key] ??= value; + } + return merged; +} + /** * Settings for connector `type` from the command line. Unset values are undefined, so the * saved settings of the connection (if any) are kept; `defaults` apply to a new connection. @@ -60,56 +136,8 @@ export function connectSettings( flags: ConnectFlags, isNew: boolean, ): Record { - const exclude = list(flags.exclude); - switch (type) { - case "notion": - return { token: flags.token, excludePages: exclude }; - case "slack": - return { - token: flags.token, - channels: list(flags.channels) ?? (isNew ? ["all"] : undefined), - excludeChannels: exclude, - threadMode: flags.threadMode ?? (isNew ? "aggregate" : undefined), - }; - case "confluence": - return { - baseUrl: isNew ? requireSource(type, source, "base URL") : source, - type: flags.server ? "server" : isNew ? "cloud" : undefined, - email: flags.email, - token: flags.token, - spaces: list(flags.spaces) ?? (isNew ? ["all"] : undefined), - excludeSpaces: exclude, - }; - case "obsidian": - return { - vaultPath: isNew - ? resolve(requireSource(type, source, "vault path")) - : source && resolve(source), - topicMapping: flags.topicMapping ?? (isNew ? "folder" : undefined), - excludePatterns: exclude ?? (isNew ? [] : undefined), - }; - case "onenote": - return { - clientId: flags.clientId ?? (isNew ? "" : undefined), - tenantId: flags.tenantId ?? (isNew ? "common" : undefined), - notebooks: flags.notebook === undefined ? (isNew ? ["all"] : undefined) : [flags.notebook], - excludeSections: isNew ? [] : undefined, - // A token given by hand cannot be refreshed. - ...(flags.token === undefined - ? {} - : { accessToken: flags.token, refreshToken: undefined, tokenExpiry: undefined }), - }; - case "docs": - return { - url: isNew ? requireSource(type, source, "site URL") : source, - type: flags.siteType, - library: flags.library, - version: flags.libVersion, - maxPages: flags.maxPages, - maxDepth: flags.maxDepth, - pathPrefix: flags.pathPrefix, - }; - } + const settings = FLAG_SETTINGS[type](source, flags); + return isNew ? newConnectionSettings(type, source, settings) : settings; } /** OneNote without a usable token: sign in with a device code (interactive). */ @@ -183,8 +211,8 @@ function registerConnect(program: Command): void { .option("--site-type ", "docs: auto, sphinx, vitepress, doxygen or generic") .option("--library ", "docs: library name for the pages") .option("--lib-version ", "docs: library version for the pages") - .option("--max-pages ", "docs: page limit (default 500)", toNumber) - .option("--max-depth ", "docs: link depth (default 10)", toNumber) + .option("--max-pages ", "docs: page limit (default 500)", Number) + .option("--max-depth ", "docs: link depth (default 10)", Number) .option("--path-prefix ", "docs: only pages under this path") .action(async (type: string, source: string | undefined, flags: ConnectFlags) => { const connector = getConnector(type); diff --git a/src/cli/commands/docs.ts b/src/cli/commands/docs.ts index 0265941a..1604b089 100644 --- a/src/cli/commands/docs.ts +++ b/src/cli/commands/docs.ts @@ -27,10 +27,9 @@ import { defined, documentInput, splitList, - toNumber, type DocumentFlags, } from "../options.js"; -import { call, plural, printList, run } from "../run.js"; +import { call, libraryLabel, plural, printList, run } from "../run.js"; /** Tags given as separate words and/or comma-separated. */ function tagList(tags: string[]): string[] { @@ -46,7 +45,7 @@ function printDocumentList(result: ListResult): void { console.log(`Documents ${result.offset + 1}-${last} of ${result.total}:\n`); for (const d of result.items) { console.log(`${d.documentId} ${d.title}`); - const lib = d.library ? ` | ${d.library}${d.version ? ` ${d.version}` : ""}` : ""; + const lib = d.library ? ` | ${libraryLabel(d.library, d.version)}` : ""; const url = d.url ? ` | ${d.url}` : ""; console.log(` ${d.sourceType}${lib}${url} | updated ${d.updatedAt}`); } @@ -57,7 +56,7 @@ function printDocument(view: DocumentView): void { console.log(`# ${d.title}\n`); console.log(`ID: ${d.documentId}`); console.log(`Type: ${d.sourceType}`); - if (d.library) console.log(`Library: ${d.library}${d.version ? ` ${d.version}` : ""}`); + if (d.library) console.log(`Library: ${libraryLabel(d.library, d.version)}`); if (d.topicId) console.log(`Topic: ${d.topicId}`); if (d.url) console.log(`URL: ${d.url}`); console.log(`Added by: ${d.submittedBy} (${d.createdAt}, updated ${d.updatedAt})`); @@ -111,7 +110,7 @@ export function updateInput(documentId: string, flags: UpdateFlags): Record", "Documents to skip (paging)", toNumber) + .option("--offset ", "Documents to skip (paging)", Number) .action(async (flags: DocumentFlags & { offset?: number }) => { await run( listDocumentsOperation, @@ -123,8 +122,8 @@ function registerViewCommands(docs: Command): void { docs .command("show ") .description("Show a document with its tags, links and ratings") - .option("--offset ", "Start the content at this character", toNumber) - .option("--max-length ", "Show at most this many characters of content", toNumber) + .option("--offset ", "Start the content at this character", Number) + .option("--max-length ", "Show at most this many characters of content", Number) .action(async (documentId: string, flags: { offset?: number; maxLength?: number }) => { await run( getDocumentOperation, @@ -304,7 +303,7 @@ function registerTagCommands(docs: Command): void { docs .command("suggest-tags ") .description("Suggest tags from a document's content") - .option("-n, --limit ", "Maximum suggestions", toNumber) + .option("-n, --limit ", "Maximum suggestions", Number) .action(async (documentId: string, flags: { limit?: number }) => { await run(suggestTagsOperation, defined({ documentId, limit: flags.limit }), (r) => printList(r.suggestions, "No suggestions.", (t) => console.log(` ${t}`)), diff --git a/src/cli/commands/pack.ts b/src/cli/commands/pack.ts index fda29a8e..63f6a37e 100644 --- a/src/cli/commands/pack.ts +++ b/src/cli/commands/pack.ts @@ -9,7 +9,7 @@ import { import type { InstallResult, InstalledPack } from "../../core/packs.js"; import type { RegistryPack } from "../../registry/types.js"; import { confirmOrCancel } from "../confirm.js"; -import { defined, splitList, toNumber } from "../options.js"; +import { defined, splitList } from "../options.js"; import { call, plural, print, printList, run } from "../run.js"; interface InstallFlags { @@ -92,9 +92,9 @@ export function register(program: Command): void { "Install a pack from the configured registries (name or name@version) or a local .json/.json.gz file", ) .option("--registry ", "Look only in this registry") - .option("--batch-size ", "Documents embedded per batch (default 10)", toNumber) - .option("--resume-from ", "Skip the first N documents (resume a partial install)", toNumber) - .option("--concurrency ", "Batches embedded in parallel (default 4)", toNumber) + .option("--batch-size ", "Documents embedded per batch (default 10)", Number) + .option("--resume-from ", "Skip the first N documents (resume a partial install)", Number) + .option("--concurrency ", "Batches embedded in parallel (default 4)", Number) .action(async (name: string, flags: InstallFlags) => { const input = defined({ pack: name, diff --git a/src/cli/commands/registry.ts b/src/cli/commands/registry.ts index 7f8f68c6..f51f80bb 100644 --- a/src/cli/commands/registry.ts +++ b/src/cli/commands/registry.ts @@ -13,7 +13,7 @@ import { import type { RegistrySearchResult, RegistrySyncStatus } from "../../registry/types.js"; import { confirmOrCancel } from "../confirm.js"; import { defined } from "../options.js"; -import { printList, run } from "../run.js"; +import { plural, printList, run } from "../run.js"; import { printWarnings } from "./pack.js"; /** Truncate a string to a max length, adding "..." if truncated. */ @@ -29,7 +29,7 @@ function padColumns(cols: string[]): string { } function packCount(packs: number | null): string { - return packs === null ? "not synced" : `${packs} pack${packs === 1 ? "" : "s"}`; + return packs === null ? "not synced" : plural(packs, "pack"); } /** One line per sync result; a registry that could not be synced sets exit code 1. */ diff --git a/src/cli/commands/search.ts b/src/cli/commands/search.ts index fcb8a794..ba7759d7 100644 --- a/src/cli/commands/search.ts +++ b/src/cli/commands/search.ts @@ -6,20 +6,14 @@ import type { AnswerResult } from "../../core/rag.js"; import type { SearchResult } from "../../core/search.js"; import { ValidationError } from "../../errors.js"; import { startInteractiveSearch } from "../repl.js"; -import { - addDocumentOptions, - defined, - documentInput, - toNumber, - type DocumentFlags, -} from "../options.js"; -import { isJsonOutput, preview, run } from "../run.js"; +import { addDocumentOptions, defined, documentInput, type DocumentFlags } from "../options.js"; +import { isJsonOutput, libraryLabel, preview, run } from "../run.js"; /** One search hit: title, score, IDs, metadata and a content preview. */ export function printSearchItem(r: SearchResult): void { console.log(`\n── ${r.title} (score ${r.score.toFixed(2)})`); console.log(` document ${r.documentId} chunk ${r.chunkId}`); - const meta = [r.library ? `${r.library}${r.version ? ` ${r.version}` : ""}` : "", r.url ?? ""]; + const meta = [r.library ? libraryLabel(r.library, r.version) : "", r.url ?? ""]; const metaLine = meta.filter(Boolean).join(" | "); if (metaLine) console.log(` ${metaLine}`); for (const c of r.contextBefore ?? []) console.log(` ↑ ${preview(c.content, 120)}`); @@ -98,10 +92,10 @@ export function register(program: Command): void { ) .option("--related ", "Find content similar to this document or chunk ID instead"); addDocumentOptions(search) - .option("--offset ", "Results to skip (paging)", toNumber) - .option("--min-rating ", "Only documents rated at least this (1-5)", toNumber) - .option("--max-per-doc ", "At most this many chunks per document", toNumber) - .option("--context ", "Neighbouring chunks to show around each result (0-2)", toNumber) + .option("--offset ", "Results to skip (paging)", Number) + .option("--min-rating ", "Only documents rated at least this (1-5)", Number) + .option("--max-per-doc ", "At most this many chunks per document", Number) + .option("--context ", "Neighbouring chunks to show around each result (0-2)", Number) .action(async (query: string | undefined, flags: SearchFlags) => { if (query === undefined && flags.related === undefined) { if (!process.stdin.isTTY || isJsonOutput()) { @@ -125,7 +119,7 @@ export function register(program: Command): void { .command("ask ") .description("Answer a question from the knowledge base with the configured LLM"); addDocumentOptions(ask, { limit: "Chunks to use as context (default 5)" }) - .option("--min-rating ", "Only documents rated at least this (1-5)", toNumber) + .option("--min-rating ", "Only documents rated at least this (1-5)", Number) .option("--model ", "LLM model for this question (default: llm.model)"); ask.action(async (question: string, flags: AskFlags) => { await run(askOperation, askInput(question, flags), printAnswer, { diff --git a/src/cli/commands/searches.ts b/src/cli/commands/searches.ts index dd3b4d1b..cde8359d 100644 --- a/src/cli/commands/searches.ts +++ b/src/cli/commands/searches.ts @@ -6,13 +6,7 @@ import { runSavedSearchOperation, saveSearchOperation, } from "../../core/operations/index.js"; -import { - addDocumentOptions, - defined, - documentInput, - toNumber, - type DocumentFlags, -} from "../options.js"; +import { addDocumentOptions, defined, documentInput, type DocumentFlags } from "../options.js"; import { printList, run } from "../run.js"; import { printSearchItem } from "./search.js"; @@ -22,7 +16,7 @@ export function register(program: Command): void { addDocumentOptions( searches.command("save ").description("Save a query and its filters"), ) - .option("--min-rating ", "Only documents rated at least this (1-5)", toNumber) + .option("--min-rating ", "Only documents rated at least this (1-5)", Number) .action(async (name: string, query: string, flags: DocumentFlags & { minRating?: number }) => { const input = { name, @@ -38,8 +32,8 @@ export function register(program: Command): void { searches .command("list") .description("List saved searches") - .option("-n, --limit ", "Maximum results", toNumber) - .option("--offset ", "Results to skip (paging)", toNumber) + .option("-n, --limit ", "Maximum results", Number) + .option("--offset ", "Results to skip (paging)", Number) .action(async (flags: { limit?: number; offset?: number }) => { await run(listSavedSearchesOperation, defined({ ...flags }), (r) => printList(r.items, "No saved searches.", (s) => { diff --git a/src/cli/commands/serve.ts b/src/cli/commands/serve.ts index 20bff5bc..be3a460c 100644 --- a/src/cli/commands/serve.ts +++ b/src/cli/commands/serve.ts @@ -7,18 +7,18 @@ import { leaveInterruptsToCommand, untilInterrupted, } from "../context.js"; -import { toNumber } from "../options.js"; const MODES = ["mcp", "api", "dashboard"] as const; async function serveMcp(): Promise { const { workspace, logLevel, verbose } = getGlobalOptions(); + const level = verbose ? "debug" : logLevel; // stdout carries the MCP protocol: print nothing here. The server handles Ctrl+C itself. leaveInterruptsToCommand(); const mcp = await import("../../mcp/server.js"); await mcp.runStdioServer({ ...(workspace ? { workspace } : {}), - ...(verbose ? { logLevel: "debug" } : logLevel ? { logLevel } : {}), + ...(level ? { logLevel: level } : {}), }); } @@ -28,7 +28,7 @@ export function register(program: Command): void { .description( "Start the MCP server on stdio (mcp, default), the REST API (api, port 3378) or the web dashboard (dashboard, port 3377)", ) - .option("--port ", "Port (api and dashboard)", toNumber) + .option("--port ", "Port (api and dashboard)", Number) .option("--host ", "Host to listen on (default localhost)") .action(async (mode: string | undefined, flags: { port?: number; host?: string }) => { const selected = mode ?? "mcp"; diff --git a/src/cli/index.ts b/src/cli/index.ts index 23031129..41420f4b 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -124,7 +124,7 @@ function isCliEntryPoint(): boolean { if (isCliEntryPoint()) { process.on("SIGINT", onInterrupt); process.on("SIGTERM", onInterrupt); - void main(); + await main(); } export { program }; diff --git a/src/cli/options.ts b/src/cli/options.ts index 3780e95c..663c6dce 100644 --- a/src/cli/options.ts +++ b/src/cli/options.ts @@ -2,11 +2,6 @@ import type { Command } from "commander"; import { SOURCE_TYPES } from "../core/indexing.js"; -/** Commander parser for numeric options; the operation schema validates the value. */ -export function toNumber(value: string): number { - return Number(value); -} - /** "a, b,c" -> ["a", "b", "c"] (empty entries dropped). */ export function splitList(value: string): string[] { return value @@ -58,7 +53,7 @@ export function addDocumentOptions(cmd: Command, config: DocumentOptionsConfig = : "Only documents with all these tags (comma-separated)", ); if (config.limit !== false) { - cmd.option("-n, --limit ", config.limit ?? "Maximum results", toNumber); + cmd.option("-n, --limit ", config.limit ?? "Maximum results", Number); } return cmd; } diff --git a/src/cli/repl.ts b/src/cli/repl.ts index 8e027717..03cc56cb 100644 --- a/src/cli/repl.ts +++ b/src/cli/repl.ts @@ -17,27 +17,35 @@ export async function startInteractiveSearch(options: InteractiveOptions): Promi console.log("LibScope interactive search (type 'quit' or 'exit', or press Ctrl+D, to leave)\n"); try { - for (;;) { - let query: string; - try { - query = await rl.question("search> "); - } catch { - // Ctrl+C / Ctrl+D or closed stream - break; - } + await promptLoop(rl, options.onQuery); + } finally { + rl.close(); + } +} - const trimmed = query.trim(); - if (!trimmed) continue; - if (trimmed === "quit" || trimmed === "exit") break; +/** Ask for one query, run it, then ask again until the user leaves. */ +async function promptLoop( + rl: readline.Interface, + onQuery: InteractiveOptions["onQuery"], +): Promise { + let query: string; + try { + query = await rl.question("search> "); + } catch { + // Ctrl+C / Ctrl+D or closed stream + return; + } + + const trimmed = query.trim(); + if (trimmed === "quit" || trimmed === "exit") return; - try { - await options.onQuery(trimmed); - } catch (err) { - console.error(`✗ ${err instanceof Error ? err.message : String(err)}`); - } - console.log(); + if (trimmed) { + try { + await onQuery(trimmed); + } catch (err) { + console.error(`✗ ${err instanceof Error ? err.message : String(err)}`); } - } finally { - rl.close(); + console.log(); } + return promptLoop(rl, onQuery); } diff --git a/src/cli/run.ts b/src/cli/run.ts index 68e8c6ae..8716f7d8 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -53,6 +53,11 @@ export function preview(text: string, max = 200): string { return flat.length > max ? `${flat.slice(0, max)}...` : flat; } +/** "library 1.2" / "library". */ +export function libraryLabel(library: string, version: string | null | undefined): string { + return version ? `${library} ${version}` : library; +} + /** "1 document" / "3 documents". */ export function plural(count: number, noun: string): string { return `${count} ${noun}${count === 1 ? "" : "s"}`; diff --git a/src/connectors/confluence.ts b/src/connectors/confluence.ts index 8ecca84b..800a249e 100644 --- a/src/connectors/confluence.ts +++ b/src/connectors/confluence.ts @@ -7,6 +7,8 @@ import { addTagsToDocument } from "../core/tags.js"; import { deleteDocument } from "../core/documents.js"; import { getLogger } from "../logger.js"; import { FetchError, ValidationError } from "../errors.js"; +import { forEachSequential } from "../utils/async.js"; +import { trimTrailingSlashes } from "../utils/strings.js"; import { fetchWithRetry } from "./http-utils.js"; import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; @@ -119,16 +121,18 @@ async function fetchAllPages( signal?: AbortSignal, ): Promise { const all: T[] = []; - let url: string | undefined = initialUrl; - while (url) { + // One page per call; follows the "next" link until there is none. + const fetchFrom = async (url: string | undefined): Promise => { + if (!url) return; signal?.throwIfAborted(); - const resp: PaginatedResponse = await confluenceFetch>(url, auth); + const resp = await confluenceFetch>(url, auth); all.push(...resp.results); const next: string | undefined = resp._links?.next; - url = next ? `${baseUrl}${next}` : undefined; - } + return fetchFrom(next ? `${baseUrl}${next}` : undefined); + }; + await fetchFrom(initialUrl); return all; } @@ -476,7 +480,7 @@ async function syncConfluenceSpace(options: SyncConfluenceSpaceOptions): Promise return; } - for (const page of pages) { + await forEachSequential(pages, async (page) => { signal?.throwIfAborted(); try { const outcome = await indexConfluencePage({ @@ -489,16 +493,24 @@ async function syncConfluenceSpace(options: SyncConfluenceSpaceOptions): Promise urls, auth, }); - if (outcome === "indexed") result.pagesIndexed++; - if (outcome === "updated") { - result.pagesUpdated++; - result.pagesIndexed++; - } + countPageOutcome(outcome, result); } catch (err) { const msg = err instanceof Error ? err.message : String(err); log.error({ pageId: page.id, title: page.title, err }, "Failed to index page"); result.errors.push({ page: page.title, error: msg }); } + }); +} + +/** Add one page's outcome to the sync counts (an updated page also counts as indexed). */ +function countPageOutcome( + outcome: Awaited>, + result: ConfluenceSyncResult, +): void { + if (outcome === "indexed") result.pagesIndexed++; + if (outcome === "updated") { + result.pagesUpdated++; + result.pagesIndexed++; } } @@ -518,13 +530,6 @@ function validateConfluenceConfig(config: ConfluenceConfig): void { } } -/** Strip trailing slashes from a Confluence base URL (loop instead of a regex). */ -function trimBaseUrl(baseUrl: string): string { - let base = baseUrl; - while (base.endsWith("/")) base = base.slice(0, -1); - return base; -} - export async function syncConfluence( db: Database.Database, provider: EmbeddingProvider, @@ -534,7 +539,7 @@ export async function syncConfluence( return trackSync( db, "confluence", - options.syncName ?? trimBaseUrl(config.baseUrl), + options.syncName ?? trimTrailingSlashes(config.baseUrl), () => runConfluenceSync(db, provider, config, options.signal), (result) => ({ added: result.pagesIndexed, @@ -557,7 +562,7 @@ async function runConfluenceSync( const confluenceType = config.type ?? "cloud"; const auth = buildAuthHeader(confluenceType, config.email, config.token); - const base = trimBaseUrl(config.baseUrl); + const base = trimTrailingSlashes(config.baseUrl); const urls = getApiUrls(base, confluenceType); const result: ConfluenceSyncResult = { @@ -580,8 +585,8 @@ async function runConfluenceSync( result.spaces = spacesToSync.length; log.info({ spaceCount: spacesToSync.length }, "Spaces to sync"); - for (const space of spacesToSync) { - await syncConfluenceSpace({ + await forEachSequential(spacesToSync, (space) => + syncConfluenceSpace({ db, provider, space, @@ -591,8 +596,8 @@ async function runConfluenceSync( auth, result, signal, - }); - } + }), + ); log.info( { diff --git a/src/connectors/docs.ts b/src/connectors/docs.ts index 4483d841..d56e7294 100644 --- a/src/connectors/docs.ts +++ b/src/connectors/docs.ts @@ -629,6 +629,70 @@ async function discoverUrls( } } +interface CrawlEntry { + url: string; + depth: number; +} + +/** State shared by the pages of one documentation crawl. */ +interface CrawlState { + page: PageContext; + fetchOptions: FetchOptions; + visited: Set; + queue: CrawlEntry[]; + maxPages: number; + maxDepth: number; + pathPrefix: string; +} + +/** Fetch and index one queued page, then queue its unvisited links within the depth budget. */ +async function crawlPage({ url, depth }: CrawlEntry, crawl: CrawlState): Promise { + const { visited, queue, maxPages, maxDepth, pathPrefix } = crawl; + if (visited.size > maxPages) return; + + let html: string; + let contentType: string; + try { + const raw = await fetchRaw(url, crawl.fetchOptions); + html = raw.body; + contentType = raw.contentType; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + getLogger().warn({ url, error: msg }, "Failed to fetch documentation page"); + crawl.page.result.errors.push({ url, error: msg }); + return; + } + + // Only process HTML pages (skip binary/asset responses that slipped through) + if (!contentType.includes("text/html") && !contentType.includes("text/plain")) { + return; + } + + await processPage(url, html, crawl.page); + + // Continue link discovery if within depth budget + if (depth < maxDepth) { + for (const link of extractDocLinks(html, url, pathPrefix)) { + if (!visited.has(link)) { + visited.add(link); + queue.push({ url: link, depth: depth + 1 }); + } + } + } +} + +/** Crawl the queue in batches of `concurrency` pages until it is empty or maxPages is reached. */ +async function crawlQueue( + crawl: CrawlState, + options: { concurrency: number; signal: AbortSignal | undefined }, +): Promise { + if (crawl.queue.length === 0 || crawl.visited.size > crawl.maxPages) return; + options.signal?.throwIfAborted(); + const batch = crawl.queue.splice(0, options.concurrency); + await Promise.allSettled(batch.map((entry) => crawlPage(entry, crawl))); + return crawlQueue(crawl, options); +} + export async function syncDocSite( db: Database.Database, provider: EmbeddingProvider, @@ -724,46 +788,13 @@ async function runDocSiteSync( await processPage(rootNormalised, rootHtml, ctx); // --- BFS crawl --- - while (queue.length > 0 && visited.size <= maxPages) { - signal?.throwIfAborted(); - const batch = queue.splice(0, concurrency); - - await Promise.allSettled( - batch.map(async ({ url, depth }) => { - if (visited.size > maxPages) return; - - let html: string; - let contentType: string; - try { - const raw = await fetchRaw(url, fetchOptions); - html = raw.body; - contentType = raw.contentType; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - log.warn({ url, error: msg }, "Failed to fetch documentation page"); - result.errors.push({ url, error: msg }); - return; - } - - // Only process HTML pages (skip binary/asset responses that slipped through) - if (!contentType.includes("text/html") && !contentType.includes("text/plain")) { - return; - } - - await processPage(url, html, ctx); - - // Continue link discovery if within depth budget - if (depth < maxDepth) { - for (const link of extractDocLinks(html, url, pathPrefix)) { - if (!visited.has(link)) { - visited.add(link); - queue.push({ url: link, depth: depth + 1 }); - } - } - } - }), - ); - } + await crawlQueue( + { page: ctx, fetchOptions, visited, queue, maxPages, maxDepth, pathPrefix }, + { + concurrency, + signal, + }, + ); log.info( { diff --git a/src/connectors/http-utils.ts b/src/connectors/http-utils.ts index b2cad595..5002dbaf 100644 --- a/src/connectors/http-utils.ts +++ b/src/connectors/http-utils.ts @@ -54,7 +54,11 @@ export async function fetchWithRetry( const dispatcher = config.indexing.allowSelfSignedCerts ? getInsecureAgent() : undefined; const fetchOptions = buildFetchOptions(options, dispatcher); - for (let attempt = 0; attempt <= maxRetries; attempt++) { + const attemptFetch = async (attempt: number): Promise => { + // False only for a negative (or NaN) maxRetries: no attempt is allowed at all. + const allowed = attempt <= maxRetries; + if (!allowed) throw new FetchError("fetchWithRetry: unexpected code path"); + const response = await fetch(url, fetchOptions); if (!isRetryableStatus(response.status)) return response; @@ -70,8 +74,8 @@ export async function fetchWithRetry( "Retrying after transient error", ); await new Promise((resolve) => setTimeout(resolve, delayMs)); - } + return attemptFetch(attempt + 1); + }; - // Unreachable, but satisfies TypeScript - throw new FetchError("fetchWithRetry: unexpected code path"); + return attemptFetch(0); } diff --git a/src/connectors/notion.ts b/src/connectors/notion.ts index 5bbc5096..de8e72d0 100644 --- a/src/connectors/notion.ts +++ b/src/connectors/notion.ts @@ -2,6 +2,7 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { getLogger } from "../logger.js"; import { ValidationError, FetchError } from "../errors.js"; +import { forEachSequential } from "../utils/async.js"; import { fetchWithRetry } from "./http-utils.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; @@ -186,12 +187,13 @@ async function fetchBlockChildren( } // Recursively fetch children - for (const block of allBlocks) { - if (block.has_children) { + await forEachSequential( + allBlocks.filter((block) => block.has_children), + async (block) => { const children = await fetchBlockChildren(token, block.id, depth + 1, maxDepth); (block as Record)["children"] = children; - } - } + }, + ); return allBlocks; } @@ -471,9 +473,9 @@ async function syncNotionDatabase( const dbTitle = extractTitle(item); const rows = await queryDatabase(token, item.id); - for (const row of rows) { + await forEachSequential(rows, async (row) => { signal?.throwIfAborted(); - if (excludeSet.has(row.id)) continue; + if (excludeSet.has(row.id)) return; const rowTitle = extractTitle(row); const tags = row.properties ? extractPropertyMetadata(row.properties) : []; @@ -484,7 +486,7 @@ async function syncNotionDatabase( const fullContent = `# ${rowTitle}${metadataSection}\n\n${content}`; await upsertNotionDocument(db, provider, row.id, `${dbTitle} — ${rowTitle}`, fullContent); - } + }); log.debug({ id: item.id, title: dbTitle, rows: rows.length }, "Indexed Notion database"); } @@ -553,11 +555,11 @@ async function runNotionSync( const searchResults = await searchNotion(config.token, config.lastSync, signal); log.info({ count: searchResults.length }, "Found Notion objects"); - for (const item of searchResults) { + await forEachSequential(searchResults, async (item) => { signal?.throwIfAborted(); if (excludeSet.has(item.id)) { log.debug({ id: item.id }, "Skipping excluded page"); - continue; + return; } try { @@ -569,7 +571,7 @@ async function runNotionSync( result.errors.push({ page: title, error: message }); log.warn({ id: item.id, err }, "Failed to index Notion item"); } - } + }); log.info( { diff --git a/src/connectors/obsidian-wikilinks.ts b/src/connectors/obsidian-wikilinks.ts new file mode 100644 index 00000000..6ed076c3 --- /dev/null +++ b/src/connectors/obsidian-wikilinks.ts @@ -0,0 +1,107 @@ +/** + * Obsidian `[[wikilink]]` scanner. + * + * Finds the same links as /(? number { + let searchedFrom = Number.POSITIVE_INFINITY; + let found = -1; + return (from) => { + const reusable = from >= searchedFrom && (found === -1 || found >= from); + if (!reusable) { + found = text.indexOf(char, from); + searchedFrom = from; + } + return found; + }; +} + +interface Scanner { + text: string; + nextClose: (from: number) => number; + nextPipe: (from: number) => number; +} + +/** `index`, or the text length when `index` is -1 (not found). */ +function orEnd(index: number, text: string): number { + return index === -1 ? text.length : index; +} + +/** + * Read the wikilink whose "[[" is at `start`. Each "[[" between `start` and the index where a + * failed scan stopped would fail the same way, so the search resumes there. + */ +function scanAt(scanner: Scanner, start: number): ScanResult { + const { text } = scanner; + if (start > 0 && text.charAt(start - 1) === "!") return { resumeAt: start + 1 }; + const linkStart = start + 2; + const pipe = scanner.nextPipe(linkStart); + const linkEnd = Math.min(orEnd(scanner.nextClose(linkStart), text), orEnd(pipe, text)); + if (linkEnd === linkStart) return { resumeAt: start + 1 }; + const link = text.slice(linkStart, linkEnd); + if (linkEnd !== pipe) { + return text.startsWith("]]", linkEnd) + ? { link: { start, end: linkEnd + 2, link, display: undefined } } + : { resumeAt: linkEnd }; + } + const displayEnd = orEnd(scanner.nextClose(pipe + 1), text); + if (!text.startsWith("]]", displayEnd)) return { resumeAt: displayEnd }; + const display = text.slice(pipe + 1, displayEnd); + return { link: { start, end: displayEnd + 2, link, display } }; +} + +/** Find every wikilink in `text`, left to right, without overlaps. */ +export function findWikilinks(text: string): Wikilink[] { + const links: Wikilink[] = []; + const scanner: Scanner = { + text, + nextClose: indexFinder(text, "]"), + nextPipe: indexFinder(text, "|"), + }; + + let start = text.indexOf("[["); + while (start !== -1) { + const scanned = scanAt(scanner, start); + let resumeAt: number; + if ("link" in scanned) { + links.push(scanned.link); + resumeAt = scanned.link.end; + } else { + resumeAt = scanned.resumeAt; + } + start = text.indexOf("[[", resumeAt); + } + return links; +} + +/** Replace each wikilink in `text` with `replace(link)`. */ +export function replaceWikilinks(text: string, replace: (link: Wikilink) => string): string { + let out = ""; + let last = 0; + for (const link of findWikilinks(text)) { + out += text.slice(last, link.start) + replace(link); + last = link.end; + } + return out + text.slice(last); +} diff --git a/src/connectors/obsidian.ts b/src/connectors/obsidian.ts index fca324bf..7a06d118 100644 --- a/src/connectors/obsidian.ts +++ b/src/connectors/obsidian.ts @@ -10,7 +10,9 @@ import { addTagsToDocument, createTag } from "../core/tags.js"; import { createLink, resolveDocumentByTitle } from "../core/links.js"; import { getLogger } from "../logger.js"; import { ValidationError } from "../errors.js"; +import { forEachSequential } from "../utils/async.js"; import { loadConnectorConfig, saveConnectorConfig } from "./index.js"; +import { findWikilinks, replaceWikilinks } from "./obsidian-wikilinks.js"; import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; export interface ObsidianConfig { @@ -131,14 +133,7 @@ function buildVaultFileMap(vaultFiles: string[]): Map { /** Collect all [[wikilinks]] from the body text. */ function collectWikilinks(body: string): string[] { - const wikilinks: string[] = []; - const wikilinkRegex = /(? w.link); } /** Apply Obsidian-specific markdown transformations: embeds, wikilinks, comments, callouts. */ @@ -152,14 +147,11 @@ function transformObsidianBody(body: string, fileMap: Map): stri }); // Resolve [[wikilinks]] - result = result.replaceAll( - /(? { - const displayText = display ?? link; - const slug = link.toLowerCase().replaceAll(/\s+/g, "-"); - return `[${displayText}](${slug})`; - }, - ); + result = replaceWikilinks(result, ({ link, display }) => { + const displayText = display ?? link; + const slug = link.toLowerCase().replaceAll(/\s+/g, "-"); + return `[${displayText}](${slug})`; + }); // Strip %%comments%% result = result.replaceAll(/%%[\s\S]*?%%/g, ""); @@ -432,7 +424,7 @@ async function syncVaultFiles(ctx: SyncVaultFilesContext): Promise { const { db, provider, config, vaultFiles, trackedFiles, newTrackedFiles, result, signal } = ctx; const log = getLogger(); const fileMap = buildVaultFileMap(vaultFiles); - for (const relPath of vaultFiles) { + await forEachSequential(vaultFiles, async (relPath) => { signal?.throwIfAborted(); try { const outcome = await processVaultFile( @@ -454,7 +446,7 @@ async function syncVaultFiles(ctx: SyncVaultFilesContext): Promise { newTrackedFiles[relPath] = tracked; } } - } + }); } export async function syncObsidianVault( diff --git a/src/connectors/onenote.ts b/src/connectors/onenote.ts index 98808539..5b756f83 100644 --- a/src/connectors/onenote.ts +++ b/src/connectors/onenote.ts @@ -4,6 +4,7 @@ import { NodeHtmlMarkdown } from "node-html-markdown"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { getLogger } from "../logger.js"; import { LibScopeError } from "../errors.js"; +import { forEachSequential } from "../utils/async.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; import { createTopic } from "../core/topics.js"; @@ -289,46 +290,60 @@ export async function authenticateDeviceCode( const interval = (dcData.interval ?? 5) * 1000; const deadline = Date.now() + dcData.expires_in * 1000; - while (Date.now() < deadline) { - await sleep(interval); - - const tokenRes = await fetch(tokenUrl, { - method: "POST", - headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: new URLSearchParams({ - grant_type: "urn:ietf:params:oauth:grant-type:device_code", - client_id: clientId, - device_code: dcData.device_code, - }), - signal: AbortSignal.timeout(30_000), - }); + return pollDeviceToken({ + tokenUrl, + clientId, + deviceCode: dcData.device_code, + interval, + deadline, + }); +} - if (tokenRes.ok) { - const tokenData = (await tokenRes.json()) as { - access_token: string; - refresh_token: string; - expires_in: number; - }; - const expiresAt = new Date(Date.now() + tokenData.expires_in * 1000).toISOString(); - return { - accessToken: tokenData.access_token, - refreshToken: tokenData.refresh_token, - expiresAt, - }; - } +/** Ask for the token every `interval` ms until the user signs in, an error, or `deadline`. */ +async function pollDeviceToken(poll: { + tokenUrl: string; + clientId: string; + deviceCode: string; + interval: number; + deadline: number; +}): Promise<{ accessToken: string; refreshToken: string; expiresAt: string }> { + if (Date.now() >= poll.deadline) { + throw new LibScopeError("Device code authentication timed out", "ONENOTE_AUTH_ERROR"); + } + await sleep(poll.interval); - const errData = (await tokenRes.json()) as { error: string }; - if (errData.error === "authorization_pending") { - continue; - } - if (errData.error === "slow_down") { - await sleep(5000); - continue; - } - throw new LibScopeError(`Authentication failed: ${errData.error}`, "ONENOTE_AUTH_ERROR"); + const tokenRes = await fetch(poll.tokenUrl, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "urn:ietf:params:oauth:grant-type:device_code", + client_id: poll.clientId, + device_code: poll.deviceCode, + }), + signal: AbortSignal.timeout(30_000), + }); + + if (tokenRes.ok) { + const tokenData = (await tokenRes.json()) as { + access_token: string; + refresh_token: string; + expires_in: number; + }; + const expiresAt = new Date(Date.now() + tokenData.expires_in * 1000).toISOString(); + return { + accessToken: tokenData.access_token, + refreshToken: tokenData.refresh_token, + expiresAt, + }; } - throw new LibScopeError("Device code authentication timed out", "ONENOTE_AUTH_ERROR"); + const errData = (await tokenRes.json()) as { error: string }; + if (errData.error === "slow_down") { + await sleep(5000); + } else if (errData.error !== "authorization_pending") { + throw new LibScopeError(`Authentication failed: ${errData.error}`, "ONENOTE_AUTH_ERROR"); + } + return pollDeviceToken(poll); } // --------------------------------------------------------------------------- @@ -473,7 +488,7 @@ async function syncOneNoteSection(options: SyncOneNoteSectionOptions): Promise { signal?.throwIfAborted(); const sourceUrl = buildSourceUrl(notebookName, section.displayName, page.title); seenSourceUrls.add(sourceUrl); @@ -495,7 +510,7 @@ async function syncOneNoteSection(options: SyncOneNoteSectionOptions): Promise !config.excludeSections.includes(s.displayName)); result.sections += filteredSections.length; - for (const section of filteredSections) { + await forEachSequential(filteredSections, async (section) => { signal?.throwIfAborted(); await syncOneNoteSection({ db, @@ -533,7 +548,7 @@ async function syncOneNoteNotebook( result, signal, }); - } + }); } /** Delete OneNote documents whose source URLs were not seen during sync. */ @@ -636,7 +651,7 @@ async function runOneNoteSync( result.notebooks = targetNotebooks.length; const seenSourceUrls = new Set(); - for (const notebook of targetNotebooks) { + await forEachSequential(targetNotebooks, async (notebook) => { signal?.throwIfAborted(); await syncOneNoteNotebook({ db, @@ -648,7 +663,7 @@ async function runOneNoteSync( result, signal, }); - } + }); result.pagesDeleted = deleteStaleOneNoteDocs(db, seenSourceUrls); diff --git a/src/connectors/slack.ts b/src/connectors/slack.ts index d0cbef36..73648b71 100644 --- a/src/connectors/slack.ts +++ b/src/connectors/slack.ts @@ -4,6 +4,7 @@ import type { EmbeddingProvider } from "../providers/embedding.js"; import { indexDocument } from "../core/indexing.js"; import { getLogger } from "../logger.js"; import { LibScopeError, ValidationError } from "../errors.js"; +import { forEachSequential, mapSequential } from "../utils/async.js"; import { fetchWithRetry } from "./http-utils.js"; import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; import { deleteDocumentRows } from "./index.js"; @@ -174,39 +175,55 @@ export async function resolveUserMentions(text: string, token: string): Promise< const mentions = [...text.matchAll(mentionRegex)]; let result = text; - for (const match of mentions) { + await forEachSequential(mentions, async (match) => { const userId = match[1]; if (userId) { const displayName = await resolveUser(token, userId); result = result.replace(match[0], `@${displayName}`); } - } + }); return result; } -async function listChannels(token: string, signal?: AbortSignal): Promise { - const channels: SlackChannel[] = []; - let cursor: string | undefined; +/** + * Call a paginated Slack method and collect `pick(response)` from every page, one page at a + * time with the rate-limit delay after each call. + */ +async function slackApiAllPages( + method: string, + token: string, + params: Record, + pick: (data: SlackApiResponse) => T[] | undefined, + signal: AbortSignal | undefined, +): Promise { + const items: T[] = []; - do { + const fetchPage = async (cursor: string | undefined): Promise => { signal?.throwIfAborted(); - const params: Record = { - types: "public_channel,private_channel", - limit: "200", - }; - if (cursor) params["cursor"] = cursor; + const pageParams = cursor ? { ...params, cursor } : params; - const data = await slackApi("conversations.list", token, params); + const data = await slackApi(method, token, pageParams); await delay(rateLimitDelayMs); - if (data.channels) { - channels.push(...data.channels); - } - cursor = data.response_metadata?.next_cursor ?? undefined; - } while (cursor); + const page = pick(data); + if (page) items.push(...page); + const next = data.response_metadata?.next_cursor ?? undefined; + if (next) await fetchPage(next); + }; + + await fetchPage(undefined); + return items; +} - return channels; +function listChannels(token: string, signal?: AbortSignal): Promise { + return slackApiAllPages( + "conversations.list", + token, + { types: "public_channel,private_channel", limit: "200" }, + (data) => data.channels, + signal, + ); } function filterChannels( @@ -229,34 +246,19 @@ function filterChannels( return filtered.filter((ch) => !excludeSet.has(ch.name.toLowerCase()) && !excludeSet.has(ch.id)); } -async function fetchMessages( +function fetchMessages( token: string, channelId: string, oldest?: string, signal?: AbortSignal, ): Promise { - const messages: SlackMessage[] = []; - let cursor: string | undefined; - - do { - signal?.throwIfAborted(); - const params: Record = { - channel: channelId, - limit: "200", - }; - if (oldest) params["oldest"] = oldest; - if (cursor) params["cursor"] = cursor; - - const data = await slackApi("conversations.history", token, params); - await delay(rateLimitDelayMs); - - if (data.messages) { - messages.push(...data.messages); - } - cursor = data.response_metadata?.next_cursor ?? undefined; - } while (cursor); + const params: Record = { + channel: channelId, + limit: "200", + }; + if (oldest) params["oldest"] = oldest; - return messages; + return slackApiAllPages("conversations.history", token, params, (data) => data.messages, signal); } async function fetchThreadReplies( @@ -300,13 +302,12 @@ async function buildThreadDocument( channelName: string, replies: SlackMessage[], ): Promise { - const lines: string[] = []; - for (const reply of replies) { + const lines = await mapSequential(replies, async (reply) => { const username = reply.user ? await resolveUser(token, reply.user) : "unknown"; const timestamp = formatTimestamp(reply.ts); const text = reply.text ? convertSlackMrkdwn(await resolveUserMentions(reply.text, token)) : ""; - lines.push(`**${username}** (${timestamp}):\n${text}`); - } + return `**${username}** (${timestamp}):\n${text}`; + }); return `# Thread in #${channelName}\n\n${lines.join("\n\n---\n\n")}`; } @@ -329,44 +330,64 @@ async function fetchAllThreadReplies( userIds: Set, ): Promise> { const threadRepliesMap = new Map(); - for (const threadParent of threadParents) { + await forEachSequential(threadParents, async (threadParent) => { const replies = await fetchThreadReplies(token, channelId, threadParent.ts); threadRepliesMap.set(threadParent.ts, replies); collectUserIds(replies, userIds); - } + }); return threadRepliesMap; } /** Batch-resolve all user IDs that are not already cached. */ async function batchResolveUsers(token: string, userIds: Set): Promise { - for (const userId of userIds) { - if (!userCache.has(userId)) { - await resolveUser(token, userId); - } - } + await forEachSequential( + [...userIds].filter((userId) => !userCache.has(userId)), + (userId) => resolveUser(token, userId), + ); } -/** Index a single standalone message. Returns true if indexed. */ -async function indexStandaloneMessage( +/** + * Index one message with text as its own document, titled + * "